diff --git a/home-mixer/README.md b/home-mixer/README.md index a1be737be..20861d7a0 100644 --- a/home-mixer/README.md +++ b/home-mixer/README.md @@ -74,7 +74,7 @@ Timeline tabs powered by Home Mixer. - ScoredTweetsRecommendationPipelineConfig (main Tweet recommendation layer) - Fetch Tweet Candidates - ScoredTweetsInNetworkCandidatePipelineConfig - - ScoredTweetsCrMixerCandidatePipelineConfig + - ScoredTweetsTweetMixerCandidatePipelineConfig - ScoredTweetsUtegCandidatePipelineConfig - ScoredTweetsFrsCandidatePipelineConfig - Feature Hydration and Scoring @@ -99,4 +99,3 @@ Timeline tabs powered by Home Mixer. - ListTweetsTimelineServiceCandidatePipelineConfig (fetch tweets from timeline service) - ConversationServiceCandidatePipelineConfig (fetch ancestors for conversation modules) - ListTweetsAdsCandidatePipelineConfig (fetch ads) - diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/BUILD.bazel index a9474ec09..103e079da 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/BUILD.bazel @@ -21,6 +21,7 @@ scala_library( "finatra/inject/inject-utils/src/main/scala", "home-mixer/server/src/main/resources", "home-mixer/server/src/main/scala/com/twitter/home_mixer/controller", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/federated", "home-mixer/server/src/main/scala/com/twitter/home_mixer/module", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product", @@ -31,6 +32,10 @@ scala_library( "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter", "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", "src/thrift/com/twitter/timelines/render:thrift-scala", + "strato/config/columns/auth-context:auth-context-strato-client", + "strato/config/columns/gizmoduck:gizmoduck-strato-client", + "strato/src/main/scala/com/twitter/strato/fed", + "strato/src/main/scala/com/twitter/strato/fed/server", "stringcenter/client", "stringcenter/client/src/main/java", "stringcenter/client/src/main/scala/com/twitter/stringcenter/client", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerHttpServerWarmupHandler.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerHttpServerWarmupHandler.scala index 16a9d9c58..e27133b23 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerHttpServerWarmupHandler.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerHttpServerWarmupHandler.scala @@ -2,7 +2,7 @@ package com.twitter.home_mixer import com.twitter.finatra.http.routing.HttpWarmup import com.twitter.finatra.httpclient.RequestBuilder._ -import com.twitter.inject.Logging +import com.twitter.util.logging.Logging import com.twitter.inject.utils.Handler import com.twitter.util.Try import javax.inject.Inject diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerServer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerServer.scala index ff7c75727..e635c7a68 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerServer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerServer.scala @@ -12,57 +12,63 @@ import com.twitter.finatra.thrift.ThriftServer import com.twitter.finatra.thrift.filters._ import com.twitter.finatra.thrift.routing.ThriftRouter import com.twitter.home_mixer.controller.HomeThriftController +import com.twitter.home_mixer.federated.HomeMixerColumn import com.twitter.home_mixer.module._ import com.twitter.home_mixer.param.GlobalParamConfigModule import com.twitter.home_mixer.product.HomeMixerProductModule import com.twitter.home_mixer.{thriftscala => st} import com.twitter.product_mixer.component_library.module.AccountRecommendationsMixerModule -import com.twitter.product_mixer.component_library.module.CrMixerClientModule import com.twitter.product_mixer.component_library.module.DarkTrafficFilterModule import com.twitter.product_mixer.component_library.module.EarlybirdModule import com.twitter.product_mixer.component_library.module.ExploreRankerClientModule import com.twitter.product_mixer.component_library.module.GizmoduckClientModule import com.twitter.product_mixer.component_library.module.OnboardingTaskServiceModule import com.twitter.product_mixer.component_library.module.SocialGraphServiceModule -import com.twitter.product_mixer.component_library.module.TimelineMixerClientModule import com.twitter.product_mixer.component_library.module.TimelineRankerClientModule import com.twitter.product_mixer.component_library.module.TimelineScorerClientModule import com.twitter.product_mixer.component_library.module.TimelineServiceClientModule import com.twitter.product_mixer.component_library.module.TweetImpressionStoreModule +import com.twitter.product_mixer.component_library.module.TweetMixerClientModule import com.twitter.product_mixer.component_library.module.UserSessionStoreModule import com.twitter.product_mixer.core.controllers.ProductMixerController import com.twitter.product_mixer.core.module.LoggingThrowableExceptionMapper import com.twitter.product_mixer.core.module.ProductMixerModule -import com.twitter.product_mixer.core.module.StratoClientModule import com.twitter.product_mixer.core.module.stringcenter.ProductScopeStringCenterModule +import com.twitter.strato.fed.StratoFed +import com.twitter.strato.fed.server.StratoFedServer object HomeMixerServerMain extends HomeMixerServer -class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMtls { +class HomeMixerServer + extends StratoFedServer + with ThriftServer + with Mtls + with HttpServer + with HttpMtls { override val name = "home-mixer-server" override val modules: Seq[Module] = Seq( AccountRecommendationsMixerModule, AdvertiserBrandSafetySettingsStoreModule, + BlenderClientModule, ClientSentImpressionsPublisherModule, ConversationServiceModule, - CrMixerClientModule, EarlybirdModule, ExploreRankerClientModule, + FeedbackHistoryClientModule, GizmoduckClientModule, GlobalParamConfigModule, HomeAdsCandidateSourceModule, HomeMixerFlagsModule, HomeMixerProductModule, HomeMixerResourcesModule, - HomeNaviModelClientModule, ImpressionBloomFilterModule, InjectionHistoryClientModule, - FeedbackHistoryClientModule, ManhattanClientsModule, ManhattanFeatureRepositoryModule, ManhattanTweetImpressionStoreModule, MemcachedFeatureRepositoryModule, + NaviModelClientModule, OnboardingTaskServiceModule, OptimizedStratoClientModule, PeopleDiscoveryServiceModule, @@ -74,24 +80,23 @@ class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMt SimClustersRecentEngagementsClientModule, SocialGraphServiceModule, StaleTweetsCacheModule, - StratoClientModule, ThriftFeatureRepositoryModule, - TimelineMixerClientModule, TimelineRankerClientModule, TimelineScorerClientModule, TimelineServiceClientModule, TimelinesPersistenceStoreClientModule, + TopicSocialProofClientModule, TweetImpressionStoreModule, - TweetyPieClientModule, + TweetMixerClientModule, + TweetypieClientModule, TweetypieStaticEntitiesCacheClientModule, - UserMetadataStoreModule, UserSessionStoreModule, new DarkTrafficFilterModule[st.HomeMixer.ReqRepServicePerEndpoint](), new MtlsThriftWebFormsModule[st.HomeMixer.MethodPerEndpoint](this), new ProductScopeStringCenterModule() ) - def configureThrift(router: ThriftRouter): Unit = { + override def configureThrift(router: ThriftRouter): Unit = { router .filter[LoggingMDCFilter] .filter[TraceIdMDCFilter] @@ -111,6 +116,11 @@ class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMt this.injector, st.HomeMixer.ExecutePipeline)) + override val dest: String = "/s/home-mixer/home-mixer:strato" + + override val columns: Seq[Class[_ <: StratoFed.Column]] = + Seq(classOf[HomeMixerColumn]) + override protected def warmup(): Unit = { handle[HomeMixerThriftServerWarmupHandler]() handle[HomeMixerHttpServerWarmupHandler]() diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerThriftServerWarmupHandler.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerThriftServerWarmupHandler.scala index df7001072..982b77487 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerThriftServerWarmupHandler.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerThriftServerWarmupHandler.scala @@ -3,7 +3,7 @@ package com.twitter.home_mixer import com.twitter.finagle.thrift.ClientId import com.twitter.finatra.thrift.routing.ThriftWarmup import com.twitter.home_mixer.{thriftscala => st} -import com.twitter.inject.Logging +import com.twitter.util.logging.Logging import com.twitter.inject.utils.Handler import com.twitter.product_mixer.core.{thriftscala => pt} import com.twitter.scrooge.Request diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/BUILD.bazel index 526efd718..3f738bcb6 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/BUILD.bazel @@ -5,19 +5,13 @@ scala_library( tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", @@ -25,10 +19,6 @@ scala_library( "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", - "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", - "timelines/src/main/scala/com/twitter/timelines/injection/scribe", - "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", ], exports = [ "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfig.scala index 8a255b7bb..d26843193 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfig.scala @@ -1,18 +1,23 @@ package com.twitter.home_mixer.candidate_pipeline +import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.SocialGraphServiceFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter -import com.twitter.home_mixer.functional_component.filter.PredicateFeatureFilter +import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature +import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSourceRequest import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata import com.twitter.product_mixer.component_library.filter.FeatureFilter +import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator @@ -33,8 +38,8 @@ import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipel class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery]( conversationServiceCandidateSource: ConversationServiceCandidateSource, tweetypieFeatureHydrator: TweetypieFeatureHydrator, - socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator, namesFeatureHydrator: NamesFeatureHydrator, + invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter, override val gates: Seq[BaseGate[Query]], override val decorator: Option[CandidateDecorator[Query, TweetCandidate]]) extends DependentCandidatePipelineConfig[ @@ -62,10 +67,10 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery]( val tweetsWithConversationMetadata = candidates.map { candidate => TweetWithConversationMetadata( tweetId = candidate.candidateIdLong, - userId = None, - sourceTweetId = None, - sourceUserId = None, - inReplyToTweetId = None, + userId = candidate.features.getOrElse(AuthorIdFeature, None), + sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None), + sourceUserId = candidate.features.getOrElse(SourceUserIdFeature, None), + inReplyToTweetId = candidate.features.getOrElse(InReplyToTweetIdFeature, None), conversationId = None, ancestors = Seq.empty ) @@ -84,7 +89,10 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery]( override val preFilterFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[Query, TweetCandidate, _] - ] = Seq(tweetypieFeatureHydrator, socialGraphServiceFeatureHydrator) + ] = Seq( + tweetypieFeatureHydrator, + InNetworkFeatureHydrator, + ) override def filters: Seq[Filter[Query, TweetCandidate]] = Seq( RetweetDeduplicationFilter, @@ -93,6 +101,7 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery]( FilterIdentifier(QuotedTweetDroppedFilterId), shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) } ), + invalidSubscriptionTweetFilter, InvalidConversationModuleFilter ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfigBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfigBuilder.scala index 219069816..bb55f85e3 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfigBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfigBuilder.scala @@ -1,9 +1,9 @@ package com.twitter.home_mixer.candidate_pipeline -import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.SocialGraphServiceFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator +import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter +import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.gate.BaseGate @@ -15,7 +15,7 @@ import javax.inject.Singleton class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery] @Inject() ( conversationServiceCandidateSource: ConversationServiceCandidateSource, tweetypieFeatureHydrator: TweetypieFeatureHydrator, - socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator, + invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter, namesFeatureHydrator: NamesFeatureHydrator) { def build( @@ -25,8 +25,8 @@ class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery] new ConversationServiceCandidatePipelineConfig( conversationServiceCandidateSource, tweetypieFeatureHydrator, - socialGraphServiceFeatureHydrator, namesFeatureHydrator, + invalidSubscriptionTweetFilter, gates, decorator ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/EditedTweetsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/EditedTweetsCandidatePipelineConfig.scala index 8f824a8ff..d9bb73695 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/EditedTweetsCandidatePipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/EditedTweetsCandidatePipelineConfig.scala @@ -1,7 +1,7 @@ package com.twitter.home_mixer.candidate_pipeline import com.twitter.home_mixer.functional_component.candidate_source.StaleTweetsCacheCandidateSource -import com.twitter.home_mixer.functional_component.decorator.HomeFeedbackActionInfoBuilder +import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator import com.twitter.home_mixer.functional_component.query_transformer.EditedTweetsCandidatePipelineQueryTransformer import com.twitter.home_mixer.service.HomeMixerAlertConfig diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/BUILD.bazel index 614dc58eb..dfaff319d 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/BUILD.bazel @@ -13,8 +13,5 @@ scala_library( "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt", "snowflake/src/main/scala/com/twitter/snowflake/id", - "src/thrift/com/twitter/context:twitter-context-scala", - "src/thrift/com/twitter/timelines/render:thrift-scala", - "twitter-context/src/main/scala", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/HomeThriftController.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/HomeThriftController.scala index fb09b405e..fc1b7770e 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/HomeThriftController.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/HomeThriftController.scala @@ -7,6 +7,7 @@ import com.twitter.home_mixer.service.ScoredTweetsService import com.twitter.home_mixer.{thriftscala => t} import com.twitter.product_mixer.core.controllers.DebugTwitterContext import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder +import com.twitter.product_mixer.core.service.debug_query.DebugQueryService import com.twitter.product_mixer.core.service.urt.UrtService import com.twitter.snowflake.id.SnowflakeId import com.twitter.stitch.Stitch diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/BUILD.bazel new file mode 100644 index 000000000..30ee81acc --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/BUILD.bazel @@ -0,0 +1,24 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", + "home-mixer/thrift/src/main/thrift:thrift-scala", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry", + "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", + "src/thrift/com/twitter/gizmoduck:thrift-scala", + "src/thrift/com/twitter/timelines/render:thrift-scala", + "stitch/stitch-repo/src/main/scala", + "strato/config/columns/auth-context:auth-context-strato-client", + "strato/config/columns/gizmoduck:gizmoduck-strato-client", + "strato/config/src/thrift/com/twitter/strato/graphql/timelines:graphql-timelines-scala", + "strato/src/main/scala/com/twitter/strato/callcontext", + "strato/src/main/scala/com/twitter/strato/fed", + "strato/src/main/scala/com/twitter/strato/fed/server", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/HomeMixerColumn.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/HomeMixerColumn.scala new file mode 100644 index 000000000..0ef27b7a0 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/HomeMixerColumn.scala @@ -0,0 +1,217 @@ +package com.twitter.home_mixer.federated + +import com.twitter.gizmoduck.{thriftscala => gd} +import com.twitter.home_mixer.marshaller.request.HomeMixerRequestUnmarshaller +import com.twitter.home_mixer.model.request.HomeMixerRequest +import com.twitter.home_mixer.{thriftscala => hm} +import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder +import com.twitter.product_mixer.core.pipeline.product.ProductPipelineRequest +import com.twitter.product_mixer.core.pipeline.product.ProductPipelineResult +import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry +import com.twitter.product_mixer.core.{thriftscala => pm} +import com.twitter.stitch.Arrow +import com.twitter.stitch.Stitch +import com.twitter.strato.callcontext.CallContext +import com.twitter.strato.catalog.OpMetadata +import com.twitter.strato.config._ +import com.twitter.strato.data._ +import com.twitter.strato.fed.StratoFed +import com.twitter.strato.generated.client.auth_context.AuditIpClientColumn +import com.twitter.strato.generated.client.gizmoduck.CompositeOnUserClientColumn +import com.twitter.strato.graphql.timelines.{thriftscala => gql} +import com.twitter.strato.thrift.ScroogeConv +import com.twitter.timelines.render.{thriftscala => tr} +import com.twitter.util.Try +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class HomeMixerColumn @Inject() ( + homeMixerRequestUnmarshaller: HomeMixerRequestUnmarshaller, + compositeOnUserClientColumn: CompositeOnUserClientColumn, + auditIpClientColumn: AuditIpClientColumn, + paramsBuilder: ParamsBuilder, + productPipelineRegistry: ProductPipelineRegistry) + extends StratoFed.Column(HomeMixerColumn.Path) + with StratoFed.Fetch.Arrow { + + override val contactInfo: ContactInfo = ContactInfo( + contactEmail = "", + ldapGroup = "", + slackRoomId = "" + ) + + override val metadata: OpMetadata = + OpMetadata( + lifecycle = Some(Lifecycle.Production), + description = + Some(Description.PlainText("Federated Strato column for Timelines served via Home Mixer")) + ) + + private val bouncerAccess: Seq[Policy] = Seq(BouncerAccess()) + private val finatraTestServiceIdentifiers: Seq[Policy] = Seq( + ServiceIdentifierPattern( + role = "", + service = "", + env = "", + zone = Seq("")) + ) + + override val policy: Policy = AnyOf(bouncerAccess ++ finatraTestServiceIdentifiers) + + override type Key = gql.TimelineKey + override type View = gql.HomeTimelineView + override type Value = tr.Timeline + + override val keyConv: Conv[Key] = ScroogeConv.fromStruct[gql.TimelineKey] + override val viewConv: Conv[View] = ScroogeConv.fromStruct[gql.HomeTimelineView] + override val valueConv: Conv[Value] = ScroogeConv.fromStruct[tr.Timeline] + + private def createHomeMixerRequestArrow( + compositeOnUserClientColumn: CompositeOnUserClientColumn, + auditIpClientColumn: AuditIpClientColumn + ): Arrow[(Key, View), hm.HomeMixerRequest] = { + + val populateUserRolesAndIp: Arrow[(Key, View), (Option[Set[String]], Option[String])] = { + val gizmoduckView: (gd.LookupContext, Set[gd.QueryFields]) = + (gd.LookupContext(), Set(gd.QueryFields.Roles)) + + val populateUserRoles = Arrow + .flatMap[(Key, View), Option[Set[String]]] { _ => + Stitch.collect { + CallContext.twitterUserId.map { userId => + compositeOnUserClientColumn.fetcher + .callStack(HomeMixerColumn.FetchCallstack) + .fetch(userId, gizmoduckView).map(_.v) + .map { + _.flatMap(_.roles.map(_.roles.toSet)).getOrElse(Set.empty) + } + } + } + } + + val populateIpAddress = Arrow + .flatMap[(Key, View), Option[String]](_ => + auditIpClientColumn.fetcher + .callStack(HomeMixerColumn.FetchCallstack) + .fetch((), ()).map(_.v)) + + Arrow.join( + populateUserRoles, + populateIpAddress + ) + } + + Arrow.zipWithArg(populateUserRolesAndIp).map { + case ((key, view), (roles, ipAddress)) => + val deviceContextOpt = Some( + hm.DeviceContext( + isPolling = CallContext.isPolling, + requestContext = view.requestContext, + latestControlAvailable = view.latestControlAvailable, + autoplayEnabled = view.autoplayEnabled + )) + val seenTweetIds = view.seenTweetIds.filter(_.nonEmpty) + + val (product, productContext) = key match { + case gql.TimelineKey.HomeTimeline(_) | gql.TimelineKey.HomeTimelineV2(_) => + ( + hm.Product.ForYou, + hm.ProductContext.ForYou( + hm.ForYou( + deviceContextOpt, + seenTweetIds, + view.dspClientContext, + view.pushToHomeTweetId + ) + )) + case gql.TimelineKey.HomeLatestTimeline(_) | gql.TimelineKey.HomeLatestTimelineV2(_) => + ( + hm.Product.Following, + hm.ProductContext.Following( + hm.Following(deviceContextOpt, seenTweetIds, view.dspClientContext))) + case gql.TimelineKey.CreatorSubscriptionsTimeline(_) => + ( + hm.Product.Subscribed, + hm.ProductContext.Subscribed(hm.Subscribed(deviceContextOpt, seenTweetIds))) + case _ => throw new UnsupportedOperationException(s"Unknown product: $key") + } + + val clientContext = pm.ClientContext( + userId = CallContext.twitterUserId, + guestId = CallContext.guestId, + guestIdAds = CallContext.guestIdAds, + guestIdMarketing = CallContext.guestIdMarketing, + appId = CallContext.clientApplicationId, + ipAddress = ipAddress, + userAgent = CallContext.userAgent, + countryCode = CallContext.requestCountryCode, + languageCode = CallContext.requestLanguageCode, + isTwoffice = CallContext.isInternalOrTwoffice, + userRoles = roles, + deviceId = CallContext.deviceId, + mobileDeviceId = CallContext.mobileDeviceId, + mobileDeviceAdId = CallContext.adId, + limitAdTracking = CallContext.limitAdTracking + ) + + hm.HomeMixerRequest( + clientContext = clientContext, + product = product, + productContext = Some(productContext), + maxResults = Try(view.count.get.toInt).toOption.orElse(HomeMixerColumn.MaxCount), + cursor = view.cursor.filter(_.nonEmpty) + ) + } + } + + override val fetch: Arrow[(Key, View), Result[Value]] = { + val transformThriftIntoPipelineRequest: Arrow[ + (Key, View), + ProductPipelineRequest[HomeMixerRequest] + ] = { + Arrow + .identity[(Key, View)] + .andThen { + createHomeMixerRequestArrow(compositeOnUserClientColumn, auditIpClientColumn) + } + .map { + case thriftRequest => + val request = homeMixerRequestUnmarshaller(thriftRequest) + val params = paramsBuilder.build( + clientContext = request.clientContext, + product = request.product, + featureOverrides = + request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty), + ) + ProductPipelineRequest(request, params) + } + } + + val underlyingProduct: Arrow[ + ProductPipelineRequest[HomeMixerRequest], + ProductPipelineResult[tr.TimelineResponse] + ] = Arrow + .identity[ProductPipelineRequest[HomeMixerRequest]] + .map { pipelineRequest => + val pipelineArrow = productPipelineRegistry + .getProductPipeline[HomeMixerRequest, tr.TimelineResponse]( + pipelineRequest.request.product) + .arrow + (pipelineArrow, pipelineRequest) + }.applyArrow + + transformThriftIntoPipelineRequest.andThen(underlyingProduct).map { + _.result match { + case Some(result) => found(result.timeline) + case _ => missing + } + } + } +} + +object HomeMixerColumn { + val Path = "home-mixer/homeMixer.Timeline" + private val FetchCallstack = s"$Path:fetch" + private val MaxCount: Option[Int] = Some(100) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/BUILD.bazel index 706f52e2e..a40bd4030 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/BUILD.bazel @@ -10,11 +10,8 @@ scala_library( "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala", "src/thrift/com/twitter/search:earlybird-scala", "stitch/stitch-timelineservice/src/main/scala", - "strato/config/columns/recommendations/similarity:similarity-strato-client", - "strato/src/main/scala/com/twitter/strato/client", ], exports = [ "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/BUILD.bazel index 3ba2d22ae..4df69c9f9 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/BUILD.bazel @@ -6,13 +6,11 @@ scala_library( dependencies = [ "finagle/finagle-core/src/main", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "joinkey/src/main/scala/com/twitter/joinkey/context", - "joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope", @@ -25,8 +23,6 @@ scala_library( "src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala", "stringcenter/client", "stringcenter/client/src/main/java", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/translation", "timelines/src/main/scala/com/twitter/timelines/injection/scribe", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeConversationServiceCandidateDecorator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeConversationServiceCandidateDecorator.scala index 9a94f1c3b..d145391d4 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeConversationServiceCandidateDecorator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeConversationServiceCandidateDecorator.scala @@ -1,6 +1,8 @@ package com.twitter.home_mixer.functional_component.decorator import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder +import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder +import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/BUILD.bazel index 66dd7fc38..cb59d1d73 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/BUILD.bazel @@ -8,7 +8,9 @@ scala_library( "finagle/finagle-core/src/main", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", + "joinkey/src/main/scala/com/twitter/joinkey/context", + "joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", @@ -18,6 +20,7 @@ scala_library( "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", "src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala", + "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", "timelines/src/main/scala/com/twitter/timelines/injection/scribe", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeAdsClientEventDetailsBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeAdsClientEventDetailsBuilder.scala similarity index 96% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeAdsClientEventDetailsBuilder.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeAdsClientEventDetailsBuilder.scala index c81223a53..eb072f135 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeAdsClientEventDetailsBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeAdsClientEventDetailsBuilder.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.decorator +package com.twitter.home_mixer.functional_component.decorator.builder import com.twitter.finagle.tracing.Trace import com.twitter.product_mixer.core.feature.featuremap.FeatureMap diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeClientEventDetailsBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventDetailsBuilder.scala similarity index 98% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeClientEventDetailsBuilder.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventDetailsBuilder.scala index 801ac0f84..2e4c60d25 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeClientEventDetailsBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventDetailsBuilder.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.decorator +package com.twitter.home_mixer.functional_component.decorator.builder import com.twitter.bijection.Base64String import com.twitter.bijection.scrooge.BinaryScalaCodec diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeTimelinesScoreInfoBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTimelinesScoreInfoBuilder.scala similarity index 93% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeTimelinesScoreInfoBuilder.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTimelinesScoreInfoBuilder.scala index bc072ec34..bc934414c 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeTimelinesScoreInfoBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTimelinesScoreInfoBuilder.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.decorator +package com.twitter.home_mixer.functional_component.decorator.builder import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.home_mixer.param.HomeGlobalParams.EnableSendScoresToClient diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeTweetTypePredicates.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTweetTypePredicates.scala similarity index 76% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeTweetTypePredicates.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTweetTypePredicates.scala index 0b06448d7..7272c360d 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeTweetTypePredicates.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTweetTypePredicates.scala @@ -1,8 +1,10 @@ -package com.twitter.home_mixer.functional_component.decorator +package com.twitter.home_mixer.functional_component.decorator.builder import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType import com.twitter.timelinemixer.injection.model.candidate.SemanticCoreFeatures import com.twitter.tweetypie.{thriftscala => tpt} @@ -25,74 +27,76 @@ object HomeTweetTypePredicates { ( "has_exclusive_conversation_author_id", _.getOrElse(ExclusiveConversationAuthorIdFeature, None).nonEmpty), - ("is_eligible_for_connect_boost", _.getOrElse(AuthorIsEligibleForConnectBoostFeature, false)), + ("is_eligible_for_connect_boost", _ => false), ("hashtag", _.getOrElse(EarlybirdFeature, None).exists(_.numHashtags > 0)), ("has_scheduled_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isScheduled)), ("has_recorded_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isRecorded)), ("is_read_from_cache", _.getOrElse(IsReadFromCacheFeature, false)), - ( - "is_self_thread_tweet", - _.getOrElse(ConversationFeature, None).exists(_.isSelfThreadTweet.contains(true))), ("get_initial", _.getOrElse(GetInitialFeature, false)), ("get_newer", _.getOrElse(GetNewerFeature, false)), ("get_middle", _.getOrElse(GetMiddleFeature, false)), ("get_older", _.getOrElse(GetOlderFeature, false)), ("pull_to_refresh", _.getOrElse(PullToRefreshFeature, false)), ("polling", _.getOrElse(PollingFeature, false)), - ("tls_size_20_plus", _ => false), - ("near_empty", _ => false), - ("ranked_request", _ => false), + ("near_empty", _.getOrElse(ServedSizeFeature, None).exists(_ < 3)), + ("is_request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)), ("mutual_follow", _.getOrElse(EarlybirdFeature, None).exists(_.fromMutualFollow)), + ( + "less_than_10_mins_since_lnpt", + _.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 10.minutes)), + ("served_in_conversation_module", _.getOrElse(ServedInConversationModuleFeature, false)), ("has_ticketed_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasTickets)), ("in_utis_top5", _.getOrElse(PositionFeature, None).exists(_ < 5)), - ("is_utis_pos0", _.getOrElse(PositionFeature, None).exists(_ == 0)), - ("is_utis_pos1", _.getOrElse(PositionFeature, None).exists(_ == 1)), - ("is_utis_pos2", _.getOrElse(PositionFeature, None).exists(_ == 2)), - ("is_utis_pos3", _.getOrElse(PositionFeature, None).exists(_ == 3)), - ("is_utis_pos4", _.getOrElse(PositionFeature, None).exists(_ == 4)), ( - "is_signup_request", - candidate => candidate.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)), - ("empty_request", _ => false), - ("served_size_less_than_5", _.getOrElse(ServedSizeFeature, None).exists(_ < 5)), - ("served_size_less_than_10", _.getOrElse(ServedSizeFeature, None).exists(_ < 10)), - ("served_size_less_than_20", _.getOrElse(ServedSizeFeature, None).exists(_ < 20)), + "conversation_module_has_2_displayed_tweets", + _.getOrElse(ConversationModule2DisplayedTweetsFeature, false)), + ("empty_request", _.getOrElse(ServedSizeFeature, None).exists(_ == 0)), ("served_size_less_than_50", _.getOrElse(ServedSizeFeature, None).exists(_ < 50)), ( "served_size_between_50_and_100", _.getOrElse(ServedSizeFeature, None).exists(size => size >= 50 && size < 100)), ("authored_by_contextual_user", _.getOrElse(AuthoredByContextualUserFeature, false)), + ( + "is_self_thread_tweet", + _.getOrElse(ConversationFeature, None).exists(_.isSelfThreadTweet.contains(true))), ("has_ancestors", _.getOrElse(AncestorsFeature, Seq.empty).nonEmpty), ("full_scoring_succeeded", _.getOrElse(FullScoringSucceededFeature, false)), + ("served_size_less_than_20", _.getOrElse(ServedSizeFeature, None).exists(_ < 20)), + ("served_size_less_than_10", _.getOrElse(ServedSizeFeature, None).exists(_ < 10)), + ("served_size_less_than_5", _.getOrElse(ServedSizeFeature, None).exists(_ < 5)), ( "account_age_less_than_30_minutes", _.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)), + ("conversation_module_has_gap", _.getOrElse(ConversationModuleHasGapFeature, false)), + ( + "directed_at_user_is_in_first_degree", + _.getOrElse(EarlybirdFeature, None).exists(_.directedAtUserIdIsInFirstDegree.contains(true))), + ( + "has_semantic_core_annotation", + _.getOrElse(EarlybirdFeature, None).exists(_.semanticCoreAnnotations.nonEmpty)), + ("is_request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false)), ( "account_age_less_than_1_day", _.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 1.day)), ( "account_age_less_than_7_days", _.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 7.days)), - ( - "directed_at_user_is_in_first_degree", - _.getOrElse(EarlybirdFeature, None).exists(_.directedAtUserIdIsInFirstDegree.contains(true))), - ("root_user_is_in_first_degree", _ => false), - ( - "has_semantic_core_annotation", - _.getOrElse(EarlybirdFeature, None).exists(_.semanticCoreAnnotations.nonEmpty)), - ("is_request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false)), ( "part_of_utt", _.getOrElse(EarlybirdFeature, None) .exists(_.semanticCoreAnnotations.exists(_.exists(annotation => annotation.domainId == SemanticCoreFeatures.UnifiedTwitterTaxonomy)))), + ( + "has_home_latest_request_past_week", + _.getOrElse(FollowingLastNonPollingTimeFeature, None).exists(_.untilNow < 7.days)), + ("is_utis_pos0", _.getOrElse(PositionFeature, None).exists(_ == 0)), + ("is_utis_pos1", _.getOrElse(PositionFeature, None).exists(_ == 1)), + ("is_utis_pos2", _.getOrElse(PositionFeature, None).exists(_ == 2)), + ("is_utis_pos3", _.getOrElse(PositionFeature, None).exists(_ == 3)), + ("is_utis_pos4", _.getOrElse(PositionFeature, None).exists(_ == 4)), ("is_random_tweet", _.getOrElse(IsRandomTweetFeature, false)), ("has_random_tweet_in_response", _.getOrElse(HasRandomTweetFeature, false)), ("is_random_tweet_above_in_utis", _.getOrElse(IsRandomTweetAboveFeature, false)), - ("is_request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)), - ("viewer_is_employee", _ => false), - ("viewer_is_timelines_employee", _ => false), - ("viewer_follows_any_topics", _.getOrElse(UserFollowedTopicsCountFeature, None).exists(_ > 0)), ( "has_ancestor_authored_by_viewer", candidate => @@ -100,11 +104,6 @@ object HomeTweetTypePredicates { .getOrElse(AncestorsFeature, Seq.empty).exists(ancestor => candidate.getOrElse(ViewerIdFeature, 0L) == ancestor.userId)), ("ancestor", _.getOrElse(IsAncestorCandidateFeature, false)), - ( - "root_ancestor", - candidate => - candidate.getOrElse(IsAncestorCandidateFeature, false) && candidate - .getOrElse(InReplyToTweetIdFeature, None).isEmpty), ( "deep_reply", candidate => @@ -119,23 +118,22 @@ object HomeTweetTypePredicates { "tweet_age_less_than_15_seconds", _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None) .exists(_.untilNow <= 15.seconds)), - ("is_followed_topic_tweet", _ => false), - ("is_recommended_topic_tweet", _ => false), - ("is_topic_tweet", _ => false), - ("preferred_language_matches_tweet_language", _ => false), + ( + "less_than_1_hour_since_lnpt", + _.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 1.hour)), + ("has_gte_10_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 10))), ( "device_language_matches_tweet_language", candidate => candidate.getOrElse(TweetLanguageFeature, None) == candidate.getOrElse(DeviceLanguageFeature, None)), + ( + "root_ancestor", + candidate => + candidate.getOrElse(IsAncestorCandidateFeature, false) && candidate + .getOrElse(InReplyToTweetIdFeature, None).isEmpty), ("question", _.getOrElse(EarlybirdFeature, None).exists(_.hasQuestion.contains(true))), - ("in_network", _.getOrElse(FromInNetworkSourceFeature, true)), - ("viewer_follows_original_author", _ => false), - ("has_account_follow_prompt", _ => false), - ("has_relevance_prompt", _ => false), - ("has_topic_annotation_haug_prompt", _ => false), - ("has_topic_annotation_random_precision_prompt", _ => false), - ("has_topic_annotation_prompt", _ => false), + ("in_network", _.getOrElse(InNetworkFeature, true)), ( "has_political_annotation", _.getOrElse(EarlybirdFeature, None).exists( @@ -153,9 +151,14 @@ object HomeTweetTypePredicates { _.getOrElse(EarlybirdFeature, None) .exists(_.conversationControl.exists(_.isInstanceOf[tpt.ConversationControl.Community]))), ("has_zero_score", _.getOrElse(ScoreFeature, None).exists(_ == 0.0)), - ("is_viewer_not_invited_to_reply", _ => false), - ("is_viewer_invited_to_reply", _ => false), - ("has_gte_10_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 10))), + ( + "is_followed_topic_tweet", + _.getOrElse(TopicContextFunctionalityTypeFeature, None) + .exists(_ == BasicTopicContextFunctionalityType)), + ( + "is_recommended_topic_tweet", + _.getOrElse(TopicContextFunctionalityTypeFeature, None) + .exists(_ == RecommendationTopicContextFunctionalityType)), ("has_gte_100_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100))), ("has_gte_1k_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 1000))), ( @@ -164,8 +167,6 @@ object HomeTweetTypePredicates { ( "has_gte_100k_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100000))), - ("above_neighbor_is_topic_tweet", _ => false), - ("is_topic_tweet_with_neighbor_below", _ => false), ("has_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasSpace)), ("has_live_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isLive)), ( @@ -187,6 +188,7 @@ object HomeTweetTypePredicates { ( "has_toxicity_score_above_threshold", _.getOrElse(EarlybirdFeature, None).exists(_.toxicityScore.exists(_ > 0.91))), + ("is_topic_tweet", _.getOrElse(TopicIdSocialContextFeature, None).isDefined), ( "text_only", candidate => @@ -204,23 +206,50 @@ object HomeTweetTypePredicates { ("has_3_images", _.getOrElse(NumImagesFeature, None).exists(_ == 3)), ("has_4_images", _.getOrElse(NumImagesFeature, None).exists(_ == 4)), ("has_card", _.getOrElse(EarlybirdFeature, None).exists(_.hasCard)), - ("3_or_more_consecutive_not_in_network", _ => false), - ("2_or_more_consecutive_not_in_network", _ => false), - ("5_out_of_7_not_in_network", _ => false), - ("7_out_of_7_not_in_network", _ => false), - ("5_out_of_5_not_in_network", _ => false), ("user_follow_count_gte_50", _.getOrElse(UserFollowingCountFeature, None).exists(_ > 50)), - ("has_liked_by_social_context", _ => false), - ("has_followed_by_social_context", _ => false), - ("has_topic_social_context", _ => false), - ("timeline_entry_has_banner", _ => false), - ("served_in_conversation_module", _.getOrElse(ServedInConversationModuleFeature, false)), ( - "conversation_module_has_2_displayed_tweets", - _.getOrElse(ConversationModule2DisplayedTweetsFeature, false)), - ("conversation_module_has_gap", _.getOrElse(ConversationModuleHasGapFeature, false)), - ("served_in_recap_tweet_candidate_module_injection", _ => false), - ("served_in_threaded_conversation_module", _ => false) + "has_liked_by_social_context", + candidateFeatures => + candidateFeatures + .getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty) + .exists(candidateFeatures + .getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Seq.empty).toSet.contains)), + ( + "has_followed_by_social_context", + _.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty).nonEmpty), + ( + "has_topic_social_context", + candidateFeatures => + candidateFeatures + .getOrElse(TopicIdSocialContextFeature, None) + .isDefined && + candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None).isDefined), + ("video_lte_10_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ <= 10000)), + ( + "video_bt_10_60_sec", + _.getOrElse(VideoDurationMsFeature, None).exists(duration => + duration > 10000 && duration <= 60000)), + ("video_gt_60_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ > 60000)), + ( + "tweet_age_lte_30_minutes", + _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None) + .exists(_.untilNow <= 30.minutes)), + ( + "tweet_age_lte_1_hour", + _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None) + .exists(_.untilNow <= 1.hour)), + ( + "tweet_age_lte_6_hours", + _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None) + .exists(_.untilNow <= 6.hours)), + ( + "tweet_age_lte_12_hours", + _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None) + .exists(_.untilNow <= 12.hours)), + ( + "tweet_age_gte_24_hours", + _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None) + .exists(_.untilNow >= 24.hours)), ) val PredicateMap = CandidatePredicates.toMap diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/ListClientEventDetailsBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/ListClientEventDetailsBuilder.scala index 4c7927cab..cfffee0ca 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/ListClientEventDetailsBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/ListClientEventDetailsBuilder.scala @@ -8,7 +8,7 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ti import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelineservice.suggests.{thriftscala => st} -object ListClientEventDetailsBuilder +case class ListClientEventDetailsBuilder(suggestType: st.SuggestType) extends BaseClientEventDetailsBuilder[PipelineQuery, UniversalNoun[Any]] { override def apply( @@ -20,7 +20,7 @@ object ListClientEventDetailsBuilder conversationDetails = None, timelinesDetails = Some( TimelinesDetails( - injectionType = Some(st.SuggestType.OrganicListTweet.name), + injectionType = Some(suggestType.name), controllerData = None, sourceData = None)), articleDetails = None, diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/AuthorChildFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AuthorChildFeedbackActionBuilder.scala similarity index 95% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/AuthorChildFeedbackActionBuilder.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AuthorChildFeedbackActionBuilder.scala index 712d85360..dee273f5f 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/AuthorChildFeedbackActionBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AuthorChildFeedbackActionBuilder.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.decorator +package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature @@ -9,7 +9,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ch import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import com.twitter.timelines.service.{thriftscala => t} - import javax.inject.Inject import javax.inject.Singleton diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BUILD.bazel index e3714868a..3a4f19592 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BUILD.bazel @@ -4,9 +4,16 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", "src/thrift/com/twitter/timelines/service:thrift-scala", "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", + "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/BlockUserChildFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BlockUserChildFeedbackActionBuilder.scala similarity index 96% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/BlockUserChildFeedbackActionBuilder.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BlockUserChildFeedbackActionBuilder.scala index 2535b24b2..a23b57dee 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/BlockUserChildFeedbackActionBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BlockUserChildFeedbackActionBuilder.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.decorator +package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature @@ -13,7 +13,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorBlockUser import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter - import javax.inject.Inject import javax.inject.Singleton diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/DontLikeFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/DontLikeFeedbackActionBuilder.scala similarity index 98% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/DontLikeFeedbackActionBuilder.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/DontLikeFeedbackActionBuilder.scala index 344ce668d..9032b53d9 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/DontLikeFeedbackActionBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/DontLikeFeedbackActionBuilder.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.decorator +package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature @@ -17,7 +17,6 @@ import com.twitter.timelines.common.{thriftscala => tlc} import com.twitter.timelineservice.model.FeedbackInfo import com.twitter.timelineservice.model.FeedbackMetadata import com.twitter.timelineservice.{thriftscala => tls} - import javax.inject.Inject import javax.inject.Singleton diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/EngagerSocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/EngagerSocialContextBuilder.scala similarity index 98% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/EngagerSocialContextBuilder.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/EngagerSocialContextBuilder.scala index 5e9f40560..950271e4a 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/EngagerSocialContextBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/EngagerSocialContextBuilder.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.decorator +package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.RealNamesFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/ExtendedReplySocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ExtendedReplySocialContextBuilder.scala similarity index 97% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/ExtendedReplySocialContextBuilder.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ExtendedReplySocialContextBuilder.scala index 88ffa1cd1..74576eae8 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/ExtendedReplySocialContextBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ExtendedReplySocialContextBuilder.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.decorator +package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.FocalTweetAuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackStrings.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackStrings.scala new file mode 100644 index 000000000..8a9c214aa --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackStrings.scala @@ -0,0 +1,18 @@ +package com.twitter.home_mixer.functional_component.decorator.urt.builder + +import com.twitter.product_mixer.core.product.guice.scope.ProductScoped +import com.twitter.stringcenter.client.ExternalStringRegistry +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +@Singleton +class FeedbackStrings @Inject() ( + @ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry]) { + private val externalStringRegistry = externalStringRegistryProvider.get() + + val seeLessOftenFeedbackString = + externalStringRegistry.createProdString("Feedback.seeLessOften") + val seeLessOftenConfirmationFeedbackString = + externalStringRegistry.createProdString("Feedback.seeLessOftenConfirmation") +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/FeedbackUtil.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackUtil.scala similarity index 96% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/FeedbackUtil.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackUtil.scala index 49e24d0b4..08534f26e 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/FeedbackUtil.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackUtil.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.decorator +package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.conversions.DurationOps._ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/FollowedBySocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FollowedBySocialContextBuilder.scala similarity index 96% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/FollowedBySocialContextBuilder.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FollowedBySocialContextBuilder.scala index 0b4882c21..7554f7b36 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/FollowedBySocialContextBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FollowedBySocialContextBuilder.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.decorator +package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeFeedbackActionInfoBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeFeedbackActionInfoBuilder.scala similarity index 96% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeFeedbackActionInfoBuilder.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeFeedbackActionInfoBuilder.scala index 032874ead..c932f6362 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeFeedbackActionInfoBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeFeedbackActionInfoBuilder.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.decorator +package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature import com.twitter.home_mixer.model.request.FollowingProduct @@ -12,7 +12,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Fe import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelines.service.{thriftscala => t} import com.twitter.timelines.util.FeedbackMetadataSerializer - import javax.inject.Inject import javax.inject.Singleton diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeTweetSocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeTweetSocialContextBuilder.scala similarity index 80% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeTweetSocialContextBuilder.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeTweetSocialContextBuilder.scala index 4f3e03f6d..5138f254a 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeTweetSocialContextBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeTweetSocialContextBuilder.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.decorator +package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleIdFeature @@ -14,10 +14,13 @@ import javax.inject.Singleton @Singleton case class HomeTweetSocialContextBuilder @Inject() ( likedBySocialContextBuilder: LikedBySocialContextBuilder, + listsSocialContextBuilder: ListsSocialContextBuilder, followedBySocialContextBuilder: FollowedBySocialContextBuilder, topicSocialContextBuilder: TopicSocialContextBuilder, extendedReplySocialContextBuilder: ExtendedReplySocialContextBuilder, - receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder) + receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder, + popularVideoSocialContextBuilder: PopularVideoSocialContextBuilder, + popularInYourAreaSocialContextBuilder: PopularInYourAreaSocialContextBuilder) extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { def apply( @@ -31,6 +34,9 @@ case class HomeTweetSocialContextBuilder @Inject() ( likedBySocialContextBuilder(query, candidate, features) .orElse(followedBySocialContextBuilder(query, candidate, features)) .orElse(topicSocialContextBuilder(query, candidate, features)) + .orElse(popularVideoSocialContextBuilder(query, candidate, features)) + .orElse(listsSocialContextBuilder(query, candidate, features)) + .orElse(popularInYourAreaSocialContextBuilder(query, candidate, features)) case Some(_) => val conversationId = features.getOrElse(ConversationModuleIdFeature, None) // Only hydrate the social context into the root tweet in a conversation module diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToFollowFeedbackActionInfoBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToFollowFeedbackActionInfoBuilder.scala index fd108dc49..49017e8cf 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToFollowFeedbackActionInfoBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToFollowFeedbackActionInfoBuilder.scala @@ -5,7 +5,6 @@ import com.twitter.product_mixer.component_library.model.candidate.UserCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.ExternalStringRegistry import com.twitter.stringcenter.client.StringCenter import javax.inject.Inject import javax.inject.Provider @@ -32,12 +31,13 @@ object HomeWhoToFollowFeedbackActionInfoBuilder { @Singleton case class HomeWhoToFollowFeedbackActionInfoBuilder @Inject() ( - @ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry], + feedbackStrings: FeedbackStrings, @ProductScoped stringCenterProvider: Provider[StringCenter]) extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] { private val whoToFollowFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder( - externalStringRegistry = externalStringRegistryProvider.get(), + seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString, + seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenConfirmationFeedbackString, stringCenter = stringCenterProvider.get(), encodedFeedbackRequest = Some(HomeWhoToFollowFeedbackActionInfoBuilder.EncodedFeedbackRequest) ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToSubscribeFeedbackActionInfoBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToSubscribeFeedbackActionInfoBuilder.scala new file mode 100644 index 000000000..270ea66f6 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToSubscribeFeedbackActionInfoBuilder.scala @@ -0,0 +1,52 @@ +package com.twitter.home_mixer.functional_component.decorator.urt.builder + +import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.WhoToFollowFeedbackActionInfoBuilder +import com.twitter.product_mixer.component_library.model.candidate.UserCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder +import com.twitter.product_mixer.core.product.guice.scope.ProductScoped +import com.twitter.stringcenter.client.StringCenter +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackActionInfo +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.timelines.service.{thriftscala => tl} +import com.twitter.timelines.util.FeedbackRequestSerializer +import com.twitter.timelineservice.suggests.thriftscala.SuggestType +import com.twitter.timelineservice.thriftscala.FeedbackType + +object HomeWhoToSubscribeFeedbackActionInfoBuilder { + private val FeedbackMetadata = tl.FeedbackMetadata( + injectionType = Some(SuggestType.WhoToSubscribe), + engagementType = None, + entityIds = Seq.empty, + ttlMs = None + ) + private val FeedbackRequest = + tl.DefaultFeedbackRequest2(FeedbackType.SeeFewer, FeedbackMetadata) + private val EncodedFeedbackRequest = + FeedbackRequestSerializer.serialize(tl.FeedbackRequest.DefaultFeedbackRequest2(FeedbackRequest)) +} + +@Singleton +case class HomeWhoToSubscribeFeedbackActionInfoBuilder @Inject() ( + feedbackStrings: FeedbackStrings, + @ProductScoped stringCenterProvider: Provider[StringCenter]) + extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] { + + private val whoToSubscribeFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder( + seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString, + seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenConfirmationFeedbackString, + stringCenter = stringCenterProvider.get(), + encodedFeedbackRequest = + Some(HomeWhoToSubscribeFeedbackActionInfoBuilder.EncodedFeedbackRequest) + ) + + override def apply( + query: PipelineQuery, + candidate: UserCandidate, + candidateFeatures: FeatureMap + ): Option[FeedbackActionInfo] = + whoToSubscribeFeedbackActionInfoBuilder.apply(query, candidate, candidateFeatures) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/LikedBySocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/LikedBySocialContextBuilder.scala similarity index 96% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/LikedBySocialContextBuilder.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/LikedBySocialContextBuilder.scala index e2e6f0a23..fbc65be13 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/LikedBySocialContextBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/LikedBySocialContextBuilder.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.decorator +package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ListsSocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ListsSocialContextBuilder.scala new file mode 100644 index 000000000..e5e746223 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ListsSocialContextBuilder.scala @@ -0,0 +1,50 @@ +package com.twitter.home_mixer.functional_component.decorator.urt.builder + +import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature +import com.twitter.home_mixer.model.HomeFeatures.UserScreenNameFeature +import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.product.guice.scope.ProductScoped +import com.twitter.stringcenter.client.StringCenter +import com.twitter.timelineservice.suggests.{thriftscala => t} +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +/** + * "Your Lists" will be rendered for the context and a url link for your lists. + */ +@Singleton +case class ListsSocialContextBuilder @Inject() ( + externalStrings: HomeMixerExternalStrings, + @ProductScoped stringCenterProvider: Provider[StringCenter]) + extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { + + private val stringCenter = stringCenterProvider.get() + private val listString = externalStrings.ownedSubscribedListsModuleHeaderString + + def apply( + query: PipelineQuery, + candidate: TweetCandidate, + candidateFeatures: FeatureMap + ): Option[SocialContext] = { + candidateFeatures.get(SuggestTypeFeature) match { + case Some(suggestType) if suggestType == t.SuggestType.RankedListTweet => + val userName = query.features.flatMap(_.getOrElse(UserScreenNameFeature, None)) + Some( + GeneralContext( + contextType = ListGeneralContextType, + text = stringCenter.prepare(listString), + url = userName.map(name => ""), + contextImageUrls = None, + landingUrl = None + )) + case _ => None + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/MuteUserChildFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/MuteUserChildFeedbackActionBuilder.scala similarity index 96% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/MuteUserChildFeedbackActionBuilder.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/MuteUserChildFeedbackActionBuilder.scala index 1c8d24ef1..542ed8a67 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/MuteUserChildFeedbackActionBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/MuteUserChildFeedbackActionBuilder.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.decorator +package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature @@ -12,7 +12,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleMuteUser import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter - import javax.inject.Inject import javax.inject.Singleton diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/NotInterestedTopicFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotInterestedTopicFeedbackActionBuilder.scala similarity index 97% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/NotInterestedTopicFeedbackActionBuilder.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotInterestedTopicFeedbackActionBuilder.scala index 7c8aebb07..2ec520e11 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/NotInterestedTopicFeedbackActionBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotInterestedTopicFeedbackActionBuilder.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.decorator +package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature @@ -15,7 +15,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorMarkNotInterestedTopic import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter - import javax.inject.Inject import javax.inject.Singleton diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/NotRelevantChildFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotRelevantChildFeedbackActionBuilder.scala similarity index 96% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/NotRelevantChildFeedbackActionBuilder.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotRelevantChildFeedbackActionBuilder.scala index 1e5536e0b..39df781ce 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/NotRelevantChildFeedbackActionBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotRelevantChildFeedbackActionBuilder.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.decorator +package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings @@ -12,7 +12,6 @@ import com.twitter.timelines.common.{thriftscala => tlc} import com.twitter.timelineservice.model.FeedbackInfo import com.twitter.timelineservice.model.FeedbackMetadata import com.twitter.timelineservice.{thriftscala => tlst} - import javax.inject.Inject import javax.inject.Singleton diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularInYourAreaSocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularInYourAreaSocialContextBuilder.scala new file mode 100644 index 000000000..3782dd115 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularInYourAreaSocialContextBuilder.scala @@ -0,0 +1,43 @@ +package com.twitter.home_mixer.functional_component.decorator.urt.builder + +import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature +import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.product.guice.scope.ProductScoped +import com.twitter.stringcenter.client.StringCenter +import com.twitter.timelineservice.suggests.{thriftscala => st} +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +@Singleton +case class PopularInYourAreaSocialContextBuilder @Inject() ( + externalStrings: HomeMixerExternalStrings, + @ProductScoped stringCenterProvider: Provider[StringCenter]) + extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { + + private val stringCenter = stringCenterProvider.get() + private val popularInYourAreaString = externalStrings.socialContextPopularInYourAreaString + + def apply( + query: PipelineQuery, + candidate: TweetCandidate, + candidateFeatures: FeatureMap + ): Option[SocialContext] = { + val suggestTypeOpt = candidateFeatures.getOrElse(SuggestTypeFeature, None) + if (suggestTypeOpt.contains(st.SuggestType.RecommendedTrendTweet)) { + Some( + GeneralContext( + contextType = LocationGeneralContextType, + text = stringCenter.prepare(popularInYourAreaString), + url = None, + contextImageUrls = None, + landingUrl = None + )) + } else None + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/YouMightLikeSocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularVideoSocialContextBuilder.scala similarity index 60% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/YouMightLikeSocialContextBuilder.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularVideoSocialContextBuilder.scala index e131a67f0..7eef7f180 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/YouMightLikeSocialContextBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularVideoSocialContextBuilder.scala @@ -1,50 +1,48 @@ -package com.twitter.home_mixer.functional_component.decorator +package com.twitter.home_mixer.functional_component.decorator.urt.builder -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature +import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter +import com.twitter.timelineservice.suggests.{thriftscala => st} import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton -/** - * Renders a fixed 'You Might Like' string above all OON Tweets. - */ @Singleton -case class YouMightLikeSocialContextBuilder @Inject() ( +case class PopularVideoSocialContextBuilder @Inject() ( externalStrings: HomeMixerExternalStrings, @ProductScoped stringCenterProvider: Provider[StringCenter]) extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { private val stringCenter = stringCenterProvider.get() - private val youMightLikeString = externalStrings.socialContextYouMightLikeString + private val popularVideoString = externalStrings.socialContextPopularVideoString def apply( query: PipelineQuery, candidate: TweetCandidate, candidateFeatures: FeatureMap ): Option[SocialContext] = { - val isInNetwork = candidateFeatures.getOrElse(InNetworkFeature, true) - val isRetweet = candidateFeatures.getOrElse(IsRetweetFeature, false) - if (!isInNetwork && !isRetweet) { + val suggestTypeOpt = candidateFeatures.getOrElse(SuggestTypeFeature, None) + if (suggestTypeOpt.contains(st.SuggestType.MediaTweet)) { Some( GeneralContext( contextType = SparkleGeneralContextType, - text = stringCenter.prepare(youMightLikeString), + text = stringCenter.prepare(popularVideoString), url = None, contextImageUrls = None, - landingUrl = None + landingUrl = Some( + Url( + urlType = DeepLink, + url = "" + ) + ) )) - } else { - None - } + } else None } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/ReceivedReplySocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReceivedReplySocialContextBuilder.scala similarity index 97% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/ReceivedReplySocialContextBuilder.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReceivedReplySocialContextBuilder.scala index a04e9b4aa..f1af07c98 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/ReceivedReplySocialContextBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReceivedReplySocialContextBuilder.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.decorator +package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/ReportTweetChildFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReportTweetChildFeedbackActionBuilder.scala similarity index 95% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/ReportTweetChildFeedbackActionBuilder.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReportTweetChildFeedbackActionBuilder.scala index 3938a476d..a4ef0cbb2 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/ReportTweetChildFeedbackActionBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReportTweetChildFeedbackActionBuilder.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.decorator +package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate @@ -8,7 +8,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorReportTweet import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter - import javax.inject.Inject import javax.inject.Singleton diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/RetweeterChildFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/RetweeterChildFeedbackActionBuilder.scala similarity index 95% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/RetweeterChildFeedbackActionBuilder.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/RetweeterChildFeedbackActionBuilder.scala index f2ec8d6f5..006e93b58 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/RetweeterChildFeedbackActionBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/RetweeterChildFeedbackActionBuilder.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.decorator +package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature @@ -10,7 +10,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ch import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import com.twitter.timelines.service.{thriftscala => t} - import javax.inject.Inject import javax.inject.Singleton diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/TopicSocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/TopicSocialContextBuilder.scala similarity index 95% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/TopicSocialContextBuilder.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/TopicSocialContextBuilder.scala index f9a8c48e6..7d01382c3 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/TopicSocialContextBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/TopicSocialContextBuilder.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.decorator +package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/UnfollowUserChildFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/UnfollowUserChildFeedbackActionBuilder.scala similarity index 97% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/UnfollowUserChildFeedbackActionBuilder.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/UnfollowUserChildFeedbackActionBuilder.scala index 8bb700b44..864ee06d9 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/UnfollowUserChildFeedbackActionBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/UnfollowUserChildFeedbackActionBuilder.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.decorator +package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature @@ -11,7 +11,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleFollowUser import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter - import javax.inject.Inject import javax.inject.Singleton diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BUILD.bazel index bc7d5247f..60c104143 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BUILD.bazel @@ -4,95 +4,56 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "3rdparty/jvm/com/twitter/storehaus:core", "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", "configapi/configapi-decider", "finatra/inject/inject-core/src/main/scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/earlybird", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/inferred_topic", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings", "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content", "joinkey/src/main/scala/com/twitter/joinkey/context", "joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/topics", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_visibility_reason", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "representation-scorer/server/src/main/scala/com/twitter/representationscorer/common", - "representation-scorer/server/src/main/thrift:thrift-scala", - "servo/repo/src/main/scala", "snowflake/src/main/scala/com/twitter/snowflake/id", - "src/java/com/twitter/ml/api/constant", "src/java/com/twitter/search/common/util/lang", - "src/scala/com/twitter/ml/api/util", - "src/scala/com/twitter/timelines/prediction/adapters/real_graph", - "src/scala/com/twitter/timelines/prediction/adapters/realtime_interaction_graph", - "src/scala/com/twitter/timelines/prediction/adapters/twistly", - "src/scala/com/twitter/timelines/prediction/adapters/two_hop_features", - "src/scala/com/twitter/timelines/prediction/common/util", - "src/scala/com/twitter/timelines/prediction/features/common", - "src/scala/com/twitter/timelines/prediction/features/realtime_interaction_graph", - "src/scala/com/twitter/timelines/prediction/features/recap", - "src/scala/com/twitter/timelines/prediction/features/time_features", + "src/scala/com/twitter/timelines/prediction/adapters/request_context", "src/thrift/com/twitter/gizmoduck:thrift-scala", - "src/thrift/com/twitter/ml/api:data-java", - "src/thrift/com/twitter/ml/api:embedding-java", - "src/thrift/com/twitter/onboarding/relevance/features:features-java", "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", "src/thrift/com/twitter/search:earlybird-scala", "src/thrift/com/twitter/search/common:constants-java", "src/thrift/com/twitter/socialgraph:thrift-scala", "src/thrift/com/twitter/spam/rtf:safety-result-scala", "src/thrift/com/twitter/timelineranker:thrift-scala", - "src/thrift/com/twitter/timelines/author_features:thrift-java", - "src/thrift/com/twitter/timelines/conversation_features:conversation_features-scala", "src/thrift/com/twitter/timelines/impression:thrift-scala", "src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala", "src/thrift/com/twitter/timelines/real_graph:real_graph-scala", "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", - "src/thrift/com/twitter/topic_recos:topic_recos-thrift-java", "src/thrift/com/twitter/tweetypie:service-scala", "src/thrift/com/twitter/tweetypie:tweet-scala", "src/thrift/com/twitter/user_session_store:thrift-java", "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", - "src/thrift/com/twitter/wtf/real_time_interaction_graph:wtf-real_time_interaction_graph-thrift-java", "stitch/stitch-core", "stitch/stitch-gizmoduck", "stitch/stitch-socialgraph", "stitch/stitch-timelineservice", "stitch/stitch-tweetypie", - "strato/config/columns/topic-signals/tsp", - "strato/config/columns/topic-signals/tsp:tsp-strato-client", "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/feedback", "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan", "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/persistence", - "timelines/src/main/scala/com/twitter/timelines/clients/strato/twistly", - "timelines/src/main/scala/com/twitter/timelines/clients/user_tweet_entity_graph", + "timelines/src/main/scala/com/twitter/timelines/clients/manhattan/store", "timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter", "timelines/src/main/scala/com/twitter/timelines/impressionstore/store", "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", - "topic-social-proof/server/src/main/thrift:thrift-scala", - "topiclisting/topiclisting-core/src/main/scala/com/twitter/topiclisting", - "tweetconvosvc/thrift/src/main/thrift:thrift-scala", - "twitter-config/yaml", "user_session_store/src/main/scala/com/twitter/user_session_store", "util/util-core", ], diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FeedbackHistoryQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FeedbackHistoryQueryFeatureHydrator.scala index aae96fe00..c0793be4b 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FeedbackHistoryQueryFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FeedbackHistoryQueryFeatureHydrator.scala @@ -1,12 +1,10 @@ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature -import com.twitter.home_mixer.param.HomeGlobalParams.EnableFeedbackFatigueParam import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch @@ -17,16 +15,12 @@ import javax.inject.Singleton @Singleton case class FeedbackHistoryQueryFeatureHydrator @Inject() ( feedbackHistoryClient: FeedbackHistoryManhattanClient) - extends QueryFeatureHydrator[PipelineQuery] - with Conditionally[PipelineQuery] { + extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FeedbackHistory") override val features: Set[Feature[_, _]] = Set(FeedbackHistoryFeature) - override def onlyIf(query: PipelineQuery): Boolean = - query.params(EnableFeedbackFatigueParam) - override def hydrate( query: PipelineQuery ): Stitch[FeatureMap] = diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FollowedTopicsQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FollowedTopicsQueryFeatureHydrator.scala deleted file mode 100644 index 5e9bae5c9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FollowedTopicsQueryFeatureHydrator.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.model.HomeFeatures.UserFollowedTopicsCountFeature -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.candidate_source.topics.FollowedTopicsCandidateSource -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyView -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class FollowedTopicsQueryFeatureHydrator @Inject() ( - followedTopicsCandidateSource: FollowedTopicsCandidateSource) - extends QueryFeatureHydrator[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FollowedTopics") - - override val features: Set[Feature[_, _]] = Set(UserFollowedTopicsCountFeature) - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - val request: StratoKeyView[Long, Unit] = StratoKeyView(query.getRequiredUserId, Unit) - followedTopicsCandidateSource(request) - .map { topics => - FeatureMapBuilder().add(UserFollowedTopicsCountFeature, Some(topics.size)).build() - }.handle { - case _ => FeatureMapBuilder().add(UserFollowedTopicsCountFeature, None).build() - } - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.9), - HomeMixerAlertConfig.BusinessHours.defaultLatencyAlert(1500.millis) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GizmoduckAuthorSafetyFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GizmoduckAuthorSafetyFeatureHydrator.scala deleted file mode 100644 index 3b13140d1..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GizmoduckAuthorSafetyFeatureHydrator.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.gizmoduck.{thriftscala => gt} -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature -import com.twitter.home_mixer.param.HomeGlobalParams.EnableGizmoduckAuthorSafetyFeatureHydratorParam -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.Conditionally -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.stitch.gizmoduck.Gizmoduck -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class GizmoduckAuthorSafetyFeatureHydrator @Inject() (gizmoduck: Gizmoduck) - extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] - with Conditionally[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("GizmoduckAuthorSafety") - - override val features: Set[Feature[_, _]] = Set(AuthorIsBlueVerifiedFeature) - - override def onlyIf(query: PipelineQuery): Boolean = - query.params(EnableGizmoduckAuthorSafetyFeatureHydratorParam) - - private val queryFields: Set[gt.QueryFields] = Set(gt.QueryFields.Safety) - - override def apply( - query: PipelineQuery, - candidate: TweetCandidate, - existingFeatures: FeatureMap - ): Stitch[FeatureMap] = { - val authorIdOption = existingFeatures.getOrElse(AuthorIdFeature, None) - - val blueVerifiedStitch = authorIdOption - .map { authorId => - gizmoduck - .getUserById( - userId = authorId, - queryFields = queryFields - ) - .map { _.safety.flatMap(_.isBlueVerified).getOrElse(false) } - }.getOrElse(Stitch.False) - - blueVerifiedStitch.map { isBlueVerified => - FeatureMapBuilder() - .add(AuthorIsBlueVerifiedFeature, isBlueVerified) - .build() - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressionBloomFilterQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressionBloomFilterQueryFeatureHydrator.scala index 19b1ab557..6ece16ce0 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressionBloomFilterQueryFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressionBloomFilterQueryFeatureHydrator.scala @@ -3,6 +3,7 @@ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature import com.twitter.home_mixer.model.request.HasSeenTweetIds +import com.twitter.home_mixer.param.HomeGlobalParams.ImpressionBloomFilterFalsePositiveRateParam import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap @@ -11,7 +12,8 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Quer import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch -import com.twitter.timelines.impressionbloomfilter.{thriftscala => t} +import com.twitter.timelines.clients.manhattan.store.ManhattanStoreClient +import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm} import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilter import javax.inject.Inject import javax.inject.Singleton @@ -19,36 +21,39 @@ import javax.inject.Singleton @Singleton case class ImpressionBloomFilterQueryFeatureHydrator[ Query <: PipelineQuery with HasSeenTweetIds] @Inject() ( - bloomFilter: ImpressionBloomFilter) - extends QueryFeatureHydrator[Query] { + bloomFilterClient: ManhattanStoreClient[ + blm.ImpressionBloomFilterKey, + blm.ImpressionBloomFilterSeq + ]) extends QueryFeatureHydrator[Query] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( "ImpressionBloomFilter") private val ImpressionBloomFilterTTL = 7.day - private val ImpressionBloomFilterFalsePositiveRate = 0.002 override val features: Set[Feature[_, _]] = Set(ImpressionBloomFilterFeature) - private val SurfaceArea = t.SurfaceArea.HomeTimeline + private val SurfaceArea = blm.SurfaceArea.HomeTimeline override def hydrate(query: Query): Stitch[FeatureMap] = { val userId = query.getRequiredUserId - bloomFilter.getBloomFilterSeq(userId, SurfaceArea).map { bloomFilterSeq => - val updatedBloomFilterSeq = - if (query.seenTweetIds.forall(_.isEmpty)) bloomFilterSeq - else { - bloomFilter.addElements( - userId = userId, - surfaceArea = SurfaceArea, - tweetIds = query.seenTweetIds.get, - bloomFilterEntrySeq = bloomFilterSeq, - timeToLive = ImpressionBloomFilterTTL, - falsePositiveRate = ImpressionBloomFilterFalsePositiveRate - ) - } - FeatureMapBuilder().add(ImpressionBloomFilterFeature, updatedBloomFilterSeq).build() - } + bloomFilterClient + .get(blm.ImpressionBloomFilterKey(userId, SurfaceArea)) + .map(_.getOrElse(blm.ImpressionBloomFilterSeq(Seq.empty))) + .map { bloomFilterSeq => + val updatedBloomFilterSeq = + if (query.seenTweetIds.forall(_.isEmpty)) bloomFilterSeq + else { + ImpressionBloomFilter.addSeenTweetIds( + surfaceArea = SurfaceArea, + tweetIds = query.seenTweetIds.get, + bloomFilterSeq = bloomFilterSeq, + timeToLive = ImpressionBloomFilterTTL, + falsePositiveRate = query.params(ImpressionBloomFilterFalsePositiveRateParam) + ) + } + FeatureMapBuilder().add(ImpressionBloomFilterFeature, updatedBloomFilterSeq).build() + } } override val alerts = Seq( diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/InNetworkFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/InNetworkFeatureHydrator.scala new file mode 100644 index 000000000..55f002b1f --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/InNetworkFeatureHydrator.scala @@ -0,0 +1,41 @@ +package com.twitter.home_mixer.functional_component.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +object InNetworkFeatureHydrator + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("InNetwork") + + override val features: Set[Feature[_, _]] = Set(InNetworkFeature) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = { + val viewerId = query.getRequiredUserId + val followedUserIds = query.features.get.get(SGSFollowedUsersFeature).toSet + + val featureMaps = candidates.map { candidate => + // We use authorId and not sourceAuthorId here so that retweets are defined as in network + val isInNetworkOpt = candidate.features.getOrElse(AuthorIdFeature, None).map { authorId => + // Users cannot follow themselves but this is in network by definition + val isSelfTweet = authorId == viewerId + isSelfTweet || followedUserIds.contains(authorId) + } + FeatureMapBuilder().add(InNetworkFeature, isInNetworkOpt.getOrElse(true)).build() + } + Stitch.value(featureMaps) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PersistenceStoreQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PersistenceStoreQueryFeatureHydrator.scala index 10f42865c..b7aae811b 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PersistenceStoreQueryFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PersistenceStoreQueryFeatureHydrator.scala @@ -2,8 +2,10 @@ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.conversions.DurationOps._ import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent +import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTweetPreviewIdsFeature import com.twitter.home_mixer.model.HomeFeatures.WhoToFollowExcludedUserIdsFeature import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.ForYouProduct @@ -27,17 +29,27 @@ import javax.inject.Singleton @Singleton case class PersistenceStoreQueryFeatureHydrator @Inject() ( - timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3]) + timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3], + statsReceiver: StatsReceiver) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("PersistenceStore") + private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val servedTweetIdsSizeStat = scopedStatsReceiver.stat("ServedTweetIdsSize") + private val WhoToFollowExcludedUserIdsLimit = 1000 - private val ServedTweetIdsDuration = 1.hour + private val ServedTweetIdsDuration = 10.minutes private val ServedTweetIdsLimit = 100 + private val ServedTweetPreviewIdsDuration = 10.hours + private val ServedTweetPreviewIdsLimit = 10 override val features: Set[Feature[_, _]] = - Set(ServedTweetIdsFeature, PersistenceEntriesFeature, WhoToFollowExcludedUserIdsFeature) + Set( + ServedTweetIdsFeature, + ServedTweetPreviewIdsFeature, + PersistenceEntriesFeature, + WhoToFollowExcludedUserIdsFeature) private val supportedClients = Seq( ClientPlatform.IPhone, @@ -80,8 +92,19 @@ case class PersistenceStoreQueryFeatureHydrator @Inject() ( .flatMap( _.entries.flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetIdsLimit)) + servedTweetIdsSizeStat.add(servedTweetIds.size) + + val servedTweetPreviewIds = timelineResponses + .filter(_.clientPlatform == clientPlatform) + .filter(_.servedTime >= Time.now - ServedTweetPreviewIdsDuration) + .sortBy(-_.servedTime.inMilliseconds) + .flatMap(_.entries + .filter(_.entityIdType == EntityIdType.TweetPreview) + .flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetPreviewIdsLimit)) + FeatureMapBuilder() .add(ServedTweetIdsFeature, servedTweetIds) + .add(ServedTweetPreviewIdsFeature, servedTweetPreviewIds) .add(PersistenceEntriesFeature, timelineResponses) .add(WhoToFollowExcludedUserIdsFeature, whoToFollowUserIds) .build() diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PerspectiveFilteredSocialContextFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PerspectiveFilteredSocialContextFeatureHydrator.scala index 1c8dedc65..602a8b2dc 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PerspectiveFilteredSocialContextFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PerspectiveFilteredSocialContextFeatureHydrator.scala @@ -10,6 +10,7 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.stitch.timelineservice.TimelineService import com.twitter.stitch.timelineservice.TimelineService.GetPerspectives @@ -37,10 +38,10 @@ class PerspectiveFilteredSocialContextFeatureHydrator @Inject() (timelineService override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { val engagingUserIdtoTweetId = candidates.flatMap { candidate => candidate.features - .get(FavoritedByUserIdsFeature).take(MaxCountUsers) + .getOrElse(FavoritedByUserIdsFeature, Seq.empty).take(MaxCountUsers) .map(favoritedBy => favoritedBy -> candidate.candidate.id) } @@ -59,7 +60,7 @@ class PerspectiveFilteredSocialContextFeatureHydrator @Inject() (timelineService candidates.map { candidate => val perspectiveFilteredFavoritedByUserIds: Seq[Long] = candidate.features - .get(FavoritedByUserIdsFeature).take(MaxCountUsers) + .getOrElse(FavoritedByUserIdsFeature, Seq.empty).take(MaxCountUsers) .filter { userId => validUserIdTweetIds.contains((userId, candidate.candidate.id)) } FeatureMapBuilder() diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphInNetworkScoresQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphInNetworkScoresQueryFeatureHydrator.scala index f04490625..92ab90f2c 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphInNetworkScoresQueryFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphInNetworkScoresQueryFeatureHydrator.scala @@ -32,11 +32,15 @@ case class RealGraphInNetworkScoresQueryFeatureHydrator @Inject() ( val realGraphScoresFeatures = realGraphFollowedUsers .getOrElse(Seq.empty) .sortBy(-_.score) - .map(candidate => candidate.userId -> candidate.score) + .map(candidate => candidate.userId -> scaleScore(candidate.score)) .take(RealGraphCandidateCount) .toMap FeatureMapBuilder().add(RealGraphInNetworkScoresFeature, realGraphScoresFeatures).build() } } + + // Rescale Real Graph v2 scores from [0,1] to the v1 scores distribution [1,2.97] + private def scaleScore(score: Double): Double = + if (score >= 0.0 && score <= 1.0) score * 1.97 + 1.0 else score } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RequestQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RequestQueryFeatureHydrator.scala index bda746816..3968c9532 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RequestQueryFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RequestQueryFeatureHydrator.scala @@ -17,9 +17,13 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.G import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor import com.twitter.product_mixer.core.pipeline.HasPipelineCursor import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest +import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure import com.twitter.search.common.util.lang.ThriftLanguageUtil import com.twitter.snowflake.id.SnowflakeId import com.twitter.stitch.Stitch +import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter.dowFromTimestamp +import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter.hourFromTimestamp import java.util.UUID import javax.inject.Inject import javax.inject.Singleton @@ -45,6 +49,9 @@ class RequestQueryFeatureHydrator[ PullToRefreshFeature, RequestJoinIdFeature, ServedRequestIdFeature, + TimestampFeature, + TimestampGMTDowFeature, + TimestampGMTHourFeature, ViewerIdFeature ) @@ -67,6 +74,7 @@ class RequestQueryFeatureHydrator[ override def hydrate(query: Query): Stitch[FeatureMap] = { val requestContext = query.deviceContext.flatMap(_.requestContextValue) val servedRequestId = UUID.randomUUID.getMostSignificantBits + val timestamp = query.queryTime.inMilliseconds val featureMap = FeatureMapBuilder() .add(AccountAgeFeature, query.getOptionalUserId.flatMap(SnowflakeId.timeFromIdOpt)) @@ -97,8 +105,15 @@ class RequestQueryFeatureHydrator[ .add(PullToRefreshFeature, requestContext.contains(RequestContext.PullToRefresh)) .add(ServedRequestIdFeature, Some(servedRequestId)) .add(RequestJoinIdFeature, getRequestJoinId(servedRequestId)) + .add(TimestampFeature, timestamp) + .add(TimestampGMTDowFeature, dowFromTimestamp(timestamp)) + .add(TimestampGMTHourFeature, hourFromTimestamp(timestamp)) .add(HasDarkRequestFeature, hasDarkRequest) - .add(ViewerIdFeature, query.getRequiredUserId) + .add( + ViewerIdFeature, + query.getOptionalUserId + .orElse(query.getGuestId).getOrElse( + throw PipelineFailure(BadRequest, "Missing viewer id"))) .build() Stitch.value(featureMap) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSFollowedUsersQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSFollowedUsersQueryFeatureHydrator.scala deleted file mode 100644 index e2aa73de8..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSFollowedUsersQueryFeatureHydrator.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.socialgraph.{thriftscala => sg} -import com.twitter.stitch.Stitch -import com.twitter.stitch.socialgraph.{SocialGraph => SocialGraphStitchClient} -import javax.inject.Inject -import javax.inject.Singleton - -object SGSFollowedUsersFeature extends Feature[PipelineQuery, Seq[Long]] - -@Singleton -case class SGSFollowedUsersQueryFeatureHydrator @Inject() ( - socialGraphStitchClient: SocialGraphStitchClient) - extends QueryFeatureHydrator[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("SGSFollowedUsers") - - override val features: Set[Feature[_, _]] = Set(SGSFollowedUsersFeature) - - private val SocialGraphLimit = 14999 - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - val userId = query.getRequiredUserId - - val request = sg.IdsRequest( - relationships = Seq( - sg.SrcRelationship(userId, sg.RelationshipType.Following, hasRelationship = true), - sg.SrcRelationship(userId, sg.RelationshipType.Muting, hasRelationship = false) - ), - pageRequest = Some(sg.PageRequest(count = Some(SocialGraphLimit))) - ) - - socialGraphStitchClient - .ids(request).map(_.ids) - .map { followedUsers => - FeatureMapBuilder().add(SGSFollowedUsersFeature, followedUsers).build() - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSValidSocialContextFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSValidSocialContextFeatureHydrator.scala index 1ba706dd1..92c351ecf 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSValidSocialContextFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSValidSocialContextFeatureHydrator.scala @@ -12,6 +12,7 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.socialgraph.{thriftscala => sg} import com.twitter.stitch.Stitch import com.twitter.stitch.socialgraph.SocialGraph @@ -41,8 +42,7 @@ class SGSValidSocialContextFeatureHydrator @Inject() ( override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { val allSocialContextUserIds = candidates.flatMap { candidate => candidate.features.getOrElse(FavoritedByUserIdsFeature, Nil).take(MaxCountUsers) ++ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SocialGraphServiceFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SocialGraphServiceFeatureHydrator.scala deleted file mode 100644 index 729a40e94..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SocialGraphServiceFeatureHydrator.scala +++ /dev/null @@ -1,67 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.socialgraph.{thriftscala => sg} -import com.twitter.stitch.Stitch -import com.twitter.stitch.socialgraph.{SocialGraph => SocialGraphStitchClient} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class SocialGraphServiceFeatureHydrator @Inject() (socialGraphStitchClient: SocialGraphStitchClient) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("SocialGraphService") - - override val features: Set[Feature[_, _]] = Set(InNetworkFeature) - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - val viewerId = query.getRequiredUserId - - // We use authorId and not sourceAuthorId here so that retweets are defined as in network - val authorIds = candidates.map(_.features.getOrElse(AuthorIdFeature, None).getOrElse(0L)) - val distinctNonSelfAuthorIds = authorIds.filter(_ != viewerId).distinct - - val idsRequest = createIdsRequest( - userId = viewerId, - relationshipTypes = Set(sg.RelationshipType.Following), - targetIds = Some(distinctNonSelfAuthorIds) - ) - - socialGraphStitchClient - .ids(request = idsRequest, requestContext = None) - .map { idResult => - authorIds.map { authorId => - // Users cannot follow themselves but this is in network by definition - val isSelfTweet = authorId == viewerId - val inNetworkAuthorIds = idResult.ids.toSet - val isInNetwork = isSelfTweet || inNetworkAuthorIds.contains(authorId) || authorId == 0L - FeatureMapBuilder().add(InNetworkFeature, isInNetwork).build() - } - } - } - - private def createIdsRequest( - userId: Long, - relationshipTypes: Set[sg.RelationshipType], - targetIds: Option[Seq[Long]] = None - ): sg.IdsRequest = sg.IdsRequest( - relationshipTypes.map { relationshipType => - sg.SrcRelationship(userId, relationshipType, targets = targetIds) - }.toSeq, - Some(sg.PageRequest(selectAll = Some(true))) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TSPInferredTopicFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TSPInferredTopicFeatureHydrator.scala deleted file mode 100644 index 2e823ebd8..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TSPInferredTopicFeatureHydrator.scala +++ /dev/null @@ -1,162 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.contentrecommender.{thriftscala => cr} -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.inferred_topic.InferredTopicAdapter -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature -import com.twitter.home_mixer.model.HomeFeatures.TSPMetricTagFeature -import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature -import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContextFunctionalityType -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.strato.generated.client.topic_signals.tsp.TopicSocialProofClientColumn -import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => sid} -import com.twitter.topiclisting.TopicListingViewerContext -import com.twitter.tsp.{thriftscala => tsp} - -import javax.inject.Inject -import javax.inject.Singleton -import scala.collection.JavaConverters._ - -object TSPInferredTopicFeature extends Feature[TweetCandidate, Map[Long, Double]] -object TSPInferredTopicDataRecordFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class TSPInferredTopicFeatureHydrator @Inject() ( - topicSocialProofClientColumn: TopicSocialProofClientColumn, - statsReceiver: StatsReceiver, -) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TSPInferredTopic") - - override val features: Set[Feature[_, _]] = - Set( - TSPInferredTopicFeature, - TSPInferredTopicDataRecordFeature, - TopicIdSocialContextFeature, - TopicContextFunctionalityTypeFeature) - - private val topK = 3 - - private val sourcesToSetSocialProof: Set[sid.CandidateTweetSourceId] = Set( - sid.CandidateTweetSourceId.Simcluster, - sid.CandidateTweetSourceId.CroonTweet - ) - - private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) - private val keyFoundCounter = scopedStatsReceiver.counter("key/found") - private val keyLossCounter = scopedStatsReceiver.counter("key/loss") - private val requestFailCounter = scopedStatsReceiver.counter("request/fail") - - private val DefaultFeatureMap = FeatureMapBuilder() - .add(TSPInferredTopicFeature, Map.empty[Long, Double]) - .add(TSPInferredTopicDataRecordFeature, new DataRecord()) - .add(TopicIdSocialContextFeature, None) - .add(TopicContextFunctionalityTypeFeature, None) - .build() - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - val tags = candidates.collect { - case candidate if candidate.features.getTry(TSPMetricTagFeature).isReturn => - candidate.candidate.id -> candidate.features - .getOrElse(TSPMetricTagFeature, Set.empty[tsp.MetricTag]) - }.toMap - - val topicSocialProofRequest = - tsp.TopicSocialProofRequest( - userId = query.getRequiredUserId, - tweetIds = candidates.map(_.candidate.id).toSet, - displayLocation = cr.DisplayLocation.HomeTimeline, - topicListingSetting = tsp.TopicListingSetting.Followable, - context = TopicListingViewerContext.fromClientContext(query.clientContext).toThrift, - bypassModes = None, - // Only CRMixer source has this data. Convert the CRMixer metric tag to tsp metric tag. - tags = if (tags.isEmpty) None else Some(tags) - ) - - topicSocialProofClientColumn.fetcher - .fetch(topicSocialProofRequest) - .map(_.v) - .map { - case Some(response) => - candidates.map { candidate => - val topicWithScores = response.socialProofs.getOrElse(candidate.candidate.id, Seq.empty) - if (topicWithScores.nonEmpty) { - keyFoundCounter.incr() - val (socialProofId, socialProofFunctionalityType) = - if (candidate.features - .getOrElse(CandidateSourceIdFeature, None) - .exists(sourcesToSetSocialProof.contains)) { - getSocialProof(topicWithScores) - } else { - (None, None) - } - val inferredTopicFeatures = convertTopicWithScores(topicWithScores) - val inferredTopicDataRecord = - InferredTopicAdapter.adaptToDataRecords(inferredTopicFeatures).asScala.head - FeatureMapBuilder() - .add(TSPInferredTopicFeature, inferredTopicFeatures) - .add(TSPInferredTopicDataRecordFeature, inferredTopicDataRecord) - .add(TopicIdSocialContextFeature, socialProofId) - .add(TopicContextFunctionalityTypeFeature, socialProofFunctionalityType) - .build() - } else { - keyLossCounter.incr() - DefaultFeatureMap - } - } - case _ => - requestFailCounter.incr() - candidates.map { _ => - DefaultFeatureMap - } - } - } - - private def getSocialProof( - topicWithScores: Seq[tsp.TopicWithScore] - ): (Option[Long], Option[TopicContextFunctionalityType]) = { - val followingTopicId = topicWithScores - .collectFirst { - case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.Following)) => - topicId - } - if (followingTopicId.nonEmpty) { - return (followingTopicId, Some(BasicTopicContextFunctionalityType)) - } - val implicitFollowingId = topicWithScores.collectFirst { - case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.ImplicitFollow)) => - topicId - } - if (implicitFollowingId.nonEmpty) { - return (implicitFollowingId, Some(RecommendationTopicContextFunctionalityType)) - } - (None, None) - } - - private def convertTopicWithScores( - topicWithScores: Seq[tsp.TopicWithScore], - ): Map[Long, Double] = { - topicWithScores.sortBy(-_.score).take(topK).map(a => (a.topicId, a.score)).toMap - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TimeFeaturesHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TimeFeaturesHydrator.scala deleted file mode 100644 index 16661188a..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TimeFeaturesHydrator.scala +++ /dev/null @@ -1,251 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature -import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.RichDataRecord -import com.twitter.ml.api.util.FDsl._ -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.search.common.features.{thriftscala => sc} -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.stitch.Stitch -import com.twitter.timelines.prediction.features.time_features.AccountAgeInterval -import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures._ -import com.twitter.timelines.prediction.features.time_features.TimeFeatures -import com.twitter.util.Duration -import scala.collection.Searching._ - -object TimeFeaturesDataRecordFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -object TimeFeaturesHydrator extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TimeFeatures") - - override val features: Set[Feature[_, _]] = Set(TimeFeaturesDataRecordFeature) - - override def apply( - query: PipelineQuery, - candidate: TweetCandidate, - existingFeatures: FeatureMap - ): Stitch[FeatureMap] = { - Stitch.value { - val richDataRecord = new RichDataRecord() - setTimeFeatures(richDataRecord, candidate, existingFeatures, query) - FeatureMapBuilder() - .add(TimeFeaturesDataRecordFeature, richDataRecord.getRecord) - .build() - } - } - - private def setTimeFeatures( - richDataRecord: RichDataRecord, - candidate: TweetCandidate, - existingFeatures: FeatureMap, - query: PipelineQuery, - ): Unit = { - val timeFeaturesOpt = getTimeFeatures(query, candidate, existingFeatures) - timeFeaturesOpt.foreach(timeFeatures => setFeatures(timeFeatures, richDataRecord)) - } - - private[feature_hydrator] def getTimeFeatures( - query: PipelineQuery, - candidate: TweetCandidate, - existingFeatures: FeatureMap, - ): Option[TimeFeatures] = { - for { - requestTimestampMs <- Some(query.queryTime.inMilliseconds) - tweetId <- Some(candidate.id) - viewerId <- query.getOptionalUserId - tweetCreationTimeMs <- timeFromTweetOrUserId(tweetId) - timeSinceTweetCreation = requestTimestampMs - tweetCreationTimeMs - accountAgeDurationOpt = timeFromTweetOrUserId(viewerId).map { viewerAccountCreationTimeMs => - Duration.fromMilliseconds(requestTimestampMs - viewerAccountCreationTimeMs) - } - timeSinceSourceTweetCreation = - existingFeatures - .getOrElse(SourceTweetIdFeature, None) - .flatMap { sourceTweetId => - timeFromTweetOrUserId(sourceTweetId).map { sourceTweetCreationTimeMs => - requestTimestampMs - sourceTweetCreationTimeMs - } - } - .getOrElse(timeSinceTweetCreation) - if (timeSinceTweetCreation > 0 && timeSinceSourceTweetCreation > 0) - } yield { - val timeFeatures = TimeFeatures( - timeSinceTweetCreation = timeSinceTweetCreation, - timeSinceSourceTweetCreation = timeSinceSourceTweetCreation, - timeSinceViewerAccountCreationSecs = accountAgeDurationOpt.map(_.inSeconds), - isDay30NewUser = accountAgeDurationOpt.map(_ < 30.days).getOrElse(false), - isMonth12NewUser = accountAgeDurationOpt.map(_ < 365.days).getOrElse(false), - accountAgeInterval = accountAgeDurationOpt.flatMap(AccountAgeInterval.fromDuration), - isTweetRecycled = false // only set in RecyclableTweetCandidateFilter, but it's not used - ) - - val timeFeaturesWithLastEngagement = addLastEngagementTimeFeatures( - existingFeatures.getOrElse(EarlybirdFeature, None), - timeFeatures, - timeSinceSourceTweetCreation - ).getOrElse(timeFeatures) - - val nonPollingTimestampsMs = - query.features.map(_.getOrElse(NonPollingTimesFeature, Seq.empty)) - val timeFeaturesWithNonPollingOpt = addNonPollingTimeFeatures( - timeFeaturesWithLastEngagement, - requestTimestampMs, - tweetCreationTimeMs, - nonPollingTimestampsMs - ) - timeFeaturesWithNonPollingOpt.getOrElse(timeFeaturesWithLastEngagement) - } - } - - private def timeFromTweetOrUserId(tweetOrUserId: Long): Option[Long] = { - if (SnowflakeId.isSnowflakeId(tweetOrUserId)) - Some(SnowflakeId(tweetOrUserId).time.inMilliseconds) - else None - } - - private def addLastEngagementTimeFeatures( - tweetFeaturesOpt: Option[sc.ThriftTweetFeatures], - timeFeatures: TimeFeatures, - timeSinceSourceTweetCreation: Long - ): Option[TimeFeatures] = { - tweetFeaturesOpt.map { tweetFeatures => - val lastFavSinceCreationHrs = tweetFeatures.lastFavSinceCreationHrs.map(_.toDouble) - val lastRetweetSinceCreationHrs = tweetFeatures.lastRetweetSinceCreationHrs.map(_.toDouble) - val lastReplySinceCreationHrs = tweetFeatures.lastReplySinceCreationHrs.map(_.toDouble) - val lastQuoteSinceCreationHrs = tweetFeatures.lastQuoteSinceCreationHrs.map(_.toDouble) - - timeFeatures.copy( - lastFavSinceCreationHrs = lastFavSinceCreationHrs, - lastRetweetSinceCreationHrs = lastRetweetSinceCreationHrs, - lastReplySinceCreationHrs = lastReplySinceCreationHrs, - lastQuoteSinceCreationHrs = lastQuoteSinceCreationHrs, - timeSinceLastFavoriteHrs = getTimeSinceLastEngagementHrs( - lastFavSinceCreationHrs, - timeSinceSourceTweetCreation - ), - timeSinceLastRetweetHrs = getTimeSinceLastEngagementHrs( - lastRetweetSinceCreationHrs, - timeSinceSourceTweetCreation - ), - timeSinceLastReplyHrs = getTimeSinceLastEngagementHrs( - lastReplySinceCreationHrs, - timeSinceSourceTweetCreation - ), - timeSinceLastQuoteHrs = getTimeSinceLastEngagementHrs( - lastQuoteSinceCreationHrs, - timeSinceSourceTweetCreation - ) - ) - } - } - - private def addNonPollingTimeFeatures( - timeFeatures: TimeFeatures, - requestTimestampMs: Long, - creationTimeMs: Long, - nonPollingTimestampsMs: Option[Seq[Long]] - ): Option[TimeFeatures] = { - for { - nonPollingTimestampsMs <- nonPollingTimestampsMs - lastNonPollingTimestampMs <- nonPollingTimestampsMs.headOption - earliestNonPollingTimestampMs <- nonPollingTimestampsMs.lastOption - } yield { - val timeSinceLastNonPollingRequest = requestTimestampMs - lastNonPollingTimestampMs - val tweetAgeRatio = timeSinceLastNonPollingRequest / math.max( - 1.0, - timeFeatures.timeSinceTweetCreation - ) - /* - * Non-polling timestamps are stored in chronological order. - * The latest timestamps occur first, therefore we need to explicitly search in reverse order. - */ - val nonPollingRequestsSinceTweetCreation = - if (nonPollingTimestampsMs.nonEmpty) { - nonPollingTimestampsMs.search(creationTimeMs)(Ordering[Long].reverse).insertionPoint - } else { - 0 - } - /* - * Calculate the average time between non-polling requests; include - * request time in this calculation as latest timestamp. - */ - val timeBetweenNonPollingRequestsAvg = - (requestTimestampMs - earliestNonPollingTimestampMs) / math - .max(1.0, nonPollingTimestampsMs.size) - val timeFeaturesWithNonPolling = timeFeatures.copy( - timeBetweenNonPollingRequestsAvg = Some(timeBetweenNonPollingRequestsAvg), - timeSinceLastNonPollingRequest = Some(timeSinceLastNonPollingRequest), - nonPollingRequestsSinceTweetCreation = Some(nonPollingRequestsSinceTweetCreation), - tweetAgeRatio = Some(tweetAgeRatio) - ) - timeFeaturesWithNonPolling - } - } - - private[this] def getTimeSinceLastEngagementHrs( - lastEngagementTimeSinceCreationHrsOpt: Option[Double], - timeSinceTweetCreation: Long - ): Option[Double] = { - lastEngagementTimeSinceCreationHrsOpt.map { lastEngagementTimeSinceCreationHrs => - val timeSinceTweetCreationHrs = (timeSinceTweetCreation / (60 * 60 * 1000)).toInt - timeSinceTweetCreationHrs - lastEngagementTimeSinceCreationHrs - } - } - - private def setFeatures(features: TimeFeatures, richDataRecord: RichDataRecord): Unit = { - val record = richDataRecord.getRecord - .setFeatureValue(IS_TWEET_RECYCLED, features.isTweetRecycled) - .setFeatureValue(TIME_SINCE_TWEET_CREATION, features.timeSinceTweetCreation) - .setFeatureValueFromOption( - TIME_SINCE_VIEWER_ACCOUNT_CREATION_SECS, - features.timeSinceViewerAccountCreationSecs) - .setFeatureValue( - USER_ID_IS_SNOWFLAKE_ID, - features.timeSinceViewerAccountCreationSecs.isDefined - ) - .setFeatureValueFromOption(ACCOUNT_AGE_INTERVAL, features.accountAgeInterval.map(_.id.toLong)) - .setFeatureValue(IS_30_DAY_NEW_USER, features.isDay30NewUser) - .setFeatureValue(IS_12_MONTH_NEW_USER, features.isMonth12NewUser) - .setFeatureValueFromOption(LAST_FAVORITE_SINCE_CREATION_HRS, features.lastFavSinceCreationHrs) - .setFeatureValueFromOption( - LAST_RETWEET_SINCE_CREATION_HRS, - features.lastRetweetSinceCreationHrs - ) - .setFeatureValueFromOption(LAST_REPLY_SINCE_CREATION_HRS, features.lastReplySinceCreationHrs) - .setFeatureValueFromOption(LAST_QUOTE_SINCE_CREATION_HRS, features.lastQuoteSinceCreationHrs) - .setFeatureValueFromOption(TIME_SINCE_LAST_FAVORITE_HRS, features.timeSinceLastFavoriteHrs) - .setFeatureValueFromOption(TIME_SINCE_LAST_RETWEET_HRS, features.timeSinceLastRetweetHrs) - .setFeatureValueFromOption(TIME_SINCE_LAST_REPLY_HRS, features.timeSinceLastReplyHrs) - .setFeatureValueFromOption(TIME_SINCE_LAST_QUOTE_HRS, features.timeSinceLastQuoteHrs) - /* - * set features whose values are optional as some users do not have non-polling timestamps - */ - features.timeBetweenNonPollingRequestsAvg.foreach( - record.setFeatureValue(TIME_BETWEEN_NON_POLLING_REQUESTS_AVG, _) - ) - features.timeSinceLastNonPollingRequest.foreach( - record.setFeatureValue(TIME_SINCE_LAST_NON_POLLING_REQUEST, _) - ) - features.nonPollingRequestsSinceTweetCreation.foreach( - record.setFeatureValue(NON_POLLING_REQUESTS_SINCE_TWEET_CREATION, _) - ) - features.tweetAgeRatio.foreach(record.setFeatureValue(TWEET_AGE_RATIO, _)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetImpressionsQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetImpressionsQueryFeatureHydrator.scala index 4e585ec45..0fc6fd1f5 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetImpressionsQueryFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetImpressionsQueryFeatureHydrator.scala @@ -24,8 +24,8 @@ case class TweetImpressionsQueryFeatureHydrator[ manhattanTweetImpressionStoreClient: ManhattanTweetImpressionStoreClient) extends QueryFeatureHydrator[Query] { - private val TweetImpressionTTL = 1.day - private val TweetImpressionCap = 3000 + private val TweetImpressionTTL = 2.days + private val TweetImpressionCap = 5000 override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetImpressions") diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetypieFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetypieFeatureHydrator.scala index 09fc62405..f2effa1b2 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetypieFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetypieFeatureHydrator.scala @@ -1,6 +1,9 @@ package com.twitter.home_mixer.functional_component.feature_hydrator +import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature import com.twitter.home_mixer.model.HomeFeatures.IsNsfwFeature @@ -10,11 +13,13 @@ import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.ForYouProduct import com.twitter.home_mixer.model.request.ListTweetsProduct import com.twitter.home_mixer.model.request.ScoredTweetsProduct +import com.twitter.home_mixer.model.request.SubscribedProduct import com.twitter.home_mixer.util.tweetypie.RequestFields import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_is_nsfw.IsNsfw import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason.VisibilityReason @@ -29,17 +34,22 @@ import com.twitter.spam.rtf.{thriftscala => rtf} import com.twitter.stitch.Stitch import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient} import com.twitter.tweetypie.{thriftscala => tp} +import com.twitter.util.logging.Logging import javax.inject.Inject import javax.inject.Singleton @Singleton -class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitchClient) - extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] { +class TweetypieFeatureHydrator @Inject() ( + tweetypieStitchClient: TweetypieStitchClient, + statsReceiver: StatsReceiver) + extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] + with Logging { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Tweetypie") override val features: Set[Feature[_, _]] = Set( AuthorIdFeature, + ExclusiveConversationAuthorIdFeature, InReplyToTweetIdFeature, IsHydratedFeature, IsNsfw, @@ -51,6 +61,7 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch SourceTweetIdFeature, SourceUserIdFeature, TweetTextFeature, + TweetLanguageFeature, VisibilityReason ) @@ -70,9 +81,12 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch ): Stitch[FeatureMap] = { val safetyLevel = query.product match { case FollowingProduct => rtf.SafetyLevel.TimelineHomeLatest - case ForYouProduct => rtf.SafetyLevel.TimelineHome + case ForYouProduct => + val inNetwork = existingFeatures.getOrElse(InNetworkFeature, true) + if (inNetwork) rtf.SafetyLevel.TimelineHome else rtf.SafetyLevel.TimelineHomeRecommendations case ScoredTweetsProduct => rtf.SafetyLevel.TimelineHome case ListTweetsProduct => rtf.SafetyLevel.TimelineLists + case SubscribedProduct => rtf.SafetyLevel.TimelineHomeSubscribed case unknown => throw new UnsupportedOperationException(s"Unknown product: $unknown") } @@ -82,9 +96,12 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch includeQuotedTweet = true, visibilityPolicy = tp.TweetVisibilityPolicy.UserVisible, safetyLevel = Some(safetyLevel), - forUserId = Some(query.getRequiredUserId) + forUserId = query.getOptionalUserId ) + val exclusiveAuthorIdOpt = + existingFeatures.getOrElse(ExclusiveConversationAuthorIdFeature, None) + tweetypieStitchClient.getTweetFields(tweetId = candidate.id, options = tweetFieldsOptions).map { case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), quoteOpt, _) => val coreData = found.tweet.coreData @@ -106,6 +123,7 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch found.retweetedTweet.exists(_.coreData.exists(data => data.nsfwAdmin || data.nsfwUser)) val tweetText = coreData.map(_.text) + val tweetLanguage = found.tweet.language.map(_.language) val tweetAuthorId = coreData.map(_.userId) val inReplyToTweetId = coreData.flatMap(_.reply.flatMap(_.inReplyToStatusId)) @@ -127,6 +145,7 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch FeatureMapBuilder() .add(AuthorIdFeature, tweetAuthorId) + .add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt) .add(InReplyToTweetIdFeature, inReplyToTweetId) .add(IsHydratedFeature, true) .add(IsNsfw, Some(isNsfw)) @@ -137,20 +156,24 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch .add(QuotedUserIdFeature, quotedTweetUserId) .add(SourceTweetIdFeature, retweetedTweetId) .add(SourceUserIdFeature, retweetedTweetUserId) + .add(TweetLanguageFeature, tweetLanguage) .add(TweetTextFeature, tweetText) .add(VisibilityReason, found.suppressReason) .build() // If no tweet result found, return default and pre-existing features case _ => - DefaultFeatureMap + - (AuthorIdFeature, existingFeatures.getOrElse(AuthorIdFeature, None)) + - (InReplyToTweetIdFeature, existingFeatures.getOrElse(InReplyToTweetIdFeature, None)) + - (IsRetweetFeature, existingFeatures.getOrElse(IsRetweetFeature, false)) + - (QuotedTweetIdFeature, existingFeatures.getOrElse(QuotedTweetIdFeature, None)) + - (QuotedUserIdFeature, existingFeatures.getOrElse(QuotedUserIdFeature, None)) + - (SourceTweetIdFeature, existingFeatures.getOrElse(SourceTweetIdFeature, None)) + - (SourceUserIdFeature, existingFeatures.getOrElse(SourceUserIdFeature, None)) + DefaultFeatureMap ++ FeatureMapBuilder() + .add(AuthorIdFeature, existingFeatures.getOrElse(AuthorIdFeature, None)) + .add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt) + .add(InReplyToTweetIdFeature, existingFeatures.getOrElse(InReplyToTweetIdFeature, None)) + .add(IsRetweetFeature, existingFeatures.getOrElse(IsRetweetFeature, false)) + .add(QuotedTweetIdFeature, existingFeatures.getOrElse(QuotedTweetIdFeature, None)) + .add(QuotedUserIdFeature, existingFeatures.getOrElse(QuotedUserIdFeature, None)) + .add(SourceTweetIdFeature, existingFeatures.getOrElse(SourceTweetIdFeature, None)) + .add(SourceUserIdFeature, existingFeatures.getOrElse(SourceUserIdFeature, None)) + .add(TweetLanguageFeature, existingFeatures.getOrElse(TweetLanguageFeature, None)) + .build() } } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features/AuthorFeaturesAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features/AuthorFeaturesAdapter.scala deleted file mode 100644 index 06fc42a53..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features/AuthorFeaturesAdapter.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.author_features - -import com.twitter.ml.api.DataRecordMerger -import com.twitter.ml.api.Feature -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.RichDataRecord -import com.twitter.ml.api.util.CompactDataRecordConverter -import com.twitter.ml.api.util.FDsl._ -import com.twitter.timelines.author_features.v1.{thriftjava => af} -import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase -import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig -import com.twitter.timelines.prediction.features.user_health.UserHealthFeatures - -object AuthorFeaturesAdapter extends TimelinesMutatingAdapterBase[Option[af.AuthorFeatures]] { - - private val originalAuthorAggregatesFeatures = - TimelinesAggregationConfig.originalAuthorReciprocalEngagementAggregates - .buildTypedAggregateGroups().flatMap(_.allOutputFeatures) - private val authorFeatures = originalAuthorAggregatesFeatures ++ - Seq( - UserHealthFeatures.AuthorState, - UserHealthFeatures.NumAuthorFollowers, - UserHealthFeatures.NumAuthorConnectDays, - UserHealthFeatures.NumAuthorConnect) - private val featureContext = new FeatureContext(authorFeatures: _*) - - override def getFeatureContext: FeatureContext = featureContext - - override val commonFeatures: Set[Feature[_]] = Set.empty - - private val compactDataRecordConverter = new CompactDataRecordConverter() - private val drMerger = new DataRecordMerger() - - override def setFeatures( - authorFeaturesOpt: Option[af.AuthorFeatures], - richDataRecord: RichDataRecord - ): Unit = { - authorFeaturesOpt.foreach { authorFeatures => - val dataRecord = richDataRecord.getRecord - - dataRecord.setFeatureValue( - UserHealthFeatures.AuthorState, - authorFeatures.user_health.user_state.getValue.toLong) - dataRecord.setFeatureValue( - UserHealthFeatures.NumAuthorFollowers, - authorFeatures.user_health.num_followers.toDouble) - dataRecord.setFeatureValue( - UserHealthFeatures.NumAuthorConnectDays, - authorFeatures.user_health.num_connect_days.toDouble) - dataRecord.setFeatureValue( - UserHealthFeatures.NumAuthorConnect, - authorFeatures.user_health.num_connect.toDouble) - - val originalAuthorAggregatesDataRecord = - compactDataRecordConverter.compactDataRecordToDataRecord(authorFeatures.aggregates) - drMerger.merge(dataRecord, originalAuthorAggregatesDataRecord) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/Phase2EdgeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/Phase2EdgeAggregateFeatureHydrator.scala deleted file mode 100644 index fb5c9e2fb..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/Phase2EdgeAggregateFeatureHydrator.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates - -import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures._ -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class Phase2EdgeAggregateFeatureHydrator @Inject() extends BaseEdgeAggregateFeatureHydrator { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("Phase2EdgeAggregate") - - override val aggregateFeatures: Set[BaseEdgeAggregateFeature] = - Set( - UserEngagerAggregateFeature, - UserEngagerGoodClickAggregateFeature, - UserInferredTopicAggregateFeature, - UserTopicAggregateFeature, - UserMediaUnderstandingAnnotationAggregateFeature - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/BUILD.bazel index 00885f70a..59284224b 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/BUILD.bazel @@ -4,6 +4,7 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", @@ -16,9 +17,11 @@ scala_library( "src/thrift/com/twitter/tweetypie:service-scala", "src/thrift/com/twitter/tweetypie:tweet-scala", "stitch/stitch-core", + "stitch/stitch-socialgraph", "stitch/stitch-tweetypie", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", + "util/util-slf4j-api/src/main/scala", ], exports = [ "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/FeedbackFatigueFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/FeedbackFatigueFilter.scala index 5b6541f51..582583e7f 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/FeedbackFatigueFilter.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/FeedbackFatigueFilter.scala @@ -72,7 +72,7 @@ object FeedbackFatigueFilter originalAuthorId.exists(authorsToFilter.contains) || (likers.nonEmpty && eligibleLikers.isEmpty) || - (followers.nonEmpty && eligibleFollowers.isEmpty) || + (followers.nonEmpty && eligibleFollowers.isEmpty && likers.isEmpty) || (candidate.features.getOrElse(IsRetweetFeature, false) && authorId.exists(retweetersToFilter.contains)) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidSubscriptionTweetFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidSubscriptionTweetFilter.scala new file mode 100644 index 000000000..285eec49f --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidSubscriptionTweetFilter.scala @@ -0,0 +1,70 @@ +package com.twitter.home_mixer.functional_component.filter + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finagle.tracing.Trace +import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.socialgraph.{thriftscala => sg} +import com.twitter.stitch.Stitch +import com.twitter.stitch.socialgraph.SocialGraph +import com.twitter.util.logging.Logging + +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Exclude invalid subscription tweets - cases where the viewer is not subscribed to the author + * + * If SGS hydration fails, `SGSInvalidSubscriptionTweetFeature` will be set to None for + * subscription tweets, so we explicitly filter those tweets out. + */ +@Singleton +case class InvalidSubscriptionTweetFilter @Inject() ( + socialGraphClient: SocialGraph, + statsReceiver: StatsReceiver) + extends Filter[PipelineQuery, TweetCandidate] + with Logging { + + override val identifier: FilterIdentifier = FilterIdentifier("InvalidSubscriptionTweet") + + private val scopedStatsReceiver = statsReceiver.scope(identifier.toString) + private val validCounter = scopedStatsReceiver.counter("validExclusiveTweet") + private val invalidCounter = scopedStatsReceiver.counter("invalidExclusiveTweet") + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = Stitch + .traverse(candidates) { candidate => + val exclusiveAuthorId = + candidate.features.getOrElse(ExclusiveConversationAuthorIdFeature, None) + + if (exclusiveAuthorId.isDefined) { + val request = sg.ExistsRequest( + source = query.getRequiredUserId, + target = exclusiveAuthorId.get, + relationships = + Seq(sg.Relationship(sg.RelationshipType.TierOneSuperFollowing, hasRelationship = true)), + ) + socialGraphClient.exists(request).map(_.exists).map { valid => + if (!valid) invalidCounter.incr() else validCounter.incr() + valid + } + } else Stitch.value(true) + }.map { validResults => + val (kept, removed) = candidates + .map(_.candidate) + .zip(validResults) + .partition { case (candidate, valid) => valid } + + val keptCandidates = kept.map { case (candidate, _) => candidate } + val removedCandidates = removed.map { case (candidate, _) => candidate } + + FilterResult(kept = keptCandidates, removed = removedCandidates) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PredicateFeatureFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PredicateFeatureFilter.scala deleted file mode 100644 index ae1fe9f49..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PredicateFeatureFilter.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.twitter.home_mixer.functional_component.filter - -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -/** - * Predicate which will be applied to each candidate. True indicates that the candidate will be - * @tparam Candidate - the type of the candidate - */ -trait ShouldKeepCandidate { - def apply(features: FeatureMap): Boolean -} - -object PredicateFeatureFilter { - - /** - * Builds a simple Filter out of a predicate function from the candidate to a boolean. For clarity, - * we recommend including the name of the shouldKeepCandidate parameter. - * - * @param identifier A FilterIdentifier for the new filter - * @param shouldKeepCandidate A predicate function. Candidates will be kept when - * this function returns True. - */ - def fromPredicate[Candidate <: UniversalNoun[Any]]( - identifier: FilterIdentifier, - shouldKeepCandidate: ShouldKeepCandidate - ): Filter[PipelineQuery, Candidate] = { - val i = identifier - - new Filter[PipelineQuery, Candidate] { - override val identifier: FilterIdentifier = i - - /** - * Filter the list of candidates - * - * @return a FilterResult including both the list of kept candidate and the list of removed candidates - */ - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[Candidate]] - ): Stitch[FilterResult[Candidate]] = { - val allowedIds = candidates - .filter(candidate => shouldKeepCandidate(candidate.features)).map(_.candidate.id).toSet - - val (keptCandidates, removedCandidates) = candidates.map(_.candidate).partition { - candidate => allowedIds.contains(candidate.id) - } - - Stitch.value(FilterResult(kept = keptCandidates, removed = removedCandidates)) - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/KeepBestOutOfNetworkTweetPerAuthorFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetPreviewsFilter.scala similarity index 55% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/KeepBestOutOfNetworkTweetPerAuthorFilter.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetPreviewsFilter.scala index 87c324b89..0ff820479 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/KeepBestOutOfNetworkTweetPerAuthorFilter.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetPreviewsFilter.scala @@ -1,8 +1,6 @@ package com.twitter.home_mixer.functional_component.filter -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedTweetPreviewIdsFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult @@ -11,24 +9,20 @@ import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch -object KeepBestOutOfNetworkTweetPerAuthorFilter extends Filter[PipelineQuery, TweetCandidate] { +object PreviouslyServedTweetPreviewsFilter extends Filter[PipelineQuery, TweetCandidate] { - override val identifier: FilterIdentifier = FilterIdentifier("KeepBestOutOfNetworkTweetPerAuthor") + override val identifier: FilterIdentifier = FilterIdentifier("PreviouslyServedTweetPreviews") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { - // Set containing best OON tweet for each authorId - val bestCandidatesForAuthorId = candidates - .filter(!_.features.getOrElse(InNetworkFeature, true)) - .groupBy(_.features.getOrElse(AuthorIdFeature, None)) - .values.map(_.maxBy(_.features.getOrElse(ScoreFeature, None))) - .toSet + + val servedTweetPreviewIds = + query.features.map(_.getOrElse(ServedTweetPreviewIdsFeature, Seq.empty)).toSeq.flatten.toSet val (removed, kept) = candidates.partition { candidate => - !candidate.features.getOrElse(InNetworkFeature, true) && - !bestCandidatesForAuthorId.contains(candidate) + servedTweetPreviewIds.contains(candidate.candidate.id) } Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ReplyFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ReplyFilter.scala new file mode 100644 index 000000000..a1b99df66 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ReplyFilter.scala @@ -0,0 +1,32 @@ +package com.twitter.home_mixer.functional_component.filter + +import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +object ReplyFilter extends Filter[PipelineQuery, TweetCandidate] { + override val identifier: FilterIdentifier = FilterIdentifier("Reply") + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + + val (kept, removed) = candidates + .partition { candidate => + candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty + } + + val filterResult = FilterResult( + kept = kept.map(_.candidate), + removed = removed.map(_.candidate) + ) + + Stitch.value(filterResult) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetFilter.scala new file mode 100644 index 000000000..0ffc4b00c --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetFilter.scala @@ -0,0 +1,32 @@ +package com.twitter.home_mixer.functional_component.filter + +import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +object RetweetFilter extends Filter[PipelineQuery, TweetCandidate] { + override val identifier: FilterIdentifier = FilterIdentifier("Retweet") + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + + val (kept, removed) = candidates + .partition { candidate => + !candidate.features.getOrElse(IsRetweetFeature, false) + } + + val filterResult = FilterResult( + kept = kept.map(_.candidate), + removed = removed.map(_.candidate) + ) + + Stitch.value(filterResult) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/BUILD.bazel index d3b77dda8..6be06dee9 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/BUILD.bazel @@ -14,7 +14,6 @@ scala_library( "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", "src/thrift/com/twitter/gizmoduck:thrift-scala", "stitch/stitch-socialgraph", - "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", ], diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/DismissFatigueGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/DismissFatigueGate.scala index c97a3931e..b35a6cda5 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/DismissFatigueGate.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/DismissFatigueGate.scala @@ -1,14 +1,14 @@ package com.twitter.home_mixer.functional_component.gate +import com.twitter.conversions.DurationOps._ +import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelinemixer.clients.manhattan.DismissInfo import com.twitter.stitch.Stitch -import com.twitter.util.Duration -import com.twitter.conversions.DurationOps._ -import com.twitter.product_mixer.core.feature.Feature +import com.twitter.timelinemixer.clients.manhattan.DismissInfo import com.twitter.timelineservice.suggests.thriftscala.SuggestType +import com.twitter.util.Duration object DismissFatigueGate { // how long a dismiss action from user needs to be respected diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/NonEmptySeqFeatureGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/NonEmptySeqFeatureGate.scala deleted file mode 100644 index ef3bc5cff..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/NonEmptySeqFeatureGate.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.home_mixer.functional_component.gate - -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import scala.reflect.runtime.universe._ - -case class NonEmptySeqFeatureGate[T: TypeTag]( - feature: Feature[PipelineQuery, Seq[T]]) - extends Gate[PipelineQuery] { - - override val identifier: GateIdentifier = GateIdentifier(s"NonEmptySeq$feature") - - override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = - Stitch.value(query.features.exists(_.get(feature).nonEmpty)) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer/EditedTweetsCandidatePipelineQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer/EditedTweetsCandidatePipelineQueryTransformer.scala index 731f7166e..8753f2f28 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer/EditedTweetsCandidatePipelineQueryTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer/EditedTweetsCandidatePipelineQueryTransformer.scala @@ -19,7 +19,7 @@ object EditedTweetsCandidatePipelineQueryTransformer override val identifier: TransformerIdentifier = TransformerIdentifier("EditedTweets") // The time window for which a tweet remains editable after creation. - private val EditTimeWindow = 30.minutes + private val EditTimeWindow = 60.minutes override def transform(query: PipelineQuery): Seq[Long] = { val applicableCandidates = getApplicableCandidates(query) @@ -29,8 +29,8 @@ object EditedTweetsCandidatePipelineQueryTransformer // Any tweets in it could have become stale since being served. val previousTimelineLoadTime = applicableCandidates.head.servedTime - // The time window for editing a tweet is 30 minutes, - // so we ignore responses older than (PTL Time - 30 mins). + // The time window for editing a tweet is 60 minutes, + // so we ignore responses older than (PTL Time - 60 mins). val inWindowCandidates: Seq[PersistenceStoreEntry] = applicableCandidates .takeWhile(_.servedTime.until(previousTimelineLoadTime) < EditTimeWindow) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/BUILD.bazel index 94a648e3a..99605204c 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/BUILD.bazel @@ -6,7 +6,6 @@ scala_library( "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/FeedbackFatigueScorer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/FeedbackFatigueScorer.scala index 98e7aeb64..ceb71139e 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/FeedbackFatigueScorer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/FeedbackFatigueScorer.scala @@ -35,8 +35,8 @@ object FeedbackFatigueScorer override def onlyIf(query: PipelineQuery): Boolean = query.features.exists(_.getOrElse(FeedbackHistoryFeature, Seq.empty).nonEmpty) - private val DurationForFiltering = 14.days - private val DurationForDiscounting = 140.days + val DurationForFiltering = 14.days + val DurationForDiscounting = 140.days private val ScoreMultiplierLowerBound = 0.2 private val ScoreMultiplierUpperBound = 1.0 private val ScoreMultiplierIncrementsCount = 4 @@ -76,42 +76,56 @@ object FeedbackFatigueScorer feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Retweet, Seq.empty)) val featureMaps = candidates.map { candidate => + val multiplier = getScoreMultiplier( + candidate, + authorsToDiscount, + likersToDiscount, + followersToDiscount, + retweetersToDiscount + ) val score = candidate.features.getOrElse(ScoreFeature, None) - - val originalAuthorId = - CandidatesUtil.getOriginalAuthorId(candidate.features).getOrElse(0L) - val originalAuthorMultiplier = authorsToDiscount.getOrElse(originalAuthorId, 1.0) - - val likers = candidate.features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty) - val likerMultipliers = likers.flatMap(likersToDiscount.get) - val likerMultiplier = - if (likerMultipliers.nonEmpty && likers.size == likerMultipliers.size) - likerMultipliers.max - else 1.0 - - val followers = candidate.features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty) - val followerMultipliers = followers.flatMap(followersToDiscount.get) - val followerMultiplier = - if (followerMultipliers.nonEmpty && followers.size == followerMultipliers.size) - followerMultipliers.max - else 1.0 - - val authorId = candidate.features.getOrElse(AuthorIdFeature, None).getOrElse(0L) - val retweeterMultiplier = - if (candidate.features.getOrElse(IsRetweetFeature, false)) - retweetersToDiscount.getOrElse(authorId, 1.0) - else 1.0 - - val multiplier = - originalAuthorMultiplier * likerMultiplier * followerMultiplier * retweeterMultiplier - FeatureMapBuilder().add(ScoreFeature, score.map(_ * multiplier)).build() } Stitch.value(featureMaps) } - private def getUserDiscounts( + def getScoreMultiplier( + candidate: CandidateWithFeatures[TweetCandidate], + authorsToDiscount: Map[Long, Double], + likersToDiscount: Map[Long, Double], + followersToDiscount: Map[Long, Double], + retweetersToDiscount: Map[Long, Double], + ): Double = { + val originalAuthorId = + CandidatesUtil.getOriginalAuthorId(candidate.features).getOrElse(0L) + val originalAuthorMultiplier = authorsToDiscount.getOrElse(originalAuthorId, 1.0) + + val likers = candidate.features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty) + val likerMultipliers = likers.flatMap(likersToDiscount.get) + val likerMultiplier = + if (likerMultipliers.nonEmpty && likers.size == likerMultipliers.size) + likerMultipliers.max + else 1.0 + + val followers = candidate.features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty) + val followerMultipliers = followers.flatMap(followersToDiscount.get) + val followerMultiplier = + if (followerMultipliers.nonEmpty && followers.size == followerMultipliers.size && + likers.isEmpty) + followerMultipliers.max + else 1.0 + + val authorId = candidate.features.getOrElse(AuthorIdFeature, None).getOrElse(0L) + val retweeterMultiplier = + if (candidate.features.getOrElse(IsRetweetFeature, false)) + retweetersToDiscount.getOrElse(authorId, 1.0) + else 1.0 + + originalAuthorMultiplier * likerMultiplier * followerMultiplier * retweeterMultiplier + } + + def getUserDiscounts( queryTime: Time, feedbackEntries: Seq[FeedbackEntry], ): Map[Long, Double] = { diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/VerifiedAuthorScalingScorer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/VerifiedAuthorScalingScorer.scala deleted file mode 100644 index 3644b7d36..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/VerifiedAuthorScalingScorer.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.home_mixer.functional_component.scorer - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature -import com.twitter.home_mixer.param.HomeGlobalParams.BlueVerifiedAuthorInNetworkMultiplierParam -import com.twitter.home_mixer.param.HomeGlobalParams.BlueVerifiedAuthorOutOfNetworkMultiplierParam -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -/** - * Scales scores of tweets whose author is Blue Verified by the provided scale factor - */ -object VerifiedAuthorScalingScorer extends Scorer[PipelineQuery, TweetCandidate] { - - override val identifier: ScorerIdentifier = ScorerIdentifier("VerifiedAuthorScaling") - - override val features: Set[Feature[_, _]] = Set(ScoreFeature) - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - Stitch.value { - candidates.map { candidate => - val score = candidate.features.getOrElse(ScoreFeature, None) - val updatedScore = getUpdatedScore(score, candidate, query) - FeatureMapBuilder().add(ScoreFeature, updatedScore).build() - } - } - } - - /** - * We should only be applying this multiplier if the author of the candidate is Blue Verified. - * We also treat In-Network vs Out-of-Network differently. - */ - private def getUpdatedScore( - score: Option[Double], - candidate: CandidateWithFeatures[TweetCandidate], - query: PipelineQuery - ): Option[Double] = { - val isAuthorBlueVerified = candidate.features.getOrElse(AuthorIsBlueVerifiedFeature, false) - - if (isAuthorBlueVerified) { - val isCandidateInNetwork = candidate.features.getOrElse(InNetworkFeature, false) - - val scaleFactor = - if (isCandidateInNetwork) query.params(BlueVerifiedAuthorInNetworkMultiplierParam) - else query.params(BlueVerifiedAuthorOutOfNetworkMultiplierParam) - - score.map(_ * scaleFactor) - } else score - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/BUILD.bazel index 0f8ac903b..3689de6ea 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/BUILD.bazel @@ -5,6 +5,7 @@ scala_library( tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateHomeClientEventDetails.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateHomeClientEventDetails.scala index 28968e64d..c7944b506 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateHomeClientEventDetails.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateHomeClientEventDetails.scala @@ -1,6 +1,6 @@ package com.twitter.home_mixer.functional_component.selector -import com.twitter.home_mixer.functional_component.decorator.HomeClientEventDetailsBuilder +import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventDetailsBuilder import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature import com.twitter.home_mixer.model.HomeFeatures.ConversationModule2DisplayedTweetsFeature import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleHasGapFeature diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/BUILD.bazel index c9c0b343c..f352ce63a 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/BUILD.bazel @@ -7,14 +7,12 @@ scala_library( "3rdparty/jvm/javax/inject:javax.inject", "eventbus/client/src/main/scala/com/twitter/eventbus/client", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging", "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "kafka/finagle-kafka/finatra-kafka/src/main/scala", @@ -22,6 +20,7 @@ scala_library( "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_subscribe_module", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", "src/scala/com/twitter/timelines/prediction/common/adapters", @@ -33,13 +32,12 @@ scala_library( "src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java", "src/thrift/com/twitter/timelines/timeline_logging:thrift-scala", "src/thrift/com/twitter/user_session_store:thrift-scala", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/core", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/uss", "timelines/ml:kafka", "timelines/ml/cont_train/common/client/src/main/scala/com/twitter/timelines/ml/cont_train/common/client/kafka", "timelines/ml/cont_train/common/domain/src/main/scala/com/twitter/timelines/ml/cont_train/common/domain/non_scalding", "timelines/src/main/scala/com/twitter/timelines/clientconfig", + "timelines/src/main/scala/com/twitter/timelines/clients/manhattan/store", "timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter", "timelines/src/main/scala/com/twitter/timelines/impressionstore/store", "timelines/src/main/scala/com/twitter/timelines/injection/scribe", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ClientEventsBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ClientEventsBuilder.scala index a0233e64e..a5cf739d3 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ClientEventsBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ClientEventsBuilder.scala @@ -2,13 +2,14 @@ package com.twitter.home_mixer.functional_component.side_effect import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.functional_component.decorator.HomeQueryTypePredicates -import com.twitter.home_mixer.functional_component.decorator.HomeTweetTypePredicates +import com.twitter.home_mixer.functional_component.decorator.builder.HomeTweetTypePredicates import com.twitter.home_mixer.model.HomeFeatures.AccountAgeFeature import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature import com.twitter.home_mixer.model.HomeFeatures.VideoDurationMsFeature import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.ForYouProduct import com.twitter.home_mixer.model.request.ListTweetsProduct +import com.twitter.home_mixer.model.request.SubscribedProduct import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.ClientEvent import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.EventNamespace import com.twitter.product_mixer.core.feature.featuremap.FeatureMap @@ -18,15 +19,17 @@ import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelines.injection.scribe.InjectionScribeUtil private[side_effect] sealed trait ClientEventsBuilder { - private val FollowingSection = Some("home_latest") + private val FollowingSection = Some("latest") private val ForYouSection = Some("home") private val ListTweetsSection = Some("list") + private val SubscribedSection = Some("subscribed") protected def section(query: PipelineQuery): Option[String] = { query.product match { case FollowingProduct => FollowingSection case ForYouProduct => ForYouSection case ListTweetsProduct => ListTweetsSection + case SubscribedProduct => SubscribedSection case other => throw new UnsupportedOperationException(s"Unknown product: $other") } } @@ -52,6 +55,7 @@ private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder { private val InjectedComponent = Some("injected") private val PromotedComponent = Some("promoted") private val WhoToFollowComponent = Some("who_to_follow") + private val WhoToSubscribeComponent = Some("who_to_subscribe") private val WithVideoDurationComponent = Some("with_video_duration") private val VideoDurationSumElement = Some("video_duration_sum") private val NumVideosElement = Some("num_videos") @@ -60,7 +64,8 @@ private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder { query: PipelineQuery, injectedTweets: Seq[ItemCandidateWithDetails], promotedTweets: Seq[ItemCandidateWithDetails], - whoToFollowUsers: Seq[ItemCandidateWithDetails] + whoToFollowUsers: Seq[ItemCandidateWithDetails], + whoToSubscribeUsers: Seq[ItemCandidateWithDetails] ): Seq[ClientEvent] = { val baseEventNamespace = EventNamespace( section = section(query), @@ -77,6 +82,9 @@ private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder { ClientEvent( baseEventNamespace.copy(component = WhoToFollowComponent, action = ServedUsersAction), eventValue = count(whoToFollowUsers)), + ClientEvent( + baseEventNamespace.copy(component = WhoToSubscribeComponent, action = ServedUsersAction), + eventValue = count(whoToSubscribeUsers)), ) val tweetTypeServedEvents = HomeTweetTypePredicates.PredicateMap.map { diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeClientEventSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeClientEventSideEffect.scala index 83f434add..a8feef707 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeClientEventSideEffect.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeClientEventSideEffect.scala @@ -5,6 +5,7 @@ import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.logpipeline.client.common.EventPublisher import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails @@ -15,16 +16,30 @@ import com.twitter.product_mixer.core.pipeline.PipelineQuery * Side effect that logs served tweet metrics to Scribe as client events. */ case class HomeScribeClientEventSideEffect( + enableScribeClientEvents: Boolean, override val logPipelinePublisher: EventPublisher[LogEvent], injectedTweetsCandidatePipelineIdentifiers: Seq[CandidatePipelineIdentifier], - adsCandidatePipelineIdentifier: CandidatePipelineIdentifier, + adsCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None, whoToFollowCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None, -) extends ScribeClientEventSideEffect[PipelineQuery, Timeline] { + whoToSubscribeCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None) + extends ScribeClientEventSideEffect[PipelineQuery, Timeline] + with PipelineResultSideEffect.Conditionally[ + PipelineQuery, + Timeline + ] { override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeClientEvent") override val page = "timelinemixer" + override def onlyIf( + query: PipelineQuery, + selectedCandidates: Seq[CandidateWithDetails], + remainingCandidates: Seq[CandidateWithDetails], + droppedCandidates: Seq[CandidateWithDetails], + response: Timeline + ): Boolean = enableScribeClientEvents + override def buildClientEvents( query: PipelineQuery, selectedCandidates: Seq[CandidateWithDetails], @@ -37,13 +52,15 @@ case class HomeScribeClientEventSideEffect( val sources = itemCandidates.groupBy(_.source) val injectedTweets = injectedTweetsCandidatePipelineIdentifiers.flatMap(sources.getOrElse(_, Seq.empty)) - val promotedTweets = sources.getOrElse(adsCandidatePipelineIdentifier, Seq.empty) + val promotedTweets = adsCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten - // WhoToFollow module is not required for all home-mixer products, e.g. list tweets timeline. + // WhoToFollow and WhoToSubscribe modules are not required for all home-mixer products, e.g. list tweets timeline. val whoToFollowUsers = whoToFollowCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten + val whoToSubscribeUsers = + whoToSubscribeCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten val servedEvents = ServedEventsBuilder - .build(query, injectedTweets, promotedTweets, whoToFollowUsers) + .build(query, injectedTweets, promotedTweets, whoToFollowUsers, whoToSubscribeUsers) val emptyTimelineEvents = EmptyTimelineEventsBuilder.build(query, injectedTweets) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeServedCandidatesSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeServedCandidatesSideEffect.scala new file mode 100644 index 000000000..3f19dfc99 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeServedCandidatesSideEffect.scala @@ -0,0 +1,245 @@ +package com.twitter.home_mixer.functional_component.side_effect + +import com.twitter.finagle.tracing.Trace +import com.twitter.home_mixer.marshaller.timeline_logging.PromotedTweetDetailsMarshaller +import com.twitter.home_mixer.marshaller.timeline_logging.TweetDetailsMarshaller +import com.twitter.home_mixer.marshaller.timeline_logging.WhoToFollowDetailsMarshaller +import com.twitter.home_mixer.model.HomeFeatures.GetInitialFeature +import com.twitter.home_mixer.model.HomeFeatures.GetMiddleFeature +import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature +import com.twitter.home_mixer.model.HomeFeatures.GetOlderFeature +import com.twitter.home_mixer.model.HomeFeatures.HasDarkRequestFeature +import com.twitter.home_mixer.model.HomeFeatures.RequestJoinIdFeature +import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature +import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature +import com.twitter.home_mixer.model.request.DeviceContext.RequestContext +import com.twitter.home_mixer.model.request.HasDeviceContext +import com.twitter.home_mixer.model.request.HasSeenTweetIds +import com.twitter.home_mixer.model.request.FollowingProduct +import com.twitter.home_mixer.model.request.ForYouProduct +import com.twitter.home_mixer.model.request.SubscribedProduct +import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeServedCandidatesFlag +import com.twitter.home_mixer.param.HomeGlobalParams.EnableScribeServedCandidatesParam +import com.twitter.home_mixer.service.HomeMixerAlertConfig +import com.twitter.inject.annotations.Flag +import com.twitter.logpipeline.client.common.EventPublisher +import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate +import com.twitter.product_mixer.component_library.model.candidate.BaseUserCandidate +import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator +import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidateDecorator +import com.twitter.product_mixer.component_library.side_effect.ScribeLogEventSideEffect +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect +import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails +import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails +import com.twitter.product_mixer.core.model.marshalling.response.urt.AddEntriesTimelineInstruction +import com.twitter.product_mixer.core.model.marshalling.response.urt.ModuleItem +import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline +import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule +import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem +import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.timelines.timeline_logging.{thriftscala => thrift} +import com.twitter.util.Time +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Side effect that logs home timeline served candidates to Scribe. + */ +@Singleton +class HomeScribeServedCandidatesSideEffect @Inject() ( + @Flag(ScribeServedCandidatesFlag) enableScribeServedCandidates: Boolean, + scribeEventPublisher: EventPublisher[thrift.ServedEntry]) + extends ScribeLogEventSideEffect[ + thrift.ServedEntry, + PipelineQuery with HasSeenTweetIds with HasDeviceContext, + Timeline + ] + with PipelineResultSideEffect.Conditionally[ + PipelineQuery with HasSeenTweetIds with HasDeviceContext, + Timeline + ] { + + override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeServedCandidates") + + override def onlyIf( + query: PipelineQuery with HasSeenTweetIds with HasDeviceContext, + selectedCandidates: Seq[CandidateWithDetails], + remainingCandidates: Seq[CandidateWithDetails], + droppedCandidates: Seq[CandidateWithDetails], + response: Timeline + ): Boolean = enableScribeServedCandidates && query.params(EnableScribeServedCandidatesParam) + + override def buildLogEvents( + query: PipelineQuery with HasSeenTweetIds with HasDeviceContext, + selectedCandidates: Seq[CandidateWithDetails], + remainingCandidates: Seq[CandidateWithDetails], + droppedCandidates: Seq[CandidateWithDetails], + response: Timeline + ): Seq[thrift.ServedEntry] = { + val timelineType = query.product match { + case FollowingProduct => thrift.TimelineType.HomeLatest + case ForYouProduct => thrift.TimelineType.Home + case SubscribedProduct => thrift.TimelineType.HomeSubscribed + case other => throw new UnsupportedOperationException(s"Unknown product: $other") + } + val requestProvenance = query.deviceContext.map { deviceContext => + deviceContext.requestContextValue match { + case RequestContext.Foreground => thrift.RequestProvenance.Foreground + case RequestContext.Launch => thrift.RequestProvenance.Launch + case RequestContext.PullToRefresh => thrift.RequestProvenance.Ptr + case _ => thrift.RequestProvenance.Other + } + } + val queryType = query.features.map { featureMap => + if (featureMap.getOrElse(GetOlderFeature, false)) thrift.QueryType.GetOlder + else if (featureMap.getOrElse(GetNewerFeature, false)) thrift.QueryType.GetNewer + else if (featureMap.getOrElse(GetMiddleFeature, false)) thrift.QueryType.GetMiddle + else if (featureMap.getOrElse(GetInitialFeature, false)) thrift.QueryType.GetInitial + else thrift.QueryType.Other + } + val requestInfo = thrift.RequestInfo( + requestTimeMs = query.queryTime.inMilliseconds, + traceId = Trace.id.traceId.toLong, + userId = query.getOptionalUserId, + clientAppId = query.clientContext.appId, + hasDarkRequest = query.features.flatMap(_.getOrElse(HasDarkRequestFeature, None)), + parentId = Some(Trace.id.parentId.toLong), + spanId = Some(Trace.id.spanId.toLong), + timelineType = Some(timelineType), + ipAddress = query.clientContext.ipAddress, + userAgent = query.clientContext.userAgent, + queryType = queryType, + requestProvenance = requestProvenance, + languageCode = query.clientContext.languageCode, + countryCode = query.clientContext.countryCode, + requestEndTimeMs = Some(Time.now.inMilliseconds), + servedRequestId = query.features.flatMap(_.getOrElse(ServedRequestIdFeature, None)), + requestJoinId = query.features.flatMap(_.getOrElse(RequestJoinIdFeature, None)) + ) + + val tweetIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] = + selectedCandidates.flatMap { + case item: ItemCandidateWithDetails if item.candidate.isInstanceOf[BaseTweetCandidate] => + Seq((item.candidateIdLong, item)) + case module: ModuleCandidateWithDetails + if module.candidates.headOption.exists(_.candidate.isInstanceOf[BaseTweetCandidate]) => + module.candidates.map(item => (item.candidateIdLong, item)) + case _ => Seq.empty + }.toMap + + val userIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] = + selectedCandidates.flatMap { + case module: ModuleCandidateWithDetails + if module.candidates.forall(_.candidate.isInstanceOf[BaseUserCandidate]) => + module.candidates.map { item => + (item.candidateIdLong, item) + } + case _ => Seq.empty + }.toMap + + response.instructions.zipWithIndex + .collect { + case (AddEntriesTimelineInstruction(entries), index) => + entries.collect { + case entry: TweetItem if entry.promotedMetadata.isDefined => + val promotedTweetDetails = PromotedTweetDetailsMarshaller(entry, index) + Seq( + thrift.EntryInfo( + id = entry.id, + position = index.shortValue(), + entryId = entry.entryIdentifier, + entryType = thrift.EntryType.PromotedTweet, + sortIndex = entry.sortIndex, + verticalSize = Some(1), + displayType = Some(entry.displayType.toString), + details = Some(thrift.ItemDetails.PromotedTweetDetails(promotedTweetDetails)) + ) + ) + case entry: TweetItem => + val candidate = tweetIdToItemCandidateMap(entry.id) + val tweetDetails = TweetDetailsMarshaller(entry, candidate) + Seq( + thrift.EntryInfo( + id = candidate.candidateIdLong, + position = index.shortValue(), + entryId = entry.entryIdentifier, + entryType = thrift.EntryType.Tweet, + sortIndex = entry.sortIndex, + verticalSize = Some(1), + score = candidate.features.getOrElse(ScoreFeature, None), + displayType = Some(entry.displayType.toString), + details = Some(thrift.ItemDetails.TweetDetails(tweetDetails)) + ) + ) + case module: TimelineModule + if module.entryNamespace.toString == WhoToFollowCandidateDecorator.EntryNamespaceString => + module.items.collect { + case ModuleItem(entry: UserItem, _, _) => + val candidate = userIdToItemCandidateMap(entry.id) + val whoToFollowDetails = WhoToFollowDetailsMarshaller(entry, candidate) + thrift.EntryInfo( + id = entry.id, + position = index.shortValue(), + entryId = module.entryIdentifier, + entryType = thrift.EntryType.WhoToFollowModule, + sortIndex = module.sortIndex, + score = candidate.features.getOrElse(ScoreFeature, None), + displayType = Some(entry.displayType.toString), + details = Some(thrift.ItemDetails.WhoToFollowDetails(whoToFollowDetails)) + ) + } + case module: TimelineModule + if module.entryNamespace.toString == WhoToSubscribeCandidateDecorator.EntryNamespaceString => + module.items.collect { + case ModuleItem(entry: UserItem, _, _) => + val candidate = userIdToItemCandidateMap(entry.id) + val whoToSubscribeDetails = WhoToFollowDetailsMarshaller(entry, candidate) + thrift.EntryInfo( + id = entry.id, + position = index.shortValue(), + entryId = module.entryIdentifier, + entryType = thrift.EntryType.WhoToSubscribeModule, + sortIndex = module.sortIndex, + score = candidate.features.getOrElse(ScoreFeature, None), + displayType = Some(entry.displayType.toString), + details = Some(thrift.ItemDetails.WhoToFollowDetails(whoToSubscribeDetails)) + ) + } + case module: TimelineModule + if module.sortIndex.isDefined && module.items.headOption.exists( + _.item.isInstanceOf[TweetItem]) => + module.items.collect { + case ModuleItem(entry: TweetItem, _, _) => + val candidate = tweetIdToItemCandidateMap(entry.id) + thrift.EntryInfo( + id = entry.id, + position = index.shortValue(), + entryId = module.entryIdentifier, + entryType = thrift.EntryType.ConversationModule, + sortIndex = module.sortIndex, + score = candidate.features.getOrElse(ScoreFeature, None), + displayType = Some(entry.displayType.toString) + ) + } + case _ => Seq.empty + }.flatten + // Other instructions + case _ => Seq.empty[thrift.EntryInfo] + }.flatten.map { entryInfo => + thrift.ServedEntry( + entry = Some(entryInfo), + request = requestInfo + ) + } + } + + override val logPipelinePublisher: EventPublisher[thrift.ServedEntry] = + scribeEventPublisher + + override val alerts = Seq( + HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert() + ) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeServedEntriesSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeServedEntriesSideEffect.scala deleted file mode 100644 index ab7436032..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeServedEntriesSideEffect.scala +++ /dev/null @@ -1,212 +0,0 @@ -package com.twitter.home_mixer.functional_component.side_effect - -import com.twitter.finagle.tracing.Trace -import com.twitter.home_mixer.marshaller.timeline_logging.ConversationEntryMarshaller -import com.twitter.home_mixer.marshaller.timeline_logging.PromotedTweetEntryMarshaller -import com.twitter.home_mixer.marshaller.timeline_logging.TweetEntryMarshaller -import com.twitter.home_mixer.marshaller.timeline_logging.WhoToFollowEntryMarshaller -import com.twitter.home_mixer.model.HomeFeatures.GetInitialFeature -import com.twitter.home_mixer.model.HomeFeatures.GetMiddleFeature -import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature -import com.twitter.home_mixer.model.HomeFeatures.GetOlderFeature -import com.twitter.home_mixer.model.HomeFeatures.HasDarkRequestFeature -import com.twitter.home_mixer.model.HomeFeatures.RequestJoinIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature -import com.twitter.home_mixer.model.request.DeviceContext.RequestContext -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.home_mixer.model.request.HasSeenTweetIds -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.logpipeline.client.common.EventPublisher -import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate -import com.twitter.product_mixer.component_library.model.candidate.BaseUserCandidate -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.AddEntriesTimelineInstruction -import com.twitter.product_mixer.core.model.marshalling.response.urt.ModuleItem -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelines.timeline_logging.{thriftscala => thrift} -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Side effect that logs home timeline served entries to Scribe. - */ -@Singleton -class HomeScribeServedEntriesSideEffect @Inject() ( - scribeEventPublisher: EventPublisher[thrift.Timeline]) - extends PipelineResultSideEffect[ - PipelineQuery with HasSeenTweetIds with HasDeviceContext, - Timeline - ] { - - override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeServedEntries") - - final override def apply( - inputs: PipelineResultSideEffect.Inputs[ - PipelineQuery with HasSeenTweetIds with HasDeviceContext, - Timeline - ] - ): Stitch[Unit] = { - val timelineThrift = buildTimeline(inputs) - Stitch.callFuture(scribeEventPublisher.publish(timelineThrift)).unit - } - - def buildTimeline( - inputs: PipelineResultSideEffect.Inputs[ - PipelineQuery with HasSeenTweetIds with HasDeviceContext, - Timeline - ] - ): thrift.Timeline = { - val timelineType = inputs.query.product match { - case FollowingProduct => thrift.TimelineType.HomeLatest - case ForYouProduct => thrift.TimelineType.Home - case other => throw new UnsupportedOperationException(s"Unknown product: $other") - } - val requestProvenance = inputs.query.deviceContext.map { deviceContext => - deviceContext.requestContextValue match { - case RequestContext.Foreground => thrift.RequestProvenance.Foreground - case RequestContext.Launch => thrift.RequestProvenance.Launch - case RequestContext.PullToRefresh => thrift.RequestProvenance.Ptr - case _ => thrift.RequestProvenance.Other - } - } - val queryType = inputs.query.features.map { featureMap => - if (featureMap.getOrElse(GetOlderFeature, false)) thrift.QueryType.GetOlder - else if (featureMap.getOrElse(GetNewerFeature, false)) thrift.QueryType.GetNewer - else if (featureMap.getOrElse(GetMiddleFeature, false)) thrift.QueryType.GetMiddle - else if (featureMap.getOrElse(GetInitialFeature, false)) thrift.QueryType.GetInitial - else thrift.QueryType.Other - } - - val tweetIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] = - inputs.selectedCandidates.flatMap { - case item: ItemCandidateWithDetails if item.candidate.isInstanceOf[BaseTweetCandidate] => - Seq((item.candidateIdLong, item)) - case module: ModuleCandidateWithDetails - if module.candidates.headOption.exists(_.candidate.isInstanceOf[BaseTweetCandidate]) => - module.candidates.map(item => (item.candidateIdLong, item)) - case _ => Seq.empty - }.toMap - - val userIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] = - inputs.selectedCandidates.flatMap { - case module: ModuleCandidateWithDetails - if module.candidates.forall(_.candidate.isInstanceOf[BaseUserCandidate]) => - module.candidates.map { item => - (item.candidateIdLong, item) - } - case _ => Seq.empty - }.toMap - - val timelineEntries = inputs.response.instructions.zipWithIndex.collect { - case (AddEntriesTimelineInstruction(entries), index) => - entries.collect { - case entry: TweetItem if entry.promotedMetadata.isDefined => - val promotedTweetEntry = PromotedTweetEntryMarshaller(entry, index) - Seq( - thrift.TimelineEntry( - content = thrift.Content.PromotedTweetEntry(promotedTweetEntry), - position = index.shortValue(), - entryId = entry.entryIdentifier, - entryType = thrift.EntryType.PromotedTweet, - sortIndex = entry.sortIndex, - verticalSize = Some(1) - ) - ) - case entry: TweetItem => - val candidate = tweetIdToItemCandidateMap(entry.id) - val tweetEntry = TweetEntryMarshaller(entry, candidate) - Seq( - thrift.TimelineEntry( - content = thrift.Content.TweetEntry(tweetEntry), - position = index.shortValue(), - entryId = entry.entryIdentifier, - entryType = thrift.EntryType.Tweet, - sortIndex = entry.sortIndex, - verticalSize = Some(1) - ) - ) - case module: TimelineModule - if module.entryNamespace.toString == WhoToFollowCandidateDecorator.EntryNamespaceString => - val whoToFollowEntries = module.items.collect { - case ModuleItem(entry: UserItem, _, _) => - val candidate = userIdToItemCandidateMap(entry.id) - val whoToFollowEntry = WhoToFollowEntryMarshaller(entry, candidate) - thrift.AtomicEntry.WtfEntry(whoToFollowEntry) - } - Seq( - thrift.TimelineEntry( - content = thrift.Content.Entries(whoToFollowEntries), - position = index.shortValue(), - entryId = module.entryIdentifier, - entryType = thrift.EntryType.WhoToFollowModule, - sortIndex = module.sortIndex - ) - ) - case module: TimelineModule - if module.sortIndex.isDefined && module.items.headOption.exists( - _.item.isInstanceOf[TweetItem]) => - val conversationTweetEntries = module.items.collect { - case ModuleItem(entry: TweetItem, _, _) => - val candidate = tweetIdToItemCandidateMap(entry.id) - val conversationEntry = ConversationEntryMarshaller(entry, candidate) - thrift.AtomicEntry.ConversationEntry(conversationEntry) - } - Seq( - thrift.TimelineEntry( - content = thrift.Content.Entries(conversationTweetEntries), - position = index.shortValue(), - entryId = module.entryIdentifier, - entryType = thrift.EntryType.ConversationModule, - sortIndex = module.sortIndex - ) - ) - case _ => Seq.empty - }.flatten - // Other instructions - case _ => Seq.empty[thrift.TimelineEntry] - }.flatten - - thrift.Timeline( - timelineEntries = timelineEntries, - requestTimeMs = inputs.query.queryTime.inMilliseconds, - traceId = Trace.id.traceId.toLong, - userId = inputs.query.getOptionalUserId, - clientAppId = inputs.query.clientContext.appId, - sourceJobInstance = None, - hasDarkRequest = inputs.query.features.flatMap(_.getOrElse(HasDarkRequestFeature, None)), - parentId = Some(Trace.id.parentId.toLong), - spanId = Some(Trace.id.spanId.toLong), - timelineType = Some(timelineType), - ipAddress = inputs.query.clientContext.ipAddress, - userAgent = inputs.query.clientContext.userAgent, - queryType = queryType, - requestProvenance = requestProvenance, - sessionId = None, - timeZone = None, - browserNotificationPermission = None, - lastNonePollingTimeMs = None, - languageCode = inputs.query.clientContext.languageCode, - countryCode = inputs.query.clientContext.countryCode, - requestEndTimeMs = Some(Time.now.inMilliseconds), - servedRequestId = inputs.query.features.flatMap(_.getOrElse(ServedRequestIdFeature, None)), - requestJoinId = inputs.query.features.flatMap(_.getOrElse(RequestJoinIdFeature, None)), - requestSeenTweetIds = inputs.query.seenTweetIds - ) - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert() - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsEventBusSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsEventBusSideEffect.scala index 3d85d1137..c0437767e 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsEventBusSideEffect.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsEventBusSideEffect.scala @@ -3,6 +3,7 @@ package com.twitter.home_mixer.functional_component.side_effect import com.twitter.eventbus.client.EventBusPublisher import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.ForYouProduct +import com.twitter.home_mixer.model.request.SubscribedProduct import com.twitter.home_mixer.model.request.HasSeenTweetIds import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect @@ -22,6 +23,7 @@ import javax.inject.Singleton object PublishClientSentImpressionsEventBusSideEffect { val HomeSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeTimeline)) val HomeLatestSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeLatestTimeline)) + val HomeSubscribedSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeSubscribed)) } /** @@ -56,6 +58,7 @@ class PublishClientSentImpressionsEventBusSideEffect @Inject() ( val surfaceArea = query.product match { case ForYouProduct => HomeSurfaceArea case FollowingProduct => HomeLatestSurfaceArea + case SubscribedProduct => HomeSubscribedSurfaceArea case _ => None } query.seenTweetIds.map { seenTweetIds => diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateImpressionBloomFilterSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishImpressionBloomFilterSideEffect.scala similarity index 56% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateImpressionBloomFilterSideEffect.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishImpressionBloomFilterSideEffect.scala index 957fbcd37..f965bafac 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateImpressionBloomFilterSideEffect.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishImpressionBloomFilterSideEffect.scala @@ -1,39 +1,46 @@ package com.twitter.home_mixer.functional_component.side_effect -import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature import com.twitter.home_mixer.model.request.HasSeenTweetIds +import com.twitter.home_mixer.param.HomeGlobalParams.EnableImpressionBloomFilter import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline +import com.twitter.product_mixer.core.model.marshalling.HasMarshalling import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch -import com.twitter.timelines.impressionbloomfilter.{thriftscala => t} -import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilter +import com.twitter.timelines.clients.manhattan.store.ManhattanStoreClient +import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm} import javax.inject.Inject import javax.inject.Singleton @Singleton -class UpdateImpressionBloomFilterSideEffect @Inject() (bloomFilter: ImpressionBloomFilter) - extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, Timeline] - with PipelineResultSideEffect.Conditionally[PipelineQuery with HasSeenTweetIds, Timeline] { - - private val SurfaceArea = t.SurfaceArea.HomeTimeline +class PublishImpressionBloomFilterSideEffect @Inject() ( + bloomFilterClient: ManhattanStoreClient[ + blm.ImpressionBloomFilterKey, + blm.ImpressionBloomFilterSeq + ]) extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, HasMarshalling] + with PipelineResultSideEffect.Conditionally[ + PipelineQuery with HasSeenTweetIds, + HasMarshalling + ] { override val identifier: SideEffectIdentifier = - SideEffectIdentifier("UpdateImpressionBloomFilter") + SideEffectIdentifier("PublishImpressionBloomFilter") + + private val SurfaceArea = blm.SurfaceArea.HomeTimeline override def onlyIf( query: PipelineQuery with HasSeenTweetIds, selectedCandidates: Seq[CandidateWithDetails], remainingCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], - response: Timeline - ): Boolean = query.seenTweetIds.exists(_.nonEmpty) + response: HasMarshalling + ): Boolean = + query.params.getBoolean(EnableImpressionBloomFilter) && query.seenTweetIds.exists(_.nonEmpty) - def buildEvents(query: PipelineQuery): Option[t.ImpressionBloomFilterSeq] = { + def buildEvents(query: PipelineQuery): Option[blm.ImpressionBloomFilterSeq] = { query.features.flatMap { featureMap => val impressionBloomFilterSeq = featureMap.get(ImpressionBloomFilterFeature) if (impressionBloomFilterSeq.entries.nonEmpty) Some(impressionBloomFilterSeq) @@ -42,19 +49,17 @@ class UpdateImpressionBloomFilterSideEffect @Inject() (bloomFilter: ImpressionBl } override def apply( - inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, Timeline] + inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, HasMarshalling] ): Stitch[Unit] = { buildEvents(inputs.query) - .map { updatedBloomFilter => - bloomFilter.writeBloomFilterSeq( - userId = inputs.query.getRequiredUserId, - surfaceArea = SurfaceArea, - impressionBloomFilterSeq = updatedBloomFilter) + .map { updatedBloomFilterSeq => + bloomFilterClient.write( + blm.ImpressionBloomFilterKey(inputs.query.getRequiredUserId, SurfaceArea), + updatedBloomFilterSeq) }.getOrElse(Stitch.Unit) } override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8), - HomeMixerAlertConfig.BusinessHours.defaultLatencyAlert(30.millis) + HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8) ) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ServedStatsSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ServedStatsSideEffect.scala deleted file mode 100644 index 3a24a5927..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ServedStatsSideEffect.scala +++ /dev/null @@ -1,80 +0,0 @@ -package com.twitter.home_mixer.functional_component.side_effect - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.param.HomeGlobalParams.AuthorListForStatsParam -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ServedStatsSideEffect @Inject() (statsReceiver: StatsReceiver) - extends PipelineResultSideEffect[PipelineQuery, Timeline] { - - override val identifier: SideEffectIdentifier = SideEffectIdentifier("ServedStats") - - private val baseStatsReceiver = statsReceiver.scope(identifier.toString) - private val authorStatsReceiver = baseStatsReceiver.scope("Author") - private val candidateSourceStatsReceiver = baseStatsReceiver.scope("CandidateSource") - private val contentBalanceStatsReceiver = baseStatsReceiver.scope("ContentBalance") - private val inNetworkStatsCounter = contentBalanceStatsReceiver.counter("InNetwork") - private val outOfNetworkStatsCounter = contentBalanceStatsReceiver.counter("OutOfNetwork") - - override def apply( - inputs: PipelineResultSideEffect.Inputs[PipelineQuery, Timeline] - ): Stitch[Unit] = { - val tweetCandidates = CandidatesUtil - .getItemCandidates(inputs.selectedCandidates).filter(_.isCandidateType[TweetCandidate]()) - - recordAuthorStats(tweetCandidates, inputs.query.params(AuthorListForStatsParam)) - recordCandidateSourceStats(tweetCandidates) - recordContentBalanceStats(tweetCandidates) - Stitch.Unit - } - - def recordAuthorStats(candidates: Seq[CandidateWithDetails], authors: Set[Long]): Unit = { - candidates - .filter { candidate => - candidate.features.getOrElse(AuthorIdFeature, None).exists(authors.contains) && - // Only include original tweets - (!candidate.features.getOrElse(IsRetweetFeature, false)) && - candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty - } - .groupBy { candidate => - (getCandidateSourceId(candidate), candidate.features.get(AuthorIdFeature).get) - } - .foreach { - case ((candidateSourceId, authorId), authorCandidates) => - authorStatsReceiver - .scope(authorId.toString).counter(candidateSourceId).incr(authorCandidates.size) - } - } - - def recordCandidateSourceStats(candidates: Seq[ItemCandidateWithDetails]): Unit = { - candidates.groupBy(getCandidateSourceId).foreach { - case (candidateSourceId, candidateSourceCandidates) => - candidateSourceStatsReceiver.counter(candidateSourceId).incr(candidateSourceCandidates.size) - } - } - - def recordContentBalanceStats(candidates: Seq[ItemCandidateWithDetails]): Unit = { - val (in, oon) = candidates.partition(_.features.getOrElse(InNetworkFeature, true)) - inNetworkStatsCounter.incr(in.size) - outOfNetworkStatsCounter.incr(oon.size) - } - - private def getCandidateSourceId(candidate: CandidateWithDetails): String = - candidate.features.getOrElse(CandidateSourceIdFeature, None).map(_.name).getOrElse("None") -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateTimelinesPersistenceStoreSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateTimelinesPersistenceStoreSideEffect.scala index ef8a737a8..5a1bb0f6b 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateTimelinesPersistenceStoreSideEffect.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateTimelinesPersistenceStoreSideEffect.scala @@ -3,8 +3,10 @@ package com.twitter.home_mixer.functional_component.side_effect import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.ForYouProduct +import com.twitter.home_mixer.model.HomeFeatures.IsTweetPreviewFeature import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator +import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidateDecorator import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier @@ -97,12 +99,14 @@ class UpdateTimelinesPersistenceStoreSideEffect @Inject() ( val entries = inputs.response.instructions.collect { case AddEntriesTimelineInstruction(entries) => entries.collect { - // includes both tweets and promoted tweets - case entry: TweetItem if entry.sortIndex.isDefined => + // includes tweets, tweet previews, and promoted tweets + case entry: TweetItem if entry.sortIndex.isDefined => { Seq( buildTweetEntryWithItemIds( tweetIdToItemCandidateMap(entry.id), - entry.sortIndex.get)) + entry.sortIndex.get + )) + } // tweet conversation modules are flattened to individual tweets in the persistence store case module: TimelineModule if module.sortIndex.isDefined && module.items.headOption.exists( @@ -125,6 +129,19 @@ class UpdateTimelinesPersistenceStoreSideEffect @Inject() ( size = module.items.size.toShort, itemIds = Some(userIds) )) + case module: TimelineModule + if module.sortIndex.isDefined && module.entryNamespace.toString == WhoToSubscribeCandidateDecorator.EntryNamespaceString => + val userIds = module.items + .map(item => + UpdateTimelinesPersistenceStoreSideEffect.EmptyItemIds.copy(userId = + Some(item.item.id.asInstanceOf[Long]))) + Seq( + EntryWithItemIds( + entityIdType = EntityIdType.WhoToSubscribe, + sortIndex = module.sortIndex.get, + size = module.items.size.toShort, + itemIds = Some(userIds) + )) }.flatten case ShowCoverInstruction(cover) => Seq( @@ -216,8 +233,11 @@ class UpdateTimelinesPersistenceStoreSideEffect @Inject() ( userId = None ) + val isPreview = features.getOrElse(IsTweetPreviewFeature, default = false) + val entityType = if (isPreview) EntityIdType.TweetPreview else EntityIdType.Tweet + EntryWithItemIds( - entityIdType = EntityIdType.Tweet, + entityIdType = entityType, sortIndex = sortIndex, size = 1.toShort, itemIds = Some(Seq(itemIds)) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/BUILD.bazel index fe1ee7190..fd35daeeb 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/BUILD.bazel @@ -4,14 +4,12 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "dspbidder/thrift/src/main/thrift/com/twitter/dspbidder/commons:thrift-scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", ], exports = [ - "dspbidder/thrift/src/main/thrift/com/twitter/dspbidder/commons:thrift-scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductContextUnmarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductContextUnmarshaller.scala index bbc93389c..ec3a183b9 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductContextUnmarshaller.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductContextUnmarshaller.scala @@ -5,9 +5,9 @@ import com.twitter.home_mixer.model.request.ForYouProductContext import com.twitter.home_mixer.model.request.ListRecommendedUsersProductContext import com.twitter.home_mixer.model.request.ListTweetsProductContext import com.twitter.home_mixer.model.request.ScoredTweetsProductContext +import com.twitter.home_mixer.model.request.SubscribedProductContext import com.twitter.home_mixer.{thriftscala => t} import com.twitter.product_mixer.core.model.marshalling.request.ProductContext - import javax.inject.Inject import javax.inject.Singleton @@ -26,15 +26,17 @@ class HomeMixerProductContextUnmarshaller @Inject() ( ForYouProductContext( deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)), seenTweetIds = p.seenTweetIds, - dspClientContext = p.dspClientContext + dspClientContext = p.dspClientContext, + pushToHomeTweetId = p.pushToHomeTweetId ) - case t.ProductContext.Realtime(p) => + case t.ProductContext.ListManagement(p) => throw new UnsupportedOperationException(s"This product is no longer used") case t.ProductContext.ScoredTweets(p) => ScoredTweetsProductContext( deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)), seenTweetIds = p.seenTweetIds, - servedTweetIds = p.servedTweetIds + servedTweetIds = p.servedTweetIds, + backfillTweetIds = p.backfillTweetIds ) case t.ProductContext.ListTweets(p) => ListTweetsProductContext( @@ -46,7 +48,13 @@ class HomeMixerProductContextUnmarshaller @Inject() ( ListRecommendedUsersProductContext( listId = p.listId, selectedUserIds = p.selectedUserIds, - excludedUserIds = p.excludedUserIds + excludedUserIds = p.excludedUserIds, + listName = p.listName + ) + case t.ProductContext.Subscribed(p) => + SubscribedProductContext( + deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)), + seenTweetIds = p.seenTweetIds, ) case t.ProductContext.UnknownUnionField(field) => throw new UnsupportedOperationException(s"Unknown display context: ${field.field.name}") diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductUnmarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductUnmarshaller.scala index 0089c5efb..f5d0d002b 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductUnmarshaller.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductUnmarshaller.scala @@ -5,9 +5,9 @@ import com.twitter.home_mixer.model.request.ForYouProduct import com.twitter.home_mixer.model.request.ListRecommendedUsersProduct import com.twitter.home_mixer.model.request.ListTweetsProduct import com.twitter.home_mixer.model.request.ScoredTweetsProduct +import com.twitter.home_mixer.model.request.SubscribedProduct import com.twitter.home_mixer.{thriftscala => t} import com.twitter.product_mixer.core.model.marshalling.request.Product - import javax.inject.Inject import javax.inject.Singleton @@ -17,11 +17,12 @@ class HomeMixerProductUnmarshaller @Inject() () { def apply(product: t.Product): Product = product match { case t.Product.Following => FollowingProduct case t.Product.ForYou => ForYouProduct - case t.Product.Realtime => + case t.Product.ListManagement => throw new UnsupportedOperationException(s"This product is no longer used") case t.Product.ScoredTweets => ScoredTweetsProduct case t.Product.ListTweets => ListTweetsProduct case t.Product.ListRecommendedUsers => ListRecommendedUsersProduct + case t.Product.Subscribed => SubscribedProduct case t.Product.EnumUnknownProduct(value) => throw new UnsupportedOperationException(s"Unknown product: $value") } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/ConversationEntryMarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/ConversationEntryMarshaller.scala deleted file mode 100644 index 8123f3597..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/ConversationEntryMarshaller.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.home_mixer.marshaller.timeline_logging - -import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog} - -object ConversationEntryMarshaller { - - def apply(entry: TweetItem, candidate: ItemCandidateWithDetails): thriftlog.ConversationEntry = - thriftlog.ConversationEntry( - displayedTweetId = entry.id, - displayType = Some(entry.displayType.toString), - score = candidate.features.getOrElse(ScoreFeature, None) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/PromotedTweetEntryMarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/PromotedTweetDetailsMarshaller.scala similarity index 56% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/PromotedTweetEntryMarshaller.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/PromotedTweetDetailsMarshaller.scala index b96bb38a2..d913f8d64 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/PromotedTweetEntryMarshaller.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/PromotedTweetDetailsMarshaller.scala @@ -3,15 +3,13 @@ package com.twitter.home_mixer.marshaller.timeline_logging import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog} -object PromotedTweetEntryMarshaller { +object PromotedTweetDetailsMarshaller { - def apply(entry: TweetItem, position: Int): thriftlog.PromotedTweetEntry = { - thriftlog.PromotedTweetEntry( - id = entry.id, - advertiserId = entry.promotedMetadata.map(_.advertiserId).getOrElse(0L), - insertPosition = position, - impressionId = entry.promotedMetadata.flatMap(_.impressionString), - displayType = Some(entry.displayType.toString) + def apply(entry: TweetItem, position: Int): thriftlog.PromotedTweetDetails = { + thriftlog.PromotedTweetDetails( + advertiserId = Some(entry.promotedMetadata.map(_.advertiserId).getOrElse(0L)), + insertPosition = Some(position), + impressionId = entry.promotedMetadata.flatMap(_.impressionString) ) } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/TweetDetailsMarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/TweetDetailsMarshaller.scala new file mode 100644 index 000000000..8e1c475d5 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/TweetDetailsMarshaller.scala @@ -0,0 +1,47 @@ +package com.twitter.home_mixer.marshaller.timeline_logging + +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature +import com.twitter.product_mixer.component_library.model.presentation.urt.UrtItemPresentation +import com.twitter.product_mixer.component_library.model.presentation.urt.UrtModulePresentation +import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.metadata.GeneralContextTypeMarshaller +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ConversationGeneralContextType +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.GeneralContext +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContext +import com.twitter.timelines.service.{thriftscala => tst} +import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog} + +object TweetDetailsMarshaller { + + private val generalContextTypeMarshaller = new GeneralContextTypeMarshaller() + + def apply(entry: TweetItem, candidate: CandidateWithDetails): thriftlog.TweetDetails = { + val socialContext = candidate.presentation.flatMap { + case _ @UrtItemPresentation(timelineItem: TweetItem, _) => timelineItem.socialContext + case _ @UrtModulePresentation(timelineModule) => + timelineModule.items.head.item match { + case timelineItem: TweetItem => timelineItem.socialContext + case _ => Some(ConversationGeneralContextType) + } + } + + val socialContextType = socialContext match { + case Some(GeneralContext(contextType, _, _, _, _)) => + Some(generalContextTypeMarshaller(contextType).value.toShort) + case Some(TopicContext(_, _)) => Some(tst.ContextType.Topic.value.toShort) + case _ => None + } + + thriftlog.TweetDetails( + sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None), + socialContextType = socialContextType, + suggestType = candidate.features.getOrElse(SuggestTypeFeature, None).map(_.name), + authorId = candidate.features.getOrElse(AuthorIdFeature, None), + sourceAuthorId = candidate.features.getOrElse(SourceUserIdFeature, None) + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/TweetEntryMarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/TweetEntryMarshaller.scala deleted file mode 100644 index ca6b08778..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/TweetEntryMarshaller.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.home_mixer.marshaller.timeline_logging - -import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature -import com.twitter.home_mixer.model.HomeFeatures.SocialContextFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.timelines.service.{thriftscala => tst} -import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog} - -object TweetEntryMarshaller { - - def apply(entry: TweetItem, candidate: CandidateWithDetails): thriftlog.TweetEntry = { - val socialContextType = candidate.features.getOrElse(SocialContextFeature, None) match { - case Some(tst.SocialContext.GeneralContext(tst.GeneralContext(contextType, _, _, _, _))) => - Some(contextType.value.toShort) - case Some(tst.SocialContext.TopicContext(_)) => - Some(tst.ContextType.Topic.value.toShort) - case _ => None - } - thriftlog.TweetEntry( - id = candidate.candidateIdLong, - sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None), - displayType = Some(entry.displayType.toString), - score = candidate.features.getOrElse(ScoreFeature, None), - socialContextType = socialContextType - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/WhoToFollowEntryMarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/WhoToFollowDetailsMarshaller.scala similarity index 62% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/WhoToFollowEntryMarshaller.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/WhoToFollowDetailsMarshaller.scala index 9a253b726..4c0b4dd1b 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/WhoToFollowEntryMarshaller.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/WhoToFollowDetailsMarshaller.scala @@ -1,17 +1,13 @@ package com.twitter.home_mixer.marshaller.timeline_logging -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.ScoreFeature import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog} -object WhoToFollowEntryMarshaller { +object WhoToFollowDetailsMarshaller { - def apply(entry: UserItem, candidate: ItemCandidateWithDetails): thriftlog.WhoToFollowEntry = - thriftlog.WhoToFollowEntry( - userId = entry.id, - displayType = Some(entry.displayType.toString), - score = candidate.features.getOrElse(ScoreFeature, None), + def apply(entry: UserItem, candidate: ItemCandidateWithDetails): thriftlog.WhoToFollowDetails = + thriftlog.WhoToFollowDetails( enableReactiveBlending = entry.enableReactiveBlending, impressionId = entry.promotedMetadata.flatMap(_.impressionString), advertiserId = entry.promotedMetadata.map(_.advertiserId) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/BUILD.bazel index e424298f0..ebf305ead 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/BUILD.bazel @@ -4,13 +4,8 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt", - "src/thrift/com/twitter/timelines/render:thrift-scala", "src/thrift/com/twitter/timelineservice:thrift-scala", - "timelineservice/common:model", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/BUILD.bazel index c5c46d2d1..65ece62a3 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/BUILD.bazel @@ -7,7 +7,6 @@ scala_library( "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord", @@ -18,22 +17,16 @@ scala_library( "src/scala/com/twitter/timelines/prediction/features/common", "src/scala/com/twitter/timelines/prediction/features/recap", "src/scala/com/twitter/timelines/prediction/features/request_context", - "src/thrift/com/twitter/dal/personal_data:personal_data-java", "src/thrift/com/twitter/escherbird:tweet-annotation-scala", - "src/thrift/com/twitter/gizmoduck:thrift-scala", - "src/thrift/com/twitter/timelines/author_features:thrift-java", + "src/thrift/com/twitter/search:earlybird-scala", "src/thrift/com/twitter/timelines/conversation_features:conversation_features-scala", "src/thrift/com/twitter/timelines/impression:thrift-scala", "src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala", "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala", - "src/thrift/com/twitter/tweetypie:media-entity-scala", "src/thrift/com/twitter/tweetypie:tweet-scala", - "src/thrift/com/twitter/user_session_store:thrift-java", "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan", "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/persistence", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", - "timelines/src/main/scala/com/twitter/timelines/model/types", "topic-social-proof/server/src/main/thrift:thrift-scala", "tweetconvosvc/common/src/main/thrift/com/twitter/tweetconvosvc/tweet_ancestor:thrift-scala", ], diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ContentFeatures.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ContentFeatures.scala index 44e8cf25e..f141d67d1 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ContentFeatures.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ContentFeatures.scala @@ -1,6 +1,7 @@ package com.twitter.home_mixer.model import com.twitter.escherbird.{thriftscala => esb} +import com.twitter.search.common.features.{thriftscala => sc} import com.twitter.tweetypie.{thriftscala => tp} object ContentFeatures { @@ -48,6 +49,53 @@ object ContentFeatures { None, None ) + + def fromThrift(ebFeatures: sc.ThriftTweetFeatures): ContentFeatures = + ContentFeatures( + length = ebFeatures.tweetLength.getOrElse(0).toShort, + hasQuestion = ebFeatures.hasQuestion.getOrElse(false), + numCaps = ebFeatures.numCaps.getOrElse(0).toShort, + numWhiteSpaces = ebFeatures.numWhitespaces.getOrElse(0).toShort, + numNewlines = ebFeatures.numNewlines, + videoDurationMs = ebFeatures.videoDurationMs, + bitRate = ebFeatures.bitRate, + aspectRatioNum = ebFeatures.aspectRatioNum, + aspectRatioDen = ebFeatures.aspectRatioDen, + widths = ebFeatures.widths.map(_.map(_.toShort)), + heights = ebFeatures.heights.map(_.map(_.toShort)), + resizeMethods = ebFeatures.resizeMethods.map(_.map(_.toShort)), + numMediaTags = ebFeatures.numMediaTags.map(_.toShort), + mediaTagScreenNames = ebFeatures.mediaTagScreenNames, + emojiTokens = ebFeatures.emojiTokens.map(_.toSet), + emoticonTokens = ebFeatures.emoticonTokens.map(_.toSet), + faceAreas = ebFeatures.faceAreas, + dominantColorRed = ebFeatures.dominantColorRed, + dominantColorBlue = ebFeatures.dominantColorBlue, + dominantColorGreen = ebFeatures.dominantColorGreen, + numColors = ebFeatures.numColors.map(_.toShort), + stickerIds = ebFeatures.stickerIds, + mediaOriginProviders = ebFeatures.mediaOriginProviders, + isManaged = ebFeatures.isManaged, + is360 = ebFeatures.is360, + viewCount = ebFeatures.viewCount, + isMonetizable = ebFeatures.isMonetizable, + isEmbeddable = ebFeatures.isEmbeddable, + hasSelectedPreviewImage = ebFeatures.hasSelectedPreviewImage, + hasTitle = ebFeatures.hasTitle, + hasDescription = ebFeatures.hasDescription, + hasVisitSiteCallToAction = ebFeatures.hasVisitSiteCallToAction, + hasAppInstallCallToAction = ebFeatures.hasAppInstallCallToAction, + hasWatchNowCallToAction = ebFeatures.hasWatchNowCallToAction, + dominantColorPercentage = ebFeatures.dominantColorPercentage, + posUnigrams = ebFeatures.posUnigrams.map(_.toSet), + posBigrams = ebFeatures.posBigrams.map(_.toSet), + semanticCoreAnnotations = ebFeatures.semanticCoreAnnotations, + tokens = ebFeatures.textTokens.map(_.toSeq), + conversationControl = ebFeatures.conversationControl, + // media and selfThreadMetadata not carried by ThriftTweetFeatures + media = None, + selfThreadMetadata = None + ) } case class ContentFeatures( diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeFeatures.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeFeatures.scala index 032623cd4..fe085b1f9 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeFeatures.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeFeatures.scala @@ -2,7 +2,6 @@ package com.twitter.home_mixer.model import com.twitter.core_workflows.user_model.{thriftscala => um} import com.twitter.dal.personal_data.{thriftjava => pd} -import com.twitter.escherbird.{thriftscala => esb} import com.twitter.gizmoduck.{thriftscala => gt} import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.ml.api.constant.SharedFeatures @@ -11,10 +10,13 @@ import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.BoolDataRecordCompatible import com.twitter.product_mixer.core.feature.datarecord.DataRecordFeature +import com.twitter.product_mixer.core.feature.datarecord.DataRecordOptionalFeature +import com.twitter.product_mixer.core.feature.datarecord.DoubleDataRecordCompatible import com.twitter.product_mixer.core.feature.datarecord.LongDiscreteDataRecordCompatible import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContextFunctionalityType import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.search.common.features.{thriftscala => sc} +import com.twitter.search.earlybird.{thriftscala => eb} import com.twitter.timelinemixer.clients.manhattan.DismissInfo import com.twitter.timelinemixer.clients.persistence.TimelineResponseV3 import com.twitter.timelinemixer.injection.model.candidate.AudioSpaceMetaData @@ -23,6 +25,7 @@ import com.twitter.timelines.impression.{thriftscala => imp} import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm} import com.twitter.timelines.model.UserId import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures +import com.twitter.timelines.prediction.features.engagement_features.EngagementDataRecordFeatures import com.twitter.timelines.prediction.features.recap.RecapFeatures import com.twitter.timelines.prediction.features.request_context.RequestContextFeatures import com.twitter.timelines.service.{thriftscala => tst} @@ -44,13 +47,19 @@ object HomeFeatures { * who created the Tweet that was retweeted. */ object AuthorIdFeature - extends Feature[TweetCandidate, Option[Long]] + extends DataRecordOptionalFeature[TweetCandidate, Long] with LongDiscreteDataRecordCompatible { override val featureName: String = SharedFeatures.AUTHOR_ID.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.UserId) } - object AuthorIsEligibleForConnectBoostFeature extends Feature[TweetCandidate, Boolean] + object AuthorIsBlueVerifiedFeature extends Feature[TweetCandidate, Boolean] + object AuthorIsGoldVerifiedFeature extends Feature[TweetCandidate, Boolean] + object AuthorIsGrayVerifiedFeature extends Feature[TweetCandidate, Boolean] + object AuthorIsLegacyVerifiedFeature extends Feature[TweetCandidate, Boolean] + object AuthorIsCreatorFeature extends Feature[TweetCandidate, Boolean] + object AuthorIsProtectedFeature extends Feature[TweetCandidate, Boolean] + object AuthoredByContextualUserFeature extends Feature[TweetCandidate, Boolean] object CachedCandidatePipelineIdentifierFeature extends Feature[TweetCandidate, Option[String]] object CandidateSourceIdFeature @@ -74,11 +83,36 @@ object HomeFeatures { object DirectedAtUserIdFeature extends Feature[TweetCandidate, Option[Long]] object EarlybirdFeature extends Feature[TweetCandidate, Option[sc.ThriftTweetFeatures]] object EarlybirdScoreFeature extends Feature[TweetCandidate, Option[Double]] + object EarlybirdSearchResultFeature extends Feature[TweetCandidate, Option[eb.ThriftSearchResult]] object EntityTokenFeature extends Feature[TweetCandidate, Option[String]] object ExclusiveConversationAuthorIdFeature extends Feature[TweetCandidate, Option[Long]] + object FavoritedByCountFeature + extends DataRecordFeature[TweetCandidate, Double] + with DoubleDataRecordCompatible { + override val featureName: String = + EngagementDataRecordFeatures.InNetworkFavoritesCount.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = + Set(pd.PersonalDataType.CountOfPrivateLikes, pd.PersonalDataType.CountOfPublicLikes) + } object FavoritedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] object FeedbackHistoryFeature extends Feature[TweetCandidate, Seq[FeedbackEntry]] + object RetweetedByCountFeature + extends DataRecordFeature[TweetCandidate, Double] + with DoubleDataRecordCompatible { + override val featureName: String = + EngagementDataRecordFeatures.InNetworkRetweetsCount.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = + Set(pd.PersonalDataType.CountOfPrivateRetweets, pd.PersonalDataType.CountOfPublicRetweets) + } object RetweetedByEngagerIdsFeature extends Feature[TweetCandidate, Seq[Long]] + object RepliedByCountFeature + extends DataRecordFeature[TweetCandidate, Double] + with DoubleDataRecordCompatible { + override val featureName: String = + EngagementDataRecordFeatures.InNetworkRepliesCount.getFeatureName + override val personalDataTypes: Set[pd.PersonalDataType] = + Set(pd.PersonalDataType.CountOfPrivateReplies, pd.PersonalDataType.CountOfPublicReplies) + } object RepliedByEngagerIdsFeature extends Feature[TweetCandidate, Seq[Long]] object FollowedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] @@ -99,7 +133,7 @@ object HomeFeatures { override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty } object IsRandomTweetFeature - extends Feature[TweetCandidate, Boolean] + extends DataRecordFeature[TweetCandidate, Boolean] with BoolDataRecordCompatible { override val featureName: String = TimelinesSharedFeatures.IS_RANDOM_TWEET.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty @@ -107,6 +141,7 @@ object HomeFeatures { object IsReadFromCacheFeature extends Feature[TweetCandidate, Boolean] object IsRetweetFeature extends Feature[TweetCandidate, Boolean] object IsRetweetedReplyFeature extends Feature[TweetCandidate, Boolean] + object IsSupportAccountReplyFeature extends Feature[TweetCandidate, Boolean] object LastScoredTimestampMsFeature extends Feature[TweetCandidate, Option[Long]] object NonSelfFavoritedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] object NumImagesFeature extends Feature[TweetCandidate, Option[Int]] @@ -124,7 +159,7 @@ object HomeFeatures { extends Feature[TweetCandidate, Map[String, Double]] object SocialContextFeature extends Feature[TweetCandidate, Option[tst.SocialContext]] object SourceTweetIdFeature - extends Feature[TweetCandidate, Option[Long]] + extends DataRecordOptionalFeature[TweetCandidate, Long] with LongDiscreteDataRecordCompatible { override val featureName: String = TimelinesSharedFeatures.SOURCE_TWEET_ID.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.TweetId) @@ -137,7 +172,7 @@ object HomeFeatures { object TweetUrlsFeature extends Feature[TweetCandidate, Seq[String]] object VideoDurationMsFeature extends Feature[TweetCandidate, Option[Int]] object ViewerIdFeature - extends Feature[TweetCandidate, Long] + extends DataRecordFeature[TweetCandidate, Long] with LongDiscreteDataRecordCompatible { override def featureName: String = SharedFeatures.USER_ID.getFeatureName override def personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.UserId) @@ -145,19 +180,19 @@ object HomeFeatures { object WeightedModelScoreFeature extends Feature[TweetCandidate, Option[Double]] object MentionUserIdFeature extends Feature[TweetCandidate, Seq[Long]] object MentionScreenNameFeature extends Feature[TweetCandidate, Seq[String]] - object SemanticAnnotationFeature extends Feature[TweetCandidate, Seq[esb.TweetEntityAnnotation]] object HasImageFeature extends Feature[TweetCandidate, Boolean] object HasVideoFeature extends Feature[TweetCandidate, Boolean] // Tweetypie VF Features - object IsHydratedFeature extends FeatureWithDefaultOnFailure[TweetCandidate, Boolean] { - override val defaultValue: Boolean = true - } + object IsHydratedFeature extends Feature[TweetCandidate, Boolean] object IsNsfwFeature extends Feature[TweetCandidate, Boolean] object QuotedTweetDroppedFeature extends Feature[TweetCandidate, Boolean] // Raw Tweet Text from Tweetypie object TweetTextFeature extends Feature[TweetCandidate, Option[String]] + object AuthorEnabledPreviewsFeature extends Feature[TweetCandidate, Boolean] + object IsTweetPreviewFeature extends Feature[TweetCandidate, Boolean] + // SGS Features /** * By convention, this is set to true for retweets of non-followed authors @@ -170,41 +205,49 @@ object HomeFeatures { // Query Features object AccountAgeFeature extends Feature[PipelineQuery, Option[Time]] object ClientIdFeature - extends Feature[PipelineQuery, Option[Long]] + extends DataRecordOptionalFeature[PipelineQuery, Long] with LongDiscreteDataRecordCompatible { override def featureName: String = SharedFeatures.CLIENT_ID.getFeatureName override def personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.ClientType) } - object CachedScoredTweetsFeature extends Feature[PipelineQuery, Seq[hmt.CachedScoredTweet]] + object CachedScoredTweetsFeature extends Feature[PipelineQuery, Seq[hmt.ScoredTweet]] object DeviceLanguageFeature extends Feature[PipelineQuery, Option[String]] object DismissInfoFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Map[st.SuggestType, Option[DismissInfo]]] { override def defaultValue: Map[st.SuggestType, Option[DismissInfo]] = Map.empty } object FollowingLastNonPollingTimeFeature extends Feature[PipelineQuery, Option[Time]] - object GetInitialFeature extends Feature[PipelineQuery, Boolean] with BoolDataRecordCompatible { + object GetInitialFeature + extends DataRecordFeature[PipelineQuery, Boolean] + with BoolDataRecordCompatible { override def featureName: String = RequestContextFeatures.IS_GET_INITIAL.getFeatureName override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty } - object GetMiddleFeature extends Feature[PipelineQuery, Boolean] with BoolDataRecordCompatible { + object GetMiddleFeature + extends DataRecordFeature[PipelineQuery, Boolean] + with BoolDataRecordCompatible { override def featureName: String = RequestContextFeatures.IS_GET_MIDDLE.getFeatureName override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty } - object GetNewerFeature extends Feature[PipelineQuery, Boolean] with BoolDataRecordCompatible { + object GetNewerFeature + extends DataRecordFeature[PipelineQuery, Boolean] + with BoolDataRecordCompatible { override def featureName: String = RequestContextFeatures.IS_GET_NEWER.getFeatureName override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty } - object GetOlderFeature extends Feature[PipelineQuery, Boolean] with BoolDataRecordCompatible { + object GetOlderFeature + extends DataRecordFeature[PipelineQuery, Boolean] + with BoolDataRecordCompatible { override def featureName: String = RequestContextFeatures.IS_GET_OLDER.getFeatureName override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty } object GuestIdFeature - extends Feature[PipelineQuery, Option[Long]] + extends DataRecordOptionalFeature[PipelineQuery, Long] with LongDiscreteDataRecordCompatible { override def featureName: String = SharedFeatures.GUEST_ID.getFeatureName override def personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.GuestId) } - object HasDarkRequestFeature extends Feature[TweetCandidate, Option[Boolean]] + object HasDarkRequestFeature extends Feature[PipelineQuery, Option[Boolean]] object ImpressionBloomFilterFeature extends FeatureWithDefaultOnFailure[PipelineQuery, blm.ImpressionBloomFilterSeq] { override def defaultValue: blm.ImpressionBloomFilterSeq = @@ -223,6 +266,26 @@ object HomeFeatures { // Internal id generated per request, mainly to deduplicate re-served cached tweets in logging object ServedRequestIdFeature extends Feature[PipelineQuery, Option[Long]] object ServedTweetIdsFeature extends Feature[PipelineQuery, Seq[Long]] + object ServedTweetPreviewIdsFeature extends Feature[PipelineQuery, Seq[Long]] + object TimelineServiceTweetsFeature extends Feature[PipelineQuery, Seq[Long]] + object TimestampFeature + extends DataRecordFeature[PipelineQuery, Long] + with LongDiscreteDataRecordCompatible { + override def featureName: String = SharedFeatures.TIMESTAMP.getFeatureName + override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty + } + object TimestampGMTDowFeature + extends DataRecordFeature[PipelineQuery, Long] + with LongDiscreteDataRecordCompatible { + override def featureName: String = RequestContextFeatures.TIMESTAMP_GMT_DOW.getFeatureName + override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty + } + object TimestampGMTHourFeature + extends DataRecordFeature[PipelineQuery, Long] + with LongDiscreteDataRecordCompatible { + override def featureName: String = RequestContextFeatures.TIMESTAMP_GMT_HOUR.getFeatureName + override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty + } object TweetImpressionsFeature extends Feature[PipelineQuery, Seq[imp.TweetImpressionsEntry]] object UserFollowedTopicsCountFeature extends Feature[PipelineQuery, Option[Int]] object UserFollowingCountFeature extends Feature[PipelineQuery, Option[Int]] diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/BUILD.bazel index 2212b8159..3883e454e 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/BUILD.bazel @@ -6,7 +6,6 @@ scala_library( dependencies = [ "dspbidder/thrift/src/main/thrift/com/twitter/dspbidder/commons:thrift-scala", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", "timelineservice/common:model", ], diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProduct.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProduct.scala index 107f4c243..7c27c50d5 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProduct.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProduct.scala @@ -33,3 +33,8 @@ case object ListRecommendedUsersProduct extends Product { override val identifier: ProductIdentifier = ProductIdentifier("ListRecommendedUsers") override val stringCenterProject: Option[String] = Some("timelinemixer") } + +case object SubscribedProduct extends Product { + override val identifier: ProductIdentifier = ProductIdentifier("Subscribed") + override val stringCenterProject: Option[String] = Some("timelinemixer") +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProductContext.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProductContext.scala index 9f3ec4cb7..dddd733f3 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProductContext.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProductContext.scala @@ -12,13 +12,15 @@ case class FollowingProductContext( case class ForYouProductContext( deviceContext: Option[DeviceContext], seenTweetIds: Option[Seq[Long]], - dspClientContext: Option[DspClientContext]) + dspClientContext: Option[DspClientContext], + pushToHomeTweetId: Option[Long]) extends ProductContext case class ScoredTweetsProductContext( deviceContext: Option[DeviceContext], seenTweetIds: Option[Seq[Long]], - servedTweetIds: Option[Seq[Long]]) + servedTweetIds: Option[Seq[Long]], + backfillTweetIds: Option[Seq[Long]]) extends ProductContext case class ListTweetsProductContext( @@ -30,5 +32,11 @@ case class ListTweetsProductContext( case class ListRecommendedUsersProductContext( listId: Long, selectedUserIds: Option[Seq[Long]], - excludedUserIds: Option[Seq[Long]]) + excludedUserIds: Option[Seq[Long]], + listName: Option[String]) + extends ProductContext + +case class SubscribedProductContext( + deviceContext: Option[DeviceContext], + seenTweetIds: Option[Seq[Long]]) extends ProductContext diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BUILD.bazel index 651d2f1d8..b7fbca0d3 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BUILD.bazel @@ -48,7 +48,9 @@ scala_library( "src/thrift/com/twitter/manhattan:v1-scala", "src/thrift/com/twitter/manhattan:v2-scala", "src/thrift/com/twitter/onboarding/relevance/features:features-java", + "src/thrift/com/twitter/search:blender-scala", "src/thrift/com/twitter/search:earlybird-scala", + "src/thrift/com/twitter/service/metastore/gen:thrift-scala", "src/thrift/com/twitter/socialgraph:thrift-scala", "src/thrift/com/twitter/timelines/author_features:thrift-java", "src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala", @@ -72,6 +74,7 @@ scala_library( "timelines/src/main/scala/com/twitter/timelines/clients/manhattan/store", "timelines/src/main/scala/com/twitter/timelines/clients/predictionservice", "timelines/src/main/scala/com/twitter/timelines/clients/strato", + "timelines/src/main/scala/com/twitter/timelines/clients/strato/topics", "timelines/src/main/scala/com/twitter/timelines/clients/strato/twistly", "timelines/src/main/scala/com/twitter/timelines/config", "timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BlenderClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BlenderClientModule.scala new file mode 100644 index 000000000..bc4045181 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BlenderClientModule.scala @@ -0,0 +1,41 @@ +package com.twitter.home_mixer.module + +import com.google.inject.Provides +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finagle.thrift.ClientId +import com.twitter.inject.TwitterModule +import com.twitter.product_mixer.shared_library.thrift_client.FinagleThriftClientBuilder +import com.twitter.product_mixer.shared_library.thrift_client.NonIdempotent +import com.twitter.search.blender.thriftscala.BlenderService +import javax.inject.Singleton + +object BlenderClientModule extends TwitterModule { + + @Singleton + @Provides + def providesBlenderClient( + serviceIdentifier: ServiceIdentifier, + statsReceiver: StatsReceiver + ): BlenderService.MethodPerEndpoint = { + val clientId = serviceIdentifier.environment.toLowerCase match { + case "prod" => ClientId("") + case _ => ClientId("") + } + + FinagleThriftClientBuilder.buildFinagleMethodPerEndpoint[ + BlenderService.ServicePerEndpoint, + BlenderService.MethodPerEndpoint + ]( + serviceIdentifier = serviceIdentifier, + clientId = clientId, + dest = "/s/blender-universal/blender", + label = "blender", + statsReceiver = statsReceiver, + idempotency = NonIdempotent, + timeoutPerRequest = 1000.milliseconds, + timeoutTotal = 1000.milliseconds, + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/FeedbackHistoryClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/FeedbackHistoryClientModule.scala index 83093de45..c065f7b35 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/FeedbackHistoryClientModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/FeedbackHistoryClientModule.scala @@ -1,21 +1,28 @@ package com.twitter.home_mixer.module import com.google.inject.Provides +import com.twitter.conversions.DurationOps._ import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule +import com.twitter.inject.annotations.Flag import com.twitter.timelinemixer.clients.feedback.FeedbackHistoryManhattanClient import com.twitter.timelinemixer.clients.feedback.FeedbackHistoryManhattanClientConfig import com.twitter.timelines.clients.manhattan.mhv3.ManhattanClientBuilder +import com.twitter.util.Duration import javax.inject.Singleton object FeedbackHistoryClientModule extends TwitterModule { private val ProdDataset = "feedback_history" private val StagingDataset = "feedback_history_nonprod" + private final val Timeout = "mh_feedback_history.timeout" + + flag[Duration](Timeout, 150.millis, "Timeout per request") @Provides @Singleton def providesFeedbackHistoryClient( + @Flag(Timeout) timeout: Duration, serviceId: ServiceIdentifier, statsReceiver: StatsReceiver ) = { @@ -28,6 +35,7 @@ object FeedbackHistoryClientModule extends TwitterModule { val dataset = manhattanDataset val isReadOnly = true val serviceIdentifier = serviceId + override val defaultMaxTimeout = timeout } new FeedbackHistoryManhattanClient( diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerFlagsModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerFlagsModule.scala index 4031596a7..e31d2d9bc 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerFlagsModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerFlagsModule.scala @@ -16,9 +16,15 @@ object HomeMixerFlagsModule extends TwitterModule { ) flag[Boolean]( - name = ScribeServedEntriesFlag, + name = ScribeServedCandidatesFlag, default = false, - help = "Toggles logging served entries to Scribe" + help = "Toggles logging served candidates to Scribe" + ) + + flag[Boolean]( + name = ScribeScoredCandidatesFlag, + default = false, + help = "Toggles logging scored candidates to Scribe" ) flag[Boolean]( diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ImpressionBloomFilterModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ImpressionBloomFilterModule.scala index cb339a1d5..f37531483 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ImpressionBloomFilterModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ImpressionBloomFilterModule.scala @@ -5,11 +5,13 @@ import com.twitter.conversions.DurationOps._ import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule +import com.twitter.inject.annotations.Flag import com.twitter.storage.client.manhattan.kv.Guarantee import com.twitter.storehaus_internal.manhattan.ManhattanClusters import com.twitter.timelines.clients.manhattan.store._ -import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilter +import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm} import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilterManhattanKeyValueDescriptor +import com.twitter.util.Duration import javax.inject.Singleton object ImpressionBloomFilterModule extends TwitterModule { @@ -20,35 +22,38 @@ object ImpressionBloomFilterModule extends TwitterModule { private val StagingDataset = "impression_bloom_filter_staging" private val ClientStatsScope = "tweetBloomFilterImpressionManhattanClient" private val DefaultTTL = 7.days + private final val Timeout = "mh_impression_store_bloom_filter.timeout" + + flag[Duration](Timeout, 150.millis, "Timeout per request") @Provides @Singleton def providesImpressionBloomFilter( + @Flag(Timeout) timeout: Duration, serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver - ): ImpressionBloomFilter = { + ): ManhattanStoreClient[blm.ImpressionBloomFilterKey, blm.ImpressionBloomFilterSeq] = { val (appId, dataset) = serviceIdentifier.environment.toLowerCase match { case "prod" => (ProdAppId, ProdDataset) case _ => (StagingAppId, StagingDataset) } - implicit val manhattanKeyValueDescriptor = ImpressionBloomFilterManhattanKeyValueDescriptor( - dataset = dataset, - ttl = DefaultTTL - ) + implicit val manhattanKeyValueDescriptor: ImpressionBloomFilterManhattanKeyValueDescriptor = + ImpressionBloomFilterManhattanKeyValueDescriptor( + dataset = dataset, + ttl = DefaultTTL + ) - val manhattanClient = ManhattanStoreClientBuilder.buildManhattanClient( + ManhattanStoreClientBuilder.buildManhattanClient( serviceIdentifier = serviceIdentifier, cluster = ManhattanClusters.nash, appId = appId, - defaultMaxTimeout = 100.milliseconds, + defaultMaxTimeout = timeout, maxRetryCount = 2, defaultGuarantee = Some(Guarantee.SoftDcReadMyWrites), isReadOnly = false, statsScope = ClientStatsScope, statsReceiver = statsReceiver ) - - ImpressionBloomFilter(manhattanClient) } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanClientsModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanClientsModule.scala index 5bdbf9e97..fc0e282af 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanClientsModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanClientsModule.scala @@ -1,10 +1,11 @@ package com.twitter.home_mixer.module import com.google.inject.Provides +import com.twitter.conversions.DurationOps._ import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealGraphManhattanEndpoint -import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserMetadataManhattanEndpoint import com.twitter.inject.TwitterModule +import com.twitter.inject.annotations.Flag import com.twitter.storage.client.manhattan.kv._ import com.twitter.timelines.config.ConfigUtils import com.twitter.util.Duration @@ -13,44 +14,28 @@ import javax.inject.Singleton object ManhattanClientsModule extends TwitterModule with ConfigUtils { - private val starbuckDest: String = "/s/manhattan/starbuck.native-thrift" - private val apolloDest: String = "/s/manhattan/apollo.native-thrift" + private val ApolloDest = "/s/manhattan/apollo.native-thrift" + private final val Timeout = "mh_real_graph.timeout" + + flag[Duration](Timeout, 150.millis, "Timeout total") @Provides @Singleton @Named(RealGraphManhattanEndpoint) def providesRealGraphManhattanEndpoint( + @Flag(Timeout) timeout: Duration, serviceIdentifier: ServiceIdentifier ): ManhattanKVEndpoint = { lazy val client = ManhattanKVClient( appId = "real_graph", - dest = apolloDest, + dest = ApolloDest, mtlsParams = ManhattanKVClientMtlsParams(serviceIdentifier = serviceIdentifier), label = "real-graph-data" ) ManhattanKVEndpointBuilder(client) .maxRetryCount(2) - .defaultMaxTimeout(Duration.fromMilliseconds(100)) - .build() - } - - @Provides - @Singleton - @Named(UserMetadataManhattanEndpoint) - def providesUserMetadataManhattanEndpoint( - serviceIdentifier: ServiceIdentifier - ): ManhattanKVEndpoint = { - val client = ManhattanKVClient( - appId = "user_metadata", - dest = starbuckDest, - mtlsParams = ManhattanKVClientMtlsParams(serviceIdentifier = serviceIdentifier), - label = "user-metadata" - ) - - ManhattanKVEndpointBuilder(client) - .maxRetryCount(1) - .defaultMaxTimeout(Duration.fromMilliseconds(70)) + .defaultMaxTimeout(timeout) .build() } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanFeatureRepositoryModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanFeatureRepositoryModule.scala index f9afc1dcb..5668ba0ee 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanFeatureRepositoryModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanFeatureRepositoryModule.scala @@ -9,17 +9,18 @@ import com.twitter.conversions.DurationOps._ import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.home_mixer.param.HomeMixerInjectionNames._ import com.twitter.home_mixer.util.InjectionTransformerImplicits._ +import com.twitter.home_mixer.util.LanguageUtil import com.twitter.home_mixer.util.TensorFlowUtil import com.twitter.inject.TwitterModule -import com.twitter.manhattan.v1.thriftscala.ManhattanCoordinator import com.twitter.manhattan.v1.{thriftscala => mh} -import com.twitter.ml.api.thriftscala.FloatTensor import com.twitter.ml.api.{thriftscala => ml} import com.twitter.ml.featurestore.lib.UserId -import com.twitter.ml.featurestore.thriftscala.EntityId +import com.twitter.ml.featurestore.{thriftscala => fs} import com.twitter.onboarding.relevance.features.{thriftjava => rf} import com.twitter.product_mixer.shared_library.manhattan_client.ManhattanClientBuilder import com.twitter.scalding_internal.multiformat.format.keyval.KeyValInjection.ScalaBinaryThrift +import com.twitter.search.common.constants.{thriftscala => scc} +import com.twitter.service.metastore.gen.{thriftscala => smg} import com.twitter.servo.cache._ import com.twitter.servo.manhattan.ManhattanKeyValueRepository import com.twitter.servo.repository.CachingKeyValueRepository @@ -28,10 +29,11 @@ import com.twitter.servo.repository.KeyValueRepository import com.twitter.servo.repository.Repository import com.twitter.servo.repository.keysAsQuery import com.twitter.servo.util.Transformer +import com.twitter.storage.client.manhattan.bijections.Bijections import com.twitter.storehaus_internal.manhattan.ManhattanClusters import com.twitter.timelines.author_features.v1.{thriftjava => af} -import com.twitter.timelines.suggests.common.dense_data_record.thriftscala.DenseFeatureMetadata -import com.twitter.user_session_store.thriftscala.UserSession +import com.twitter.timelines.suggests.common.dense_data_record.{thriftscala => ddr} +import com.twitter.user_session_store.{thriftscala => uss_scala} import com.twitter.user_session_store.{thriftjava => uss} import com.twitter.util.Duration import com.twitter.util.Try @@ -46,24 +48,40 @@ object ManhattanFeatureRepositoryModule extends TwitterModule { private val DEFAULT_RPC_CHUNK_SIZE = 50 - private val ThriftEntityIdInjection = ScalaBinaryThrift(EntityId) + private val ThriftEntityIdInjection = ScalaBinaryThrift(fs.EntityId) - val UserIdKeyTransformer = new Transformer[Long, ByteBuffer] { + private val FeatureStoreUserIdKeyTransformer = new Transformer[Long, ByteBuffer] { override def to(userId: Long): Try[ByteBuffer] = { Try(ByteBuffer.wrap(ThriftEntityIdInjection.apply(UserId(userId).toThrift))) } override def from(b: ByteBuffer): Try[Long] = ??? } - val FloatTensorTransformer = new Transformer[ByteBuffer, FloatTensor] { - override def to(input: ByteBuffer): Try[FloatTensor] = { + private val FloatTensorTransformer = new Transformer[ByteBuffer, ml.FloatTensor] { + override def to(input: ByteBuffer): Try[ml.FloatTensor] = { val floatTensor = TensorFlowUtil.embeddingByteBufferToFloatTensor(input) Try(floatTensor) } - override def from(b: FloatTensor): Try[ByteBuffer] = ??? + override def from(b: ml.FloatTensor): Try[ByteBuffer] = ??? } + private val LanguageTransformer = new Transformer[ByteBuffer, Seq[scc.ThriftLanguage]] { + override def to(input: ByteBuffer): Try[Seq[scc.ThriftLanguage]] = { + Try.fromScala( + Bijections + .BinaryScalaInjection(smg.UserLanguages) + .andThen(Bijections.byteBuffer2Buf.inverse) + .invert(input).map(LanguageUtil.computeLanguages(_))) + } + + override def from(b: Seq[scc.ThriftLanguage]): Try[ByteBuffer] = ??? + } + + private val LongKeyTransformer = Injection + .connect[Long, Array[Byte]] + .toByteBufferTransformer() + // manhattan clients @Provides @@ -127,18 +145,14 @@ object ManhattanFeatureRepositoryModule extends TwitterModule { @Named(ManhattanStarbuckClient) client: mh.ManhattanCoordinator.MethodPerEndpoint ): KeyValueRepository[Seq[Long], Long, rf.MCUserCountingFeatures] = { - val keyTransformer = Injection - .connect[Long, Array[Byte]] - .toByteBufferTransformer() - val valueTransformer = ThriftCodec - .toCompact[rf.MCUserCountingFeatures] + .toBinary[rf.MCUserCountingFeatures] .toByteBufferTransformer() .flip batchedManhattanKeyValueRepository[Long, rf.MCUserCountingFeatures]( client = client, - keyTransformer = keyTransformer, + keyTransformer = LongKeyTransformer, valueTransformer = valueTransformer, appId = "wtf_ml", dataset = "mc_user_counting_features_v0_starbuck", @@ -158,25 +172,25 @@ object ManhattanFeatureRepositoryModule extends TwitterModule { @Named(TimelineAggregateMetadataRepository) def providesTimelineAggregateMetadataRepository( @Named(ManhattanAthenaClient) client: mh.ManhattanCoordinator.MethodPerEndpoint - ): Repository[Int, Option[DenseFeatureMetadata]] = { + ): Repository[Int, Option[ddr.DenseFeatureMetadata]] = { val keyTransformer = Injection .connect[Int, Array[Byte]] .toByteBufferTransformer() - val valueTransformer = new Transformer[ByteBuffer, DenseFeatureMetadata] { + val valueTransformer = new Transformer[ByteBuffer, ddr.DenseFeatureMetadata] { private val compactProtocolFactory = new TCompactProtocol.Factory - def to(buffer: ByteBuffer): Try[DenseFeatureMetadata] = Try { + def to(buffer: ByteBuffer): Try[ddr.DenseFeatureMetadata] = Try { val transport = transportFromByteBuffer(buffer) - DenseFeatureMetadata.decode(compactProtocolFactory.getProtocol(transport)) + ddr.DenseFeatureMetadata.decode(compactProtocolFactory.getProtocol(transport)) } // Encoding intentionally not implemented as it is never used - def from(metadata: DenseFeatureMetadata): Try[ByteBuffer] = ??? + def from(metadata: ddr.DenseFeatureMetadata): Try[ByteBuffer] = ??? } - val inProcessCache: Cache[Int, Cached[DenseFeatureMetadata]] = InProcessLruCacheFactory( + val inProcessCache: Cache[Int, Cached[ddr.DenseFeatureMetadata]] = InProcessLruCacheFactory( ttl = Duration.fromMinutes(20), lruSize = 30 ).apply(serializer = Transformer(_ => ???, _ => ???)) // Serialization is not necessary here. @@ -192,7 +206,7 @@ object ManhattanFeatureRepositoryModule extends TwitterModule { KeyValueRepository .singular( - new CachingKeyValueRepository[Seq[Int], Int, DenseFeatureMetadata]( + new CachingKeyValueRepository[Seq[Int], Int, ddr.DenseFeatureMetadata]( keyValueRepository, new NonLockingCache(inProcessCache), keysAsQuery[Int] @@ -204,17 +218,17 @@ object ManhattanFeatureRepositoryModule extends TwitterModule { @Singleton @Named(RealGraphFeatureRepository) def providesRealGraphFeatureRepository( - @Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint - ): Repository[Long, Option[UserSession]] = { - val valueTransformer = CompactScalaCodec(UserSession).toByteBufferTransformer().flip + @Named(ManhattanAthenaClient) client: mh.ManhattanCoordinator.MethodPerEndpoint + ): Repository[Long, Option[uss_scala.UserSession]] = { + val valueTransformer = CompactScalaCodec(uss_scala.UserSession).toByteBufferTransformer().flip KeyValueRepository.singular( new ManhattanKeyValueRepository( client = client, - keyTransformer = UserIdKeyTransformer, + keyTransformer = LongKeyTransformer, valueTransformer = valueTransformer, appId = "real_graph", - dataset = "real_graph_user_features", + dataset = "split_real_graph_features", timeoutInMillis = 100, ) ) @@ -230,16 +244,12 @@ object ManhattanFeatureRepositoryModule extends TwitterModule { @Named(HomeAuthorFeaturesCacheClient) cacheClient: Memcache ): KeyValueRepository[Seq[Long], Long, af.AuthorFeatures] = { - val keyTransformer = Injection - .connect[Long, Array[Byte]] - .toByteBufferTransformer() - val valueInjection = ThriftCodec .toCompact[af.AuthorFeatures] val keyValueRepository = batchedManhattanKeyValueRepository( client = client, - keyTransformer = keyTransformer, + keyTransformer = LongKeyTransformer, valueTransformer = valueInjection.toByteBufferTransformer().flip, appId = "timelines_author_feature_store_athena", dataset = "timelines_author_features", @@ -263,47 +273,58 @@ object ManhattanFeatureRepositoryModule extends TwitterModule { @Provides @Singleton - @Named(TwhinAuthorFollow20200101FeatureRepository) - def providesTwhinAuthorFollow20200101FeatureRepository( + @Named(TwhinAuthorFollowFeatureRepository) + def providesTwhinAuthorFollowFeatureRepository( @Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint, - @Named(TwhinAuthorFollow20200101FeatureCacheClient) cacheClient: Memcache - ): KeyValueRepository[Seq[Long], Long, ml.Embedding] = { + @Named(TwhinAuthorFollowFeatureCacheClient) cacheClient: Memcache + ): KeyValueRepository[Seq[Long], Long, ml.FloatTensor] = { + val keyValueRepository = + batchedManhattanKeyValueRepository( + client = client, + keyTransformer = FeatureStoreUserIdKeyTransformer, + valueTransformer = FloatTensorTransformer, + appId = "ml_features_apollo", + dataset = "twhin_author_follow_embedding_fsv1__v1_thrift__embedding", + timeoutInMillis = 100 + ) - val keyTransformer = Injection - .connect[Long, Array[Byte]] - .toByteBufferTransformer() - - val valueInjection: Injection[ml.Embedding, Array[Byte]] = - BinaryScalaCodec(ml.Embedding) - - val keyValueRepository = batchedManhattanKeyValueRepository( - client = client, - keyTransformer = keyTransformer, - valueTransformer = valueInjection.toByteBufferTransformer().flip, - appId = "twhin", - dataset = "twhinauthor_follow_0101", - timeoutInMillis = 100 - ) + val valueInjection: Injection[ml.FloatTensor, Array[Byte]] = + BinaryScalaCodec(ml.FloatTensor) buildMemCachedRepository( keyValueRepository = keyValueRepository, cacheClient = cacheClient, - cachePrefix = "TwhinAuthorFollow20200101FeatureHydrator", - ttl = 48.hours, + cachePrefix = "twhinAuthorFollows", + ttl = 24.hours, valueInjection = valueInjection ) } + @Provides + @Singleton + @Named(UserLanguagesRepository) + def providesUserLanguagesFeatureRepository( + @Named(ManhattanStarbuckClient) client: mh.ManhattanCoordinator.MethodPerEndpoint + ): KeyValueRepository[Seq[Long], Long, Seq[scc.ThriftLanguage]] = { + batchedManhattanKeyValueRepository( + client = client, + keyTransformer = LongKeyTransformer, + valueTransformer = LanguageTransformer, + appId = "user_metadata", + dataset = "languages", + timeoutInMillis = 70 + ) + } + @Provides @Singleton @Named(TwhinUserFollowFeatureRepository) def providesTwhinUserFollowFeatureRepository( @Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint - ): KeyValueRepository[Seq[Long], Long, FloatTensor] = { - + ): KeyValueRepository[Seq[Long], Long, ml.FloatTensor] = { batchedManhattanKeyValueRepository( client = client, - keyTransformer = UserIdKeyTransformer, + keyTransformer = FeatureStoreUserIdKeyTransformer, valueTransformer = FloatTensorTransformer, appId = "ml_features_apollo", dataset = "twhin_user_follow_embedding_fsv1__v1_thrift__embedding", @@ -340,11 +361,11 @@ object ManhattanFeatureRepositoryModule extends TwitterModule { @Named(TwhinUserEngagementFeatureRepository) def providesTwhinUserEngagementFeatureRepository( @Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint - ): KeyValueRepository[Seq[Long], Long, FloatTensor] = { + ): KeyValueRepository[Seq[Long], Long, ml.FloatTensor] = { batchedManhattanKeyValueRepository( client = client, - keyTransformer = UserIdKeyTransformer, + keyTransformer = FeatureStoreUserIdKeyTransformer, valueTransformer = FloatTensorTransformer, appId = "ml_features_apollo", dataset = "twhin_user_engagement_embedding_fsv1__v1_thrift__embedding", @@ -399,7 +420,7 @@ object ManhattanFeatureRepositoryModule extends TwitterModule { } private def batchedManhattanKeyValueRepository[K, V]( - client: ManhattanCoordinator.MethodPerEndpoint, + client: mh.ManhattanCoordinator.MethodPerEndpoint, keyTransformer: Transformer[K, ByteBuffer], valueTransformer: Transformer[ByteBuffer, V], appId: String, @@ -430,24 +451,18 @@ object ManhattanFeatureRepositoryModule extends TwitterModule { mhDataset: String, mhAppId: String ): Repository[Long, Option[uss.UserSession]] = { - val keyTransformer = Injection - .connect[Long, Array[Byte]] - .toByteBufferTransformer() - val valueInjection = ThriftCodec .toCompact[uss.UserSession] KeyValueRepository.singular( new ManhattanKeyValueRepository( client = mhClient, - keyTransformer = keyTransformer, + keyTransformer = LongKeyTransformer, valueTransformer = valueInjection.toByteBufferTransformer().flip, appId = mhAppId, dataset = mhDataset, timeoutInMillis = 100 ) ) - } - } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanTweetImpressionStoreModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanTweetImpressionStoreModule.scala index 1f6ae824a..c6782665a 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanTweetImpressionStoreModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanTweetImpressionStoreModule.scala @@ -5,11 +5,13 @@ import com.twitter.conversions.DurationOps._ import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule +import com.twitter.inject.annotations.Flag import com.twitter.storage.client.manhattan.kv.Guarantee import com.twitter.storehaus_internal.manhattan.ManhattanClusters import com.twitter.timelines.clients.manhattan.mhv3.ManhattanClientBuilder import com.twitter.timelines.impressionstore.store.ManhattanTweetImpressionStoreClientConfig import com.twitter.timelines.impressionstore.store.ManhattanTweetImpressionStoreClient +import com.twitter.util.Duration import javax.inject.Singleton object ManhattanTweetImpressionStoreModule extends TwitterModule { @@ -20,10 +22,14 @@ object ManhattanTweetImpressionStoreModule extends TwitterModule { private val StagingDataset = "timelines_tweet_impressions_staging" private val StatsScope = "manhattanTweetImpressionStoreClient" private val DefaultTTL = 2.days + private final val Timeout = "mh_impression_store.timeout" + + flag[Duration](Timeout, 150.millis, "Timeout per request") @Provides @Singleton def providesManhattanTweetImpressionStoreClient( + @Flag(Timeout) timeout: Duration, serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): ManhattanTweetImpressionStoreClient = { @@ -39,7 +45,7 @@ object ManhattanTweetImpressionStoreModule extends TwitterModule { dataset = dataset, statsScope = StatsScope, defaultGuarantee = Guarantee.SoftDcReadMyWrites, - defaultMaxTimeout = 100.milliseconds, + defaultMaxTimeout = timeout, maxRetryCount = 2, isReadOnly = false, serviceIdentifier = serviceIdentifier, diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MemcachedFeatureRepositoryModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MemcachedFeatureRepositoryModule.scala index 2f5ce1f5d..8afafbfb7 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MemcachedFeatureRepositoryModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MemcachedFeatureRepositoryModule.scala @@ -8,7 +8,7 @@ import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.HomeAuthorFeaturesCacheClient import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealTimeInteractionGraphUserVertexClient import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelinesRealTimeAggregateClient -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinAuthorFollow20200101FeatureCacheClient +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinAuthorFollowFeatureCacheClient import com.twitter.inject.TwitterModule import com.twitter.product_mixer.shared_library.memcached_client.MemcachedClientBuilder import com.twitter.servo.cache.FinagleMemcacheFactory @@ -31,9 +31,10 @@ object MemcachedFeatureRepositoryModule extends TwitterModule { statsReceiver: StatsReceiver ): Memcache = { val rawClient = MemcachedClientBuilder.buildRawMemcachedClient( - numTries = 1, - requestTimeout = 150.milliseconds, - globalTimeout = 150.milliseconds, + numTries = 3, + numConnections = 1, + requestTimeout = 100.milliseconds, + globalTimeout = 300.milliseconds, connectTimeout = 200.milliseconds, acquisitionTimeout = 200.milliseconds, serviceIdentifier = serviceIdentifier, @@ -51,9 +52,10 @@ object MemcachedFeatureRepositoryModule extends TwitterModule { statsReceiver: StatsReceiver ): Memcache = { val cacheClient = MemcachedClientBuilder.buildRawMemcachedClient( - numTries = 1, - requestTimeout = 50.milliseconds, - globalTimeout = 50.milliseconds, + numTries = 2, + numConnections = 1, + requestTimeout = 150.milliseconds, + globalTimeout = 300.milliseconds, connectTimeout = 200.milliseconds, acquisitionTimeout = 200.milliseconds, serviceIdentifier = serviceIdentifier, @@ -65,15 +67,16 @@ object MemcachedFeatureRepositoryModule extends TwitterModule { @Provides @Singleton - @Named(TwhinAuthorFollow20200101FeatureCacheClient) - def providesTwhinAuthorFollow20200101FeatureCacheClient( + @Named(TwhinAuthorFollowFeatureCacheClient) + def providesTwhinAuthorFollowFeatureCacheClient( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): Memcache = { val cacheClient = MemcachedClientBuilder.buildRawMemcachedClient( - numTries = 1, - requestTimeout = 50.milliseconds, - globalTimeout = 50.milliseconds, + numTries = 2, + numConnections = 1, + requestTimeout = 150.milliseconds, + globalTimeout = 300.milliseconds, connectTimeout = 200.milliseconds, acquisitionTimeout = 200.milliseconds, serviceIdentifier = serviceIdentifier, @@ -91,9 +94,10 @@ object MemcachedFeatureRepositoryModule extends TwitterModule { statsReceiver: StatsReceiver ): Memcache = { val cacheClient = MemcachedClientBuilder.buildRawMemcachedClient( - numTries = 1, - requestTimeout = 100.milliseconds, - globalTimeout = 100.milliseconds, + numTries = 2, + numConnections = 1, + requestTimeout = 150.milliseconds, + globalTimeout = 300.milliseconds, connectTimeout = 200.milliseconds, acquisitionTimeout = 200.milliseconds, serviceIdentifier = serviceIdentifier, diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeNaviModelClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/NaviModelClientModule.scala similarity index 88% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeNaviModelClientModule.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/module/NaviModelClientModule.scala index df70592c2..60d580a73 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeNaviModelClientModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/NaviModelClientModule.scala @@ -10,10 +10,9 @@ import com.twitter.inject.TwitterModule import com.twitter.timelines.clients.predictionservice.PredictionGRPCService import com.twitter.util.Duration import io.grpc.ManagedChannel - import javax.inject.Singleton -object HomeNaviModelClientModule extends TwitterModule { +object NaviModelClientModule extends TwitterModule { @Singleton @Provides @@ -23,10 +22,9 @@ object HomeNaviModelClientModule extends TwitterModule { // Wily path to the ML Model service (e.g. /s/ml-serving/navi-explore-ranker). val modelPath = "/s/ml-serving/navi_home_recap_onnx" - // timeout for prediction service requests. - val MaxPredictionTimeoutMs: Duration = 300.millis + val MaxPredictionTimeoutMs: Duration = 500.millis val ConnectTimeoutMs: Duration = 200.millis - val AcquisitionTimeoutMs: Duration = 20000.millis + val AcquisitionTimeoutMs: Duration = 500.millis val MaxRetryAttempts: Int = 2 val client = Http.client diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/OptimizedStratoClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/OptimizedStratoClientModule.scala index 7575685f1..b9e315acc 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/OptimizedStratoClientModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/OptimizedStratoClientModule.scala @@ -17,7 +17,7 @@ import javax.inject.Singleton object OptimizedStratoClientModule extends TwitterModule { - private val ModerateStratoServerClientRequestTimeout = 150.millis + private val ModerateStratoServerClientRequestTimeout = 500.millis private val DefaultRetryPartialFunction: PartialFunction[Try[Nothing], Boolean] = RetryPolicy.TimeoutAndWriteExceptionsOnly @@ -35,7 +35,7 @@ object OptimizedStratoClientModule extends TwitterModule { ): Client = { Strato.client .withMutualTls(serviceIdentifier, opportunisticLevel = OpportunisticTls.Required) - .withSession.acquisitionTimeout(150.milliseconds) + .withSession.acquisitionTimeout(500.milliseconds) .withRequestTimeout(ModerateStratoServerClientRequestTimeout) .withPerRequestTimeout(ModerateStratoServerClientRequestTimeout) .withRpcBatchSize(5) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PipelineFailureExceptionMapper.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PipelineFailureExceptionMapper.scala index 65f0ac7bd..ad15988cb 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PipelineFailureExceptionMapper.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PipelineFailureExceptionMapper.scala @@ -2,7 +2,7 @@ package com.twitter.home_mixer.module import com.twitter.finatra.thrift.exceptions.ExceptionMapper import com.twitter.home_mixer.{thriftscala => t} -import com.twitter.inject.Logging +import com.twitter.util.logging.Logging import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure import com.twitter.product_mixer.core.pipeline.pipeline_failure.ProductDisabled import com.twitter.scrooge.ThriftException diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealtimeAggregateFeatureRepositoryModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealtimeAggregateFeatureRepositoryModule.scala index 19512e663..c3c545819 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealtimeAggregateFeatureRepositoryModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealtimeAggregateFeatureRepositoryModule.scala @@ -39,12 +39,12 @@ object RealtimeAggregateFeatureRepositoryModule extends TwitterModule with RealtimeAggregateHelpers { - private val authorIdFeature = new Feature.Discrete("entities.source_author_id") - private val countryCodeFeature = new Feature.Text("geo.user_location.country_code") - private val listIdFeature = new Feature.Discrete("list.id") - private val userIdFeature = new Feature.Discrete("meta.user_id") - private val topicIdFeature = new Feature.Discrete("entities.topic_id") - private val tweetIdFeature = new Feature.Discrete("entities.source_tweet_id") + private val authorIdFeature = new Feature.Discrete("entities.source_author_id").getFeatureId + private val countryCodeFeature = new Feature.Text("geo.user_location.country_code").getFeatureId + private val listIdFeature = new Feature.Discrete("list.id").getFeatureId + private val userIdFeature = new Feature.Discrete("meta.user_id").getFeatureId + private val topicIdFeature = new Feature.Discrete("entities.topic_id").getFeatureId + private val tweetIdFeature = new Feature.Discrete("entities.source_tweet_id").getFeatureId @Provides @Singleton @@ -208,42 +208,20 @@ trait RealtimeAggregateHelpers { .compose((k: AggregationKey) => (k, defaultBatchID)) } - protected def keyTransformD1(f1: Feature.Discrete)(key: Long): String = { - val aggregationKey = AggregationKey( - Map(f1.getFeatureId -> key), - Map.empty - ) - + protected def keyTransformD1(f1: Long)(key: Long): String = { + val aggregationKey = AggregationKey(Map(f1 -> key), Map.empty) keyEncoder(aggregationKey) } - protected def keyTransformD2( - f1: Feature.Discrete, - f2: Feature.Discrete - )( - keys: (Long, Long) - ): String = { + protected def keyTransformD2(f1: Long, f2: Long)(keys: (Long, Long)): String = { val (k1, k2) = keys - val aggregationKey = AggregationKey( - Map(f1.getFeatureId -> k1, f2.getFeatureId -> k2), - Map.empty - ) - + val aggregationKey = AggregationKey(Map(f1 -> k1, f2 -> k2), Map.empty) keyEncoder(aggregationKey) } - protected def keyTransformD1T1( - f1: Feature.Discrete, - f2: Feature.Text - )( - keys: (Long, String) - ): String = { + protected def keyTransformD1T1(f1: Long, f2: Long)(keys: (Long, String)): String = { val (k1, k2) = keys - val aggregationKey = AggregationKey( - Map(f1.getFeatureId -> k1), - Map(f2.getFeatureId -> k2) - ) - + val aggregationKey = AggregationKey(Map(f1 -> k1), Map(f2 -> k2)) keyEncoder(aggregationKey) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScoredTweetsMemcacheModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScoredTweetsMemcacheModule.scala index b8ac940b1..4bed31c5c 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScoredTweetsMemcacheModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScoredTweetsMemcacheModule.scala @@ -10,7 +10,6 @@ import com.twitter.product_mixer.shared_library.memcached_client.MemcachedClient import com.twitter.servo.cache.FinagleMemcache import com.twitter.servo.cache.KeyTransformer import com.twitter.servo.cache.KeyValueTransformingTtlCache -import com.twitter.servo.cache.ObservableTtlCache import com.twitter.servo.cache.Serializer import com.twitter.servo.cache.ThriftSerializer import com.twitter.servo.cache.TtlCache @@ -24,8 +23,10 @@ object ScoredTweetsMemcacheModule extends TwitterModule { private val ScopeName = "ScoredTweetsCache" private val ProdDestName = "/srv#/prod/local/cache/home_scored_tweets:twemcaches" private val StagingDestName = "/srv#/test/local/cache/twemcache_home_scored_tweets:twemcaches" - private val cachedScoredTweetsSerializer: Serializer[t.CachedScoredTweets] = - new ThriftSerializer[t.CachedScoredTweets](t.CachedScoredTweets, new TCompactProtocol.Factory()) + private val scoredTweetsSerializer: Serializer[t.ScoredTweetsResponse] = + new ThriftSerializer[t.ScoredTweetsResponse]( + t.ScoredTweetsResponse, + new TCompactProtocol.Factory()) private val userIdKeyTransformer: KeyTransformer[UserId] = (userId: UserId) => userId.toString @Singleton @@ -33,7 +34,7 @@ object ScoredTweetsMemcacheModule extends TwitterModule { def providesScoredTweetsCache( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver - ): TtlCache[UserId, t.CachedScoredTweets] = { + ): TtlCache[UserId, t.ScoredTweetsResponse] = { val destName = serviceIdentifier.environment.toLowerCase match { case "prod" => ProdDestName case _ => StagingDestName @@ -41,6 +42,7 @@ object ScoredTweetsMemcacheModule extends TwitterModule { val client = MemcachedClientBuilder.buildMemcachedClient( destName = destName, numTries = 2, + numConnections = 1, requestTimeout = 200.milliseconds, globalTimeout = 400.milliseconds, connectTimeout = 100.milliseconds, @@ -49,17 +51,11 @@ object ScoredTweetsMemcacheModule extends TwitterModule { statsReceiver = statsReceiver.scope(ScopeName) ) val underlyingCache = new FinagleMemcache(client) - val baseCache: KeyValueTransformingTtlCache[UserId, String, t.CachedScoredTweets, Array[Byte]] = - new KeyValueTransformingTtlCache( - underlyingCache = underlyingCache, - transformer = cachedScoredTweetsSerializer, - underlyingKey = userIdKeyTransformer - ) - ObservableTtlCache( - underlyingCache = baseCache, - statsReceiver = statsReceiver, - windowSize = 1000L, - name = ScopeName + + new KeyValueTransformingTtlCache( + underlyingCache = underlyingCache, + transformer = scoredTweetsSerializer, + underlyingKey = userIdKeyTransformer ) } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScribeEventPublisherModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScribeEventPublisherModule.scala index 9c74fb1c3..99bf61630 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScribeEventPublisherModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScribeEventPublisherModule.scala @@ -2,124 +2,76 @@ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.clientapp.{thriftscala => ca} -import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeClientEventsFlag -import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeServedCommonFeaturesAndCandidateFeaturesFlag -import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeServedEntriesFlag import com.twitter.home_mixer.param.HomeMixerInjectionNames.CandidateFeaturesScribeEventPublisher import com.twitter.home_mixer.param.HomeMixerInjectionNames.CommonFeaturesScribeEventPublisher import com.twitter.home_mixer.param.HomeMixerInjectionNames.MinimumFeaturesScribeEventPublisher import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag import com.twitter.logpipeline.client.EventPublisherManager import com.twitter.logpipeline.client.common.EventPublisher import com.twitter.logpipeline.client.serializers.EventLogMsgTBinarySerializer import com.twitter.logpipeline.client.serializers.EventLogMsgThriftStructSerializer import com.twitter.timelines.suggests.common.poly_data_record.{thriftjava => pldr} import com.twitter.timelines.timeline_logging.{thriftscala => tl} - import javax.inject.Named import javax.inject.Singleton object ScribeEventPublisherModule extends TwitterModule { - val InMemoryBufferSize = 10000 val ClientEventLogCategory = "client_event" - val ServedEntriesLogCategory = "home_timeline_served_entries" + val ServedCandidatesLogCategory = "home_timeline_served_candidates_flattened" + val ScoredCandidatesLogCategory = "home_timeline_scored_candidates" val ServedCommonFeaturesLogCategory = "tq_served_common_features_offline" val ServedCandidateFeaturesLogCategory = "tq_served_candidate_features_offline" val ServedMinimumFeaturesLogCategory = "tq_served_minimum_features_offline" @Provides @Singleton - def providesClientEventsScribeEventPublisher( - @Flag(ScribeClientEventsFlag) sendToScribe: Boolean - ): EventPublisher[ca.LogEvent] = { + def providesClientEventsScribeEventPublisher: EventPublisher[ca.LogEvent] = { val serializer = EventLogMsgThriftStructSerializer.getNewSerializer[ca.LogEvent]() - - if (sendToScribe) - EventPublisherManager.buildScribeLogPipelinePublisher(ClientEventLogCategory, serializer) - else - EventPublisherManager.buildInMemoryPublisher( - ClientEventLogCategory, - serializer, - InMemoryBufferSize - ) + EventPublisherManager.buildScribeLogPipelinePublisher(ClientEventLogCategory, serializer) } @Provides @Singleton @Named(CommonFeaturesScribeEventPublisher) - def providesCommonFeaturesScribeEventPublisher( - @Flag(ScribeServedCommonFeaturesAndCandidateFeaturesFlag) sendToScribe: Boolean - ): EventPublisher[pldr.PolyDataRecord] = { + def providesCommonFeaturesScribeEventPublisher: EventPublisher[pldr.PolyDataRecord] = { val serializer = EventLogMsgTBinarySerializer.getNewSerializer - - if (sendToScribe) - EventPublisherManager.buildScribeLogPipelinePublisher( - ServedCommonFeaturesLogCategory, - serializer) - else - EventPublisherManager.buildInMemoryPublisher( - ServedCommonFeaturesLogCategory, - serializer, - InMemoryBufferSize - ) + EventPublisherManager.buildScribeLogPipelinePublisher( + ServedCommonFeaturesLogCategory, + serializer) } @Provides @Singleton @Named(CandidateFeaturesScribeEventPublisher) - def providesCandidateFeaturesScribeEventPublisher( - @Flag(ScribeServedCommonFeaturesAndCandidateFeaturesFlag) sendToScribe: Boolean - ): EventPublisher[pldr.PolyDataRecord] = { + def providesCandidateFeaturesScribeEventPublisher: EventPublisher[pldr.PolyDataRecord] = { val serializer = EventLogMsgTBinarySerializer.getNewSerializer - - if (sendToScribe) - EventPublisherManager.buildScribeLogPipelinePublisher( - ServedCandidateFeaturesLogCategory, - serializer) - else - EventPublisherManager.buildInMemoryPublisher( - ServedCandidateFeaturesLogCategory, - serializer, - InMemoryBufferSize - ) + EventPublisherManager.buildScribeLogPipelinePublisher( + ServedCandidateFeaturesLogCategory, + serializer) } @Provides @Singleton @Named(MinimumFeaturesScribeEventPublisher) - def providesMinimumFeaturesScribeEventPublisher( - @Flag(ScribeServedCommonFeaturesAndCandidateFeaturesFlag) sendToScribe: Boolean - ): EventPublisher[pldr.PolyDataRecord] = { + def providesMinimumFeaturesScribeEventPublisher: EventPublisher[pldr.PolyDataRecord] = { val serializer = EventLogMsgTBinarySerializer.getNewSerializer - - if (sendToScribe) - EventPublisherManager.buildScribeLogPipelinePublisher( - ServedMinimumFeaturesLogCategory, - serializer) - else - EventPublisherManager.buildInMemoryPublisher( - ServedMinimumFeaturesLogCategory, - serializer, - InMemoryBufferSize - ) + EventPublisherManager.buildScribeLogPipelinePublisher( + ServedMinimumFeaturesLogCategory, + serializer) } @Provides @Singleton - def providesServedEntriesScribeEventPublisher( - @Flag(ScribeServedEntriesFlag) sendToScribe: Boolean - ): EventPublisher[tl.Timeline] = { - val serializer = EventLogMsgThriftStructSerializer.getNewSerializer[tl.Timeline]() + def providesServedCandidatesScribeEventPublisher: EventPublisher[tl.ServedEntry] = { + val serializer = EventLogMsgThriftStructSerializer.getNewSerializer[tl.ServedEntry]() + EventPublisherManager.buildScribeLogPipelinePublisher(ServedCandidatesLogCategory, serializer) + } - if (sendToScribe) - EventPublisherManager.buildScribeLogPipelinePublisher(ServedEntriesLogCategory, serializer) - else - EventPublisherManager.buildInMemoryPublisher( - ServedEntriesLogCategory, - serializer, - InMemoryBufferSize - ) + @Provides + @Singleton + def provideScoredCandidatesScribeEventPublisher: EventPublisher[tl.ScoredCandidate] = { + val serializer = EventLogMsgThriftStructSerializer.getNewSerializer[tl.ScoredCandidate]() + EventPublisherManager.buildScribeLogPipelinePublisher(ScoredCandidatesLogCategory, serializer) } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/StaleTweetsCacheModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/StaleTweetsCacheModule.scala index 070de320a..301dc51c2 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/StaleTweetsCacheModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/StaleTweetsCacheModule.scala @@ -24,6 +24,7 @@ object StaleTweetsCacheModule extends TwitterModule { MemcachedClientBuilder.buildMemcachedClient( destName = "/srv#/prod/local/cache/staletweetscache:twemcaches", numTries = 3, + numConnections = 1, requestTimeout = 200.milliseconds, globalTimeout = 500.milliseconds, connectTimeout = 200.milliseconds, diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ThriftFeatureRepositoryModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ThriftFeatureRepositoryModule.scala index 5a2a8701a..dcbf62451 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ThriftFeatureRepositoryModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ThriftFeatureRepositoryModule.scala @@ -69,8 +69,8 @@ object ThriftFeatureRepositoryModule extends TwitterModule { label = "interests", statsReceiver = statsReceiver, idempotency = Idempotent(1.percent), - timeoutPerRequest = 100.milliseconds, - timeoutTotal = 100.milliseconds + timeoutPerRequest = 350.milliseconds, + timeoutTotal = 350.milliseconds ) } @@ -166,8 +166,8 @@ object ThriftFeatureRepositoryModule extends TwitterModule { label = "tweetypie-content-repo", statsReceiver = statsReceiver, idempotency = Idempotent(1.percent), - timeoutPerRequest = 150.milliseconds, - timeoutTotal = 250.milliseconds + timeoutPerRequest = 300.milliseconds, + timeoutTotal = 500.milliseconds ) def lookup(tweetIds: Seq[Long]): Future[Seq[Option[tp.Tweet]]] = { @@ -176,15 +176,12 @@ object ThriftFeatureRepositoryModule extends TwitterModule { includeRetweetedTweet = false, includeQuotedTweet = false, forUserId = None, - // Service needs to be whitelisted - // We rely on the VF at the end of serving. No need to filter now. safetyLevel = Some(sp.SafetyLevel.FilterNone), visibilityPolicy = tp.TweetVisibilityPolicy.NoFiltering ) - val request = tp.GetTweetFieldsRequest( - tweetIds = tweetIds, - options = getTweetFieldsOptions - ) + + val request = tp.GetTweetFieldsRequest(tweetIds = tweetIds, options = getTweetFieldsOptions) + client.getTweetFields(request).map { results => results.map { case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), _, _) => @@ -196,16 +193,17 @@ object ThriftFeatureRepositoryModule extends TwitterModule { val keyValueRepository = toRepositoryBatch(lookup, chunkSize = 20) - // Cache val cacheClient = MemcachedClientBuilder.buildRawMemcachedClient( numTries = 1, - requestTimeout = 100.milliseconds, - globalTimeout = 100.milliseconds, + numConnections = 1, + requestTimeout = 200.milliseconds, + globalTimeout = 200.milliseconds, connectTimeout = 200.milliseconds, acquisitionTimeout = 200.milliseconds, serviceIdentifier = serviceIdentifier, statsReceiver = statsReceiver ) + val finagleMemcacheFactory = FinagleMemcacheFactory(cacheClient, "/s/cache/home_content_features:twemcaches") val cacheValueTransformer = @@ -281,10 +279,12 @@ object ThriftFeatureRepositoryModule extends TwitterModule { tweetIds: Seq[Long], viewerId: Long ): Future[Seq[Option[eb.ThriftSearchResult]]] = { - val request = EarlybirdRequestUtil.getTweetsEBFeaturesRequest( + val request = EarlybirdRequestUtil.getTweetsFeaturesRequest( userId = Some(viewerId), tweetIds = Some(tweetIds), - clientId = Some(clientId.name) + clientId = Some(clientId.name), + authorScoreMap = None, + tensorflowModel = Some("timelines_rectweet_replica") ) client diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TimelinesPersistenceStoreClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TimelinesPersistenceStoreClientModule.scala index 4292b36be..4af39fd32 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TimelinesPersistenceStoreClientModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TimelinesPersistenceStoreClientModule.scala @@ -5,34 +5,40 @@ import com.twitter.conversions.DurationOps._ import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule +import com.twitter.inject.annotations.Flag import com.twitter.timelinemixer.clients.persistence.TimelinePersistenceManhattanClientBuilder import com.twitter.timelinemixer.clients.persistence.TimelinePersistenceManhattanClientConfig import com.twitter.timelinemixer.clients.persistence.TimelineResponseBatchesClient import com.twitter.timelinemixer.clients.persistence.TimelineResponseV3 +import com.twitter.util.Duration import javax.inject.Singleton object TimelinesPersistenceStoreClientModule extends TwitterModule { private val StagingDataset = "timeline_response_batches_v5_nonprod" private val ProdDataset = "timeline_response_batches_v5" + private final val Timeout = "mh_persistence_store.timeout" + + flag[Duration](Timeout, 300.millis, "Timeout per request") @Provides @Singleton def providesTimelinesPersistenceStoreClient( + @Flag(Timeout) timeout: Duration, injectedServiceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): TimelineResponseBatchesClient[TimelineResponseV3] = { - val (timelineResponseBatchesDataset, manhattanReadOnly) = + val timelineResponseBatchesDataset = injectedServiceIdentifier.environment.toLowerCase match { - case "prod" => (ProdDataset, false) - case _ => (StagingDataset, true) + case "prod" => ProdDataset + case _ => StagingDataset } val timelineResponseBatchesConfig = new TimelinePersistenceManhattanClientConfig { val dataset = timelineResponseBatchesDataset - val isReadOnly = manhattanReadOnly + val isReadOnly = false val serviceIdentifier = injectedServiceIdentifier - override val defaultMaxTimeout = 300.milliseconds - override val maxRetryCount = 1 + override val defaultMaxTimeout = timeout + override val maxRetryCount = 2 } TimelinePersistenceManhattanClientBuilder.buildTimelineResponseV3BatchesClient( diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TopicSocialProofClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TopicSocialProofClientModule.scala new file mode 100644 index 000000000..9333e0f84 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TopicSocialProofClientModule.scala @@ -0,0 +1,22 @@ +package com.twitter.home_mixer.module + +import com.google.inject.Provides +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithModerateTimeout +import com.twitter.inject.TwitterModule +import com.twitter.strato.client.Client +import com.twitter.timelines.clients.strato.topics.TopicSocialProofClient +import com.twitter.timelines.clients.strato.topics.TopicSocialProofClientImpl +import javax.inject.Named +import javax.inject.Singleton + +object TopicSocialProofClientModule extends TwitterModule { + + @Singleton + @Provides + def providesSimilarityClient( + @Named(BatchedStratoClientWithModerateTimeout) + stratoClient: Client, + statsReceiver: StatsReceiver + ): TopicSocialProofClient = new TopicSocialProofClientImpl(stratoClient, statsReceiver) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetyPieClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetyPieClientModule.scala index 84b634cc3..1eb49206c 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetyPieClientModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetyPieClientModule.scala @@ -7,6 +7,7 @@ import com.twitter.finagle.thrift.ClientId import com.twitter.finagle.thriftmux.MethodBuilder import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient import com.twitter.inject.Injector +import com.twitter.inject.annotations.Flags import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule import com.twitter.stitch.tweetypie.TweetyPie import com.twitter.tweetypie.thriftscala.TweetService @@ -14,14 +15,21 @@ import com.twitter.util.Duration import javax.inject.Singleton /** - * Idempotent TweetyPie Thrift and Stitch client. + * Idempotent Tweetypie Thrift and Stitch client. */ -object TweetyPieClientModule +object TweetypieClientModule extends ThriftMethodBuilderClientModule[ TweetService.ServicePerEndpoint, TweetService.MethodPerEndpoint ] with MtlsClient { + + private val TimeoutRequest = "tweetypie.timeout_request" + private val TimeoutTotal = "tweetypie.timeout_total" + + flag[Duration](TimeoutRequest, 1000.millis, "Timeout per request") + flag[Duration](TimeoutTotal, 1000.millis, "Total timeout") + override val label: String = "tweetypie" override val dest: String = "/s/tweetypie/tweetypie" @@ -42,10 +50,14 @@ object TweetyPieClientModule override protected def configureMethodBuilder( injector: Injector, methodBuilder: MethodBuilder - ): MethodBuilder = - methodBuilder - .withTimeoutPerRequest(500.milliseconds) - .withTimeoutTotal(500.milliseconds) + ): MethodBuilder = { + val timeoutRequest = injector.instance[Duration](Flags.named(TimeoutRequest)) + val timeoutTotal = injector.instance[Duration](Flags.named(TimeoutTotal)) - override protected def sessionAcquisitionTimeout: Duration = 250.milliseconds + methodBuilder + .withTimeoutPerRequest(timeoutRequest) + .withTimeoutTotal(timeoutTotal) + } + + override protected def sessionAcquisitionTimeout: Duration = 500.millis } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetypieStaticEntitiesCacheClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetypieStaticEntitiesCacheClientModule.scala index 2d0d5e08b..13283e4b3 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetypieStaticEntitiesCacheClientModule.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetypieStaticEntitiesCacheClientModule.scala @@ -39,6 +39,7 @@ object TweetypieStaticEntitiesCacheClientModule extends TwitterModule { val memCacheClient = MemcachedClientBuilder.buildMemcachedClient( destName = ProdDest, numTries = 1, + numConnections = 1, requestTimeout = 50.milliseconds, globalTimeout = 100.milliseconds, connectTimeout = 100.milliseconds, diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/UserMetadataStoreModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/UserMetadataStoreModule.scala deleted file mode 100644 index 7a450ac81..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/UserMetadataStoreModule.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.google.inject.name.Named -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserMetadataManhattanEndpoint -import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserLanguagesStore -import com.twitter.home_mixer.store.UserLanguagesStore -import com.twitter.inject.TwitterModule -import com.twitter.search.common.constants.{thriftscala => scc} -import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpoint -import com.twitter.storehaus.ReadableStore -import javax.inject.Singleton - -object UserMetadataStoreModule extends TwitterModule { - - @Provides - @Singleton - @Named(UserLanguagesStore) - def providesUserLanguagesFeaturesStore( - @Named(UserMetadataManhattanEndpoint) UserMetadataManhattanKVEndpoint: ManhattanKVEndpoint, - statsReceiver: StatsReceiver - ): ReadableStore[Long, Seq[scc.ThriftLanguage]] = { - new UserLanguagesStore(UserMetadataManhattanKVEndpoint, statsReceiver) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParamConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParamConfig.scala index a998aa9ca..6fbd28fce 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParamConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParamConfig.scala @@ -14,14 +14,13 @@ class HomeGlobalParamConfig @Inject() () extends GlobalParamConfig { override val booleanFSOverrides = Seq( AdsDisableInjectionBasedOnUserRoleParam, - EnableSendScoresToClient, + EnableAdvertiserBrandSafetySettingsFeatureHydratorParam, + EnableImpressionBloomFilter, EnableNahFeedbackInfoParam, EnableNewTweetsPillAvatarsParam, - EnableServedCandidateKafkaPublishingParam, + EnableScribeServedCandidatesParam, + EnableSendScoresToClient, EnableSocialContextParam, - EnableGizmoduckAuthorSafetyFeatureHydratorParam, - EnableAdvertiserBrandSafetySettingsFeatureHydratorParam, - EnableFeedbackFatigueParam ) override val boundedIntFSOverrides = Seq( @@ -30,11 +29,6 @@ class HomeGlobalParamConfig @Inject() () extends GlobalParamConfig { ) override val boundedDoubleFSOverrides = Seq( - BlueVerifiedAuthorInNetworkMultiplierParam, - BlueVerifiedAuthorOutOfNetworkMultiplierParam - ) - - override val longSetFSOverrides = Seq( - AuthorListForStatsParam + ImpressionBloomFilterFalsePositiveRateParam ) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParams.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParams.scala index 0bd5dd1f0..f19817bc9 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParams.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParams.scala @@ -52,59 +52,35 @@ object HomeGlobalParams { default = true ) - object EnableServedCandidateKafkaPublishingParam - extends FSParam[Boolean]( - name = "home_mixer_enable_served_candidate_kafka_publishing", - default = true - ) - - /** - * This author ID list is used purely for realtime metrics collection around how often we - * are serving Tweets from these authors and which candidate sources they are coming from. - */ - object AuthorListForStatsParam - extends FSParam[Set[Long]]( - name = "home_mixer_author_list_for_stats", - default = Set.empty - ) - object EnableSocialContextParam extends FSParam[Boolean]( name = "home_mixer_enable_social_context", - default = false - ) - - object EnableGizmoduckAuthorSafetyFeatureHydratorParam - extends FSParam[Boolean]( - name = "home_mixer_enable_gizmoduck_author_safety_feature_hydrator", default = true ) - object EnableFeedbackFatigueParam - extends FSParam[Boolean]( - name = "home_mixer_enable_feedback_fatigue", - default = true - ) - - object BlueVerifiedAuthorInNetworkMultiplierParam - extends FSBoundedParam[Double]( - name = "home_mixer_blue_verified_author_in_network_multiplier", - default = 4.0, - min = 0.0, - max = 100.0 - ) - - object BlueVerifiedAuthorOutOfNetworkMultiplierParam - extends FSBoundedParam[Double]( - name = "home_mixer_blue_verified_author_out_of_network_multiplier", - default = 2.0, - min = 0.0, - max = 100.0 - ) - object EnableAdvertiserBrandSafetySettingsFeatureHydratorParam extends FSParam[Boolean]( name = "home_mixer_enable_advertiser_brand_safety_settings_feature_hydrator", default = true ) + + object EnableImpressionBloomFilter + extends FSParam[Boolean]( + name = "home_mixer_enable_impression_bloom_filter", + default = false + ) + + object ImpressionBloomFilterFalsePositiveRateParam + extends FSBoundedParam[Double]( + name = "home_mixer_impression_bloom_filter_false_positive_rate", + default = 0.005, + min = 0.001, + max = 0.01 + ) + + object EnableScribeServedCandidatesParam + extends FSParam[Boolean]( + name = "home_mixer_served_tweets_enable_scribing", + default = true + ) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerFlagName.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerFlagName.scala index afe23c35a..dc5f5513f 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerFlagName.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerFlagName.scala @@ -2,7 +2,8 @@ package com.twitter.home_mixer.param object HomeMixerFlagName { final val ScribeClientEventsFlag = "scribe.client_events" - final val ScribeServedEntriesFlag = "scribe.served_entries" + final val ScribeServedCandidatesFlag = "scribe.served_candidates" + final val ScribeScoredCandidatesFlag = "scribe.scored_candidates" final val ScribeServedCommonFeaturesAndCandidateFeaturesFlag = "scribe.served_common_features_and_candidate_features" final val DataRecordMetadataStoreConfigsYmlFlag = "data.record.metadata.store.configs.yml" diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerInjectionNames.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerInjectionNames.scala index 739c48a77..5ea87f5ab 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerInjectionNames.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerInjectionNames.scala @@ -32,18 +32,15 @@ object HomeMixerInjectionNames { final val TweetEngagementCache = "TweetEngagementCache" final val TweetypieContentRepository = "TweetypieContentRepository" final val TweetypieStaticEntitiesCache = "TweetypieStaticEntitiesCache" - final val TwhinAuthorFollow20200101FeatureCacheClient = - "TwhinAuthorFollow20200101FeatureCacheClient" - final val TwhinAuthorFollow20200101FeatureRepository = - "TwhinAuthorFollow20200101FeatureRepository" + final val TwhinAuthorFollowFeatureCacheClient = "TwhinAuthorFollowFeatureCacheClient" + final val TwhinAuthorFollowFeatureRepository = "TwhinAuthorFollowFeatureRepository" final val TwhinUserEngagementFeatureRepository = "TwhinUserEngagementFeatureRepository" final val TwhinUserFollowFeatureRepository = "TwhinUserFollowFeatureRepository" final val TwitterListEngagementCache = "TwitterListEngagementCache" final val UserAuthorEngagementCache = "UserAuthorEngagementCache" final val UserEngagementCache = "UserEngagementCache" final val UserFollowedTopicIdsRepository = "UserFollowedTopicIdsRepository" - final val UserLanguagesStore = "UserLanguagesStore" - final val UserMetadataManhattanEndpoint = "UserMetadataManhattanEndpoint" + final val UserLanguagesRepository = "UserLanguagesRepository" final val UserTopicEngagementForNewUserCache = "UserTopicEngagementForNewUserCache" final val UtegSocialProofRepository = "UtegSocialProofRepository" } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/DeciderKey.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/DeciderKey.scala index b81ea760a..91f7646d9 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/DeciderKey.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/DeciderKey.scala @@ -20,21 +20,33 @@ object DeciderKey extends DeciderKeyEnum { val EnableListRecommendedUsersProduct = Value("enable_list_recommended_users_product") + val EnableSubscribedProduct = Value("enable_subscribed_product") + // Candidate Pipelines - val EnableForYouScoredTweetsCandidatePipeline = Value( - "enable_for_you_scored_tweets_candidate_pipeline") + val EnableForYouScoredTweetsCandidatePipeline = + Value("enable_for_you_scored_tweets_candidate_pipeline") - val EnableScoredTweetsCrMixerCandidatePipeline = Value( - "enable_scored_tweets_cr_mixer_candidate_pipeline") + val EnableScoredTweetsTweetMixerCandidatePipeline = + Value("enable_scored_tweets_tweet_mixer_candidate_pipeline") - val EnableScoredTweetsFrsCandidatePipeline = Value("enable_scored_tweets_frs_candidate_pipeline") + val EnableScoredTweetsInNetworkCandidatePipeline = + Value("enable_scored_tweets_in_network_candidate_pipeline") - val EnableScoredTweetsInNetworkCandidatePipeline = Value( - "enable_scored_tweets_in_network_candidate_pipeline") + val EnableScoredTweetsUtegCandidatePipeline = + Value("enable_scored_tweets_uteg_candidate_pipeline") - val EnableScoredTweetsUtegCandidatePipeline = Value( - "enable_scored_tweets_uteg_candidate_pipeline") + val EnableScoredTweetsFrsCandidatePipeline = + Value("enable_scored_tweets_frs_candidate_pipeline") - val EnableSimClustersSimilarityFeatureHydration = Value( - "enable_simclusters_similarity_feature_hydration") + val EnableScoredTweetsListsCandidatePipeline = + Value("enable_scored_tweets_lists_candidate_pipeline") + + val EnableScoredTweetsPopularVideosCandidatePipeline = + Value("enable_scored_tweets_popular_videos_candidate_pipeline") + + val EnableScoredTweetsBackfillCandidatePipeline = + Value("enable_scored_tweets_backfill_candidate_pipeline") + + val EnableSimClustersSimilarityFeatureHydration = + Value("enable_simclusters_similarity_feature_hydration") } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/BUILD.bazel index b59b7d9a3..e4fa669d2 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/BUILD.bazel @@ -17,6 +17,8 @@ scala_library( "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/model", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/HomeProductPipelineRegistryConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/HomeProductPipelineRegistryConfig.scala index 213143351..5db4bd7f6 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/HomeProductPipelineRegistryConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/HomeProductPipelineRegistryConfig.scala @@ -5,15 +5,16 @@ import com.twitter.home_mixer.model.request.ForYouProduct import com.twitter.home_mixer.model.request.ListRecommendedUsersProduct import com.twitter.home_mixer.model.request.ListTweetsProduct import com.twitter.home_mixer.model.request.ScoredTweetsProduct +import com.twitter.home_mixer.model.request.SubscribedProduct import com.twitter.home_mixer.product.following.FollowingProductPipelineConfig import com.twitter.home_mixer.product.for_you.ForYouProductPipelineConfig import com.twitter.home_mixer.product.list_recommended_users.ListRecommendedUsersProductPipelineConfig import com.twitter.home_mixer.product.scored_tweets.ScoredTweetsProductPipelineConfig import com.twitter.home_mixer.product.list_tweets.ListTweetsProductPipelineConfig +import com.twitter.home_mixer.product.subscribed.SubscribedProductPipelineConfig import com.twitter.inject.Injector import com.twitter.product_mixer.core.product.guice.ProductScope import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistryConfig - import javax.inject.Inject import javax.inject.Singleton @@ -44,11 +45,16 @@ class HomeProductPipelineRegistryConfig @Inject() ( injector.instance[ListRecommendedUsersProductPipelineConfig] } + private val subscribedProductPipelineConfig = productScope.let(SubscribedProduct) { + injector.instance[SubscribedProductPipelineConfig] + } + override val productPipelineConfigs = Seq( followingProductPipelineConfig, forYouProductPipelineConfig, scoredTweetsProductPipelineConfig, listTweetsProductPipelineConfig, - listRecommendedUsersProductPipelineConfig + listRecommendedUsersProductPipelineConfig, + subscribedProductPipelineConfig, ) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/BUILD.bazel index 3d3273daf..4771fe655 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/BUILD.bazel @@ -30,12 +30,14 @@ scala_library( "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/account_recommendations_mixer", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/hermit", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/people_discovery", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/async", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/param_gated", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", @@ -81,9 +83,7 @@ scala_library( "stitch/stitch-tweetypie", "stringcenter/client", "stringcenter/client/src/main/java", - "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", "timelines/src/main/scala/com/twitter/timelines/clients/relevance_search", "timelines/src/main/scala/com/twitter/timelines/injection/scribe", ], diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingAdsCandidatePipelineBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingAdsCandidatePipelineBuilder.scala index 77477652f..d4898eb31 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingAdsCandidatePipelineBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingAdsCandidatePipelineBuilder.scala @@ -1,8 +1,10 @@ package com.twitter.home_mixer.product.following import com.twitter.adserver.{thriftscala => ads} -import com.twitter.home_mixer.functional_component.decorator.HomeAdsClientEventDetailsBuilder +import com.twitter.home_mixer.functional_component.decorator.builder.HomeAdsClientEventDetailsBuilder import com.twitter.home_mixer.functional_component.gate.ExcludeSoftUserGate +import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature import com.twitter.home_mixer.param.HomeGlobalParams import com.twitter.home_mixer.param.HomeGlobalParams.EnableAdvertiserBrandSafetySettingsFeatureHydratorParam import com.twitter.home_mixer.product.following.model.FollowingQuery @@ -21,7 +23,7 @@ import com.twitter.product_mixer.component_library.model.candidate.ads.AdsCandid import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsDependentCandidatePipelineConfig import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsDependentCandidatePipelineConfigBuilder import com.twitter.product_mixer.component_library.pipeline.candidate.ads.CountCandidatesFromPipelines -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.PipelineScopedOrganicItemIds +import com.twitter.product_mixer.component_library.pipeline.candidate.ads.PipelineScopedOrganicItems import com.twitter.product_mixer.component_library.pipeline.candidate.ads.ValidAdImpressionIdFilter import com.twitter.product_mixer.core.functional_component.common.CandidateScope import com.twitter.product_mixer.core.gate.ParamNotGate @@ -77,7 +79,11 @@ class FollowingAdsCandidatePipelineBuilder @Inject() ( adsDisplayLocationBuilder = query => if (query.params.getBoolean(EnableFastAds)) ads.DisplayLocation.TimelineHomeReverseChron else ads.DisplayLocation.TimelineHome, - getOrganicItemIds = PipelineScopedOrganicItemIds(organicCandidatePipelines), + getOrganicItems = PipelineScopedOrganicItems( + pipelines = organicCandidatePipelines, + textFeature = TweetTextFeature, + languageFeature = TweetLanguageFeature + ), countNumOrganicItems = CountCandidatesFromPipelines(organicCandidatePipelines), supportedClientParam = Some(EnableAdsCandidatePipelineParam), gates = Seq( diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdCandidatePipelineConfig.scala index ea26ca19e..addb298c2 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdCandidatePipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdCandidatePipelineConfig.scala @@ -2,9 +2,9 @@ package com.twitter.home_mixer.product.following import com.twitter.home_mixer.candidate_pipeline.FollowingEarlybirdResponseFeatureTransformer import com.twitter.home_mixer.functional_component.candidate_source.EarlybirdCandidateSource -import com.twitter.home_mixer.functional_component.feature_hydrator.SGSFollowedUsersFeature -import com.twitter.home_mixer.functional_component.gate.NonEmptySeqFeatureGate import com.twitter.home_mixer.product.following.model.FollowingQuery +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature +import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.gate.Gate diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdQueryTransformer.scala index 9f7dd306e..c388cfa92 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdQueryTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdQueryTransformer.scala @@ -2,10 +2,10 @@ package com.twitter.home_mixer.product.following import com.twitter.finagle.thrift.ClientId import com.twitter.finagle.tracing.Trace -import com.twitter.home_mixer.functional_component.feature_hydrator.SGSFollowedUsersFeature import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature import com.twitter.home_mixer.product.following.model.FollowingQuery import com.twitter.home_mixer.product.following.param.FollowingParam.ServerMaxResultsParam +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.BottomCursor import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor @@ -30,7 +30,7 @@ case class FollowingEarlybirdQueryTransformer @Inject() (clientId: ClientId) val realGraphInNetworkFollowedUserIds = query.features.map(_.get(RealGraphInNetworkScoresFeature)).getOrElse(Map.empty).keySet val userId = query.getRequiredUserId - val combinedUserIds = userId +: (followedUserIds ++ realGraphInNetworkFollowedUserIds).toSeq + val combinedUserIds = userId +: followedUserIds.toSeq val baseFollowedUsersSearchOperator = new SearchOperator.Builder() .setType(SearchOperator.Type.FEATURE_VALUE_IN_ACCEPT_LIST_OR_UNSET) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingMixerPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingMixerPipelineConfig.scala index 23b4164a5..efda03595 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingMixerPipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingMixerPipelineConfig.scala @@ -6,14 +6,17 @@ import com.twitter.home_mixer.candidate_pipeline.ConversationServiceCandidatePip import com.twitter.home_mixer.candidate_pipeline.EditedTweetsCandidatePipelineConfig import com.twitter.home_mixer.candidate_pipeline.NewTweetsPillCandidatePipelineConfig import com.twitter.home_mixer.functional_component.decorator.HomeConversationServiceCandidateDecorator -import com.twitter.home_mixer.functional_component.decorator.HomeFeedbackActionInfoBuilder import com.twitter.home_mixer.functional_component.decorator.urt.builder.AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder +import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder import com.twitter.home_mixer.functional_component.feature_hydrator._ import com.twitter.home_mixer.functional_component.selector.UpdateHomeClientEventDetails import com.twitter.home_mixer.functional_component.selector.UpdateNewTweetsPillDecoration import com.twitter.home_mixer.functional_component.side_effect._ import com.twitter.home_mixer.model.GapIncludeInstruction +import com.twitter.home_mixer.param.HomeGlobalParams.EnableImpressionBloomFilter import com.twitter.home_mixer.param.HomeGlobalParams.MaxNumberReplaceInstructionsParam +import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeClientEventsFlag +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator import com.twitter.home_mixer.product.following.model.FollowingQuery import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.home_mixer.product.following.param.FollowingParam.EnableFlipInjectionModuleCandidatePipelineParam @@ -21,8 +24,11 @@ import com.twitter.home_mixer.product.following.param.FollowingParam.FlipInlineI import com.twitter.home_mixer.product.following.param.FollowingParam.ServerMaxResultsParam import com.twitter.home_mixer.product.following.param.FollowingParam.WhoToFollowPositionParam import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.inject.annotations.Flag import com.twitter.logpipeline.client.common.EventPublisher import com.twitter.product_mixer.component_library.feature_hydrator.query.async.AsyncQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweetsQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.AsyncParamGatedQueryFeatureHydrator import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.FlipPromptDependentCandidatePipelineConfigBuilder @@ -81,7 +87,7 @@ class FollowingMixerPipelineConfig @Inject() ( ], homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder, followingAdsCandidatePipelineBuilder: FollowingAdsCandidatePipelineBuilder, - followingWhoToFollowArmCandidatePipelineConfigBuilder: FollowingWhoToFollowArmCandidatePipelineConfigBuilder, + followingWhoToFollowCandidatePipelineConfigBuilder: FollowingWhoToFollowCandidatePipelineConfigBuilder, flipPromptDependentCandidatePipelineConfigBuilder: FlipPromptDependentCandidatePipelineConfigBuilder, editedTweetsCandidatePipelineConfig: EditedTweetsCandidatePipelineConfig, newTweetsPillCandidatePipelineConfig: NewTweetsPillCandidatePipelineConfig[FollowingQuery], @@ -91,19 +97,27 @@ class FollowingMixerPipelineConfig @Inject() ( realGraphInNetworkSourceQueryHydrator: RealGraphInNetworkScoresQueryFeatureHydrator, requestQueryFeatureHydrator: RequestQueryFeatureHydrator[FollowingQuery], sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, - tweetImpressionsQueryFeatureHydrator: TweetImpressionsQueryFeatureHydrator[FollowingQuery], + impressionBloomFilterQueryFeatureHydrator: ImpressionBloomFilterQueryFeatureHydrator[ + FollowingQuery + ], + manhattanTweetImpressionsQueryFeatureHydrator: TweetImpressionsQueryFeatureHydrator[ + FollowingQuery + ], + memcacheTweetImpressionsQueryFeatureHydrator: ImpressedTweetsQueryFeatureHydrator, lastNonPollingTimeQueryFeatureHydrator: LastNonPollingTimeQueryFeatureHydrator, adsInjector: AdsInjector, updateLastNonPollingTimeSideEffect: UpdateLastNonPollingTimeSideEffect[FollowingQuery, Timeline], publishClientSentImpressionsEventBusSideEffect: PublishClientSentImpressionsEventBusSideEffect, publishClientSentImpressionsManhattanSideEffect: PublishClientSentImpressionsManhattanSideEffect, + publishImpressionBloomFilterSideEffect: PublishImpressionBloomFilterSideEffect, updateTimelinesPersistenceStoreSideEffect: UpdateTimelinesPersistenceStoreSideEffect, truncateTimelinesPersistenceStoreSideEffect: TruncateTimelinesPersistenceStoreSideEffect, - homeTimelineServedEntriesSideEffect: HomeScribeServedEntriesSideEffect, + homeTimelineServedCandidatesSideEffect: HomeScribeServedCandidatesSideEffect, clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent], externalStrings: HomeMixerExternalStrings, @ProductScoped stringCenterProvider: Provider[StringCenter], - urtTransportMarshaller: UrtTransportMarshaller) + urtTransportMarshaller: UrtTransportMarshaller, + @Flag(ScribeClientEventsFlag) enableScribeClientEvents: Boolean) extends MixerPipelineConfig[FollowingQuery, Timeline, urt.TimelineResponse] { override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("Following") @@ -119,7 +133,12 @@ class FollowingMixerPipelineConfig @Inject() ( AsyncQueryFeatureHydrator(dependentCandidatesStep, gizmoduckUserQueryFeatureHydrator), AsyncQueryFeatureHydrator(dependentCandidatesStep, persistenceStoreQueryFeatureHydrator), AsyncQueryFeatureHydrator(dependentCandidatesStep, lastNonPollingTimeQueryFeatureHydrator), - AsyncQueryFeatureHydrator(resultSelectorsStep, tweetImpressionsQueryFeatureHydrator), + AsyncParamGatedQueryFeatureHydrator( + EnableImpressionBloomFilter, + resultSelectorsStep, + impressionBloomFilterQueryFeatureHydrator), + AsyncQueryFeatureHydrator(resultSelectorsStep, manhattanTweetImpressionsQueryFeatureHydrator), + AsyncQueryFeatureHydrator(resultSelectorsStep, memcacheTweetImpressionsQueryFeatureHydrator) ) private val earlybirdCandidatePipelineScope = @@ -134,8 +153,8 @@ class FollowingMixerPipelineConfig @Inject() ( private val followingAdsCandidatePipelineConfig = followingAdsCandidatePipelineBuilder.build(earlybirdCandidatePipelineScope) - private val followingWhoToFollowArmCandidatePipelineConfig = - followingWhoToFollowArmCandidatePipelineConfigBuilder.build(earlybirdCandidatePipelineScope) + private val followingWhoToFollowCandidatePipelineConfig = + followingWhoToFollowCandidatePipelineConfigBuilder.build(earlybirdCandidatePipelineScope) private val flipPromptCandidatePipelineConfig = flipPromptDependentCandidatePipelineConfigBuilder.build[FollowingQuery]( @@ -150,7 +169,7 @@ class FollowingMixerPipelineConfig @Inject() ( ] = Seq( conversationServiceCandidatePipelineConfig, followingAdsCandidatePipelineConfig, - followingWhoToFollowArmCandidatePipelineConfig, + followingWhoToFollowCandidatePipelineConfig, flipPromptCandidatePipelineConfig, editedTweetsCandidatePipelineConfig, newTweetsPillCandidatePipelineConfig @@ -158,7 +177,7 @@ class FollowingMixerPipelineConfig @Inject() ( override val failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = Map( followingAdsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - followingWhoToFollowArmCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + followingWhoToFollowCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, flipPromptCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, editedTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, newTweetsPillCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, @@ -178,16 +197,16 @@ class FollowingMixerPipelineConfig @Inject() ( maxSelectionsParam = ServerMaxResultsParam ), DropModuleTooFewModuleItemResults( - candidatePipeline = followingWhoToFollowArmCandidatePipelineConfig.identifier, + candidatePipeline = followingWhoToFollowCandidatePipelineConfig.identifier, minModuleItemsParam = StaticParam(WhoToFollowArmCandidatePipelineConfig.MinCandidatesSize) ), DropMaxModuleItemCandidates( - candidatePipeline = followingWhoToFollowArmCandidatePipelineConfig.identifier, + candidatePipeline = followingWhoToFollowCandidatePipelineConfig.identifier, maxModuleItemsParam = StaticParam(WhoToFollowArmCandidatePipelineConfig.MaxCandidatesSize) ), InsertAppendResults(candidatePipeline = conversationServiceCandidatePipelineConfig.identifier), InsertFixedPositionResults( - candidatePipeline = followingWhoToFollowArmCandidatePipelineConfig.identifier, + candidatePipeline = followingWhoToFollowCandidatePipelineConfig.identifier, positionParam = WhoToFollowPositionParam ), InsertFixedPositionResults( @@ -221,22 +240,24 @@ class FollowingMixerPipelineConfig @Inject() ( ) private val homeScribeClientEventSideEffect = HomeScribeClientEventSideEffect( + enableScribeClientEvents = enableScribeClientEvents, logPipelinePublisher = clientEventsScribeEventPublisher, injectedTweetsCandidatePipelineIdentifiers = Seq(conversationServiceCandidatePipelineConfig.identifier), - adsCandidatePipelineIdentifier = followingAdsCandidatePipelineConfig.identifier, + adsCandidatePipelineIdentifier = Some(followingAdsCandidatePipelineConfig.identifier), whoToFollowCandidatePipelineIdentifier = - Some(followingWhoToFollowArmCandidatePipelineConfig.identifier), + Some(followingWhoToFollowCandidatePipelineConfig.identifier), ) override val resultSideEffects: Seq[PipelineResultSideEffect[FollowingQuery, Timeline]] = Seq( - updateLastNonPollingTimeSideEffect, + homeScribeClientEventSideEffect, + homeTimelineServedCandidatesSideEffect, publishClientSentImpressionsEventBusSideEffect, publishClientSentImpressionsManhattanSideEffect, - homeScribeClientEventSideEffect, - updateTimelinesPersistenceStoreSideEffect, + publishImpressionBloomFilterSideEffect, truncateTimelinesPersistenceStoreSideEffect, - homeTimelineServedEntriesSideEffect + updateLastNonPollingTimeSideEffect, + updateTimelinesPersistenceStoreSideEffect, ) override val domainMarshaller: DomainMarshaller[FollowingQuery, Timeline] = { diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingWhoToFollowArmCandidatePipelineConfigBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingWhoToFollowCandidatePipelineConfigBuilder.scala similarity index 98% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingWhoToFollowArmCandidatePipelineConfigBuilder.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingWhoToFollowCandidatePipelineConfigBuilder.scala index 484a72d5f..cd5b1795d 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingWhoToFollowArmCandidatePipelineConfigBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingWhoToFollowCandidatePipelineConfigBuilder.scala @@ -27,7 +27,7 @@ import javax.inject.Inject import javax.inject.Singleton @Singleton -class FollowingWhoToFollowArmCandidatePipelineConfigBuilder @Inject() ( +class FollowingWhoToFollowCandidatePipelineConfigBuilder @Inject() ( whoToFollowArmDependentCandidatePipelineConfigBuilder: WhoToFollowArmDependentCandidatePipelineConfigBuilder, homeWhoToFollowFeedbackActionInfoBuilder: HomeWhoToFollowFeedbackActionInfoBuilder) { diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/FollowingQuery.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/FollowingQuery.scala index 39442a61b..c45c7cf68 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/FollowingQuery.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/FollowingQuery.scala @@ -47,4 +47,5 @@ case class FollowingQuery( override val isEmptyState: Option[Boolean] = None override val isFirstRequestAfterSignup: Option[Boolean] = None override val isEndOfTimeline: Option[Boolean] = None + override val timelineId: Option[Long] = None } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/HomeMixerExternalStrings.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/HomeMixerExternalStrings.scala index fb07bc12c..9c6faafa7 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/HomeMixerExternalStrings.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/HomeMixerExternalStrings.scala @@ -63,4 +63,25 @@ class HomeMixerExternalStrings @Inject() ( externalStringRegistryProvider.get().createProdString("SocialContext.extendedReply") val socialContextReceivedReply = externalStringRegistryProvider.get().createProdString("SocialContext.receivedReply") + + val socialContextPopularVideoString = + externalStringRegistryProvider.get().createProdString("SocialContext.popularVideo") + + val socialContextPopularInYourAreaString = + externalStringRegistryProvider.get().createProdString("PopgeoTweet.socialProof") + + val listToFollowModuleHeaderString = + externalStringRegistryProvider.get().createProdString("ListToFollowModule.header") + val listToFollowModuleFooterString = + externalStringRegistryProvider.get().createProdString("ListToFollowModule.footer") + val pinnedListsModuleHeaderString = + externalStringRegistryProvider.get().createProdString("PinnedListModule.header") + val pinnedListsModuleEmptyStateMessageString = + externalStringRegistryProvider.get().createProdString("PinnedListModule.emptyStateMessage") + + val ownedSubscribedListsModuleHeaderString = + externalStringRegistryProvider.get().createProdString("OwnedSubscribedListModule.header") + val ownedSubscribedListsModuleEmptyStateMessageString = + externalStringRegistryProvider + .get().createProdString("OwnedSubscribedListModule.emptyStateMessage") } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/BUILD.bazel index 8371cbdf1..5f597c78b 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/BUILD.bazel @@ -26,27 +26,39 @@ scala_library( "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/scorer", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/people_discovery", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/earlybird", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_scorer", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_service", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_tweetypie", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/async", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/param_gated", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_subscribe_module", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tweet_tlx", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", @@ -73,10 +85,12 @@ scala_library( "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor", "src/java/com/twitter/search/common/util/lang", "src/scala/com/twitter/suggests/controller_data", + "src/thrift/com/twitter/search:earlybird-scala", "src/thrift/com/twitter/search/common:constants-java", "src/thrift/com/twitter/timelines/render:thrift-scala", "src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java", "src/thrift/com/twitter/timelineservice:thrift-scala", + "stitch/stitch-tweetypie", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", "timelines/src/main/scala/com/twitter/timelines/injection/scribe", "timelines/src/main/scala/com/twitter/timelines/model/candidate", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsCandidatePipelineBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsCandidatePipelineBuilder.scala index 82963f5bb..99ed0f584 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsCandidatePipelineBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsCandidatePipelineBuilder.scala @@ -2,7 +2,7 @@ package com.twitter.home_mixer.product.for_you import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedCandidateFeatureHydrator import com.twitter.adserver.{thriftscala => ads} -import com.twitter.home_mixer.functional_component.decorator.HomeAdsClientEventDetailsBuilder +import com.twitter.home_mixer.functional_component.decorator.builder.HomeAdsClientEventDetailsBuilder import com.twitter.home_mixer.functional_component.gate.ExcludeSoftUserGate import com.twitter.home_mixer.param.HomeGlobalParams import com.twitter.home_mixer.param.HomeGlobalParams.EnableAdvertiserBrandSafetySettingsFeatureHydratorParam diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsDependentCandidatePipelineBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsDependentCandidatePipelineBuilder.scala new file mode 100644 index 000000000..ef65d7062 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsDependentCandidatePipelineBuilder.scala @@ -0,0 +1,111 @@ +package com.twitter.home_mixer.product.for_you + +import com.twitter.adserver.{thriftscala => ads} +import com.twitter.home_mixer.functional_component.decorator.builder.HomeAdsClientEventDetailsBuilder +import com.twitter.home_mixer.functional_component.gate.ExcludeSoftUserGate +import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature +import com.twitter.home_mixer.param.HomeGlobalParams +import com.twitter.home_mixer.param.HomeGlobalParams.EnableAdvertiserBrandSafetySettingsFeatureHydratorParam +import com.twitter.home_mixer.product.for_you.model.ForYouQuery +import com.twitter.home_mixer.service.HomeMixerAlertConfig +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.product_mixer.component_library.candidate_source.ads.AdsProdThriftCandidateSource +import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator +import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.item.ad.AdsCandidateUrtItemBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.ads.AdvertiserBrandSafetySettingsFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedCandidateFeatureHydrator +import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate +import com.twitter.product_mixer.component_library.model.candidate.ads.AdsCandidate +import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsDependentCandidatePipelineConfig +import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsDependentCandidatePipelineConfigBuilder +import com.twitter.product_mixer.component_library.pipeline.candidate.ads.CountTruncatedItemCandidatesFromPipelines +import com.twitter.product_mixer.component_library.pipeline.candidate.ads.StaticAdsDisplayLocationBuilder +import com.twitter.product_mixer.component_library.pipeline.candidate.ads.TruncatedPipelineScopedOrganicItems +import com.twitter.product_mixer.component_library.pipeline.candidate.ads.ValidAdImpressionIdFilter +import com.twitter.product_mixer.core.functional_component.common.CandidateScope +import com.twitter.product_mixer.core.gate.ParamNotGate +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.marshalling.response.rtf.safety_level.TimelineHomePromotedHydrationSafetyLevel +import com.twitter.product_mixer.core.model.marshalling.response.urt.contextual_ref.TweetHydrationContext +import com.twitter.timelines.injection.scribe.InjectionScribeUtil +import com.twitter.timelineservice.suggests.{thriftscala => st} +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ForYouAdsDependentCandidatePipelineBuilder @Inject() ( + adsCandidatePipelineConfigBuilder: AdsDependentCandidatePipelineConfigBuilder, + adsCandidateSource: AdsProdThriftCandidateSource, + advertiserBrandSafetySettingsFeatureHydrator: AdvertiserBrandSafetySettingsFeatureHydrator[ + ForYouQuery, + AdsCandidate + ]) { + + private val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier("ForYouAdsDependent") + + private val suggestType = st.SuggestType.Promoted + + private val MaxOrganicTweets = 35 + + private val clientEventInfoBuilder = ClientEventInfoBuilder( + component = InjectionScribeUtil.scribeComponent(suggestType).get, + detailsBuilder = Some(HomeAdsClientEventDetailsBuilder(Some(suggestType.name))) + ) + + private val contextualTweetRefBuilder = ContextualTweetRefBuilder( + TweetHydrationContext( + safetyLevelOverride = Some(TimelineHomePromotedHydrationSafetyLevel), + outerTweetContext = None + )) + + private val decorator = UrtItemCandidateDecorator( + AdsCandidateUrtItemBuilder( + tweetClientEventInfoBuilder = Some(clientEventInfoBuilder), + contextualTweetRefBuilder = Some(contextualTweetRefBuilder) + )) + + private val alerts = Seq( + HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), + HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() + ) + + def build( + organicCandidatePipelines: CandidateScope + ): AdsDependentCandidatePipelineConfig[ForYouQuery] = + adsCandidatePipelineConfigBuilder.build[ForYouQuery]( + adsCandidateSource = adsCandidateSource, + identifier = identifier, + adsDisplayLocationBuilder = StaticAdsDisplayLocationBuilder(ads.DisplayLocation.TimelineHome), + getOrganicItems = TruncatedPipelineScopedOrganicItems( + pipelines = organicCandidatePipelines, + textFeature = TweetTextFeature, + languageFeature = TweetLanguageFeature, + ordering = CandidatesUtil.scoreOrdering, + maxCount = MaxOrganicTweets + ), + countNumOrganicItems = + CountTruncatedItemCandidatesFromPipelines(organicCandidatePipelines, MaxOrganicTweets), + gates = Seq( + ParamNotGate( + name = "AdsDisableInjectionBasedOnUserRole", + param = HomeGlobalParams.AdsDisableInjectionBasedOnUserRoleParam + ), + ExcludeSoftUserGate, + NonEmptyCandidatesGate(organicCandidatePipelines) + ), + filters = Seq(ValidAdImpressionIdFilter), + postFilterFeatureHydration = Seq( + ParamGatedCandidateFeatureHydrator( + EnableAdvertiserBrandSafetySettingsFeatureHydratorParam, + advertiserBrandSafetySettingsFeatureHydrator + ) + ), + decorator = Some(decorator), + alerts = alerts, + urtRequest = Some(true) + ) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouConversationServiceCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouConversationServiceCandidatePipelineConfig.scala index 3f69ad888..494d3a584 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouConversationServiceCandidatePipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouConversationServiceCandidatePipelineConfig.scala @@ -2,26 +2,26 @@ package com.twitter.home_mixer.product.for_you import com.twitter.home_mixer.candidate_pipeline.ConversationServiceResponseFeatureTransformer import com.twitter.home_mixer.functional_component.decorator.HomeConversationServiceCandidateDecorator -import com.twitter.home_mixer.functional_component.decorator.HomeFeedbackActionInfoBuilder +import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder +import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.SocialGraphServiceFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.TimelineServiceTweetsFeature import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter -import com.twitter.home_mixer.functional_component.filter.PredicateFeatureFilter -import com.twitter.home_mixer.functional_component.filter.PreviouslySeenTweetsFilter +import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter import com.twitter.home_mixer.functional_component.filter.PreviouslyServedTweetsFilter import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter -import com.twitter.home_mixer.functional_component.gate.NonEmptySeqFeatureGate import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature +import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSourceRequest import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata import com.twitter.product_mixer.component_library.filter.FeatureFilter +import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter import com.twitter.product_mixer.component_library.gate.NoCandidatesGate +import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines @@ -47,8 +47,8 @@ class ForYouConversationServiceCandidatePipelineConfig @Inject() ( forYouTimelineScorerCandidatePipelineConfig: ForYouTimelineScorerCandidatePipelineConfig, conversationServiceCandidateSource: ConversationServiceCandidateSource, tweetypieFeatureHydrator: TweetypieFeatureHydrator, - socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator, namesFeatureHydrator: NamesFeatureHydrator, + invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter, homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder) extends DependentCandidatePipelineConfig[ ForYouQuery, @@ -107,17 +107,20 @@ class ForYouConversationServiceCandidatePipelineConfig @Inject() ( override val preFilterFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] - ] = Seq(tweetypieFeatureHydrator, socialGraphServiceFeatureHydrator) + ] = Seq( + InNetworkFeatureHydrator, + tweetypieFeatureHydrator + ) override def filters: Seq[Filter[ForYouQuery, TweetCandidate]] = Seq( PreviouslyServedTweetsFilter, - PreviouslySeenTweetsFilter, RetweetDeduplicationFilter, FeatureFilter.fromFeature(FilterIdentifier("TweetypieHydrated"), IsHydratedFeature), PredicateFeatureFilter.fromPredicate( FilterIdentifier("QuotedTweetDropped"), shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) } ), + invalidSubscriptionTweetFilter, InvalidConversationModuleFilter ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouProductPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouProductPipelineConfig.scala index e84b05389..e1cf2161c 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouProductPipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouProductPipelineConfig.scala @@ -2,10 +2,11 @@ package com.twitter.home_mixer.product.for_you import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.marshaller.timelines.ChronologicalCursorUnmarshaller -import com.twitter.home_mixer.model.request.HomeMixerRequest import com.twitter.home_mixer.model.request.ForYouProduct import com.twitter.home_mixer.model.request.ForYouProductContext +import com.twitter.home_mixer.model.request.HomeMixerRequest import com.twitter.home_mixer.product.for_you.model.ForYouQuery +import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnablePushToHomeMixerPipelineParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableScoredTweetsMixerPipelineParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.ServerMaxResultsParam import com.twitter.home_mixer.product.for_you.param.ForYouParamConfig @@ -45,6 +46,7 @@ import javax.inject.Singleton class ForYouProductPipelineConfig @Inject() ( forYouTimelineScorerMixerPipelineConfig: ForYouTimelineScorerMixerPipelineConfig, forYouScoredTweetsMixerPipelineConfig: ForYouScoredTweetsMixerPipelineConfig, + forYouPushToHomeMixerPipelineConfig: ForYouPushToHomeMixerPipelineConfig, forYouParamConfig: ForYouParamConfig) extends ProductPipelineConfig[HomeMixerRequest, ForYouQuery, urt.TimelineResponse] { @@ -91,20 +93,25 @@ class ForYouProductPipelineConfig @Inject() ( debugOptions = debugOptions, deviceContext = context.deviceContext, seenTweetIds = context.seenTweetIds, - dspClientContext = context.dspClientContext + dspClientContext = context.dspClientContext, + pushToHomeTweetId = context.pushToHomeTweetId ) } - override val pipelines: Seq[PipelineConfig] = - Seq(forYouTimelineScorerMixerPipelineConfig, forYouScoredTweetsMixerPipelineConfig) + override val pipelines: Seq[PipelineConfig] = Seq( + forYouTimelineScorerMixerPipelineConfig, + forYouScoredTweetsMixerPipelineConfig, + forYouPushToHomeMixerPipelineConfig + ) override def pipelineSelector( query: ForYouQuery ): ComponentIdentifier = { - if (query.params.getBoolean(EnableScoredTweetsMixerPipelineParam)) + if (query.pushToHomeTweetId.isDefined && query.params(EnablePushToHomeMixerPipelineParam)) + forYouPushToHomeMixerPipelineConfig.identifier + else if (query.params(EnableScoredTweetsMixerPipelineParam)) forYouScoredTweetsMixerPipelineConfig.identifier - else - forYouTimelineScorerMixerPipelineConfig.identifier + else forYouTimelineScorerMixerPipelineConfig.identifier } override val alerts: Seq[Alert] = Seq( @@ -116,8 +123,8 @@ class ForYouProductPipelineConfig @Inject() ( LatencyAlert( notificationGroup = DefaultNotificationGroup, percentile = P99, - warnPredicate = TriggerIfLatencyAbove(2000.millis, 15, 30), - criticalPredicate = TriggerIfLatencyAbove(2100.millis, 15, 30) + warnPredicate = TriggerIfLatencyAbove(2300.millis, 15, 30), + criticalPredicate = TriggerIfLatencyAbove(2800.millis, 15, 30) ), ThroughputAlert( notificationGroup = DefaultNotificationGroup, diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPushToHomeMixerPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPushToHomeMixerPipelineConfig.scala new file mode 100644 index 000000000..ac529f582 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPushToHomeMixerPipelineConfig.scala @@ -0,0 +1,74 @@ +package com.twitter.home_mixer.product.for_you + +import com.twitter.home_mixer.product.for_you.model.ForYouQuery +import com.twitter.home_mixer.product.for_you.param.ForYouParam +import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.AddEntriesInstructionBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ClearCacheInstructionBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedBottomCursorBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedTopCursorBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ParamGatedIncludeInstruction +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.StaticTimelineScribeConfigBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtMetadataBuilder +import com.twitter.product_mixer.component_library.selector.InsertAppendResults +import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller +import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.UrtTransportMarshaller +import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller +import com.twitter.product_mixer.core.functional_component.selector.Selector +import com.twitter.product_mixer.core.model.common.UniversalNoun +import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier +import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline +import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig +import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig +import com.twitter.timelines.render.{thriftscala => urt} +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ForYouPushToHomeMixerPipelineConfig @Inject() ( + forYouPushToHomeTweetCandidatePipelineConfig: ForYouPushToHomeTweetCandidatePipelineConfig, + urtTransportMarshaller: UrtTransportMarshaller) + extends MixerPipelineConfig[ForYouQuery, Timeline, urt.TimelineResponse] { + + override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("ForYouPushToHome") + + override val candidatePipelines: Seq[CandidatePipelineConfig[ForYouQuery, _, _, _]] = + Seq(forYouPushToHomeTweetCandidatePipelineConfig) + + override val resultSelectors: Seq[Selector[ForYouQuery]] = + Seq(InsertAppendResults(forYouPushToHomeTweetCandidatePipelineConfig.identifier)) + + override val domainMarshaller: DomainMarshaller[ForYouQuery, Timeline] = { + val instructionBuilders = Seq( + ClearCacheInstructionBuilder( + ParamGatedIncludeInstruction(ForYouParam.EnableClearCacheOnPushToHome)), + AddEntriesInstructionBuilder()) + + val idSelector: PartialFunction[UniversalNoun[_], Long] = { case item: TweetItem => item.id } + val topCursorBuilder = OrderedTopCursorBuilder(idSelector) + val bottomCursorBuilder = OrderedBottomCursorBuilder(idSelector) + + val metadataBuilder = UrtMetadataBuilder( + title = None, + scribeConfigBuilder = Some( + StaticTimelineScribeConfigBuilder( + TimelineScribeConfig( + page = Some("for_you_push_to_home"), + section = None, + entityToken = None) + ) + ) + ) + + UrtDomainMarshaller( + instructionBuilders = instructionBuilders, + metadataBuilder = Some(metadataBuilder), + cursorBuilders = Seq(topCursorBuilder, bottomCursorBuilder) + ) + } + + override val transportMarshaller: TransportMarshaller[Timeline, urt.TimelineResponse] = + urtTransportMarshaller +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPushToHomeTweetCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPushToHomeTweetCandidatePipelineConfig.scala new file mode 100644 index 000000000..49df4fdf8 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPushToHomeTweetCandidatePipelineConfig.scala @@ -0,0 +1,81 @@ +package com.twitter.home_mixer.product.for_you + +import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventInfoBuilder +import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature +import com.twitter.home_mixer.product.for_you.functional_component.gate.PushToHomeRequestGate +import com.twitter.home_mixer.product.for_you.model.ForYouQuery +import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator +import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.candidate_source.PassthroughCandidateSource +import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.timelineservice.suggests.{thriftscala => st} + +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ForYouPushToHomeTweetCandidatePipelineConfig @Inject() () + extends CandidatePipelineConfig[ + ForYouQuery, + ForYouQuery, + TweetCandidate, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier("ForYouPushToHomeTweet") + + override val gates: Seq[Gate[ForYouQuery]] = Seq(PushToHomeRequestGate) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + ForYouQuery, + ForYouQuery + ] = identity + + override val featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[TweetCandidate] + ] = Seq(new CandidateFeatureTransformer[TweetCandidate] { + override def features: Set[Feature[_, _]] = Set(SuggestTypeFeature) + + override val identifier: TransformerIdentifier = + TransformerIdentifier("ForYouPushToHomeTweet") + + override def transform(input: TweetCandidate): FeatureMap = + FeatureMapBuilder().add(SuggestTypeFeature, Some(st.SuggestType.Magicrec)).build() + }) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + TweetCandidate, + TweetCandidate + ] = identity + + override val candidateSource: CandidateSource[ + ForYouQuery, + TweetCandidate + ] = PassthroughCandidateSource( + CandidateSourceIdentifier("PushToHomeTweet"), + { query => query.pushToHomeTweetId.toSeq.map(TweetCandidate(_)) } + ) + + override val decorator: Option[ + CandidateDecorator[ForYouQuery, TweetCandidate] + ] = { + val tweetItemBuilder = TweetCandidateUrtItemBuilder( + clientEventInfoBuilder = HomeClientEventInfoBuilder() + ) + Some(UrtItemCandidateDecorator(tweetItemBuilder)) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsCandidatePipelineConfig.scala index a8e2c36da..d7716e190 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsCandidatePipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsCandidatePipelineConfig.scala @@ -1,21 +1,16 @@ package com.twitter.home_mixer.product.for_you -import com.twitter.home_mixer.functional_component.decorator.HomeFeedbackActionInfoBuilder -import com.twitter.home_mixer.functional_component.decorator.HomeTimelinesScoreInfoBuilder -import com.twitter.home_mixer.functional_component.decorator.HomeTweetSocialContextBuilder import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventInfoBuilder import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder -import com.twitter.home_mixer.functional_component.feature_hydrator.FocalTweetFeatureHydrator +import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder +import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder +import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeTweetSocialContextBuilder +import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.PerspectiveFilteredSocialContextFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.SGSValidSocialContextFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator -import com.twitter.home_mixer.functional_component.filter.FeedbackFatigueFilter import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter -import com.twitter.home_mixer.functional_component.filter.PredicateFeatureFilter -import com.twitter.home_mixer.functional_component.filter.SocialContextFilter +import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter import com.twitter.home_mixer.functional_component.gate.SupportedLanguagesGate -import com.twitter.home_mixer.functional_component.scorer.FeedbackFatigueScorer import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature @@ -23,6 +18,8 @@ import com.twitter.home_mixer.model.HomeFeatures.IsNsfwFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature import com.twitter.home_mixer.product.for_you.candidate_source.ScoredTweetWithConversationMetadata import com.twitter.home_mixer.product.for_you.candidate_source.ScoredTweetsProductCandidateSource +import com.twitter.home_mixer.product.for_you.feature_hydrator.FocalTweetFeatureHydrator +import com.twitter.home_mixer.product.for_you.filter.SocialContextFilter import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableScoredTweetsCandidatePipelineParam import com.twitter.home_mixer.service.HomeMixerAlertConfig @@ -33,13 +30,13 @@ import com.twitter.product_mixer.component_library.decorator.urt.builder.timelin import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.StaticModuleDisplayTypeBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder import com.twitter.product_mixer.component_library.filter.FeatureFilter +import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.scorer.Scorer import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer @@ -55,11 +52,10 @@ import javax.inject.Singleton @Singleton class ForYouScoredTweetsCandidatePipelineConfig @Inject() ( scoredTweetsProductCandidateSource: ScoredTweetsProductCandidateSource, - tweetypieFeatureHydrator: TweetypieFeatureHydrator, - namesFeatureHydrator: NamesFeatureHydrator, - sgsValidSocialContextFeatureHydrator: SGSValidSocialContextFeatureHydrator, - perspectiveFilteredSocialContextFeatureHydrator: PerspectiveFilteredSocialContextFeatureHydrator, focalTweetFeatureHydrator: FocalTweetFeatureHydrator, + namesFeatureHydrator: NamesFeatureHydrator, + tweetypieFeatureHydrator: TweetypieFeatureHydrator, + invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter, homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder, homeTweetSocialContextBuilder: HomeTweetSocialContextBuilder) extends CandidatePipelineConfig[ @@ -99,12 +95,7 @@ class ForYouScoredTweetsCandidatePipelineConfig @Inject() ( override val preFilterFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] - ] = Seq( - namesFeatureHydrator, - tweetypieFeatureHydrator, - sgsValidSocialContextFeatureHydrator, - perspectiveFilteredSocialContextFeatureHydrator, - ) + ] = Seq(InNetworkFeatureHydrator, namesFeatureHydrator, tweetypieFeatureHydrator) override val filters: Seq[Filter[ForYouQuery, TweetCandidate]] = Seq( FeatureFilter.fromFeature(FilterIdentifier(TweetypieHydratedFilterId), IsHydratedFeature), @@ -119,8 +110,8 @@ class ForYouScoredTweetsCandidatePipelineConfig @Inject() ( !features.getOrElse(IsNsfwFeature, false) } ), - FeedbackFatigueFilter, SocialContextFilter, + invalidSubscriptionTweetFilter, InvalidConversationModuleFilter ) @@ -128,8 +119,6 @@ class ForYouScoredTweetsCandidatePipelineConfig @Inject() ( BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] ] = Seq(focalTweetFeatureHydrator) - override val scorers: Seq[Scorer[ForYouQuery, TweetCandidate]] = Seq(FeedbackFatigueScorer) - override val decorator: Option[CandidateDecorator[ForYouQuery, TweetCandidate]] = { val clientEventInfoBuilder = HomeClientEventInfoBuilder() diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsMixerPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsMixerPipelineConfig.scala index 29cb63678..2137c5267 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsMixerPipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsMixerPipelineConfig.scala @@ -14,19 +14,30 @@ import com.twitter.home_mixer.functional_component.side_effect._ import com.twitter.home_mixer.model.ClearCacheIncludeInstruction import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.param.HomeGlobalParams.MaxNumberReplaceInstructionsParam +import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeClientEventsFlag import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings +import com.twitter.home_mixer.product.for_you.feature_hydrator.TimelineServiceTweetsQueryFeatureHydrator import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.product.for_you.param.ForYouParam.ClearCacheOnPtr import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableFlipInjectionModuleCandidatePipelineParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.FlipInlineInjectionModulePosition import com.twitter.home_mixer.product.for_you.param.ForYouParam.ServerMaxResultsParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.TweetPreviewsPositionParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToFollowPositionParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToSubscribePositionParam +import com.twitter.home_mixer.product.for_you.side_effect.ServedCandidateFeatureKeysKafkaSideEffectBuilder +import com.twitter.home_mixer.product.for_you.side_effect.ServedCandidateKeysKafkaSideEffectBuilder +import com.twitter.home_mixer.product.for_you.side_effect.ServedStatsSideEffect import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.inject.annotations.Flag import com.twitter.logpipeline.client.common.EventPublisher import com.twitter.product_mixer.component_library.feature_hydrator.query.async.AsyncQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.PreviewCreatorsQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.FlipPromptCandidatePipelineConfigBuilder import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidatePipelineConfig +import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidatePipelineConfig import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ClearCacheInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedBottomCursorBuilder @@ -78,52 +89,67 @@ import javax.inject.Singleton @Singleton class ForYouScoredTweetsMixerPipelineConfig @Inject() ( - forYouScoredTweetsCandidatePipelineConfig: ForYouScoredTweetsCandidatePipelineConfig, + forYouAdsDependentCandidatePipelineBuilder: ForYouAdsDependentCandidatePipelineBuilder, forYouConversationServiceCandidatePipelineConfig: ForYouConversationServiceCandidatePipelineConfig, - forYouAdsCandidatePipelineBuilder: ForYouAdsCandidatePipelineBuilder, + forYouPushToHomeTweetCandidatePipelineConfig: ForYouPushToHomeTweetCandidatePipelineConfig, + forYouScoredTweetsCandidatePipelineConfig: ForYouScoredTweetsCandidatePipelineConfig, forYouWhoToFollowCandidatePipelineConfigBuilder: ForYouWhoToFollowCandidatePipelineConfigBuilder, + forYouWhoToSubscribeCandidatePipelineConfigBuilder: ForYouWhoToSubscribeCandidatePipelineConfigBuilder, flipPromptCandidatePipelineConfigBuilder: FlipPromptCandidatePipelineConfigBuilder, editedTweetsCandidatePipelineConfig: EditedTweetsCandidatePipelineConfig, newTweetsPillCandidatePipelineConfig: NewTweetsPillCandidatePipelineConfig[ForYouQuery], + forYouTweetPreviewsCandidatePipelineConfig: ForYouTweetPreviewsCandidatePipelineConfig, dismissInfoQueryFeatureHydrator: DismissInfoQueryFeatureHydrator, gizmoduckUserQueryFeatureHydrator: GizmoduckUserQueryFeatureHydrator, persistenceStoreQueryFeatureHydrator: PersistenceStoreQueryFeatureHydrator, requestQueryFeatureHydrator: RequestQueryFeatureHydrator[ForYouQuery], - feedbackHistoryQueryFeatureHydrator: FeedbackHistoryQueryFeatureHydrator, timelineServiceTweetsQueryFeatureHydrator: TimelineServiceTweetsQueryFeatureHydrator, + previewCreatorsQueryFeatureHydrator: PreviewCreatorsQueryFeatureHydrator, + sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, adsInjector: AdsInjector, servedCandidateKeysKafkaSideEffectBuilder: ServedCandidateKeysKafkaSideEffectBuilder, servedCandidateFeatureKeysKafkaSideEffectBuilder: ServedCandidateFeatureKeysKafkaSideEffectBuilder, updateTimelinesPersistenceStoreSideEffect: UpdateTimelinesPersistenceStoreSideEffect, truncateTimelinesPersistenceStoreSideEffect: TruncateTimelinesPersistenceStoreSideEffect, - homeScribeServedEntriesSideEffect: HomeScribeServedEntriesSideEffect, + homeScribeServedCandidatesSideEffect: HomeScribeServedCandidatesSideEffect, servedStatsSideEffect: ServedStatsSideEffect, clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent], externalStrings: HomeMixerExternalStrings, @ProductScoped stringCenterProvider: Provider[StringCenter], - urtTransportMarshaller: UrtTransportMarshaller) + urtTransportMarshaller: UrtTransportMarshaller, + @Flag(ScribeClientEventsFlag) enableScribeClientEvents: Boolean) extends MixerPipelineConfig[ForYouQuery, Timeline, urt.TimelineResponse] { override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("ForYouScoredTweets") private val MaxConsecutiveOutOfNetworkCandidates = 2 + private val PushToHomeTweetPosition = 0 + private val dependentCandidatesStep = MixerPipelineConfig.dependentCandidatePipelinesStep override val fetchQueryFeatures: Seq[QueryFeatureHydrator[ForYouQuery]] = Seq( requestQueryFeatureHydrator, persistenceStoreQueryFeatureHydrator, timelineServiceTweetsQueryFeatureHydrator, - feedbackHistoryQueryFeatureHydrator, + previewCreatorsQueryFeatureHydrator, + sgsFollowedUsersQueryFeatureHydrator, AsyncQueryFeatureHydrator(dependentCandidatesStep, dismissInfoQueryFeatureHydrator), AsyncQueryFeatureHydrator(dependentCandidatesStep, gizmoduckUserQueryFeatureHydrator), ) - private val forYouAdsCandidatePipelineConfig = forYouAdsCandidatePipelineBuilder.build() + private val scoredTweetsCandidatePipelineScope = + SpecificPipeline(forYouScoredTweetsCandidatePipelineConfig.identifier) + + private val forYouAdsCandidatePipelineConfig = forYouAdsDependentCandidatePipelineBuilder + .build(scoredTweetsCandidatePipelineScope) private val forYouWhoToFollowCandidatePipelineConfig = forYouWhoToFollowCandidatePipelineConfigBuilder.build() + private val forYouWhoToSubscribeCandidatePipelineConfig = + forYouWhoToSubscribeCandidatePipelineConfigBuilder.build() + private val flipPromptCandidatePipelineConfig = flipPromptCandidatePipelineConfigBuilder.build[ForYouQuery]( supportedClientParam = Some(EnableFlipInjectionModuleCandidatePipelineParam) @@ -131,14 +157,17 @@ class ForYouScoredTweetsMixerPipelineConfig @Inject() ( override val candidatePipelines: Seq[CandidatePipelineConfig[ForYouQuery, _, _, _]] = Seq( forYouScoredTweetsCandidatePipelineConfig, - forYouAdsCandidatePipelineConfig, + forYouPushToHomeTweetCandidatePipelineConfig, forYouWhoToFollowCandidatePipelineConfig, + forYouWhoToSubscribeCandidatePipelineConfig, + forYouTweetPreviewsCandidatePipelineConfig, flipPromptCandidatePipelineConfig ) override val dependentCandidatePipelines: Seq[ DependentCandidatePipelineConfig[ForYouQuery, _, _, _] ] = Seq( + forYouAdsCandidatePipelineConfig, forYouConversationServiceCandidatePipelineConfig, editedTweetsCandidatePipelineConfig, newTweetsPillCandidatePipelineConfig @@ -148,6 +177,8 @@ class ForYouScoredTweetsMixerPipelineConfig @Inject() ( forYouScoredTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouAdsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouWhoToFollowCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + forYouWhoToSubscribeCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + forYouTweetPreviewsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, flipPromptCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, editedTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, newTweetsPillCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, @@ -191,32 +222,56 @@ class ForYouScoredTweetsMixerPipelineConfig @Inject() ( candidatePipeline = editedTweetsCandidatePipelineConfig.identifier, maxSelectionsParam = MaxNumberReplaceInstructionsParam ), - DropModuleTooFewModuleItemResults( - candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, - minModuleItemsParam = StaticParam(WhoToFollowCandidatePipelineConfig.MinCandidatesSize) - ), DropMaxModuleItemCandidates( candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, maxModuleItemsParam = StaticParam(WhoToFollowCandidatePipelineConfig.MaxCandidatesSize) ), + DropMaxModuleItemCandidates( + candidatePipeline = forYouWhoToSubscribeCandidatePipelineConfig.identifier, + maxModuleItemsParam = StaticParam(WhoToSubscribeCandidatePipelineConfig.MaxCandidatesSize) + ), // The Conversation Service pipeline will only run if the Scored Tweets pipeline returned nothing - InsertAppendResults(candidatePipeline = - forYouConversationServiceCandidatePipelineConfig.identifier), - InsertAppendResults(candidatePipeline = forYouScoredTweetsCandidatePipelineConfig.identifier), + InsertAppendResults( + candidatePipeline = forYouConversationServiceCandidatePipelineConfig.identifier + ), + InsertAppendResults( + candidatePipeline = forYouScoredTweetsCandidatePipelineConfig.identifier + ), + InsertFixedPositionResults( + candidatePipeline = forYouTweetPreviewsCandidatePipelineConfig.identifier, + positionParam = TweetPreviewsPositionParam + ), InsertFixedPositionResults( candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, positionParam = WhoToFollowPositionParam ), + InsertFixedPositionResults( + candidatePipeline = forYouWhoToSubscribeCandidatePipelineConfig.identifier, + positionParam = WhoToSubscribePositionParam + ), InsertFixedPositionResults( candidatePipeline = flipPromptCandidatePipelineConfig.identifier, positionParam = FlipInlineInjectionModulePosition ), + // Insert Push To Home Tweet at top of Timeline + InsertFixedPositionResults( + candidatePipeline = forYouPushToHomeTweetCandidatePipelineConfig.identifier, + positionParam = StaticParam(PushToHomeTweetPosition) + ), InsertAdResults( surfaceAreaName = AdsInjectionSurfaceAreas.HomeTimeline, adsInjector = adsInjector.forSurfaceArea(AdsInjectionSurfaceAreas.HomeTimeline), adsCandidatePipeline = forYouAdsCandidatePipelineConfig.identifier ), // This selector must come after the tweets are inserted into the results + DropModuleTooFewModuleItemResults( + candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, + minModuleItemsParam = StaticParam(WhoToFollowCandidatePipelineConfig.MinCandidatesSize) + ), + DropModuleTooFewModuleItemResults( + candidatePipeline = forYouWhoToSubscribeCandidatePipelineConfig.identifier, + minModuleItemsParam = StaticParam(WhoToSubscribeCandidatePipelineConfig.MinCandidatesSize) + ), UpdateNewTweetsPillDecoration( pipelineScope = SpecificPipelines( forYouConversationServiceCandidatePipelineConfig.identifier, @@ -250,14 +305,18 @@ class ForYouScoredTweetsMixerPipelineConfig @Inject() ( Set(forYouScoredTweetsCandidatePipelineConfig.identifier)) private val homeScribeClientEventSideEffect = HomeScribeClientEventSideEffect( + enableScribeClientEvents = enableScribeClientEvents, logPipelinePublisher = clientEventsScribeEventPublisher, injectedTweetsCandidatePipelineIdentifiers = Seq( forYouScoredTweetsCandidatePipelineConfig.identifier, forYouConversationServiceCandidatePipelineConfig.identifier ), - adsCandidatePipelineIdentifier = forYouAdsCandidatePipelineConfig.identifier, - whoToFollowCandidatePipelineIdentifier = - Some(forYouWhoToFollowCandidatePipelineConfig.identifier), + adsCandidatePipelineIdentifier = Some(forYouAdsCandidatePipelineConfig.identifier), + whoToFollowCandidatePipelineIdentifier = Some( + forYouWhoToFollowCandidatePipelineConfig.identifier + ), + whoToSubscribeCandidatePipelineIdentifier = + Some(forYouWhoToSubscribeCandidatePipelineConfig.identifier) ) override val resultSideEffects: Seq[PipelineResultSideEffect[ForYouQuery, Timeline]] = Seq( @@ -266,7 +325,7 @@ class ForYouScoredTweetsMixerPipelineConfig @Inject() ( updateTimelinesPersistenceStoreSideEffect, truncateTimelinesPersistenceStoreSideEffect, homeScribeClientEventSideEffect, - homeScribeServedEntriesSideEffect, + homeScribeServedCandidatesSideEffect, servedStatsSideEffect ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsResponseFeatureTransformer.scala index 9d8b43f73..eafc09352 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsResponseFeatureTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsResponseFeatureTransformer.scala @@ -1,6 +1,5 @@ package com.twitter.home_mixer.product.for_you -import com.twitter.timelines.render.{thriftscala => tl} import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.home_mixer.product.for_you.candidate_source.ScoredTweetWithConversationMetadata import com.twitter.product_mixer.core.feature.Feature @@ -11,6 +10,8 @@ import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentif import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecWithEducationTopicContextFunctionalityType import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType +import com.twitter.timelines.render.{thriftscala => tl} +import com.twitter.timelineservice.suggests.{thriftscala => tls} object ForYouScoredTweetsResponseFeatureTransformer extends CandidateFeatureTransformer[ScoredTweetWithConversationMetadata] { @@ -21,18 +22,29 @@ object ForYouScoredTweetsResponseFeatureTransformer override val features: Set[Feature[_, _]] = Set( AncestorsFeature, AuthorIdFeature, - ConversationModuleIdFeature, + AuthorIsBlueVerifiedFeature, + AuthorIsCreatorFeature, + AuthorIsGoldVerifiedFeature, + AuthorIsGrayVerifiedFeature, + AuthorIsLegacyVerifiedFeature, ConversationModuleFocalTweetIdFeature, + ConversationModuleIdFeature, DirectedAtUserIdFeature, + ExclusiveConversationAuthorIdFeature, + FullScoringSucceededFeature, FavoritedByUserIdsFeature, FollowedByUserIdsFeature, InNetworkFeature, InReplyToTweetIdFeature, InReplyToUserIdFeature, + IsAncestorCandidateFeature, IsReadFromCacheFeature, IsRetweetFeature, + PerspectiveFilteredLikedByUserIdsFeature, QuotedTweetIdFeature, QuotedUserIdFeature, + SGSValidFollowedByUserIdsFeature, + SGSValidLikedByUserIdsFeature, ScoreFeature, SourceTweetIdFeature, SourceUserIdFeature, @@ -46,23 +58,36 @@ object ForYouScoredTweetsResponseFeatureTransformer FeatureMapBuilder() .add(AncestorsFeature, input.ancestors.getOrElse(Seq.empty)) .add(AuthorIdFeature, Some(input.authorId)) + .add(AuthorIsBlueVerifiedFeature, input.authorIsBlueVerified.getOrElse(false)) + .add(AuthorIsGoldVerifiedFeature, input.authorIsGoldVerified.getOrElse(false)) + .add(AuthorIsGrayVerifiedFeature, input.authorIsGrayVerified.getOrElse(false)) + .add(AuthorIsLegacyVerifiedFeature, input.authorIsLegacyVerified.getOrElse(false)) + .add(AuthorIsCreatorFeature, input.authorIsCreator.getOrElse(false)) .add(ConversationModuleIdFeature, input.conversationId) .add(ConversationModuleFocalTweetIdFeature, input.conversationFocalTweetId) .add(DirectedAtUserIdFeature, input.directedAtUserId) - .add(FavoritedByUserIdsFeature, input.favoritedByUserIds.getOrElse(Seq.empty)) - .add(FollowedByUserIdsFeature, input.followedByUserIds.getOrElse(Seq.empty)) - .add(InNetworkFeature, input.inNetwork.getOrElse(false)) + .add(ExclusiveConversationAuthorIdFeature, input.exclusiveConversationAuthorId) + .add(SGSValidLikedByUserIdsFeature, input.sgsValidLikedByUserIds.getOrElse(Seq.empty)) + .add(SGSValidFollowedByUserIdsFeature, input.sgsValidFollowedByUserIds.getOrElse(Seq.empty)) + .add(FavoritedByUserIdsFeature, input.sgsValidLikedByUserIds.getOrElse(Seq.empty)) + .add(FollowedByUserIdsFeature, input.sgsValidFollowedByUserIds.getOrElse(Seq.empty)) + .add(FullScoringSucceededFeature, true) + .add(InNetworkFeature, input.inNetwork.getOrElse(true)) .add(InReplyToTweetIdFeature, input.inReplyToTweetId) .add(InReplyToUserIdFeature, input.inReplyToUserId) + .add(IsAncestorCandidateFeature, input.conversationFocalTweetId.exists(_ != input.tweetId)) .add(IsReadFromCacheFeature, input.isReadFromCache.getOrElse(false)) .add(IsRetweetFeature, input.sourceTweetId.isDefined) + .add( + PerspectiveFilteredLikedByUserIdsFeature, + input.perspectiveFilteredLikedByUserIds.getOrElse(Seq.empty)) .add(QuotedTweetIdFeature, input.quotedTweetId) .add(QuotedUserIdFeature, input.quotedUserId) .add(ScoreFeature, input.score) .add(SourceTweetIdFeature, input.sourceTweetId) .add(SourceUserIdFeature, input.sourceUserId) .add(StreamToKafkaFeature, input.streamToKafka.getOrElse(false)) - .add(SuggestTypeFeature, input.suggestType) + .add(SuggestTypeFeature, input.suggestType.orElse(Some(tls.SuggestType.Undefined))) .add( TopicContextFunctionalityTypeFeature, input.topicFunctionalityType.collect { diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerCandidatePipelineConfig.scala index 37f93c2a1..bb93a4110 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerCandidatePipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerCandidatePipelineConfig.scala @@ -1,23 +1,20 @@ package com.twitter.home_mixer.product.for_you -import com.twitter.home_mixer.functional_component.decorator.HomeFeedbackActionInfoBuilder -import com.twitter.home_mixer.functional_component.decorator.HomeTimelinesScoreInfoBuilder -import com.twitter.home_mixer.functional_component.decorator.HomeTweetSocialContextBuilder import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventInfoBuilder import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder -import com.twitter.home_mixer.functional_component.feature_hydrator.FocalTweetFeatureHydrator +import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder +import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder +import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeTweetSocialContextBuilder +import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.PerspectiveFilteredSocialContextFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.SGSValidSocialContextFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.SocialGraphServiceFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.TimelineServiceTweetsFeature import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator import com.twitter.home_mixer.functional_component.filter.FeedbackFatigueFilter import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter -import com.twitter.home_mixer.functional_component.filter.PredicateFeatureFilter +import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter import com.twitter.home_mixer.functional_component.filter.RejectTweetFromViewerFilter import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter -import com.twitter.home_mixer.functional_component.filter.SocialContextFilter import com.twitter.home_mixer.functional_component.scorer.FeedbackFatigueScorer import com.twitter.home_mixer.functional_component.scorer.OONTweetScalingScorer import com.twitter.home_mixer.marshaller.timelines.DeviceContextMarshaller @@ -27,7 +24,10 @@ import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature import com.twitter.home_mixer.model.HomeFeatures.IsNsfwFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature +import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature import com.twitter.home_mixer.model.request.DeviceContext +import com.twitter.home_mixer.product.for_you.feature_hydrator.FocalTweetFeatureHydrator +import com.twitter.home_mixer.product.for_you.filter.SocialContextFilter import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableTimelineScorerCandidatePipelineParam import com.twitter.home_mixer.service.HomeMixerAlertConfig @@ -40,6 +40,7 @@ import com.twitter.product_mixer.component_library.decorator.urt.builder.timelin import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.StaticModuleDisplayTypeBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder import com.twitter.product_mixer.component_library.filter.FeatureFilter +import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.common.alert.Alert @@ -71,11 +72,11 @@ class ForYouTimelineScorerCandidatePipelineConfig @Inject() ( timelineScorerCandidateSource: TimelineScorerCandidateSource, deviceContextMarshaller: DeviceContextMarshaller, tweetypieFeatureHydrator: TweetypieFeatureHydrator, - sgsFeatureHydrator: SocialGraphServiceFeatureHydrator, sgsValidSocialContextFeatureHydrator: SGSValidSocialContextFeatureHydrator, perspectiveFilteredSocialContextFeatureHydrator: PerspectiveFilteredSocialContextFeatureHydrator, namesFeatureHydrator: NamesFeatureHydrator, focalTweetFeatureHydrator: FocalTweetFeatureHydrator, + invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter, homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder, homeTweetSocialContextBuilder: HomeTweetSocialContextBuilder) extends CandidatePipelineConfig[ @@ -124,7 +125,9 @@ class ForYouTimelineScorerCandidatePipelineConfig @Inject() ( CandidateTweetSourceId.CroonTweet, CandidateTweetSourceId.RecommendedTweet, CandidateTweetSourceId.FrsTweet, - CandidateTweetSourceId.ListTweet + CandidateTweetSourceId.ListTweet, + CandidateTweetSourceId.RecommendedTrendTweet, + CandidateTweetSourceId.PopularTopicTweet ) val timelineServiceTweets = @@ -167,11 +170,11 @@ class ForYouTimelineScorerCandidatePipelineConfig @Inject() ( override val preFilterFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] ] = Seq( + namesFeatureHydrator, tweetypieFeatureHydrator, - sgsFeatureHydrator, + InNetworkFeatureHydrator, sgsValidSocialContextFeatureHydrator, perspectiveFilteredSocialContextFeatureHydrator, - namesFeatureHydrator ) override def filters: Seq[Filter[ForYouQuery, TweetCandidate]] = Seq( @@ -191,6 +194,7 @@ class ForYouTimelineScorerCandidatePipelineConfig @Inject() ( FeedbackFatigueFilter, RejectTweetFromViewerFilter, SocialContextFilter, + invalidSubscriptionTweetFilter, InvalidConversationModuleFilter ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerMixerPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerMixerPipelineConfig.scala index b43c6ed67..441e5796d 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerMixerPipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerMixerPipelineConfig.scala @@ -5,6 +5,7 @@ import com.twitter.goldfinch.api.AdsInjectionSurfaceAreas import com.twitter.home_mixer.candidate_pipeline.EditedTweetsCandidatePipelineConfig import com.twitter.home_mixer.candidate_pipeline.NewTweetsPillCandidatePipelineConfig import com.twitter.home_mixer.functional_component.decorator.urt.builder.AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder +import com.twitter.home_mixer.functional_component.feature_hydrator.FeedbackHistoryQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator._ import com.twitter.home_mixer.functional_component.selector.DebunchCandidates import com.twitter.home_mixer.functional_component.selector.UpdateConversationModuleId @@ -13,21 +14,34 @@ import com.twitter.home_mixer.functional_component.selector.UpdateNewTweetsPillD import com.twitter.home_mixer.functional_component.side_effect._ import com.twitter.home_mixer.model.ClearCacheIncludeInstruction import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature +import com.twitter.home_mixer.param.HomeGlobalParams.EnableImpressionBloomFilter import com.twitter.home_mixer.param.HomeGlobalParams.MaxNumberReplaceInstructionsParam +import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeClientEventsFlag import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings +import com.twitter.home_mixer.product.for_you.feature_hydrator.TimelineServiceTweetsQueryFeatureHydrator import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.product.for_you.param.ForYouParam.ClearCacheOnPtr import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableFlipInjectionModuleCandidatePipelineParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.FlipInlineInjectionModulePosition import com.twitter.home_mixer.product.for_you.param.ForYouParam.ServerMaxResultsParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.TweetPreviewsPositionParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToFollowPositionParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToSubscribePositionParam +import com.twitter.home_mixer.product.for_you.side_effect.ServedCandidateFeatureKeysKafkaSideEffectBuilder +import com.twitter.home_mixer.product.for_you.side_effect.ServedCandidateKeysKafkaSideEffectBuilder +import com.twitter.home_mixer.product.for_you.side_effect.ServedStatsSideEffect import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.inject.annotations.Flag import com.twitter.logpipeline.client.common.EventPublisher import com.twitter.product_mixer.component_library.feature_hydrator.query.async.AsyncQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweetsQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.AsyncParamGatedQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.PreviewCreatorsQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.FlipPromptCandidatePipelineConfigBuilder -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidatePipelineConfig +import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmCandidatePipelineConfig +import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidatePipelineConfig import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ClearCacheInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedBottomCursorBuilder @@ -80,14 +94,18 @@ import javax.inject.Singleton @Singleton class ForYouTimelineScorerMixerPipelineConfig @Inject() ( forYouTimelineScorerCandidatePipelineConfig: ForYouTimelineScorerCandidatePipelineConfig, + forYouPushToHomeTweetCandidatePipelineConfig: ForYouPushToHomeTweetCandidatePipelineConfig, forYouConversationServiceCandidatePipelineConfig: ForYouConversationServiceCandidatePipelineConfig, - forYouAdsCandidatePipelineBuilder: ForYouAdsCandidatePipelineBuilder, + forYouAdsDependentCandidatePipelineBuilder: ForYouAdsDependentCandidatePipelineBuilder, forYouWhoToFollowCandidatePipelineConfigBuilder: ForYouWhoToFollowCandidatePipelineConfigBuilder, + forYouWhoToSubscribeCandidatePipelineConfigBuilder: ForYouWhoToSubscribeCandidatePipelineConfigBuilder, flipPromptCandidatePipelineConfigBuilder: FlipPromptCandidatePipelineConfigBuilder, editedTweetsCandidatePipelineConfig: EditedTweetsCandidatePipelineConfig, newTweetsPillCandidatePipelineConfig: NewTweetsPillCandidatePipelineConfig[ForYouQuery], + forYouTweetPreviewsCandidatePipelineConfig: ForYouTweetPreviewsCandidatePipelineConfig, dismissInfoQueryFeatureHydrator: DismissInfoQueryFeatureHydrator, gizmoduckUserQueryFeatureHydrator: GizmoduckUserQueryFeatureHydrator, + impressionBloomFilterQueryFeatureHydrator: ImpressionBloomFilterQueryFeatureHydrator[ForYouQuery], manhattanTweetImpressionsQueryFeatureHydrator: TweetImpressionsQueryFeatureHydrator[ForYouQuery], memcacheTweetImpressionsQueryFeatureHydrator: ImpressedTweetsQueryFeatureHydrator, persistenceStoreQueryFeatureHydrator: PersistenceStoreQueryFeatureHydrator, @@ -95,26 +113,32 @@ class ForYouTimelineScorerMixerPipelineConfig @Inject() ( timelineServiceTweetsQueryFeatureHydrator: TimelineServiceTweetsQueryFeatureHydrator, lastNonPollingTimeQueryFeatureHydrator: LastNonPollingTimeQueryFeatureHydrator, feedbackHistoryQueryFeatureHydrator: FeedbackHistoryQueryFeatureHydrator, + previewCreatorsQueryFeatureHydrator: PreviewCreatorsQueryFeatureHydrator, + sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, adsInjector: AdsInjector, servedCandidateKeysKafkaSideEffectBuilder: ServedCandidateKeysKafkaSideEffectBuilder, servedCandidateFeatureKeysKafkaSideEffectBuilder: ServedCandidateFeatureKeysKafkaSideEffectBuilder, updateLastNonPollingTimeSideEffect: UpdateLastNonPollingTimeSideEffect[ForYouQuery, Timeline], publishClientSentImpressionsEventBusSideEffect: PublishClientSentImpressionsEventBusSideEffect, publishClientSentImpressionsManhattanSideEffect: PublishClientSentImpressionsManhattanSideEffect, + publishImpressionBloomFilterSideEffect: PublishImpressionBloomFilterSideEffect, updateTimelinesPersistenceStoreSideEffect: UpdateTimelinesPersistenceStoreSideEffect, truncateTimelinesPersistenceStoreSideEffect: TruncateTimelinesPersistenceStoreSideEffect, - homeScribeServedEntriesSideEffect: HomeScribeServedEntriesSideEffect, + homeScribeServedCandidatesSideEffect: HomeScribeServedCandidatesSideEffect, servedStatsSideEffect: ServedStatsSideEffect, clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent], externalStrings: HomeMixerExternalStrings, @ProductScoped stringCenterProvider: Provider[StringCenter], - urtTransportMarshaller: UrtTransportMarshaller) + urtTransportMarshaller: UrtTransportMarshaller, + @Flag(ScribeClientEventsFlag) enableScribeClientEvents: Boolean) extends MixerPipelineConfig[ForYouQuery, Timeline, urt.TimelineResponse] { override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("ForYouTimelineScorer") private val MaxConsecutiveOutOfNetworkCandidates = 2 + private val PushToHomeTweetPosition = 0 + private val dependentCandidatesStep = MixerPipelineConfig.dependentCandidatePipelinesStep private val resultSelectorsStep = MixerPipelineConfig.resultSelectorsStep @@ -123,18 +147,31 @@ class ForYouTimelineScorerMixerPipelineConfig @Inject() ( persistenceStoreQueryFeatureHydrator, timelineServiceTweetsQueryFeatureHydrator, feedbackHistoryQueryFeatureHydrator, + previewCreatorsQueryFeatureHydrator, + sgsFollowedUsersQueryFeatureHydrator, AsyncQueryFeatureHydrator(dependentCandidatesStep, dismissInfoQueryFeatureHydrator), AsyncQueryFeatureHydrator(dependentCandidatesStep, gizmoduckUserQueryFeatureHydrator), AsyncQueryFeatureHydrator(dependentCandidatesStep, lastNonPollingTimeQueryFeatureHydrator), + AsyncParamGatedQueryFeatureHydrator( + EnableImpressionBloomFilter, + resultSelectorsStep, + impressionBloomFilterQueryFeatureHydrator), AsyncQueryFeatureHydrator(resultSelectorsStep, manhattanTweetImpressionsQueryFeatureHydrator), AsyncQueryFeatureHydrator(resultSelectorsStep, memcacheTweetImpressionsQueryFeatureHydrator) ) - private val forYouAdsCandidatePipelineConfig = forYouAdsCandidatePipelineBuilder.build() + private val timelineScorerCandidatePipelineScope = + SpecificPipeline(forYouTimelineScorerCandidatePipelineConfig.identifier) + + private val forYouAdsCandidatePipelineConfig = forYouAdsDependentCandidatePipelineBuilder + .build(timelineScorerCandidatePipelineScope) private val forYouWhoToFollowCandidatePipelineConfig = forYouWhoToFollowCandidatePipelineConfigBuilder.build() + private val forYouWhoToSubscribeCandidatePipelineConfig = + forYouWhoToSubscribeCandidatePipelineConfigBuilder.build() + private val flipPromptCandidatePipelineConfig = flipPromptCandidatePipelineConfigBuilder.build[ForYouQuery]( supportedClientParam = Some(EnableFlipInjectionModuleCandidatePipelineParam) @@ -142,14 +179,17 @@ class ForYouTimelineScorerMixerPipelineConfig @Inject() ( override val candidatePipelines: Seq[CandidatePipelineConfig[ForYouQuery, _, _, _]] = Seq( forYouTimelineScorerCandidatePipelineConfig, - forYouAdsCandidatePipelineConfig, + forYouPushToHomeTweetCandidatePipelineConfig, forYouWhoToFollowCandidatePipelineConfig, + forYouWhoToSubscribeCandidatePipelineConfig, + forYouTweetPreviewsCandidatePipelineConfig, flipPromptCandidatePipelineConfig ) override val dependentCandidatePipelines: Seq[ DependentCandidatePipelineConfig[ForYouQuery, _, _, _] ] = Seq( + forYouAdsCandidatePipelineConfig, forYouConversationServiceCandidatePipelineConfig, editedTweetsCandidatePipelineConfig, newTweetsPillCandidatePipelineConfig @@ -159,6 +199,8 @@ class ForYouTimelineScorerMixerPipelineConfig @Inject() ( forYouTimelineScorerCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouAdsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouWhoToFollowCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + forYouWhoToSubscribeCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + forYouTweetPreviewsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, flipPromptCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, editedTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, newTweetsPillCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, @@ -202,13 +244,17 @@ class ForYouTimelineScorerMixerPipelineConfig @Inject() ( candidatePipeline = editedTweetsCandidatePipelineConfig.identifier, maxSelectionsParam = MaxNumberReplaceInstructionsParam ), - DropModuleTooFewModuleItemResults( - candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, - minModuleItemsParam = StaticParam(WhoToFollowCandidatePipelineConfig.MinCandidatesSize) - ), DropMaxModuleItemCandidates( candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, - maxModuleItemsParam = StaticParam(WhoToFollowCandidatePipelineConfig.MaxCandidatesSize) + maxModuleItemsParam = StaticParam(WhoToFollowArmCandidatePipelineConfig.MaxCandidatesSize) + ), + DropModuleTooFewModuleItemResults( + candidatePipeline = forYouWhoToSubscribeCandidatePipelineConfig.identifier, + minModuleItemsParam = StaticParam(WhoToSubscribeCandidatePipelineConfig.MinCandidatesSize) + ), + DropMaxModuleItemCandidates( + candidatePipeline = forYouWhoToSubscribeCandidatePipelineConfig.identifier, + maxModuleItemsParam = StaticParam(WhoToSubscribeCandidatePipelineConfig.MaxCandidatesSize) ), // Add Conversation Service tweets to results only if the scored pipeline doesn't return any SelectConditionally( @@ -219,20 +265,37 @@ class ForYouTimelineScorerMixerPipelineConfig @Inject() ( forYouTimelineScorerCandidatePipelineConfig.identifier == candidate.source) ), InsertAppendResults(candidatePipeline = forYouTimelineScorerCandidatePipelineConfig.identifier), + InsertFixedPositionResults( + candidatePipeline = forYouTweetPreviewsCandidatePipelineConfig.identifier, + positionParam = TweetPreviewsPositionParam + ), InsertFixedPositionResults( candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, positionParam = WhoToFollowPositionParam ), + InsertFixedPositionResults( + candidatePipeline = forYouWhoToSubscribeCandidatePipelineConfig.identifier, + positionParam = WhoToSubscribePositionParam + ), InsertFixedPositionResults( candidatePipeline = flipPromptCandidatePipelineConfig.identifier, positionParam = FlipInlineInjectionModulePosition ), + // Insert Push To Home Tweet at top of Timeline + InsertFixedPositionResults( + candidatePipeline = forYouPushToHomeTweetCandidatePipelineConfig.identifier, + positionParam = StaticParam(PushToHomeTweetPosition) + ), InsertAdResults( surfaceAreaName = AdsInjectionSurfaceAreas.HomeTimeline, adsInjector = adsInjector.forSurfaceArea(AdsInjectionSurfaceAreas.HomeTimeline), adsCandidatePipeline = forYouAdsCandidatePipelineConfig.identifier ), // This selector must come after the tweets are inserted into the results + DropModuleTooFewModuleItemResults( + candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, + minModuleItemsParam = StaticParam(WhoToFollowArmCandidatePipelineConfig.MinCandidatesSize) + ), UpdateNewTweetsPillDecoration( pipelineScope = SpecificPipelines( forYouConversationServiceCandidatePipelineConfig.identifier, @@ -266,27 +329,31 @@ class ForYouTimelineScorerMixerPipelineConfig @Inject() ( Set(forYouTimelineScorerCandidatePipelineConfig.identifier)) private val homeScribeClientEventSideEffect = HomeScribeClientEventSideEffect( + enableScribeClientEvents = enableScribeClientEvents, logPipelinePublisher = clientEventsScribeEventPublisher, injectedTweetsCandidatePipelineIdentifiers = Seq( forYouTimelineScorerCandidatePipelineConfig.identifier, forYouConversationServiceCandidatePipelineConfig.identifier ), - adsCandidatePipelineIdentifier = forYouAdsCandidatePipelineConfig.identifier, + adsCandidatePipelineIdentifier = Some(forYouAdsCandidatePipelineConfig.identifier), whoToFollowCandidatePipelineIdentifier = Some(forYouWhoToFollowCandidatePipelineConfig.identifier), + whoToSubscribeCandidatePipelineIdentifier = + Some(forYouWhoToSubscribeCandidatePipelineConfig.identifier) ) override val resultSideEffects: Seq[PipelineResultSideEffect[ForYouQuery, Timeline]] = Seq( - servedCandidateKeysKafkaSideEffect, - servedCandidateFeatureKeysKafkaSideEffect, - updateLastNonPollingTimeSideEffect, + homeScribeClientEventSideEffect, + homeScribeServedCandidatesSideEffect, publishClientSentImpressionsEventBusSideEffect, publishClientSentImpressionsManhattanSideEffect, - updateTimelinesPersistenceStoreSideEffect, + publishImpressionBloomFilterSideEffect, + servedCandidateKeysKafkaSideEffect, + servedCandidateFeatureKeysKafkaSideEffect, + servedStatsSideEffect, truncateTimelinesPersistenceStoreSideEffect, - homeScribeClientEventSideEffect, - homeScribeServedEntriesSideEffect, - servedStatsSideEffect + updateLastNonPollingTimeSideEffect, + updateTimelinesPersistenceStoreSideEffect ) override val domainMarshaller: DomainMarshaller[ForYouQuery, Timeline] = { diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerResponseFeatureTransformer.scala index dfae6a10f..236ca9a26 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerResponseFeatureTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerResponseFeatureTransformer.scala @@ -30,7 +30,11 @@ object ForYouTimelineScorerResponseFeatureTransformer AncestorsFeature, AudioSpaceMetaDataFeature, AuthorIdFeature, - AuthorIsEligibleForConnectBoostFeature, + AuthorIsBlueVerifiedFeature, + AuthorIsCreatorFeature, + AuthorIsGoldVerifiedFeature, + AuthorIsGrayVerifiedFeature, + AuthorIsLegacyVerifiedFeature, AuthoredByContextualUserFeature, CandidateSourceIdFeature, ConversationFeature, @@ -47,6 +51,7 @@ object ForYouTimelineScorerResponseFeatureTransformer FromInNetworkSourceFeature, FullScoringSucceededFeature, HasDisplayedTextFeature, + InNetworkFeature, InReplyToTweetIdFeature, IsAncestorCandidateFeature, IsExtendedReplyFeature, @@ -105,8 +110,7 @@ object ForYouTimelineScorerResponseFeatureTransformer val tcoLengthsPlusSpaces = 23 * numMedia + (if (numMedia > 0) numMedia - 1 else 0) length > tcoLengthsPlusSpaces })) - val suggestType = Some( - candidate.overrideSuggestType.getOrElse(tls.SuggestType.RankedTimelineTweet)) + val suggestType = candidate.overrideSuggestType.orElse(Some(tls.SuggestType.Undefined)) val topicSocialProofMetadataOpt = candidate.entityData.flatMap(_.topicSocialProofMetadata) val topicIdSocialContextOpt = topicSocialProofMetadataOpt.map(_.topicId) @@ -129,9 +133,14 @@ object ForYouTimelineScorerResponseFeatureTransformer AudioSpaceMetaDataFeature, candidate.audioSpaceMetaDatalist.map(_.head).map(AudioSpaceMetaData.fromThrift)) .add(AuthorIdFeature, Some(candidate.authorId)) + .add(AuthorIsBlueVerifiedFeature, candidate.authorIsBlueVerified.getOrElse(false)) .add( - AuthorIsEligibleForConnectBoostFeature, - candidate.authorIsEligibleForConnectBoost.getOrElse(false)) + AuthorIsCreatorFeature, + candidate.authorIsCreator.getOrElse(false) + ) + .add(AuthorIsGoldVerifiedFeature, candidate.authorIsGoldVerified.getOrElse(false)) + .add(AuthorIsGrayVerifiedFeature, candidate.authorIsGrayVerified.getOrElse(false)) + .add(AuthorIsLegacyVerifiedFeature, candidate.authorIsLegacyVerified.getOrElse(false)) .add( AuthoredByContextualUserFeature, candidate.viewerId.contains(candidate.authorId) || @@ -151,6 +160,7 @@ object ForYouTimelineScorerResponseFeatureTransformer .add(TopicContextFunctionalityTypeFeature, topicContextFunctionalityTypeOpt) .add(FullScoringSucceededFeature, candidate.fullScoringSucceeded.getOrElse(false)) .add(HasDisplayedTextFeature, hasDisplayedText) + .add(InNetworkFeature, candidate.isInNetwork.getOrElse(true)) .add(InReplyToTweetIdFeature, candidate.inReplyToTweetId) .add(IsAncestorCandidateFeature, candidate.isAncestorCandidate.getOrElse(false)) .add( diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTweetPreviewsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTweetPreviewsCandidatePipelineConfig.scala new file mode 100644 index 000000000..c60278bfb --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTweetPreviewsCandidatePipelineConfig.scala @@ -0,0 +1,143 @@ +package com.twitter.home_mixer.product.for_you + +import com.twitter.home_mixer.functional_component.candidate_source.EarlybirdCandidateSource +import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder +import com.twitter.home_mixer.functional_component.filter.DropMaxCandidatesFilter +import com.twitter.home_mixer.functional_component.filter.PreviouslyServedTweetPreviewsFilter +import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate +import com.twitter.home_mixer.model.HomeFeatures.AuthorEnabledPreviewsFeature +import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature +import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature +import com.twitter.home_mixer.product.for_you.feature_hydrator.AuthorEnabledPreviewsFeatureHydrator +import com.twitter.home_mixer.product.for_you.feature_hydrator.TweetPreviewTweetypieCandidateFeatureHydrator +import com.twitter.home_mixer.product.for_you.filter.TweetPreviewTextFilter +import com.twitter.home_mixer.product.for_you.model.ForYouQuery +import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableTweetPreviewsCandidatePipelineParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.TweetPreviewsMaxCandidatesParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.TweetPreviewsMinInjectionIntervalParam +import com.twitter.home_mixer.product.for_you.query_transformer.TweetPreviewsQueryTransformer +import com.twitter.home_mixer.product.for_you.response_transformer.TweetPreviewResponseFeatureTransformer +import com.twitter.home_mixer.service.HomeMixerAlertConfig +import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator +import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.PreviewCreatorsFeature +import com.twitter.product_mixer.component_library.filter.FeatureFilter +import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSourceWithExtractedFeatures +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.model.marshalling.response.rtf.safety_level.TimelineHomeTweetPreviewHydrationSafetyLevel +import com.twitter.product_mixer.core.model.marshalling.response.urt.contextual_ref.TweetHydrationContext +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.search.earlybird.{thriftscala => eb} +import com.twitter.timelines.configapi.FSParam +import com.twitter.timelines.injection.scribe.InjectionScribeUtil +import com.twitter.timelineservice.model.rich.EntityIdType +import com.twitter.timelineservice.suggests.{thriftscala => st} + +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ForYouTweetPreviewsCandidatePipelineConfig @Inject() ( + earlybirdCandidateSource: EarlybirdCandidateSource, + authorEnabledPreviewsFeatureHydrator: AuthorEnabledPreviewsFeatureHydrator, + tweetPreviewsQueryTransformer: TweetPreviewsQueryTransformer, + tweetPreviewTweetypieCandidateFeatureHydrator: TweetPreviewTweetypieCandidateFeatureHydrator, + homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder) + extends CandidatePipelineConfig[ + ForYouQuery, + eb.EarlybirdRequest, + eb.ThriftSearchResult, + TweetCandidate + ] { + + val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ForYouTweetPreviews") + + override val supportedClientParam: Option[FSParam[Boolean]] = + Some(EnableTweetPreviewsCandidatePipelineParam) + + override val gates: Seq[Gate[ForYouQuery]] = { + Seq( + TimelinesPersistenceStoreLastInjectionGate( + TweetPreviewsMinInjectionIntervalParam, + PersistenceEntriesFeature, + EntityIdType.TweetPreview + ), + NonEmptySeqFeatureGate(PreviewCreatorsFeature) + ) + } + + override val queryTransformer: CandidatePipelineQueryTransformer[ + PipelineQuery, + eb.EarlybirdRequest + ] = tweetPreviewsQueryTransformer + + override val candidateSource: CandidateSourceWithExtractedFeatures[ + eb.EarlybirdRequest, + eb.ThriftSearchResult + ] = earlybirdCandidateSource + + override val featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[eb.ThriftSearchResult] + ] = Seq(TweetPreviewResponseFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + eb.ThriftSearchResult, + TweetCandidate + ] = { tweet => TweetCandidate(tweet.id) } + + override val preFilterFeatureHydrationPhase1: Seq[ + BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] + ] = Seq( + authorEnabledPreviewsFeatureHydrator, + tweetPreviewTweetypieCandidateFeatureHydrator, + ) + + override val filters: Seq[ + Filter[ForYouQuery, TweetCandidate] + ] = Seq( + PreviouslyServedTweetPreviewsFilter, + FeatureFilter + .fromFeature(FilterIdentifier("TweetPreviewVisibilityFiltering"), IsHydratedFeature), + FeatureFilter + .fromFeature(FilterIdentifier("AuthorEnabledPreviews"), AuthorEnabledPreviewsFeature), + TweetPreviewTextFilter, + DropMaxCandidatesFilter(TweetPreviewsMaxCandidatesParam) + ) + + override val decorator: Option[CandidateDecorator[PipelineQuery, TweetCandidate]] = { + val component = InjectionScribeUtil.scribeComponent(st.SuggestType.TweetPreview).get + val clientEventInfoBuilder = ClientEventInfoBuilder[PipelineQuery, TweetCandidate](component) + + val tweetItemBuilder = TweetCandidateUrtItemBuilder[PipelineQuery, TweetCandidate]( + clientEventInfoBuilder = clientEventInfoBuilder, + contextualTweetRefBuilder = Some( + ContextualTweetRefBuilder( + TweetHydrationContext( + safetyLevelOverride = Some(TimelineHomeTweetPreviewHydrationSafetyLevel), + outerTweetContext = None + ) + ) + ), + feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder), + ) + + Some(UrtItemCandidateDecorator(tweetItemBuilder)) + } + + override val alerts: Seq[Alert] = Seq( + HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(95)) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToFollowCandidatePipelineConfigBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToFollowCandidatePipelineConfigBuilder.scala index 0f01c9e14..856f1fa6b 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToFollowCandidatePipelineConfigBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToFollowCandidatePipelineConfigBuilder.scala @@ -8,13 +8,13 @@ import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature import com.twitter.home_mixer.model.HomeFeatures.WhoToFollowExcludedUserIdsFeature import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableWhoToFollowCandidatePipelineParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToFollowDisplayLocationParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToFollowDisplayTypeIdParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToFollowMinInjectionIntervalParam import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ParamWhoToFollowModuleDisplayTypeBuilder -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidatePipelineConfig -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidatePipelineConfigBuilder -import com.twitter.product_mixer.core.functional_component.configapi.StaticParam +import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmCandidatePipelineConfig +import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmCandidatePipelineConfigBuilder import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.timelineservice.model.rich.EntityIdType import com.twitter.timelineservice.suggests.thriftscala.SuggestType @@ -23,13 +23,10 @@ import javax.inject.Singleton @Singleton class ForYouWhoToFollowCandidatePipelineConfigBuilder @Inject() ( - whoToFollowCandidatePipelineConfigBuilder: WhoToFollowCandidatePipelineConfigBuilder, + whoToFollowArmCandidatePipelineConfigBuilder: WhoToFollowArmCandidatePipelineConfigBuilder, homeWhoToFollowFeedbackActionInfoBuilder: HomeWhoToFollowFeedbackActionInfoBuilder) { - // People Discovery module timeout is set to 350ms currently so use faster display location here - private val DisplayLocation = "timeline_reverse_chron" - - def build(): WhoToFollowCandidatePipelineConfig[ForYouQuery] = { + def build(): WhoToFollowArmCandidatePipelineConfig[ForYouQuery] = { val gates: Seq[Gate[ForYouQuery]] = Seq( TimelinesPersistenceStoreLastInjectionGate( WhoToFollowMinInjectionIntervalParam, @@ -39,8 +36,8 @@ class ForYouWhoToFollowCandidatePipelineConfigBuilder @Inject() ( DismissFatigueGate(SuggestType.WhoToFollow, DismissInfoFeature) ) - whoToFollowCandidatePipelineConfigBuilder.build[ForYouQuery]( - identifier = WhoToFollowCandidatePipelineConfig.identifier, + whoToFollowArmCandidatePipelineConfigBuilder.build[ForYouQuery]( + identifier = WhoToFollowArmCandidatePipelineConfig.identifier, supportedClientParam = Some(EnableWhoToFollowCandidatePipelineParam), alerts = alerts, gates = gates, @@ -48,7 +45,7 @@ class ForYouWhoToFollowCandidatePipelineConfigBuilder @Inject() ( ParamWhoToFollowModuleDisplayTypeBuilder(WhoToFollowDisplayTypeIdParam), feedbackActionInfoBuilder = Some(homeWhoToFollowFeedbackActionInfoBuilder), excludedUserIdsFeature = Some(WhoToFollowExcludedUserIdsFeature), - displayLocationParam = StaticParam(DisplayLocation) + displayLocationParam = WhoToFollowDisplayLocationParam ) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToSubscribeCandidatePipelineConfigBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToSubscribeCandidatePipelineConfigBuilder.scala new file mode 100644 index 000000000..bd4437b7c --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToSubscribeCandidatePipelineConfigBuilder.scala @@ -0,0 +1,52 @@ +package com.twitter.home_mixer.product.for_you + +import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeWhoToSubscribeFeedbackActionInfoBuilder +import com.twitter.home_mixer.functional_component.gate.DismissFatigueGate +import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate +import com.twitter.home_mixer.model.HomeFeatures.DismissInfoFeature +import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature +import com.twitter.home_mixer.product.for_you.model.ForYouQuery +import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableWhoToSubscribeCandidatePipelineParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToSubscribeDisplayTypeIdParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToSubscribeMinInjectionIntervalParam +import com.twitter.home_mixer.service.HomeMixerAlertConfig +import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ParamWhoToFollowModuleDisplayTypeBuilder +import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidatePipelineConfig +import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidatePipelineConfigBuilder +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.timelineservice.model.rich.EntityIdType +import com.twitter.timelineservice.suggests.thriftscala.SuggestType +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ForYouWhoToSubscribeCandidatePipelineConfigBuilder @Inject() ( + whoToSubscribeCandidatePipelineConfigBuilder: WhoToSubscribeCandidatePipelineConfigBuilder, + homeWhoToSubscribeFeedbackActionInfoBuilder: HomeWhoToSubscribeFeedbackActionInfoBuilder) { + + def build(): WhoToSubscribeCandidatePipelineConfig[ForYouQuery] = { + val gates: Seq[Gate[ForYouQuery]] = Seq( + TimelinesPersistenceStoreLastInjectionGate( + WhoToSubscribeMinInjectionIntervalParam, + PersistenceEntriesFeature, + EntityIdType.WhoToSubscribe + ), + DismissFatigueGate(SuggestType.WhoToSubscribe, DismissInfoFeature) + ) + + whoToSubscribeCandidatePipelineConfigBuilder.build[ForYouQuery]( + identifier = WhoToSubscribeCandidatePipelineConfig.identifier, + supportedClientParam = Some(EnableWhoToSubscribeCandidatePipelineParam), + alerts = alerts, + gates = gates, + moduleDisplayTypeBuilder = + ParamWhoToFollowModuleDisplayTypeBuilder(WhoToSubscribeDisplayTypeIdParam), + feedbackActionInfoBuilder = Some(homeWhoToSubscribeFeedbackActionInfoBuilder) + ) + } + + private val alerts = Seq( + HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(70), + HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() + ) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/BUILD.bazel index ed317eb09..540823800 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/BUILD.bazel @@ -8,15 +8,14 @@ scala_library( "finatra/inject/inject-core/src/main/scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/product_pipeline", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/timelinemixer:thrift-scala", "stitch/stitch-timelineservice/src/main/scala", ], exports = [ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/ScoredTweetsProductCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/ScoredTweetsProductCandidateSource.scala index db15a2cac..d1daeb93e 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/ScoredTweetsProductCandidateSource.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/ScoredTweetsProductCandidateSource.scala @@ -2,11 +2,13 @@ package com.twitter.home_mixer.product.for_you.candidate_source import com.google.inject.Provider import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature import com.twitter.home_mixer.model.request.HomeMixerRequest import com.twitter.home_mixer.model.request.ScoredTweetsProduct import com.twitter.home_mixer.model.request.ScoredTweetsProductContext import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.{thriftscala => t} +import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer import com.twitter.product_mixer.core.functional_component.candidate_source.product_pipeline.ProductPipelineCandidateSource import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier @@ -33,15 +35,22 @@ case class ScoredTweetWithConversationMetadata( inReplyToUserId: Option[Long] = None, directedAtUserId: Option[Long] = None, inNetwork: Option[Boolean] = None, - favoritedByUserIds: Option[Seq[Long]] = None, - followedByUserIds: Option[Seq[Long]] = None, + sgsValidLikedByUserIds: Option[Seq[Long]] = None, + sgsValidFollowedByUserIds: Option[Seq[Long]] = None, ancestors: Option[Seq[ta.TweetAncestor]] = None, topicId: Option[Long] = None, topicFunctionalityType: Option[tl.TopicContextFunctionalityType] = None, conversationId: Option[Long] = None, conversationFocalTweetId: Option[Long] = None, isReadFromCache: Option[Boolean] = None, - streamToKafka: Option[Boolean] = None) + streamToKafka: Option[Boolean] = None, + exclusiveConversationAuthorId: Option[Long] = None, + authorIsBlueVerified: Option[Boolean] = None, + authorIsGoldVerified: Option[Boolean] = None, + authorIsGrayVerified: Option[Boolean] = None, + authorIsLegacyVerified: Option[Boolean] = None, + authorIsCreator: Option[Boolean] = None, + perspectiveFilteredLikedByUserIds: Option[Seq[Long]] = None) @Singleton class ScoredTweetsProductCandidateSource @Inject() ( @@ -68,9 +77,11 @@ class ScoredTweetsProductCandidateSource @Inject() ( ScoredTweetsProductContext( productPipelineQuery.deviceContext, productPipelineQuery.seenTweetIds, - productPipelineQuery.features.map(_.getOrElse(ServedTweetIdsFeature, Seq.empty)) + productPipelineQuery.features.map(_.getOrElse(ServedTweetIdsFeature, Seq.empty)), + productPipelineQuery.features.map(_.getOrElse(TimelineServiceTweetsFeature, Seq.empty)) )), - serializedRequestCursor = None, + serializedRequestCursor = + productPipelineQuery.pipelineCursor.map(UrtCursorSerializer.serializeCursor), maxResults = productPipelineQuery.requestedMaxResults, debugParams = None, homeRequestParam = false @@ -91,7 +102,8 @@ class ScoredTweetsProductCandidateSource @Inject() ( authorId = ancestor.userId, suggestType = focalTweet.suggestType, conversationId = Some(ancestor.tweetId), - conversationFocalTweetId = Some(focalTweet.tweetId) + conversationFocalTweetId = Some(focalTweet.tweetId), + exclusiveConversationAuthorId = focalTweet.exclusiveConversationAuthorId ) } val conversationId = rootScoredTweet.headOption.map(_.tweetId) @@ -107,7 +119,8 @@ class ScoredTweetsProductCandidateSource @Inject() ( suggestType = focalTweet.suggestType, inReplyToTweetId = tweetsToParents.get(ancestor).map(_.tweetId), conversationId = conversationId, - conversationFocalTweetId = Some(focalTweet.tweetId) + conversationFocalTweetId = Some(focalTweet.tweetId), + exclusiveConversationAuthorId = focalTweet.exclusiveConversationAuthorId ) } val parentScoredTweets = rootScoredTweet ++ intermediateScoredTweets @@ -128,15 +141,22 @@ class ScoredTweetsProductCandidateSource @Inject() ( inReplyToUserId = focalTweet.inReplyToUserId, directedAtUserId = focalTweet.directedAtUserId, inNetwork = focalTweet.inNetwork, - favoritedByUserIds = focalTweet.favoritedByUserIds, - followedByUserIds = focalTweet.followedByUserIds, + sgsValidLikedByUserIds = focalTweet.sgsValidLikedByUserIds, + sgsValidFollowedByUserIds = focalTweet.sgsValidFollowedByUserIds, topicId = focalTweet.topicId, topicFunctionalityType = focalTweet.topicFunctionalityType, ancestors = focalTweet.ancestors, conversationId = conversationId, conversationFocalTweetId = conversationFocalTweetId, isReadFromCache = focalTweet.isReadFromCache, - streamToKafka = focalTweet.streamToKafka + streamToKafka = focalTweet.streamToKafka, + exclusiveConversationAuthorId = focalTweet.exclusiveConversationAuthorId, + authorIsBlueVerified = focalTweet.authorMetadata.map(_.blueVerified), + authorIsGoldVerified = focalTweet.authorMetadata.map(_.goldVerified), + authorIsGrayVerified = focalTweet.authorMetadata.map(_.grayVerified), + authorIsLegacyVerified = focalTweet.authorMetadata.map(_.legacyVerified), + authorIsCreator = focalTweet.authorMetadata.map(_.creator), + perspectiveFilteredLikedByUserIds = focalTweet.perspectiveFilteredLikedByUserIds ) parentScoredTweets :+ focalScoredTweet diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/AuthorEnabledPreviewsFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/AuthorEnabledPreviewsFeatureHydrator.scala new file mode 100644 index 000000000..f93a5041f --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/AuthorEnabledPreviewsFeatureHydrator.scala @@ -0,0 +1,72 @@ +package com.twitter.home_mixer.product.for_you.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.AuthorEnabledPreviewsFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.audiencerewards.audienceRewardsService.GetCreatorPreferencesOnUserClientColumn + +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Hydrates the `AuthorEnabledPreviews` feature for tweets authored by creators by querying the + * `GetCreatorPreferences` Strato column. This feature corresponds to the `previews_enabled` field of that column. + * Given a tweet from a creator, this feature indicates whether that creator has enabled previews + * on their profile. + */ +@Singleton +class AuthorEnabledPreviewsFeatureHydrator @Inject() ( + getCreatorPreferencesOnUserClientColumn: GetCreatorPreferencesOnUserClientColumn) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("AuthorEnabledPreviews") + + override val features: Set[Feature[_, _]] = Set(AuthorEnabledPreviewsFeature) + + private val fetcher = getCreatorPreferencesOnUserClientColumn.fetcher + + private val DefaultAuthorEnabledPreviewsValue = true + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = { + val candidateAuthors = candidates + .map(_.features.getOrElse(AuthorIdFeature, None)) + .toSet + .flatten + + // Build a map of creator -> authorEnabledPreviews, then use it to populate candidate features + val authorIdToFeatureStitch = Stitch.collect { + candidateAuthors + .map { author => + val isAuthorEnabledPreviews = fetcher.fetch(author).map { + _.v.map(_.previewsEnabled).getOrElse(DefaultAuthorEnabledPreviewsValue) + } + (author, isAuthorEnabledPreviews) + }.toMap + } + + authorIdToFeatureStitch.map { authorIdToFeatureMap => + candidates.map { + _.features.getOrElse(AuthorIdFeature, None) match { + case Some(authorId) => FeatureMapBuilder() + .add(AuthorEnabledPreviewsFeature, authorIdToFeatureMap(authorId)) + .build() + case _ => FeatureMapBuilder() + .add(AuthorEnabledPreviewsFeature, DefaultAuthorEnabledPreviewsValue) + .build() + } + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/BUILD.bazel new file mode 100644 index 000000000..87a260b7e --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/BUILD.bazel @@ -0,0 +1,28 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "audience-rewards/thrift/src/main/thrift/common:thrift-scala", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_tweetypie", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", + "src/thrift/com/twitter/socialgraph:thrift-scala", + "stitch/stitch-core", + "stitch/stitch-socialgraph", + "stitch/stitch-timelineservice", + "strato/config/columns/audiencerewards/audienceRewardsService:audienceRewardsService-strato-client", + "strato/src/main/scala/com/twitter/strato/client", + "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/feedback", + "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", + "util/util-core", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FocalTweetFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/FocalTweetFeatureHydrator.scala similarity index 98% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FocalTweetFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/FocalTweetFeatureHydrator.scala index 79500a7b4..0270a81ab 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FocalTweetFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/FocalTweetFeatureHydrator.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator +package com.twitter.home_mixer.product.for_you.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TimelineServiceTweetsQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TimelineServiceTweetsQueryFeatureHydrator.scala similarity index 94% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TimelineServiceTweetsQueryFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TimelineServiceTweetsQueryFeatureHydrator.scala index fc9727cb7..e0ae2207e 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TimelineServiceTweetsQueryFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TimelineServiceTweetsQueryFeatureHydrator.scala @@ -1,6 +1,7 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator +package com.twitter.home_mixer.product.for_you.feature_hydrator import com.twitter.home_mixer.marshaller.timelines.DeviceContextMarshaller +import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature import com.twitter.home_mixer.model.request.DeviceContext import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.home_mixer.service.HomeMixerAlertConfig @@ -16,8 +17,6 @@ import com.twitter.timelineservice.{thriftscala => t} import javax.inject.Inject import javax.inject.Singleton -object TimelineServiceTweetsFeature extends Feature[PipelineQuery, Seq[Long]] - @Singleton case class TimelineServiceTweetsQueryFeatureHydrator @Inject() ( timelineService: TimelineService, diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetPreviewTweetypieCandidateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetPreviewTweetypieCandidateFeatureHydrator.scala new file mode 100644 index 000000000..07f3ae0e9 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetPreviewTweetypieCandidateFeatureHydrator.scala @@ -0,0 +1,72 @@ +package com.twitter.home_mixer.product.for_you.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature +import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.spam.rtf.{thriftscala => rtf} +import com.twitter.stitch.Stitch +import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient} +import com.twitter.tweetypie.{thriftscala => TP} +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class TweetPreviewTweetypieCandidateFeatureHydrator @Inject() ( + tweetypieStitchClient: TweetypieStitchClient) + extends CandidateFeatureHydrator[PipelineQuery, BaseTweetCandidate] { + + private val CoreTweetFields: Set[TP.TweetInclude] = Set[TP.TweetInclude]( + TP.TweetInclude.TweetFieldId(TP.Tweet.IdField.id), + TP.TweetInclude.TweetFieldId(TP.Tweet.CoreDataField.id) + ) + + private val DefaultFeatureMap = FeatureMapBuilder() + .add(TweetTextFeature, None) + .add(IsHydratedFeature, false) + .add(AuthorIdFeature, None) + .build() + + override val features: Set[Feature[_, _]] = + Set(TweetTextFeature, IsHydratedFeature, AuthorIdFeature) + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("TweetPreviewTweetypie") + + override def apply( + query: PipelineQuery, + candidate: BaseTweetCandidate, + existingFeatures: FeatureMap + ): Stitch[FeatureMap] = { + tweetypieStitchClient + .getTweetFields( + tweetId = candidate.id, + options = TP.GetTweetFieldsOptions( + tweetIncludes = CoreTweetFields, + includeRetweetedTweet = false, + includeQuotedTweet = false, + visibilityPolicy = TP.TweetVisibilityPolicy.UserVisible, + safetyLevel = Some(rtf.SafetyLevel.TimelineHomeTweetPreview), + forUserId = query.getOptionalUserId + ) + ).map { + case TP.GetTweetFieldsResult(_, TP.TweetFieldsResultState.Found(found), quoteOpt, _) => + val tweetText = found.tweet.coreData.map(_.text) + FeatureMapBuilder() + .add(TweetTextFeature, tweetText) + .add(IsHydratedFeature, true) + .add(AuthorIdFeature, found.tweet.coreData.map(_.userId)) + .build() + // If no tweet result found, return default features + case _ => + DefaultFeatureMap + } + + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/BUILD.bazel new file mode 100644 index 000000000..0a875371d --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/BUILD.bazel @@ -0,0 +1,15 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter", + "stitch/stitch-core", + "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/SocialContextFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/SocialContextFilter.scala new file mode 100644 index 000000000..5b007969b --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/SocialContextFilter.scala @@ -0,0 +1,76 @@ +package com.twitter.home_mixer.product.for_you.filter + +import com.twitter.home_mixer.model.HomeFeatures._ +import com.twitter.home_mixer.product.for_you.param.ForYouParam +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import com.twitter.timelineservice.suggests.{thriftscala => st} + +object SocialContextFilter extends Filter[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("SocialContext") + + // Tweets from candidate sources which don't need generic like/follow/topic proof + private val AllowedSources: Set[st.SuggestType] = Set( + st.SuggestType.RankedListTweet, + st.SuggestType.RecommendedTrendTweet, + st.SuggestType.MediaTweet + ) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val enableIsVerifiedAuthorFilter = + query.params(ForYouParam.EnableVerifiedAuthorSocialContextBypassParam) + + val enableTopicSocialContextFilter = + query.params(ForYouParam.EnableTopicSocialContextFilterParam) + + val validTweetIds = candidates + .filter { candidate => + candidate.features.getOrElse(InNetworkFeature, true) || + candidate.features.getOrElse(SuggestTypeFeature, None).exists(AllowedSources.contains) || + candidate.features.getOrElse(ConversationModuleFocalTweetIdFeature, None).isDefined || + (enableIsVerifiedAuthorFilter && isVerifiedAuthor(candidate.features)) || + hasLikedBySocialContext(candidate.features) || + hasFollowedBySocialContext(candidate.features) || + (enableTopicSocialContextFilter && hasTopicSocialContext(candidate.features)) + }.map(_.candidate.id).toSet + + val (kept, removed) = + candidates.map(_.candidate).partition(candidate => validTweetIds.contains(candidate.id)) + + Stitch.value(FilterResult(kept = kept, removed = removed)) + } + + private def isVerifiedAuthor(candidateFeatures: FeatureMap): Boolean = { + candidateFeatures.getOrElse(AuthorIsBlueVerifiedFeature, false) || + candidateFeatures.getOrElse(AuthorIsGoldVerifiedFeature, false) || + candidateFeatures.getOrElse(AuthorIsGrayVerifiedFeature, false) || + candidateFeatures.getOrElse(AuthorIsLegacyVerifiedFeature, false) + } + + private def hasLikedBySocialContext(candidateFeatures: FeatureMap): Boolean = + candidateFeatures + .getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty) + .exists( + candidateFeatures + .getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Seq.empty) + .toSet.contains + ) + + private def hasFollowedBySocialContext(candidateFeatures: FeatureMap): Boolean = + candidateFeatures.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty).nonEmpty + + private def hasTopicSocialContext(candidateFeatures: FeatureMap): Boolean = { + candidateFeatures.getOrElse(TopicIdSocialContextFeature, None).isDefined && + candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None).isDefined + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/TweetPreviewTextFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/TweetPreviewTextFilter.scala new file mode 100644 index 000000000..61125fcd3 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/TweetPreviewTextFilter.scala @@ -0,0 +1,44 @@ +package com.twitter.home_mixer.product.for_you.filter + +import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +object TweetPreviewTextFilter extends Filter[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("TweetPreviewText") + + private val PreviewTextLength = 50 + private val MinTweetLength = PreviewTextLength * 2 + private val MaxNewlines = 2 + private val HttpPrefix = "http://" + private val HttpsPrefix = "https://" + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + + val (kept, removed) = candidates + .partition { candidate => + val text = candidate.features.get(TweetTextFeature).getOrElse("") + + text.length > MinTweetLength && + text.take(PreviewTextLength).count(_ == '\n') <= MaxNewlines && + !(text.startsWith(HttpPrefix) || text.startsWith(HttpsPrefix)) + } + + val filterResult = FilterResult( + kept = kept.map(_.candidate), + removed = removed.map(_.candidate) + ) + + Stitch.value(filterResult) + } + +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate/BUILD.bazel new file mode 100644 index 000000000..9a1d186dd --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate/BUILD.bazel @@ -0,0 +1,13 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate/PushToHomeRequestGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate/PushToHomeRequestGate.scala new file mode 100644 index 000000000..4c9d81021 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate/PushToHomeRequestGate.scala @@ -0,0 +1,16 @@ +package com.twitter.home_mixer.product.for_you.functional_component.gate + +import com.twitter.home_mixer.product.for_you.model.ForYouQuery +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier +import com.twitter.stitch.Stitch + +/** + * Continues when the request is a Push-To-Home notification request + */ +object PushToHomeRequestGate extends Gate[ForYouQuery] { + override val identifier: GateIdentifier = GateIdentifier("PushToHomeRequest") + + override def shouldContinue(query: ForYouQuery): Stitch[Boolean] = + Stitch.value(query.pushToHomeTweetId.isDefined) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/ForYouQuery.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/ForYouQuery.scala index ec701ac60..dda427350 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/ForYouQuery.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/ForYouQuery.scala @@ -26,7 +26,8 @@ case class ForYouQuery( override val features: Option[FeatureMap], override val deviceContext: Option[DeviceContext], override val seenTweetIds: Option[Seq[Long]], - override val dspClientContext: Option[dsp.DspClientContext]) + override val dspClientContext: Option[dsp.DspClientContext], + pushToHomeTweetId: Option[Long]) extends PipelineQuery with HasPipelineCursor[UrtOrderedCursor] with HasDeviceContext @@ -47,4 +48,5 @@ case class ForYouQuery( override val isEmptyState: Option[Boolean] = None override val isFirstRequestAfterSignup: Option[Boolean] = None override val isEndOfTimeline: Option[Boolean] = None + override val timelineId: Option[Long] = None } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParam.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParam.scala index 0ed56a3f8..5d117199f 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParam.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParam.scala @@ -14,10 +14,22 @@ import com.twitter.util.Duration object ForYouParam { val SupportedClientFSName = "for_you_supported_client" + object EnableTopicSocialContextFilterParam + extends FSParam[Boolean]( + name = "for_you_enable_topic_social_context_filter", + default = true + ) + + object EnableVerifiedAuthorSocialContextBypassParam + extends FSParam[Boolean]( + name = "for_you_enable_verified_author_social_context_bypass", + default = true + ) + object EnableTimelineScorerCandidatePipelineParam extends FSParam[Boolean]( name = "for_you_enable_timeline_scorer_candidate_pipeline", - default = true + default = false ) object EnableScoredTweetsCandidatePipelineParam @@ -29,6 +41,24 @@ object ForYouParam { default = true ) + object EnableWhoToSubscribeCandidatePipelineParam + extends FSParam[Boolean]( + name = "for_you_enable_who_to_subscribe", + default = true + ) + + object EnableTweetPreviewsCandidatePipelineParam + extends FSParam[Boolean]( + name = "for_you_enable_tweet_previews_candidate_pipeline", + default = true + ) + + object EnablePushToHomeMixerPipelineParam + extends FSParam[Boolean]( + name = "for_you_enable_push_to_home_mixer_pipeline", + default = false + ) + object EnableScoredTweetsMixerPipelineParam extends FSParam[Boolean]( name = "for_you_enable_scored_tweets_mixer_pipeline", @@ -43,14 +73,6 @@ object ForYouParam { max = 500 ) - object TimelineServiceMaxResultsParam - extends FSBoundedParam[Int]( - name = "for_you_timeline_service_max_results", - default = 800, - min = 1, - max = 800 - ) - object AdsNumOrganicItemsParam extends FSBoundedParam[Int]( name = "for_you_ads_num_organic_items", @@ -84,6 +106,65 @@ object ForYouParam { enum = WhoToFollowModuleDisplayType ) + object WhoToFollowDisplayLocationParam + extends FSParam[String]( + name = "for_you_who_to_follow_display_location", + default = "timeline" + ) + + object WhoToSubscribePositionParam + extends FSBoundedParam[Int]( + name = "for_you_who_to_subscribe_position", + default = 7, + min = 0, + max = 99 + ) + + object WhoToSubscribeMinInjectionIntervalParam + extends FSBoundedParam[Duration]( + "for_you_who_to_subscribe_min_injection_interval_in_minutes", + default = 1800.minutes, + min = 0.minutes, + max = 6000.minutes) + with HasDurationConversion { + override val durationConversion: DurationConversion = DurationConversion.FromMinutes + } + + object WhoToSubscribeDisplayTypeIdParam + extends FSEnumParam[WhoToFollowModuleDisplayType.type]( + name = "for_you_enable_who_to_subscribe_display_type_id", + default = WhoToFollowModuleDisplayType.Vertical, + enum = WhoToFollowModuleDisplayType + ) + + object TweetPreviewsPositionParam + extends FSBoundedParam[Int]( + name = "for_you_tweet_previews_position", + default = 3, + min = 0, + max = 99 + ) + + object TweetPreviewsMinInjectionIntervalParam + extends FSBoundedParam[Duration]( + "for_you_tweet_previews_min_injection_interval_in_minutes", + default = 2.hours, + min = 0.minutes, + max = 600.minutes) + with HasDurationConversion { + override val durationConversion: DurationConversion = DurationConversion.FromMinutes + } + + object TweetPreviewsMaxCandidatesParam + extends FSBoundedParam[Int]( + name = "for_you_tweet_previews_max_candidates", + default = 1, + min = 0, + // NOTE: previews are injected at a fixed position, so max candidates = 1 + // to avoid bunching of previews. + max = 1 + ) + object EnableFlipInjectionModuleCandidatePipelineParam extends FSParam[Boolean]( name = "for_you_enable_flip_inline_injection_module", @@ -99,7 +180,6 @@ object ForYouParam { ) object ClearCacheOnPtr { - object EnableParam extends FSParam[Boolean]( name = "for_you_clear_cache_ptr_enable", @@ -114,4 +194,22 @@ object ForYouParam { max = 35 ) } + + object EnableClearCacheOnPushToHome + extends FSParam[Boolean]( + name = "for_you_enable_clear_cache_push_to_home", + default = false + ) + + object EnableServedCandidateKafkaPublishingParam + extends FSParam[Boolean]( + name = "for_you_enable_served_candidate_kafka_publishing", + default = true + ) + + object ExperimentStatsParam + extends FSParam[String]( + name = "for_you_experiment_stats", + default = "" + ) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParamConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParamConfig.scala index ea8e389cc..001ee57ad 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParamConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParamConfig.scala @@ -17,27 +17,44 @@ class ForYouParamConfig @Inject() () extends ProductParamConfig { ) override val booleanFSOverrides = Seq( - EnableFlipInjectionModuleCandidatePipelineParam, - EnableWhoToFollowCandidatePipelineParam, - EnableScoredTweetsMixerPipelineParam, ClearCacheOnPtr.EnableParam, - EnableTimelineScorerCandidatePipelineParam + EnableFlipInjectionModuleCandidatePipelineParam, + EnablePushToHomeMixerPipelineParam, + EnableScoredTweetsMixerPipelineParam, + EnableServedCandidateKafkaPublishingParam, + EnableTimelineScorerCandidatePipelineParam, + EnableTopicSocialContextFilterParam, + EnableVerifiedAuthorSocialContextBypassParam, + EnableWhoToFollowCandidatePipelineParam, + EnableWhoToSubscribeCandidatePipelineParam, + EnableTweetPreviewsCandidatePipelineParam, + EnableClearCacheOnPushToHome ) override val boundedIntFSOverrides = Seq( + AdsNumOrganicItemsParam, + ClearCacheOnPtr.MinEntriesParam, + FlipInlineInjectionModulePosition, ServerMaxResultsParam, WhoToFollowPositionParam, - FlipInlineInjectionModulePosition, - TimelineServiceMaxResultsParam, - AdsNumOrganicItemsParam, - ClearCacheOnPtr.MinEntriesParam + WhoToSubscribePositionParam, + TweetPreviewsPositionParam, + TweetPreviewsMaxCandidatesParam + ) + + override val stringFSOverrides = Seq( + WhoToFollowDisplayLocationParam, + ExperimentStatsParam ) override val boundedDurationFSOverrides = Seq( - WhoToFollowMinInjectionIntervalParam + WhoToFollowMinInjectionIntervalParam, + WhoToSubscribeMinInjectionIntervalParam, + TweetPreviewsMinInjectionIntervalParam ) override val enumFSOverrides = Seq( - WhoToFollowDisplayTypeIdParam + WhoToFollowDisplayTypeIdParam, + WhoToSubscribeDisplayTypeIdParam ) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/BUILD.bazel new file mode 100644 index 000000000..17e99c46c --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/BUILD.bazel @@ -0,0 +1,16 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "3rdparty/jvm/javax/inject:javax.inject", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", + "src/java/com/twitter/search/common/schema/earlybird", + "src/java/com/twitter/search/queryparser/query:core-query-nodes", + "src/java/com/twitter/search/queryparser/query/search:search-query-nodes", + "src/thrift/com/twitter/search:earlybird-scala", + "src/thrift/com/twitter/socialgraph:thrift-scala", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/TweetPreviewsQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/TweetPreviewsQueryTransformer.scala new file mode 100644 index 000000000..41940b823 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/TweetPreviewsQueryTransformer.scala @@ -0,0 +1,92 @@ +package com.twitter.home_mixer.product.for_you.query_transformer + +import com.twitter.conversions.DurationOps.richDurationFromInt +import com.twitter.finagle.thrift.ClientId +import com.twitter.finagle.tracing.Trace +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.PreviewCreatorsFeature +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.search.common.ranking.{thriftscala => scr} +import com.twitter.search.common.schema.earlybird.EarlybirdFieldConstants.EarlybirdFieldConstant +import com.twitter.search.earlybird.{thriftscala => t} +import com.twitter.search.queryparser.query.Conjunction +import com.twitter.search.queryparser.query.Query +import com.twitter.search.queryparser.query.search.SearchOperator +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class TweetPreviewsQueryTransformer @Inject() (clientId: ClientId) + extends CandidatePipelineQueryTransformer[PipelineQuery, t.EarlybirdRequest] { + + private val MaxPreviewTweets = 200 + private val EarlybirdRelevanceTensorflowModel = "timelines_rectweet_replica" + private val SinceDuration = 7.days + + val MetadataOptions = t.ThriftSearchResultMetadataOptions( + getReferencedTweetAuthorId = true, + getFromUserId = true + ) + + override def transform(query: PipelineQuery): t.EarlybirdRequest = { + val candidatePreviewCreatorIds = + query.features.map(_.get(PreviewCreatorsFeature)).getOrElse(Seq.empty) + + val searchQuery = new Conjunction( + // Include subscriber only (aka exclusive) Tweets + new SearchOperator.Builder() + .setType(SearchOperator.Type.FILTER) + .addOperand(EarlybirdFieldConstant.EXCLUSIVE_FILTER_TERM) + .build(), + // Include only original Tweets + new SearchOperator.Builder() + .setType(SearchOperator.Type.FILTER) + .addOperand(EarlybirdFieldConstant.NATIVE_RETWEETS_FILTER_TERM) + .setOccur(Query.Occur.MUST_NOT) + .build(), + new SearchOperator.Builder() + .setType(SearchOperator.Type.FILTER) + .addOperand(EarlybirdFieldConstant.REPLIES_FILTER_TERM) + .setOccur(Query.Occur.MUST_NOT) + .build(), + new SearchOperator.Builder() + .setType(SearchOperator.Type.FILTER) + .addOperand(EarlybirdFieldConstant.QUOTE_FILTER_TERM) + .setOccur(Query.Occur.MUST_NOT) + .build(), + new SearchOperator(SearchOperator.Type.SINCE_TIME, SinceDuration.ago.inSeconds.toString) + ) + + t.EarlybirdRequest( + searchQuery = t.ThriftSearchQuery( + serializedQuery = Some(searchQuery.serialize), + fromUserIDFilter64 = Some(candidatePreviewCreatorIds), + numResults = MaxPreviewTweets, + rankingMode = t.ThriftSearchRankingMode.Relevance, + relevanceOptions = Some( + t.ThriftSearchRelevanceOptions( + filterDups = true, + keepDupWithHigherScore = true, + proximityScoring = true, + maxConsecutiveSameUser = Some(5), + rankingParams = Some( + scr.ThriftRankingParams( + `type` = Some(scr.ThriftScoringFunctionType.TensorflowBased), + selectedTensorflowModel = Some(EarlybirdRelevanceTensorflowModel), + minScore = -1.0e100, + applyBoosts = false, + ) + ), + ), + ), + resultMetadataOptions = Some(MetadataOptions), + searcherId = query.getOptionalUserId, + ), + getOlderResults = Some(true), // needed for archive access to older tweets + clientRequestID = Some(s"${Trace.id.traceId}"), + followedUserIds = Some(candidatePreviewCreatorIds.toSeq), + numResultsToReturnAtRoot = Some(MaxPreviewTweets), + clientId = Some(clientId.name), + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/BUILD.bazel new file mode 100644 index 000000000..cf629b665 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/BUILD.bazel @@ -0,0 +1,13 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "3rdparty/jvm/javax/inject:javax.inject", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", + "src/java/com/twitter/search/common/schema/earlybird", + "src/thrift/com/twitter/search:earlybird-scala", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/TweetPreviewResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/TweetPreviewResponseFeatureTransformer.scala new file mode 100644 index 000000000..8b783db38 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/TweetPreviewResponseFeatureTransformer.scala @@ -0,0 +1,32 @@ +package com.twitter.home_mixer.product.for_you.response_transformer + +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.IsTweetPreviewFeature +import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.timelineservice.suggests.{thriftscala => st} +import com.twitter.search.earlybird.{thriftscala => eb} + +object TweetPreviewResponseFeatureTransformer + extends CandidateFeatureTransformer[eb.ThriftSearchResult] { + + override val identifier: TransformerIdentifier = + TransformerIdentifier("TweetPreviewResponse") + + override val features: Set[Feature[_, _]] = + Set(AuthorIdFeature, IsTweetPreviewFeature, SuggestTypeFeature) + + def transform( + input: eb.ThriftSearchResult + ): FeatureMap = { + FeatureMapBuilder() + .add(IsTweetPreviewFeature, true) + .add(SuggestTypeFeature, Some(st.SuggestType.TweetPreview)) + .add(AuthorIdFeature, input.metadata.map(_.fromUserId)) + .build() + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/scorer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/scorer/BUILD.bazel new file mode 100644 index 000000000..c320c8838 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/scorer/BUILD.bazel @@ -0,0 +1,10 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", + "timelineservice/common:model", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/BUILD.bazel new file mode 100644 index 000000000..90faf3d45 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/BUILD.bazel @@ -0,0 +1,24 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", + "kafka/finagle-kafka/finatra-kafka/src/main/scala", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", + "src/scala/com/twitter/timelines/prediction/common/adapters", + "src/scala/com/twitter/timelines/prediction/features/common", + "src/thrift/com/twitter/timelines/served_candidates_logging:served_candidates_logging-scala", + "src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java", + "timelines/ml:kafka", + "timelines/ml/cont_train/common/client/src/main/scala/com/twitter/timelines/ml/cont_train/common/client/kafka", + "timelines/ml/cont_train/common/domain/src/main/scala/com/twitter/timelines/ml/cont_train/common/domain/non_scalding", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ServedCandidateFeatureKeysKafkaSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffect.scala similarity index 96% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ServedCandidateFeatureKeysKafkaSideEffect.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffect.scala index 52cf0d720..bfa7dd52e 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ServedCandidateFeatureKeysKafkaSideEffect.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffect.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.side_effect +package com.twitter.home_mixer.product.for_you.side_effect import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature @@ -7,7 +7,7 @@ import com.twitter.home_mixer.model.HomeFeatures.PredictionRequestIdFeature import com.twitter.home_mixer.model.HomeFeatures.ServedIdFeature import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.param.HomeGlobalParams.EnableServedCandidateKafkaPublishingParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableServedCandidateKafkaPublishingParam import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.component_library.side_effect.KafkaPublishingSideEffect import com.twitter.product_mixer.core.feature.featuremap.FeatureMap diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ServedCandidateFeatureKeysKafkaSideEffectBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffectBuilder.scala similarity index 92% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ServedCandidateFeatureKeysKafkaSideEffectBuilder.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffectBuilder.scala index c84071986..5e4164276 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ServedCandidateFeatureKeysKafkaSideEffectBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffectBuilder.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.side_effect +package com.twitter.home_mixer.product.for_you.side_effect import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ServedCandidateKafkaSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKafkaSideEffect.scala similarity index 97% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ServedCandidateKafkaSideEffect.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKafkaSideEffect.scala index a59a2219a..7da751764 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ServedCandidateKafkaSideEffect.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKafkaSideEffect.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.side_effect +package com.twitter.home_mixer.product.for_you.side_effect import com.twitter.home_mixer.model.HomeFeatures.IsReadFromCacheFeature import com.twitter.home_mixer.model.HomeFeatures.PredictionRequestIdFeature diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ServedCandidateKeysKafkaSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKeysKafkaSideEffect.scala similarity index 96% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ServedCandidateKeysKafkaSideEffect.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKeysKafkaSideEffect.scala index 28db059c8..aa1247fbb 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ServedCandidateKeysKafkaSideEffect.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKeysKafkaSideEffect.scala @@ -1,10 +1,10 @@ -package com.twitter.home_mixer.functional_component.side_effect +package com.twitter.home_mixer.product.for_you.side_effect import com.twitter.home_mixer.model.HomeFeatures.IsReadFromCacheFeature import com.twitter.home_mixer.model.HomeFeatures.PredictionRequestIdFeature import com.twitter.home_mixer.model.HomeFeatures.ServedIdFeature import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature -import com.twitter.home_mixer.param.HomeGlobalParams.EnableServedCandidateKafkaPublishingParam +import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableServedCandidateKafkaPublishingParam import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.ml.api.DataRecord import com.twitter.ml.api.util.SRichDataRecord diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ServedCandidateKeysKafkaSideEffectBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKeysKafkaSideEffectBuilder.scala similarity index 91% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ServedCandidateKeysKafkaSideEffectBuilder.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKeysKafkaSideEffectBuilder.scala index 5e86fdada..31ee389b9 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ServedCandidateKeysKafkaSideEffectBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKeysKafkaSideEffectBuilder.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.side_effect +package com.twitter.home_mixer.product.for_you.side_effect import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedStatsSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedStatsSideEffect.scala new file mode 100644 index 000000000..be7c2a533 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedStatsSideEffect.scala @@ -0,0 +1,90 @@ +package com.twitter.home_mixer.product.for_you.side_effect + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature +import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature +import com.twitter.home_mixer.product.for_you.param.ForYouParam.ExperimentStatsParam +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect +import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails +import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ServedStatsSideEffect @Inject() (statsReceiver: StatsReceiver) + extends PipelineResultSideEffect[PipelineQuery, Timeline] { + + override val identifier: SideEffectIdentifier = SideEffectIdentifier("ServedStats") + + private val baseStatsReceiver = statsReceiver.scope(identifier.toString) + private val suggestTypeStatsReceiver = baseStatsReceiver.scope("SuggestType") + private val responseSizeStatsReceiver = baseStatsReceiver.scope("ResponseSize") + private val contentBalanceStatsReceiver = baseStatsReceiver.scope("ContentBalance") + + private val inNetworkStatsReceiver = contentBalanceStatsReceiver.scope("InNetwork") + private val outOfNetworkStatsReceiver = contentBalanceStatsReceiver.scope("OutOfNetwork") + private val replyStatsReceiver = contentBalanceStatsReceiver.scope("Reply") + private val originalStatsReceiver = contentBalanceStatsReceiver.scope("Original") + + private val emptyStatsReceiver = responseSizeStatsReceiver.scope("Empty") + private val lessThan5StatsReceiver = responseSizeStatsReceiver.scope("LessThan5") + private val lessThan10StatsReceiver = responseSizeStatsReceiver.scope("LessThan10") + + override def apply( + inputs: PipelineResultSideEffect.Inputs[PipelineQuery, Timeline] + ): Stitch[Unit] = { + val tweetCandidates = CandidatesUtil + .getItemCandidates(inputs.selectedCandidates).filter(_.isCandidateType[TweetCandidate]()) + + val expBucket = inputs.query.params(ExperimentStatsParam) + + recordSuggestTypeStats(tweetCandidates, expBucket) + recordContentBalanceStats(tweetCandidates, expBucket) + recordResponseSizeStats(tweetCandidates, expBucket) + Stitch.Unit + } + + def recordSuggestTypeStats( + candidates: Seq[ItemCandidateWithDetails], + expBucket: String + ): Unit = { + candidates.groupBy(getSuggestType).foreach { + case (suggestType, suggestTypeCandidates) => + suggestTypeStatsReceiver + .scope(expBucket).counter(suggestType).incr(suggestTypeCandidates.size) + } + } + + def recordContentBalanceStats( + candidates: Seq[ItemCandidateWithDetails], + expBucket: String + ): Unit = { + val (in, oon) = candidates.partition(_.features.getOrElse(InNetworkFeature, true)) + inNetworkStatsReceiver.counter(expBucket).incr(in.size) + outOfNetworkStatsReceiver.counter(expBucket).incr(oon.size) + + val (reply, original) = + candidates.partition(_.features.getOrElse(InReplyToTweetIdFeature, None).isDefined) + replyStatsReceiver.counter(expBucket).incr(reply.size) + originalStatsReceiver.counter(expBucket).incr(original.size) + } + + def recordResponseSizeStats( + candidates: Seq[ItemCandidateWithDetails], + expBucket: String + ): Unit = { + if (candidates.size == 0) emptyStatsReceiver.counter(expBucket).incr() + if (candidates.size < 5) lessThan5StatsReceiver.counter(expBucket).incr() + if (candidates.size < 10) lessThan10StatsReceiver.counter(expBucket).incr() + } + + private def getSuggestType(candidate: CandidateWithDetails): String = + candidate.features.getOrElse(SuggestTypeFeature, None).map(_.name).getOrElse("None") +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BUILD.bazel index 4c69111d1..3a52b15dc 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BUILD.bazel @@ -19,12 +19,15 @@ scala_library( "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope", @@ -41,6 +44,7 @@ scala_library( "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", "src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala", + "src/thrift/com/twitter/search:blender-scala", "src/thrift/com/twitter/timelines/render:thrift-scala", ], exports = [ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BlenderUsersCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BlenderUsersCandidatePipelineConfig.scala new file mode 100644 index 000000000..e0919ae9d --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BlenderUsersCandidatePipelineConfig.scala @@ -0,0 +1,68 @@ +package com.twitter.home_mixer.product.list_recommended_users + +import com.twitter.home_mixer.product.list_recommended_users.candidate_source.BlenderUsersCandidateSource +import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.IsGizmoduckValidUserFeatureHydrator +import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.IsSGSValidUserFeatureHydrator +import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.RecentListMembersFeature +import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersQuery +import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator +import com.twitter.product_mixer.component_library.decorator.urt.builder.item.user.UserCandidateUrtItemBuilder +import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder +import com.twitter.product_mixer.component_library.gate.EmptySeqFeatureGate +import com.twitter.product_mixer.component_library.model.candidate.UserCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource +import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.search.blender.thriftscala.ThriftBlenderRequest +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class BlenderUsersCandidatePipelineConfig @Inject() ( + blenderUsersCandidateSource: BlenderUsersCandidateSource, + isGizmoduckValidUserFeatureHydrator: IsGizmoduckValidUserFeatureHydrator, + isSGSValidUserFeatureHydrator: IsSGSValidUserFeatureHydrator) + extends CandidatePipelineConfig[ + ListRecommendedUsersQuery, + ThriftBlenderRequest, + Long, + UserCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier("BlenderUsers") + + override val gates: Seq[Gate[ListRecommendedUsersQuery]] = + Seq(EmptySeqFeatureGate(RecentListMembersFeature)) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + ListRecommendedUsersQuery, + ThriftBlenderRequest + ] = BlenderUsersCandidatePipelineQueryTransformer + + override val candidateSource: BaseCandidateSource[ThriftBlenderRequest, Long] = + blenderUsersCandidateSource + + override val resultTransformer: CandidatePipelineResultsTransformer[ + Long, + UserCandidate + ] = { candidate => UserCandidate(id = candidate) } + + override val preFilterFeatureHydrationPhase1: Seq[ + BaseCandidateFeatureHydrator[ListRecommendedUsersQuery, UserCandidate, _] + ] = Seq( + isGizmoduckValidUserFeatureHydrator, + isSGSValidUserFeatureHydrator + ) + + override val decorator: Option[CandidateDecorator[ListRecommendedUsersQuery, UserCandidate]] = { + val clientEventInfoBuilder = ClientEventInfoBuilder("user") + val userItemBuilder = UserCandidateUrtItemBuilder(clientEventInfoBuilder) + Some(UrtItemCandidateDecorator(userItemBuilder)) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BlenderUsersCandidatePipelineQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BlenderUsersCandidatePipelineQueryTransformer.scala new file mode 100644 index 000000000..03844fd38 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BlenderUsersCandidatePipelineQueryTransformer.scala @@ -0,0 +1,49 @@ +package com.twitter.home_mixer.product.list_recommended_users + +import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersQuery +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.search.adaptive.adaptive_results.thriftscala.ResultType +import com.twitter.search.blender.adaptive_search.thriftscala.AdaptiveSearchRequest +import com.twitter.search.blender.thriftscala.ThriftBlenderRequest +import com.twitter.search.blender.thriftscala.ThriftBlenderTweetypieOptions +import com.twitter.search.blender.thriftscala.ThriftBlenderWorkflowID +import com.twitter.search.common.constants.thriftscala.ThriftQuerySource +import com.twitter.spam.rtf.thriftscala.SafetyLevel + +object BlenderUsersCandidatePipelineQueryTransformer + extends CandidatePipelineQueryTransformer[ListRecommendedUsersQuery, ThriftBlenderRequest] { + + override val identifier: TransformerIdentifier = TransformerIdentifier("BlenderUsers") + + /** + * This is a user-defined descriptor used by Blender to track the source of traffic, and it + * is different from a client id, which is set during Finagle client construction. + */ + private val ClientAppName = "timelinemixer.list_recommended_users" + + override def transform(query: ListRecommendedUsersQuery): ThriftBlenderRequest = { + + ThriftBlenderRequest( + workflowID = Some(ThriftBlenderWorkflowID.AdaptiveSearch), + userID = Some(query.getRequiredUserId), // perspectival + uiLang = query.clientContext.languageCode, // perspectival + clientAppName = Some(ClientAppName), + adaptiveSearchRequest = Some( + AdaptiveSearchRequest( + rawQuery = query.listName, + numResults = 40, + getPromotedContent = false, + resultFilter = Some(ResultType.User), + ) + ), + querySource = Some(ThriftQuerySource.TypedQuery), + getCorrections = true, + tweetypieOptions = Some( + ThriftBlenderTweetypieOptions( + safetyLevel = Some(SafetyLevel.Recommendations) + ) + ) + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListMemberBasedUsersCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListMemberBasedUsersCandidatePipelineConfig.scala index 267c0b4f3..756c58d72 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListMemberBasedUsersCandidatePipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListMemberBasedUsersCandidatePipelineConfig.scala @@ -1,38 +1,41 @@ package com.twitter.home_mixer.product.list_recommended_users import com.twitter.hermit.candidate.{thriftscala => t} -import com.twitter.home_mixer.functional_component.candidate_source.SimilarityBasedUsersCandidateSource -import com.twitter.home_mixer.functional_component.feature_hydrator.ListMembersFeature -import com.twitter.home_mixer.functional_component.filter.PredicateFeatureFilter -import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.GizmoduckUserFeatureHydrator +import com.twitter.home_mixer.product.list_recommended_users.candidate_source.SimilarityBasedUsersCandidateSource +import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.IsGizmoduckValidUserFeatureHydrator import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.IsListMemberFeatureHydrator -import com.twitter.home_mixer.product.list_recommended_users.filter.DropMaxCandidatesByScoreFilter +import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.IsSGSValidUserFeatureHydrator +import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.RecentListMembersFeature +import com.twitter.home_mixer.product.list_recommended_users.filter.DropMaxCandidatesByAggregatedScoreFilter import com.twitter.home_mixer.product.list_recommended_users.filter.PreviouslyServedUsersFilter -import com.twitter.home_mixer.product.list_recommended_users.model.ListFeatures.IsListMemberFeature +import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.IsListMemberFeature import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersQuery import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.item.user.UserCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder +import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter +import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate import com.twitter.product_mixer.component_library.model.candidate.UserCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig - import javax.inject.Inject import javax.inject.Singleton @Singleton class ListMemberBasedUsersCandidatePipelineConfig @Inject() ( similarityBasedUsersCandidateSource: SimilarityBasedUsersCandidateSource, - gizmoduckUserFeatureHydrator: GizmoduckUserFeatureHydrator, - isListMemberFeatureHydrator: IsListMemberFeatureHydrator) + isGizmoduckValidUserFeatureHydrator: IsGizmoduckValidUserFeatureHydrator, + isListMemberFeatureHydrator: IsListMemberFeatureHydrator, + isSGSValidUserFeatureHydrator: IsSGSValidUserFeatureHydrator) extends CandidatePipelineConfig[ ListRecommendedUsersQuery, Seq[Long], @@ -43,10 +46,13 @@ class ListMemberBasedUsersCandidatePipelineConfig @Inject() ( override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ListMemberBasedUsers") + override val gates: Seq[Gate[ListRecommendedUsersQuery]] = + Seq(NonEmptySeqFeatureGate(RecentListMembersFeature)) + override val queryTransformer: CandidatePipelineQueryTransformer[ListRecommendedUsersQuery, Seq[ Long ]] = { query => - query.features.map(_.getOrElse(ListMembersFeature, Seq.empty)).getOrElse(Seq.empty) + query.features.map(_.getOrElse(RecentListMembersFeature, Seq.empty)).getOrElse(Seq.empty) } override val candidateSource: BaseCandidateSource[Seq[Long], t.Candidate] = @@ -74,12 +80,15 @@ class ListMemberBasedUsersCandidatePipelineConfig @Inject() ( FilterIdentifier("IsListMember"), shouldKeepCandidate = { features => !features.getOrElse(IsListMemberFeature, false) } ), - DropMaxCandidatesByScoreFilter + DropMaxCandidatesByAggregatedScoreFilter ) override val postFilterFeatureHydration: Seq[ BaseCandidateFeatureHydrator[ListRecommendedUsersQuery, UserCandidate, _] - ] = Seq(gizmoduckUserFeatureHydrator) + ] = Seq( + isGizmoduckValidUserFeatureHydrator, + isSGSValidUserFeatureHydrator + ) override val decorator: Option[CandidateDecorator[ListRecommendedUsersQuery, UserCandidate]] = { val clientEventInfoBuilder = ClientEventInfoBuilder("user") diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListMemberBasedUsersResponseFeatureTransfromer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListMemberBasedUsersResponseFeatureTransfromer.scala index bce2ed457..7153d1f06 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListMemberBasedUsersResponseFeatureTransfromer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListMemberBasedUsersResponseFeatureTransfromer.scala @@ -1,7 +1,7 @@ package com.twitter.home_mixer.product.list_recommended_users import com.twitter.hermit.candidate.{thriftscala => t} -import com.twitter.home_mixer.product.list_recommended_users.model.ListFeatures.ScoreFeature +import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.ScoreFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListRecommendedUsersMixerPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListRecommendedUsersMixerPipelineConfig.scala index d370c1057..8c72aefec 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListRecommendedUsersMixerPipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListRecommendedUsersMixerPipelineConfig.scala @@ -1,8 +1,9 @@ package com.twitter.home_mixer.product.list_recommended_users -import com.twitter.home_mixer.functional_component.feature_hydrator.ListMembersQueryFeatureHydrator -import com.twitter.home_mixer.functional_component.gate.ViewerIsListOwnerGate -import com.twitter.home_mixer.product.list_recommended_users.model.ListFeatures.GizmoduckUserFeature +import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.RecentListMembersQueryFeatureHydrator +import com.twitter.home_mixer.product.list_recommended_users.gate.ViewerIsListOwnerGate +import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.IsGizmoduckValidUserFeature +import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.IsSGSValidUserFeature import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersQuery import com.twitter.home_mixer.product.list_recommended_users.param.ListRecommendedUsersParam.ExcludedIdsMaxLengthParam import com.twitter.home_mixer.product.list_recommended_users.param.ListRecommendedUsersParam.ServerMaxResultsParam @@ -36,8 +37,9 @@ import javax.inject.Singleton @Singleton class ListRecommendedUsersMixerPipelineConfig @Inject() ( listMemberBasedUsersCandidatePipelineConfig: ListMemberBasedUsersCandidatePipelineConfig, + blenderUsersCandidatePipelineConfig: BlenderUsersCandidatePipelineConfig, viewerIsListOwnerGate: ViewerIsListOwnerGate, - listMembersQueryFeatureHydrator: ListMembersQueryFeatureHydrator, + recentListMembersQueryFeatureHydrator: RecentListMembersQueryFeatureHydrator, urtTransportMarshaller: UrtTransportMarshaller) extends MixerPipelineConfig[ListRecommendedUsersQuery, Timeline, urt.TimelineResponse] { @@ -46,22 +48,31 @@ class ListRecommendedUsersMixerPipelineConfig @Inject() ( override val gates = Seq(viewerIsListOwnerGate) override val fetchQueryFeatures: Seq[QueryFeatureHydrator[ListRecommendedUsersQuery]] = - Seq(listMembersQueryFeatureHydrator) + Seq(recentListMembersQueryFeatureHydrator) override val candidatePipelines: Seq[ CandidatePipelineConfig[ListRecommendedUsersQuery, _, _, _] - ] = - Seq(listMemberBasedUsersCandidatePipelineConfig) + ] = Seq( + listMemberBasedUsersCandidatePipelineConfig, + blenderUsersCandidatePipelineConfig + ) + + private val candidatePipelineIdentifiers = Set( + listMemberBasedUsersCandidatePipelineConfig.identifier, + blenderUsersCandidatePipelineConfig.identifier + ) override val resultSelectors: Seq[Selector[ListRecommendedUsersQuery]] = Seq( DropFilteredCandidates( - candidatePipeline = listMemberBasedUsersCandidatePipelineConfig.identifier, - filter = candidate => candidate.features.getOrElse(GizmoduckUserFeature, None).isDefined + candidatePipelines = candidatePipelineIdentifiers, + filter = candidate => + candidate.features.getOrElse(IsSGSValidUserFeature, false) && + candidate.features.getOrElse(IsGizmoduckValidUserFeature, false) ), DropMaxCandidates( - candidatePipeline = listMemberBasedUsersCandidatePipelineConfig.identifier, + candidatePipelines = candidatePipelineIdentifiers, maxSelectionsParam = ServerMaxResultsParam), - InsertAppendResults(listMemberBasedUsersCandidatePipelineConfig.identifier) + InsertAppendResults(candidatePipelineIdentifiers) ) override val domainMarshaller: DomainMarshaller[ListRecommendedUsersQuery, Timeline] = { diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListRecommendedUsersProductPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListRecommendedUsersProductPipelineConfig.scala index e20bb8aec..f0c2ac70e 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListRecommendedUsersProductPipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListRecommendedUsersProductPipelineConfig.scala @@ -1,5 +1,6 @@ package com.twitter.home_mixer.product.list_recommended_users +import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.marshaller.timelines.RecommendedUsersCursorUnmarshaller import com.twitter.home_mixer.model.request.HomeMixerRequest import com.twitter.home_mixer.model.request.ListRecommendedUsersProduct @@ -8,8 +9,15 @@ import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommend import com.twitter.home_mixer.product.list_recommended_users.param.ListRecommendedUsersParam.ServerMaxResultsParam import com.twitter.home_mixer.product.list_recommended_users.param.ListRecommendedUsersParamConfig import com.twitter.home_mixer.service.HomeMixerAccessPolicy.DefaultHomeMixerAccessPolicy +import com.twitter.home_mixer.service.HomeMixerAlertConfig.DefaultNotificationGroup import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy +import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfBelow +import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfLatencyAbove +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.common.alert.LatencyAlert +import com.twitter.product_mixer.core.functional_component.common.alert.P99 +import com.twitter.product_mixer.core.functional_component.common.alert.SuccessRateAlert import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier import com.twitter.product_mixer.core.model.marshalling.request @@ -66,7 +74,8 @@ class ListRecommendedUsersProductPipelineConfig @Inject() ( requestedMaxResults = Some(params(ServerMaxResultsParam)), debugOptions = debugOptions, selectedUserIds = context.selectedUserIds, - excludedUserIds = context.excludedUserIds + excludedUserIds = context.excludedUserIds, + listName = context.listName ) } @@ -75,5 +84,19 @@ class ListRecommendedUsersProductPipelineConfig @Inject() ( override def pipelineSelector(query: ListRecommendedUsersQuery): ComponentIdentifier = listRecommendedUsersMixerPipelineConfig.identifier + override val alerts: Seq[Alert] = Seq( + SuccessRateAlert( + notificationGroup = DefaultNotificationGroup, + warnPredicate = TriggerIfBelow(99.9, 20, 30), + criticalPredicate = TriggerIfBelow(99.9, 30, 30), + ), + LatencyAlert( + notificationGroup = DefaultNotificationGroup, + percentile = P99, + warnPredicate = TriggerIfLatencyAbove(1000.millis, 15, 30), + criticalPredicate = TriggerIfLatencyAbove(1500.millis, 15, 30) + ) + ) + override val debugAccessPolicies: Set[AccessPolicy] = DefaultHomeMixerAccessPolicy } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/BUILD.bazel new file mode 100644 index 000000000..e1fa1de1c --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/BUILD.bazel @@ -0,0 +1,13 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", + "src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala", + "src/thrift/com/twitter/search:blender-scala", + "strato/config/columns/recommendations/similarity:similarity-strato-client", + "strato/src/main/scala/com/twitter/strato/client", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/BlenderUsersCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/BlenderUsersCandidateSource.scala new file mode 100644 index 000000000..3b7ac39fc --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/BlenderUsersCandidateSource.scala @@ -0,0 +1,45 @@ +package com.twitter.home_mixer.product.list_recommended_users.candidate_source + +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.search.adaptive.adaptive_results.thriftscala.AdaptiveSearchResultData +import com.twitter.search.adaptive.adaptive_results.thriftscala.Result +import com.twitter.search.adaptive.adaptive_results.thriftscala.ResultData +import com.twitter.search.blender.adaptive_search.thriftscala.AdaptiveSearchResponse +import com.twitter.search.blender.adaptive_search.thriftscala.Container +import com.twitter.search.blender.thriftscala.BlenderService +import com.twitter.search.blender.thriftscala.ThriftBlenderRequest +import com.twitter.stitch.Stitch +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class BlenderUsersCandidateSource @Inject() ( + blenderClient: BlenderService.MethodPerEndpoint) + extends CandidateSource[ThriftBlenderRequest, Long] { + + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("BlenderUsers") + + override def apply(request: ThriftBlenderRequest): Stitch[Seq[Long]] = { + Stitch.callFuture( + blenderClient.serveV2(request).map { response => + val userIdsOpt = + response.adaptiveSearchResponse.map(extractUserIdsFromAdaptiveSearchResponse) + userIdsOpt.getOrElse(Seq.empty) + } + ) + } + + private def extractUserIdsFromAdaptiveSearchResponse( + response: AdaptiveSearchResponse + ): Seq[Long] = { + response match { + case AdaptiveSearchResponse(Some(Seq(Container(Some(results), _))), _, _) => + results.map(_.data).collect { + case AdaptiveSearchResultData.Result(Result(ResultData.User(user), _)) => + user.id + } + case _ => Seq.empty + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/SimilarityBasedUsersCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/SimilarityBasedUsersCandidateSource.scala similarity index 71% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/SimilarityBasedUsersCandidateSource.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/SimilarityBasedUsersCandidateSource.scala index f117e91f9..f5428d032 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/SimilarityBasedUsersCandidateSource.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/SimilarityBasedUsersCandidateSource.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.candidate_source +package com.twitter.home_mixer.product.list_recommended_users.candidate_source import com.twitter.hermit.candidate.{thriftscala => t} import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource @@ -21,13 +21,19 @@ class SimilarityBasedUsersCandidateSource @Inject() ( private val fetcher: Fetcher[Long, Unit, t.Candidates] = similarUsersBySimsOnUserClientColumn.fetcher + private val MaxCandidatesToKeep = 4000 + override def apply(request: Seq[Long]): Stitch[Seq[t.Candidate]] = { Stitch .collect { request.map { userId => - fetcher.fetch(userId, Unit).map { result => - result.v.map(_.candidates).getOrElse(Seq.empty) - } + fetcher + .fetch(userId, Unit).map { result => + result.v.map(_.candidates).getOrElse(Seq.empty) + }.map { candidates => + val sortedCandidates = candidates.sortBy(-_.score) + sortedCandidates.take(MaxCandidatesToKeep) + } } }.map(_.flatten) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/BUILD.bazel index 326d086a3..bdd2de735 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/BUILD.bazel @@ -12,5 +12,4 @@ scala_library( "stitch/stitch-gizmoduck", "stitch/stitch-socialgraph", ], - exports = [], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/GizmoduckUserFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsGizmoduckValidUserFeatureHydrator.scala similarity index 71% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/GizmoduckUserFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsGizmoduckValidUserFeatureHydrator.scala index d1db2c348..80a41adf0 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/GizmoduckUserFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsGizmoduckValidUserFeatureHydrator.scala @@ -1,7 +1,7 @@ package com.twitter.home_mixer.product.list_recommended_users.feature_hydrator import com.twitter.gizmoduck.{thriftscala => gt} -import com.twitter.home_mixer.product.list_recommended_users.model.ListFeatures.GizmoduckUserFeature +import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.IsGizmoduckValidUserFeature import com.twitter.product_mixer.component_library.model.candidate.UserCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap @@ -19,13 +19,13 @@ import javax.inject.Inject import javax.inject.Singleton @Singleton -class GizmoduckUserFeatureHydrator @Inject() (gizmoduck: Gizmoduck) +class IsGizmoduckValidUserFeatureHydrator @Inject() (gizmoduck: Gizmoduck) extends BulkCandidateFeatureHydrator[PipelineQuery, UserCandidate] { override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("GizmoduckUser") + FeatureHydratorIdentifier("IsGizmoduckValidUser") - override val features: Set[Feature[_, _]] = Set(GizmoduckUserFeature) + override val features: Set[Feature[_, _]] = Set(IsGizmoduckValidUserFeature) private val queryFields: Set[gt.QueryFields] = Set(gt.QueryFields.Safety) @@ -44,14 +44,21 @@ class GizmoduckUserFeatureHydrator @Inject() (gizmoduck: Gizmoduck) .collectToTry( userIds.map(userId => gizmoduck.getUserById(userId, queryFields, context))).map { userResults => - val idToUserMap = userResults + val idToUserSafetyMap = userResults .collect { case Return(user) => user - }.map(user => user.id -> user).toMap + }.map(user => user.id -> user.safety).toMap candidates.map { candidate => + val safety = idToUserSafetyMap.getOrElse(candidate.candidate.id, None) + val isValidUser = safety.isDefined && + !safety.exists(_.deactivated) && + !safety.exists(_.suspended) && + !safety.exists(_.isProtected) && + !safety.flatMap(_.offboarded).getOrElse(false) + FeatureMapBuilder() - .add(GizmoduckUserFeature, idToUserMap.get(candidate.candidate.id)) + .add(IsGizmoduckValidUserFeature, isValidUser) .build() } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsListMemberFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsListMemberFeatureHydrator.scala index a7c51b4b1..c9e529de5 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsListMemberFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsListMemberFeatureHydrator.scala @@ -1,7 +1,7 @@ package com.twitter.home_mixer.product.list_recommended_users.feature_hydrator import com.twitter.home_mixer.model.request.HasListId -import com.twitter.home_mixer.product.list_recommended_users.model.ListFeatures.IsListMemberFeature +import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.IsListMemberFeature import com.twitter.product_mixer.component_library.model.candidate.UserCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsSGSValidUserFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsSGSValidUserFeatureHydrator.scala new file mode 100644 index 000000000..de70d45d8 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsSGSValidUserFeatureHydrator.scala @@ -0,0 +1,65 @@ +package com.twitter.home_mixer.product.list_recommended_users.feature_hydrator + +import com.twitter.home_mixer.model.request.HasListId +import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.IsSGSValidUserFeature +import com.twitter.product_mixer.component_library.model.candidate.UserCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.socialgraph.{thriftscala => sg} +import com.twitter.stitch.Stitch +import com.twitter.stitch.socialgraph.SocialGraph + +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class IsSGSValidUserFeatureHydrator @Inject() (socialGraph: SocialGraph) + extends BulkCandidateFeatureHydrator[PipelineQuery with HasListId, UserCandidate] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("IsSGSValidUser") + + override def features: Set[Feature[_, _]] = Set(IsSGSValidUserFeature) + + override def apply( + query: PipelineQuery with HasListId, + candidates: Seq[CandidateWithFeatures[UserCandidate]] + ): Stitch[Seq[FeatureMap]] = { + val sourceId = query.getRequiredUserId + val targetUserIds = candidates.map(_.candidate.id) + val request = sg.IdsRequest( + relationships = Seq( + sg.SrcRelationship( + source = sourceId, + relationshipType = sg.RelationshipType.Blocking, + hasRelationship = true, + targets = Some(targetUserIds)), + sg.SrcRelationship( + source = sourceId, + relationshipType = sg.RelationshipType.BlockedBy, + hasRelationship = true, + targets = Some(targetUserIds)), + sg.SrcRelationship( + source = sourceId, + relationshipType = sg.RelationshipType.Muting, + hasRelationship = true, + targets = Some(targetUserIds)) + ), + pageRequest = Some(sg.PageRequest(selectAll = Some(true))), + context = Some(sg.LookupContext(performUnion = Some(true))) + ) + + socialGraph.ids(request).map(_.ids).map(_.toSet).map { hasRelationshipUserIds => + candidates.map { candidate => + FeatureMapBuilder() + .add(IsSGSValidUserFeature, !hasRelationshipUserIds.contains(candidate.candidate.id)) + .build() + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ListMembersQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/RecentListMembersQueryFeatureHydrator.scala similarity index 72% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ListMembersQueryFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/RecentListMembersQueryFeatureHydrator.scala index a6d3324d3..81a5ce504 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ListMembersQueryFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/RecentListMembersQueryFeatureHydrator.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator +package com.twitter.home_mixer.product.list_recommended_users.feature_hydrator import com.twitter.home_mixer.model.request.HasListId import com.twitter.product_mixer.core.feature.featuremap.FeatureMap @@ -15,17 +15,18 @@ import com.twitter.stitch.socialgraph.SocialGraph import javax.inject.Inject import javax.inject.Singleton -case object ListMembersFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] { +case object RecentListMembersFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] { override val defaultValue: Seq[Long] = Seq.empty } @Singleton -class ListMembersQueryFeatureHydrator @Inject() (socialGraph: SocialGraph) +class RecentListMembersQueryFeatureHydrator @Inject() (socialGraph: SocialGraph) extends QueryFeatureHydrator[PipelineQuery with HasListId] { - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ListMembers") + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("RecentListMembers") - override val features: Set[Feature[_, _]] = Set(ListMembersFeature) + override val features: Set[Feature[_, _]] = Set(RecentListMembersFeature) private val MaxRecentMembers = 10 @@ -36,7 +37,7 @@ class ListMembersQueryFeatureHydrator @Inject() (socialGraph: SocialGraph) pageRequest = Some(sg.PageRequest(selectAll = Some(true), count = Some(MaxRecentMembers))) ) socialGraph.ids(request).map(_.ids).map { listMembers => - FeatureMapBuilder().add(ListMembersFeature, listMembers).build() + FeatureMapBuilder().add(RecentListMembersFeature, listMembers).build() } } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/BUILD.bazel index c54e5a7b3..72a5f251f 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/BUILD.bazel @@ -4,8 +4,8 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter", ], diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/DropMaxCandidatesByScoreFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/DropMaxCandidatesByAggregatedScoreFilter.scala similarity index 61% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/DropMaxCandidatesByScoreFilter.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/DropMaxCandidatesByAggregatedScoreFilter.scala index d75b301e3..a68c37450 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/DropMaxCandidatesByScoreFilter.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/DropMaxCandidatesByAggregatedScoreFilter.scala @@ -1,6 +1,6 @@ package com.twitter.home_mixer.product.list_recommended_users.filter -import com.twitter.home_mixer.product.list_recommended_users.model.ListFeatures.ScoreFeature +import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.ScoreFeature import com.twitter.product_mixer.component_library.model.candidate.UserCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult @@ -9,18 +9,26 @@ import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch -object DropMaxCandidatesByScoreFilter extends Filter[PipelineQuery, UserCandidate] { +object DropMaxCandidatesByAggregatedScoreFilter extends Filter[PipelineQuery, UserCandidate] { - override val identifier: FilterIdentifier = FilterIdentifier("DropMaxCandidatesByScore") + override val identifier: FilterIdentifier = FilterIdentifier("DropMaxCandidatesByAggregatedScore") - private val MaxSimilarUserCandidates = 1000 + private val MaxSimilarUserCandidates = 150 override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[UserCandidate]] ): Stitch[FilterResult[UserCandidate]] = { + val userIdToAggregatedScoreMap = candidates + .groupBy(_.candidate.id) + .map { + case (userId, candidates) => + val aggregatedScore = candidates.map(_.features.getOrElse(ScoreFeature, 0.0)).sum + (userId, aggregatedScore) + } - val sortedCandidates = candidates.sortBy(-_.features.getOrElse(ScoreFeature, 0.0)) + val sortedCandidates = candidates.sortBy(candidate => + -userIdToAggregatedScoreMap.getOrElse(candidate.candidate.id, 0.0)) val (kept, removed) = sortedCandidates.map(_.candidate).splitAt(MaxSimilarUserCandidates) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/PreviouslyServedUsersFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/PreviouslyServedUsersFilter.scala index 97cd6a5c1..ac8c3107d 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/PreviouslyServedUsersFilter.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/PreviouslyServedUsersFilter.scala @@ -1,6 +1,6 @@ package com.twitter.home_mixer.product.list_recommended_users.filter -import com.twitter.home_mixer.functional_component.feature_hydrator.ListMembersFeature +import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.RecentListMembersFeature import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersQuery import com.twitter.product_mixer.component_library.model.candidate.UserCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter @@ -18,7 +18,7 @@ object PreviouslyServedUsersFilter extends Filter[ListRecommendedUsersQuery, Use candidates: Seq[CandidateWithFeatures[UserCandidate]] ): Stitch[FilterResult[UserCandidate]] = { - val recentListMembers = query.features.map(_.getOrElse(ListMembersFeature, Seq.empty)) + val recentListMembers = query.features.map(_.getOrElse(RecentListMembersFeature, Seq.empty)) val servedUserIds = query.pipelineCursor.map(_.excludedIds) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate/BUILD.bazel new file mode 100644 index 000000000..f8a7083b9 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate/BUILD.bazel @@ -0,0 +1,12 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", + "src/thrift/com/twitter/socialgraph:thrift-scala", + "stitch/stitch-socialgraph", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/ViewerIsListOwnerGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate/ViewerIsListOwnerGate.scala similarity index 94% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/ViewerIsListOwnerGate.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate/ViewerIsListOwnerGate.scala index 487487e92..e7ab3084e 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/ViewerIsListOwnerGate.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate/ViewerIsListOwnerGate.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.gate +package com.twitter.home_mixer.product.list_recommended_users.gate import com.twitter.home_mixer.model.request.HasListId import com.twitter.product_mixer.core.functional_component.gate.Gate diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListFeatures.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListRecommendedUsersFeatures.scala similarity index 65% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListFeatures.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListRecommendedUsersFeatures.scala index 9abcd6fb2..baed8171b 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListFeatures.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListRecommendedUsersFeatures.scala @@ -1,12 +1,12 @@ package com.twitter.home_mixer.product.list_recommended_users.model -import com.twitter.gizmoduck.{thriftscala => gt} import com.twitter.product_mixer.component_library.model.candidate.UserCandidate import com.twitter.product_mixer.core.feature.Feature -object ListFeatures { +object ListRecommendedUsersFeatures { // Candidate features - object GizmoduckUserFeature extends Feature[UserCandidate, Option[gt.User]] + object IsGizmoduckValidUserFeature extends Feature[UserCandidate, Boolean] object IsListMemberFeature extends Feature[UserCandidate, Boolean] + object IsSGSValidUserFeature extends Feature[UserCandidate, Boolean] object ScoreFeature extends Feature[UserCandidate, Double] } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListRecommendedUsersQuery.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListRecommendedUsersQuery.scala index 8d8c2e6ea..15b16c25b 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListRecommendedUsersQuery.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListRecommendedUsersQuery.scala @@ -18,7 +18,8 @@ case class ListRecommendedUsersQuery( override val debugOptions: Option[DebugOptions], override val features: Option[FeatureMap], selectedUserIds: Option[Seq[Long]], - excludedUserIds: Option[Seq[Long]]) + excludedUserIds: Option[Seq[Long]], + listName: Option[String]) extends PipelineQuery with HasPipelineCursor[UrtUnorderedExcludeIdsCursor] with HasListId { diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/BUILD.bazel index e60fc064e..a7d22148a 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/BUILD.bazel @@ -22,6 +22,7 @@ scala_library( "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", @@ -30,6 +31,7 @@ scala_library( "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_service", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/ads", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsAdsCandidatePipelineBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsAdsCandidatePipelineBuilder.scala index 77c61710a..9e5a4e541 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsAdsCandidatePipelineBuilder.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsAdsCandidatePipelineBuilder.scala @@ -1,7 +1,7 @@ package com.twitter.home_mixer.product.list_tweets import com.twitter.adserver.{thriftscala => ads} -import com.twitter.home_mixer.functional_component.decorator.HomeAdsClientEventDetailsBuilder +import com.twitter.home_mixer.functional_component.decorator.builder.HomeAdsClientEventDetailsBuilder import com.twitter.home_mixer.functional_component.gate.ExcludeSoftUserGate import com.twitter.home_mixer.param.HomeGlobalParams import com.twitter.home_mixer.param.HomeGlobalParams.EnableAdvertiserBrandSafetySettingsFeatureHydratorParam diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsMixerPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsMixerPipelineConfig.scala index f4e20ce7e..2d7dd3b31 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsMixerPipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsMixerPipelineConfig.scala @@ -3,13 +3,16 @@ package com.twitter.home_mixer.product.list_tweets import com.twitter.clientapp.{thriftscala => ca} import com.twitter.goldfinch.api.AdsInjectionSurfaceAreas import com.twitter.home_mixer.candidate_pipeline.ConversationServiceCandidatePipelineConfigBuilder -import com.twitter.home_mixer.functional_component.decorator.ListConversationServiceCandidateDecorator import com.twitter.home_mixer.functional_component.feature_hydrator.RequestQueryFeatureHydrator import com.twitter.home_mixer.functional_component.side_effect.HomeScribeClientEventSideEffect import com.twitter.home_mixer.model.GapIncludeInstruction +import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeClientEventsFlag +import com.twitter.home_mixer.product.list_tweets.decorator.ListConversationServiceCandidateDecorator import com.twitter.home_mixer.product.list_tweets.model.ListTweetsQuery import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.inject.annotations.Flag import com.twitter.logpipeline.client.common.EventPublisher +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller import com.twitter.product_mixer.component_library.premarshaller.urt.builder.AddEntriesWithReplaceAndShowAlertInstructionBuilder @@ -55,9 +58,11 @@ class ListTweetsMixerPipelineConfig @Inject() ( ], listTweetsAdsCandidatePipelineBuilder: ListTweetsAdsCandidatePipelineBuilder, requestQueryFeatureHydrator: RequestQueryFeatureHydrator[ListTweetsQuery], + sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, adsInjector: AdsInjector, clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent], - urtTransportMarshaller: UrtTransportMarshaller) + urtTransportMarshaller: UrtTransportMarshaller, + @Flag(ScribeClientEventsFlag) enableScribeClientEvents: Boolean) extends MixerPipelineConfig[ListTweetsQuery, Timeline, urt.TimelineResponse] { override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("ListTweets") @@ -101,14 +106,16 @@ class ListTweetsMixerPipelineConfig @Inject() ( ) override val fetchQueryFeatures: Seq[QueryFeatureHydrator[ListTweetsQuery]] = Seq( - requestQueryFeatureHydrator + requestQueryFeatureHydrator, + sgsFollowedUsersQueryFeatureHydrator ) private val homeScribeClientEventSideEffect = HomeScribeClientEventSideEffect( + enableScribeClientEvents = enableScribeClientEvents, logPipelinePublisher = clientEventsScribeEventPublisher, injectedTweetsCandidatePipelineIdentifiers = Seq(conversationServiceCandidatePipelineConfig.identifier), - adsCandidatePipelineIdentifier = listTweetsAdsCandidatePipelineConfig.identifier, + adsCandidatePipelineIdentifier = Some(listTweetsAdsCandidatePipelineConfig.identifier), ) override val resultSideEffects: Seq[PipelineResultSideEffect[ListTweetsQuery, Timeline]] = diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsProductPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsProductPipelineConfig.scala index 06afd9a92..b4b5b1924 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsProductPipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsProductPipelineConfig.scala @@ -1,5 +1,6 @@ package com.twitter.home_mixer.product.list_tweets +import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.marshaller.timelines.ChronologicalCursorUnmarshaller import com.twitter.home_mixer.model.request.HomeMixerRequest import com.twitter.home_mixer.model.request.ListTweetsProduct @@ -8,9 +9,19 @@ import com.twitter.home_mixer.product.list_tweets.model.ListTweetsQuery import com.twitter.home_mixer.product.list_tweets.param.ListTweetsParam.ServerMaxResultsParam import com.twitter.home_mixer.product.list_tweets.param.ListTweetsParamConfig import com.twitter.home_mixer.service.HomeMixerAccessPolicy.DefaultHomeMixerAccessPolicy +import com.twitter.home_mixer.service.HomeMixerAlertConfig.DefaultNotificationGroup import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.common.alert.EmptyResponseRateAlert +import com.twitter.product_mixer.core.functional_component.common.alert.LatencyAlert +import com.twitter.product_mixer.core.functional_component.common.alert.P99 +import com.twitter.product_mixer.core.functional_component.common.alert.SuccessRateAlert +import com.twitter.product_mixer.core.functional_component.common.alert.ThroughputAlert +import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfAbove +import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfBelow +import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfLatencyAbove import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier import com.twitter.product_mixer.core.model.marshalling.request @@ -40,6 +51,7 @@ class ListTweetsProductPipelineConfig @Inject() ( override val identifier: ProductPipelineIdentifier = ProductPipelineIdentifier("ListTweets") override val product: request.Product = ListTweetsProduct override val paramConfig: ProductParamConfig = listTweetsParamConfig + override val denyLoggedOutUsers: Boolean = false override def pipelineQueryTransformer( request: HomeMixerRequest, @@ -90,5 +102,29 @@ class ListTweetsProductPipelineConfig @Inject() ( override def pipelineSelector(query: ListTweetsQuery): ComponentIdentifier = listTweetsMixerPipelineConfig.identifier + override val alerts: Seq[Alert] = Seq( + SuccessRateAlert( + notificationGroup = DefaultNotificationGroup, + warnPredicate = TriggerIfBelow(99.9, 20, 30), + criticalPredicate = TriggerIfBelow(99.9, 30, 30), + ), + LatencyAlert( + notificationGroup = DefaultNotificationGroup, + percentile = P99, + warnPredicate = TriggerIfLatencyAbove(300.millis, 15, 30), + criticalPredicate = TriggerIfLatencyAbove(400.millis, 15, 30) + ), + ThroughputAlert( + notificationGroup = DefaultNotificationGroup, + warnPredicate = TriggerIfAbove(3000), + criticalPredicate = TriggerIfAbove(4000) + ), + EmptyResponseRateAlert( + notificationGroup = DefaultNotificationGroup, + warnPredicate = TriggerIfAbove(65), + criticalPredicate = TriggerIfAbove(80) + ) + ) + override val debugAccessPolicies: Set[AccessPolicy] = DefaultHomeMixerAccessPolicy } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsTimelineServiceCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsTimelineServiceCandidatePipelineConfig.scala index aba9f47e0..1af95e873 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsTimelineServiceCandidatePipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsTimelineServiceCandidatePipelineConfig.scala @@ -4,6 +4,7 @@ import com.twitter.home_mixer.candidate_pipeline.TimelineServiceResponseFeatureT import com.twitter.home_mixer.marshaller.timelines.TimelineServiceCursorMarshaller import com.twitter.home_mixer.product.list_tweets.model.ListTweetsQuery import com.twitter.home_mixer.product.list_tweets.param.ListTweetsParam.ServerMaxResultsParam +import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.component_library.candidate_source.timeline_service.TimelineServiceTweetCandidateSource import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource @@ -51,4 +52,8 @@ class ListTweetsTimelineServiceCandidatePipelineConfig @Inject() ( override val featuresFromCandidateSourceTransformers: Seq[CandidateFeatureTransformer[t.Tweet]] = Seq(TimelineServiceResponseFeatureTransformer) + + override val alerts = Seq( + HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.7) + ) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/BUILD.bazel new file mode 100644 index 000000000..4258ffc36 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/BUILD.bazel @@ -0,0 +1,15 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/builder", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", + "timelines/src/main/scala/com/twitter/timelines/injection/scribe", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/ListConversationServiceCandidateDecorator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/ListConversationServiceCandidateDecorator.scala similarity index 91% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/ListConversationServiceCandidateDecorator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/ListConversationServiceCandidateDecorator.scala index a8df05029..a02b628e8 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/ListConversationServiceCandidateDecorator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/ListConversationServiceCandidateDecorator.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.decorator +package com.twitter.home_mixer.product.list_tweets.decorator import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder import com.twitter.home_mixer.functional_component.decorator.builder.ListClientEventDetailsBuilder @@ -23,8 +23,10 @@ object ListConversationServiceCandidateDecorator { def apply(): Some[UrtMultipleModulesDecorator[PipelineQuery, TweetCandidate, Long]] = { val suggestType = st.SuggestType.OrganicListTweet val component = InjectionScribeUtil.scribeComponent(suggestType).get - val clientEventInfoBuilder = - ClientEventInfoBuilder(component, Some(ListClientEventDetailsBuilder)) + val clientEventInfoBuilder = ClientEventInfoBuilder( + component = component, + detailsBuilder = Some(ListClientEventDetailsBuilder(st.SuggestType.OrganicListTweet)) + ) val tweetItemBuilder = TweetCandidateUrtItemBuilder( clientEventInfoBuilder = clientEventInfoBuilder ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/builder/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/builder/BUILD.bazel new file mode 100644 index 000000000..3ed81a196 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/builder/BUILD.bazel @@ -0,0 +1,10 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/BUILD.bazel index e016b2a82..4bbbec94d 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/BUILD.bazel @@ -4,54 +4,37 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline", + "explore/explore-ranker/thrift/src/main/thrift:thrift-scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/module", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/selector", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/async", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/param_gated", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tweet_tlx", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", - "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsProductPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsProductPipelineConfig.scala index 9870030b8..27278db0c 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsProductPipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsProductPipelineConfig.scala @@ -1,6 +1,7 @@ package com.twitter.home_mixer.product.scored_tweets import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature import com.twitter.home_mixer.model.request.HomeMixerRequest import com.twitter.home_mixer.model.request.ScoredTweetsProduct import com.twitter.home_mixer.model.request.ScoredTweetsProductContext @@ -9,6 +10,7 @@ import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.Serv import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParamConfig import com.twitter.home_mixer.service.HomeMixerAccessPolicy.DefaultHomeMixerAccessPolicy import com.twitter.home_mixer.{thriftscala => t} +import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier @@ -44,19 +46,19 @@ class ScoredTweetsProductPipelineConfig @Inject() ( case _ => throw PipelineFailure(BadRequest, "ScoredTweetsProductContext not found") } - val featureMap = context.servedTweetIds.map { servedTweets => - FeatureMapBuilder() - .add(ServedTweetIdsFeature, servedTweets) - .build() - } + val featureMap = FeatureMapBuilder() + .add(ServedTweetIdsFeature, context.servedTweetIds.getOrElse(Seq.empty)) + .add(TimelineServiceTweetsFeature, context.backfillTweetIds.getOrElse(Seq.empty)) + .build() ScoredTweetsQuery( params = params, clientContext = request.clientContext, - features = featureMap, - pipelineCursor = None, + pipelineCursor = + request.serializedRequestCursor.flatMap(UrtCursorSerializer.deserializeOrderedCursor), requestedMaxResults = Some(params(ServerMaxResultsParam)), debugOptions = request.debugParams.flatMap(_.debugOptions), + features = Some(featureMap), deviceContext = context.deviceContext, seenTweetIds = context.seenTweetIds, qualityFactorStatus = None diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsRecommendationPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsRecommendationPipelineConfig.scala index 8d64a2976..e3d50040b 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsRecommendationPipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsRecommendationPipelineConfig.scala @@ -1,63 +1,83 @@ package com.twitter.home_mixer.product.scored_tweets import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.functional_component.feature_hydrator.LastNonPollingTimeQueryFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.FeedbackHistoryQueryFeatureHydrator +import com.twitter.home_mixer.functional_component.feature_hydrator.ImpressionBloomFilterQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.RealGraphInNetworkScoresQueryFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.RealGraphQueryFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.RealTimeInteractionGraphUserVertexQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.RequestQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.TweetImpressionsQueryFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.TwhinUserEngagementQueryFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.TwhinUserFollowQueryFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.UserLanguagesFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.UserStateQueryFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.PartAAggregateQueryFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.PartBAggregateQueryFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates.UserEngagementRealTimeAggregatesFeatureHydrator -import com.twitter.home_mixer.functional_component.filter.KeepBestOutOfNetworkTweetPerAuthorFilter -import com.twitter.home_mixer.functional_component.filter.OutOfNetworkCompetitorFilter -import com.twitter.home_mixer.functional_component.filter.OutOfNetworkCompetitorURLFilter +import com.twitter.home_mixer.functional_component.filter.FeedbackFatigueFilter import com.twitter.home_mixer.functional_component.filter.PreviouslySeenTweetsFilter import com.twitter.home_mixer.functional_component.filter.PreviouslyServedTweetsFilter import com.twitter.home_mixer.functional_component.filter.RejectTweetFromViewerFilter import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter import com.twitter.home_mixer.functional_component.side_effect.PublishClientSentImpressionsEventBusSideEffect import com.twitter.home_mixer.functional_component.side_effect.PublishClientSentImpressionsManhattanSideEffect +import com.twitter.home_mixer.functional_component.side_effect.PublishImpressionBloomFilterSideEffect import com.twitter.home_mixer.functional_component.side_effect.UpdateLastNonPollingTimeSideEffect +import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature +import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.IsSupportAccountReplyFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature +import com.twitter.home_mixer.param.HomeGlobalParams.EnableImpressionBloomFilter import com.twitter.home_mixer.param.HomeMixerFlagName.TargetFetchLatency import com.twitter.home_mixer.param.HomeMixerFlagName.TargetScoringLatency import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.CachedScoredTweetsCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsCrMixerCandidatePipelineConfig +import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsBackfillCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsFrsCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsInNetworkCandidatePipelineConfig +import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsListsCandidatePipelineConfig +import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsPopularVideosCandidatePipelineConfig +import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsTweetMixerCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsUtegCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.CachedScoredTweetsQueryFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.ListIdsQueryFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RealGraphQueryFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RealTimeInteractionGraphUserVertexQueryFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RequestTimeQueryFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TwhinUserEngagementQueryFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TwhinUserFollowQueryFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.UserLanguagesFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.UserStateQueryFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.PartAAggregateQueryFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.PartBAggregateQueryFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates.UserEngagementRealTimeAggregatesFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.filter.DuplicateConversationTweetsFilter +import com.twitter.home_mixer.product.scored_tweets.filter.OutOfNetworkCompetitorFilter +import com.twitter.home_mixer.product.scored_tweets.filter.OutOfNetworkCompetitorURLFilter +import com.twitter.home_mixer.product.scored_tweets.filter.ScoredTweetsSocialContextFilter import com.twitter.home_mixer.product.scored_tweets.marshaller.ScoredTweetsResponseDomainMarshaller import com.twitter.home_mixer.product.scored_tweets.marshaller.ScoredTweetsResponseTransportMarshaller import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.ServerMaxResultsParam -import com.twitter.home_mixer.product.scored_tweets.scoring_pipeline.ScoredTweetsDiversityScoringPipelineConfig -import com.twitter.home_mixer.product.scored_tweets.scoring_pipeline.ScoredTweetsRescoreOONScoringPipelineConfig -import com.twitter.home_mixer.product.scored_tweets.scoring_pipeline.ScoredTweetsRescoreVerifiedAuthorScoringPipelineConfig -import com.twitter.home_mixer.product.scored_tweets.scoring_pipeline.ScoredTweetsScoringPipelineConfig -import com.twitter.home_mixer.product.scored_tweets.scoring_pipeline.ScoredTweetsWeightedScoresSumScoringPipelineConfig +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.MaxInNetworkResultsParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.MaxOutOfNetworkResultsParam +import com.twitter.home_mixer.product.scored_tweets.scoring_pipeline.ScoredTweetsHeuristicScoringPipelineConfig +import com.twitter.home_mixer.product.scored_tweets.scoring_pipeline.ScoredTweetsModelScoringPipelineConfig +import com.twitter.home_mixer.product.scored_tweets.selector.KeepBestOutOfNetworkCandidatePerAuthorPerSuggestType import com.twitter.home_mixer.product.scored_tweets.side_effect.CachedScoredTweetsSideEffect +import com.twitter.home_mixer.product.scored_tweets.side_effect.ScribeScoredCandidatesSideEffect import com.twitter.home_mixer.product.scored_tweets.side_effect.ScribeServedCommonFeaturesAndCandidateFeaturesSideEffect import com.twitter.home_mixer.{thriftscala => t} import com.twitter.inject.annotations.Flag import com.twitter.product_mixer.component_library.feature_hydrator.query.async.AsyncQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweetsQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.ParamGatedQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator +import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.selector.DropDuplicateCandidates +import com.twitter.product_mixer.component_library.selector.DropFilteredMaxCandidates import com.twitter.product_mixer.component_library.selector.DropMaxCandidates import com.twitter.product_mixer.component_library.selector.IdAndClassDuplicationKey import com.twitter.product_mixer.component_library.selector.InsertAppendResults import com.twitter.product_mixer.component_library.selector.PickFirstCandidateMerger import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates import com.twitter.product_mixer.component_library.selector.sorter.FeatureValueSorter +import com.twitter.product_mixer.core.functional_component.common.AllExceptPipelines import com.twitter.product_mixer.core.functional_component.common.AllPipelines +import com.twitter.product_mixer.core.functional_component.configapi.StaticParam import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller @@ -66,8 +86,10 @@ import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.model.common.identifier.RecommendationPipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier +import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails import com.twitter.product_mixer.core.pipeline.FailOpenPolicy import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.product_mixer.core.pipeline.recommendation.RecommendationPipelineConfig @@ -76,6 +98,7 @@ import com.twitter.product_mixer.core.quality_factor.BoundsWithDefault import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactorConfig import com.twitter.product_mixer.core.quality_factor.QualityFactorConfig import com.twitter.util.Duration + import javax.inject.Inject import javax.inject.Singleton @@ -83,31 +106,41 @@ import javax.inject.Singleton class ScoredTweetsRecommendationPipelineConfig @Inject() ( scoredTweetsInNetworkCandidatePipelineConfig: ScoredTweetsInNetworkCandidatePipelineConfig, scoredTweetsUtegCandidatePipelineConfig: ScoredTweetsUtegCandidatePipelineConfig, - scoredTweetsCrMixerCandidatePipelineConfig: ScoredTweetsCrMixerCandidatePipelineConfig, + scoredTweetsTweetMixerCandidatePipelineConfig: ScoredTweetsTweetMixerCandidatePipelineConfig, scoredTweetsFrsCandidatePipelineConfig: ScoredTweetsFrsCandidatePipelineConfig, + scoredTweetsListsCandidatePipelineConfig: ScoredTweetsListsCandidatePipelineConfig, + scoredTweetsPopularVideosCandidatePipelineConfig: ScoredTweetsPopularVideosCandidatePipelineConfig, + scoredTweetsBackfillCandidatePipelineConfig: ScoredTweetsBackfillCandidatePipelineConfig, cachedScoredTweetsCandidatePipelineConfig: CachedScoredTweetsCandidatePipelineConfig, requestQueryFeatureHydrator: RequestQueryFeatureHydrator[ScoredTweetsQuery], - lastNonPollingTimeQueryFeatureHydrator: LastNonPollingTimeQueryFeatureHydrator, + requestTimeQueryFeatureHydrator: RequestTimeQueryFeatureHydrator, realTimeInteractionGraphUserVertexQueryFeatureHydrator: RealTimeInteractionGraphUserVertexQueryFeatureHydrator, userStateQueryFeatureHydrator: UserStateQueryFeatureHydrator, userEngagementRealTimeAggregatesFeatureHydrator: UserEngagementRealTimeAggregatesFeatureHydrator, twhinUserEngagementQueryFeatureHydrator: TwhinUserEngagementQueryFeatureHydrator, twhinUserFollowQueryFeatureHydrator: TwhinUserFollowQueryFeatureHydrator, cachedScoredTweetsQueryFeatureHydrator: CachedScoredTweetsQueryFeatureHydrator, - scoredTweetsScoringPipelineConfig: ScoredTweetsScoringPipelineConfig, - scoredTweetsWeightedScoresSumScoringPipelineConfig: ScoredTweetsWeightedScoresSumScoringPipelineConfig, + sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, + scoredTweetsModelScoringPipelineConfig: ScoredTweetsModelScoringPipelineConfig, + impressionBloomFilterQueryFeatureHydrator: ImpressionBloomFilterQueryFeatureHydrator[ + ScoredTweetsQuery + ], manhattanTweetImpressionsQueryFeatureHydrator: TweetImpressionsQueryFeatureHydrator[ ScoredTweetsQuery ], memcacheTweetImpressionsQueryFeatureHydrator: ImpressedTweetsQueryFeatureHydrator, + listIdsQueryFeatureHydrator: ListIdsQueryFeatureHydrator, + feedbackHistoryQueryFeatureHydrator: FeedbackHistoryQueryFeatureHydrator, publishClientSentImpressionsEventBusSideEffect: PublishClientSentImpressionsEventBusSideEffect, publishClientSentImpressionsManhattanSideEffect: PublishClientSentImpressionsManhattanSideEffect, + publishImpressionBloomFilterSideEffect: PublishImpressionBloomFilterSideEffect, realGraphInNetworkScoresQueryFeatureHydrator: RealGraphInNetworkScoresQueryFeatureHydrator, realGraphQueryFeatureHydrator: RealGraphQueryFeatureHydrator, userLanguagesFeatureHydrator: UserLanguagesFeatureHydrator, partAAggregateQueryFeatureHydrator: PartAAggregateQueryFeatureHydrator, partBAggregateQueryFeatureHydrator: PartBAggregateQueryFeatureHydrator, cachedScoredTweetsSideEffect: CachedScoredTweetsSideEffect, + scribeScoredCandidatesSideEffect: ScribeScoredCandidatesSideEffect, scribeServedCommonFeaturesAndCandidateFeaturesSideEffect: ScribeServedCommonFeaturesAndCandidateFeaturesSideEffect, updateLastNonPollingTimeSideEffect: UpdateLastNonPollingTimeSideEffect[ ScoredTweetsQuery, @@ -125,17 +158,27 @@ class ScoredTweetsRecommendationPipelineConfig @Inject() ( override val identifier: RecommendationPipelineIdentifier = RecommendationPipelineIdentifier("ScoredTweets") + private val SubscriptionReplyFilterId = "SubscriptionReply" + private val MaxBackfillTweets = 50 + private val scoringStep = RecommendationPipelineConfig.scoringPipelinesStep override val fetchQueryFeatures: Seq[QueryFeatureHydrator[ScoredTweetsQuery]] = Seq( requestQueryFeatureHydrator, realGraphInNetworkScoresQueryFeatureHydrator, cachedScoredTweetsQueryFeatureHydrator, + sgsFollowedUsersQueryFeatureHydrator, + ParamGatedQueryFeatureHydrator( + EnableImpressionBloomFilter, + impressionBloomFilterQueryFeatureHydrator + ), manhattanTweetImpressionsQueryFeatureHydrator, memcacheTweetImpressionsQueryFeatureHydrator, + listIdsQueryFeatureHydrator, + userStateQueryFeatureHydrator, + AsyncQueryFeatureHydrator(scoringStep, feedbackHistoryQueryFeatureHydrator), AsyncQueryFeatureHydrator(scoringStep, realGraphQueryFeatureHydrator), - AsyncQueryFeatureHydrator(scoringStep, lastNonPollingTimeQueryFeatureHydrator), - AsyncQueryFeatureHydrator(scoringStep, userStateQueryFeatureHydrator), + AsyncQueryFeatureHydrator(scoringStep, requestTimeQueryFeatureHydrator), AsyncQueryFeatureHydrator(scoringStep, userLanguagesFeatureHydrator), AsyncQueryFeatureHydrator(scoringStep, userEngagementRealTimeAggregatesFeatureHydrator), AsyncQueryFeatureHydrator(scoringStep, realTimeInteractionGraphUserVertexQueryFeatureHydrator), @@ -151,8 +194,11 @@ class ScoredTweetsRecommendationPipelineConfig @Inject() ( cachedScoredTweetsCandidatePipelineConfig, scoredTweetsInNetworkCandidatePipelineConfig, scoredTweetsUtegCandidatePipelineConfig, - scoredTweetsCrMixerCandidatePipelineConfig, - scoredTweetsFrsCandidatePipelineConfig + scoredTweetsTweetMixerCandidatePipelineConfig, + scoredTweetsFrsCandidatePipelineConfig, + scoredTweetsListsCandidatePipelineConfig, + scoredTweetsPopularVideosCandidatePipelineConfig, + scoredTweetsBackfillCandidatePipelineConfig ) override val postCandidatePipelinesSelectors: Seq[Selector[ScoredTweetsQuery]] = Seq( @@ -168,9 +214,16 @@ class ScoredTweetsRecommendationPipelineConfig @Inject() ( // sort these to have the "cheaper" filters run first RejectTweetFromViewerFilter, RetweetDeduplicationFilter, - PreviouslyServedTweetsFilter, PreviouslySeenTweetsFilter, - OutOfNetworkCompetitorFilter + PreviouslyServedTweetsFilter, + PredicateFeatureFilter.fromPredicate( + FilterIdentifier(SubscriptionReplyFilterId), + shouldKeepCandidate = { features => + features.getOrElse(InReplyToTweetIdFeature, None).isEmpty || + features.getOrElse(ExclusiveConversationAuthorIdFeature, None).isEmpty + } + ), + FeedbackFatigueFilter ) override val candidatePipelineFailOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = @@ -178,19 +231,20 @@ class ScoredTweetsRecommendationPipelineConfig @Inject() ( cachedScoredTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, scoredTweetsInNetworkCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, scoredTweetsUtegCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - scoredTweetsCrMixerCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - scoredTweetsFrsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always + scoredTweetsTweetMixerCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + scoredTweetsFrsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + scoredTweetsListsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + scoredTweetsPopularVideosCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + scoredTweetsBackfillCandidatePipelineConfig.identifier -> FailOpenPolicy.Always ) override val scoringPipelineFailOpenPolicies: Map[ScoringPipelineIdentifier, FailOpenPolicy] = Map( - ScoredTweetsRescoreOONScoringPipelineConfig.identifier -> FailOpenPolicy.Always, - ScoredTweetsRescoreVerifiedAuthorScoringPipelineConfig.identifier -> FailOpenPolicy.Always, - ScoredTweetsDiversityScoringPipelineConfig.identifier -> FailOpenPolicy.Always + ScoredTweetsHeuristicScoringPipelineConfig.identifier -> FailOpenPolicy.Always ) private val candidatePipelineQualityFactorConfig = LinearLatencyQualityFactorConfig( - qualityFactorBounds = BoundsWithDefault(minInclusive = 0.1, maxInclusive = 1.0, default = 0.4), + qualityFactorBounds = BoundsWithDefault(minInclusive = 0.1, maxInclusive = 1.0, default = 0.95), initialDelay = 60.seconds, targetLatency = targetFetchLatency, targetLatencyPercentile = 95.0, @@ -204,41 +258,73 @@ class ScoredTweetsRecommendationPipelineConfig @Inject() ( // candidate pipelines scoredTweetsInNetworkCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, scoredTweetsUtegCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, - scoredTweetsCrMixerCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, + scoredTweetsTweetMixerCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, scoredTweetsFrsCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, + scoredTweetsListsCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, + scoredTweetsPopularVideosCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, + scoredTweetsBackfillCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, // scoring pipelines - scoredTweetsScoringPipelineConfig.identifier -> scoringPipelineQualityFactorConfig, + scoredTweetsModelScoringPipelineConfig.identifier -> scoringPipelineQualityFactorConfig, ) override val scoringPipelines: Seq[ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate]] = Seq( - // scoring pipielines - run on non-cached candidates only since cached ones are already scored - scoredTweetsScoringPipelineConfig, - scoredTweetsWeightedScoresSumScoringPipelineConfig, - // re-scoring pipielines - run on all candidates since these are request specific - ScoredTweetsRescoreOONScoringPipelineConfig, - ScoredTweetsRescoreVerifiedAuthorScoringPipelineConfig, - ScoredTweetsDiversityScoringPipelineConfig + // scoring pipeline - run on non-cached candidates only since cached ones are already scored + scoredTweetsModelScoringPipelineConfig, + // re-scoring pipeline - run on all candidates since these are request specific + ScoredTweetsHeuristicScoringPipelineConfig ) - override val resultSelectors: Seq[Selector[ScoredTweetsQuery]] = Seq( - UpdateSortCandidates(AllPipelines, FeatureValueSorter.descending(ScoreFeature)), - DropMaxCandidates(AllPipelines, ServerMaxResultsParam), - InsertAppendResults(AllPipelines) + override val postScoringFilters = Seq( + ScoredTweetsSocialContextFilter, + OutOfNetworkCompetitorFilter, + OutOfNetworkCompetitorURLFilter, + DuplicateConversationTweetsFilter, + PredicateFeatureFilter.fromPredicate( + FilterIdentifier("IsSupportAccountReply"), + shouldKeepCandidate = { features => + !features.getOrElse(IsSupportAccountReplyFeature, false) + }) ) - override val postSelectionFilters = Seq( - OutOfNetworkCompetitorURLFilter, - KeepBestOutOfNetworkTweetPerAuthorFilter, + override val resultSelectors: Seq[Selector[ScoredTweetsQuery]] = Seq( + KeepBestOutOfNetworkCandidatePerAuthorPerSuggestType(AllPipelines), + UpdateSortCandidates(AllPipelines, FeatureValueSorter.descending(ScoreFeature)), + DropFilteredMaxCandidates( + pipelineScope = + AllExceptPipelines(Set(scoredTweetsBackfillCandidatePipelineConfig.identifier)), + filter = { + case ItemCandidateWithDetails(_, _, features) => + features.getOrElse(InNetworkFeature, false) + case _ => false + }, + maxSelectionsParam = MaxInNetworkResultsParam + ), + DropFilteredMaxCandidates( + pipelineScope = AllPipelines, + filter = { + case ItemCandidateWithDetails(_, _, features) => + !features.getOrElse(InNetworkFeature, false) + case _ => false + }, + maxSelectionsParam = MaxOutOfNetworkResultsParam + ), + DropMaxCandidates( + candidatePipeline = scoredTweetsBackfillCandidatePipelineConfig.identifier, + maxSelectionsParam = StaticParam(MaxBackfillTweets) + ), + InsertAppendResults(AllPipelines) ) override val resultSideEffects: Seq[ PipelineResultSideEffect[ScoredTweetsQuery, ScoredTweetsResponse] ] = Seq( cachedScoredTweetsSideEffect, - scribeServedCommonFeaturesAndCandidateFeaturesSideEffect, publishClientSentImpressionsEventBusSideEffect, publishClientSentImpressionsManhattanSideEffect, + publishImpressionBloomFilterSideEffect, + scribeScoredCandidatesSideEffect, + scribeServedCommonFeaturesAndCandidateFeaturesSideEffect, updateLastNonPollingTimeSideEffect ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/BUILD.bazel index c5cec9df1..c5bb294a1 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/BUILD.bazel @@ -4,10 +4,7 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "cr-mixer/thrift/src/main/thrift:thrift-scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", + "explore/explore-ranker/thrift/src/main/thrift:thrift-scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", @@ -15,27 +12,26 @@ scala_library( "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/cr_mixer", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/explore_ranker", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweet_mixer", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", - "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor", "src/thrift/com/twitter/timelineranker:thrift-scala", + "tweet-mixer/thrift/src/main/thrift:thrift-scala", ], exports = [ - "cr-mixer/thrift/src/main/thrift:thrift-scala", "src/thrift/com/twitter/timelineranker:thrift-scala", + "tweet-mixer/thrift/src/main/thrift:thrift-scala", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/CachedScoredTweetsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/CachedScoredTweetsCandidatePipelineConfig.scala index 95cc79aad..65a42cbee 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/CachedScoredTweetsCandidatePipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/CachedScoredTweetsCandidatePipelineConfig.scala @@ -24,7 +24,7 @@ class CachedScoredTweetsCandidatePipelineConfig @Inject() ( extends CandidatePipelineConfig[ ScoredTweetsQuery, ScoredTweetsQuery, - hmt.CachedScoredTweet, + hmt.ScoredTweet, TweetCandidate ] { @@ -35,15 +35,15 @@ class CachedScoredTweetsCandidatePipelineConfig @Inject() ( ScoredTweetsQuery ] = identity - override val candidateSource: BaseCandidateSource[ScoredTweetsQuery, hmt.CachedScoredTweet] = + override val candidateSource: BaseCandidateSource[ScoredTweetsQuery, hmt.ScoredTweet] = cachedScoredTweetsCandidateSource override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[hmt.CachedScoredTweet] + CandidateFeatureTransformer[hmt.ScoredTweet] ] = Seq(CachedScoredTweetsResponseFeatureTransformer) override val resultTransformer: CandidatePipelineResultsTransformer[ - hmt.CachedScoredTweet, + hmt.ScoredTweet, TweetCandidate ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsBackfillCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsBackfillCandidatePipelineConfig.scala new file mode 100644 index 000000000..c34031c78 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsBackfillCandidatePipelineConfig.scala @@ -0,0 +1,94 @@ +package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline + +import com.twitter.home_mixer.functional_component.filter.ReplyFilter +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TweetypieStaticEntitiesFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate +import com.twitter.home_mixer.product.scored_tweets.gate.MinTimeSinceLastRequestGate +import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidatePipeline +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableBackfillCandidatePipelineParam +import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsBackfillResponseFeatureTransformer +import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter +import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.candidate_source.PassthroughCandidateSource +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.timelines.configapi.FSParam +import com.twitter.timelines.configapi.decider.DeciderParam +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ScoredTweetsBackfillCandidatePipelineConfig @Inject() ( + tweetypieStaticEntitiesHydrator: TweetypieStaticEntitiesFeatureHydrator) + extends CandidatePipelineConfig[ + ScoredTweetsQuery, + ScoredTweetsQuery, + Long, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier("ScoredTweetsBackfill") + + private val HasAuthorFilterId = "HasAuthor" + + override val enabledDeciderParam: Option[DeciderParam[Boolean]] = + Some(CandidatePipeline.EnableBackfillParam) + + override val supportedClientParam: Option[FSParam[Boolean]] = + Some(EnableBackfillCandidatePipelineParam) + + override val gates: Seq[Gate[ScoredTweetsQuery]] = + Seq( + MinTimeSinceLastRequestGate, + NonEmptySeqFeatureGate(TimelineServiceTweetsFeature), + MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + ScoredTweetsQuery, + ScoredTweetsQuery + ] = identity + + override def candidateSource: CandidateSource[ScoredTweetsQuery, Long] = + PassthroughCandidateSource( + identifier = CandidateSourceIdentifier("ScoredTweetsBackfill"), + candidateExtractor = { query => + query.features.map(_.getOrElse(TimelineServiceTweetsFeature, Seq.empty)).toSeq.flatten + } + ) + + override val featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[Long] + ] = Seq(ScoredTweetsBackfillResponseFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[Long, TweetCandidate] = { + sourceResult => TweetCandidate(id = sourceResult) + } + + override val preFilterFeatureHydrationPhase1: Seq[ + BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] + ] = Seq(tweetypieStaticEntitiesHydrator) + + override val filters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq( + ReplyFilter, + PredicateFeatureFilter.fromPredicate( + FilterIdentifier(HasAuthorFilterId), + shouldKeepCandidate = _.getOrElse(AuthorIdFeature, None).isDefined + ) + ) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsFrsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsFrsCandidatePipelineConfig.scala index 018eaa688..f25a839b7 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsFrsCandidatePipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsFrsCandidatePipelineConfig.scala @@ -1,9 +1,10 @@ package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline -import com.twitter.home_mixer.functional_component.gate.MinCachedTweetsGate +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.FrsSeedUsersQueryFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets -import com.twitter.home_mixer.product.scored_tweets.query_feature_hydrator.FrsSeedUsersQueryFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidatePipeline import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerFrsQueryTransformer import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsFrsResponseFeatureTransformer import com.twitter.product_mixer.component_library.candidate_source.timeline_ranker.TimelineRankerRecapCandidateSource @@ -17,6 +18,7 @@ import com.twitter.product_mixer.core.functional_component.transformer.Candidate import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.timelineranker.{thriftscala => tlr} +import com.twitter.timelines.configapi.decider.DeciderParam import javax.inject.Inject import javax.inject.Singleton @@ -40,6 +42,9 @@ class ScoredTweetsFrsCandidatePipelineConfig @Inject() ( override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ScoredTweetsFrs") + override val enabledDeciderParam: Option[DeciderParam[Boolean]] = + Some(CandidatePipeline.EnableFrsParam) + override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsInNetworkCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsInNetworkCandidatePipelineConfig.scala index 28118238d..63bd1add8 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsInNetworkCandidatePipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsInNetworkCandidatePipelineConfig.scala @@ -1,12 +1,13 @@ package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline -import com.twitter.home_mixer.functional_component.feature_hydrator.ReplyFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.RetweetSourceTweetFeatureHydrator -import com.twitter.home_mixer.functional_component.filter.RetweetSourceTweetRemovingFilter -import com.twitter.home_mixer.functional_component.gate.MinCachedTweetsGate +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.IsExtendedReplyFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.ReplyFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RetweetSourceTweetFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.filter.RetweetSourceTweetRemovingFilter +import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.InNetworkSource +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidatePipeline import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerInNetworkQueryTransformer import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsInNetworkResponseFeatureTransformer import com.twitter.product_mixer.component_library.candidate_source.timeline_ranker.TimelineRankerInNetworkCandidateSource @@ -23,7 +24,6 @@ import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.timelineranker.{thriftscala => t} import com.twitter.timelines.configapi.decider.DeciderParam - import javax.inject.Inject import javax.inject.Singleton @@ -45,7 +45,7 @@ class ScoredTweetsInNetworkCandidatePipelineConfig @Inject() ( CandidatePipelineIdentifier("ScoredTweetsInNetwork") override val enabledDeciderParam: Option[DeciderParam[Boolean]] = - Some(InNetworkSource.EnableCandidatePipelineParam) + Some(CandidatePipeline.EnableInNetworkParam) override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) @@ -69,7 +69,7 @@ class ScoredTweetsInNetworkCandidatePipelineConfig @Inject() ( override val postFilterFeatureHydration: Seq[ BaseCandidateFeatureHydrator[PipelineQuery, TweetCandidate, _] - ] = Seq(replyFeatureHydrator) + ] = Seq(IsExtendedReplyFeatureHydrator, replyFeatureHydrator) override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[t.CandidateTweet] diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsListsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsListsCandidatePipelineConfig.scala new file mode 100644 index 000000000..1161a8278 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsListsCandidatePipelineConfig.scala @@ -0,0 +1,86 @@ +package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline + +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TweetypieStaticEntitiesFeatureHydrator +import com.twitter.home_mixer.functional_component.filter.ReplyFilter +import com.twitter.home_mixer.functional_component.filter.RetweetFilter +import com.twitter.home_mixer.product.scored_tweets.candidate_source.ListsCandidateSource +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.ListIdsFeature +import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate +import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidatePipeline +import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsListsResponseFeatureTransformer +import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.timelines.configapi.decider.DeciderParam +import com.twitter.timelineservice.{thriftscala => t} +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ScoredTweetsListsCandidatePipelineConfig @Inject() ( + listsCandidateSource: ListsCandidateSource, + tweetypieStaticEntitiesHydrator: TweetypieStaticEntitiesFeatureHydrator) + extends CandidatePipelineConfig[ + ScoredTweetsQuery, + Seq[t.TimelineQuery], + t.Tweet, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier("ScoredTweetsLists") + + private val MaxTweetsToFetchPerList = 20 + + override val enabledDeciderParam: Option[DeciderParam[Boolean]] = + Some(CandidatePipeline.EnableListsParam) + + override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( + NonEmptySeqFeatureGate(ListIdsFeature), + MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + ScoredTweetsQuery, + Seq[t.TimelineQuery] + ] = { query => + val listIds = query.features.map(_.get(ListIdsFeature)).get + listIds.map { listId => + t.TimelineQuery( + timelineType = t.TimelineType.List, + timelineId = listId, + maxCount = MaxTweetsToFetchPerList.toShort, + options = Some(t.TimelineQueryOptions(query.clientContext.userId)), + timelineId2 = Some(t.TimelineId(t.TimelineType.List, listId, None)) + ) + } + } + + override def candidateSource: CandidateSource[Seq[t.TimelineQuery], t.Tweet] = + listsCandidateSource + + override val featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[t.Tweet] + ] = Seq(ScoredTweetsListsResponseFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[t.Tweet, TweetCandidate] = { + sourceResult => TweetCandidate(id = sourceResult.statusId) + } + + override val preFilterFeatureHydrationPhase1: Seq[ + BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] + ] = Seq(tweetypieStaticEntitiesHydrator) + + override val filters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = + Seq(ReplyFilter, RetweetFilter) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsPopularVideosCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsPopularVideosCandidatePipelineConfig.scala new file mode 100644 index 000000000..c8b28b0ff --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsPopularVideosCandidatePipelineConfig.scala @@ -0,0 +1,82 @@ +package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline + +import com.twitter.explore_ranker.{thriftscala => ert} +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TweetypieStaticEntitiesFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate +import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidatePipeline +import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsPopularVideosResponseFeatureTransformer +import com.twitter.home_mixer.util.CachedScoredTweetsHelper +import com.twitter.product_mixer.component_library.candidate_source.explore_ranker.ExploreRankerImmersiveRecsCandidateSource +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.marshaller.request.ClientContextMarshaller +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.timelines.configapi.decider.DeciderParam +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ScoredTweetsPopularVideosCandidatePipelineConfig @Inject() ( + exploreRankerCandidateSource: ExploreRankerImmersiveRecsCandidateSource, + tweetypieStaticEntitiesFeatureHydrator: TweetypieStaticEntitiesFeatureHydrator) + extends CandidatePipelineConfig[ + ScoredTweetsQuery, + ert.ExploreRankerRequest, + ert.ExploreTweetRecommendation, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier("ScoredTweetsPopularVideos") + + private val MaxTweetsToFetch = 40 + + override val enabledDeciderParam: Option[DeciderParam[Boolean]] = + Some(CandidatePipeline.EnablePopularVideosParam) + + override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( + MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + ScoredTweetsQuery, + ert.ExploreRankerRequest + ] = { query => + val excludedTweetIds = query.features.map( + CachedScoredTweetsHelper.tweetImpressionsAndCachedScoredTweets(_, identifier)) + + ert.ExploreRankerRequest( + clientContext = ClientContextMarshaller(query.clientContext), + product = ert.Product.HomeTimelineVideoInline, + productContext = Some( + ert.ProductContext.HomeTimelineVideoInline(ert.HomeTimelineVideoInline(excludedTweetIds))), + maxResults = Some(MaxTweetsToFetch) + ) + } + + override def candidateSource: BaseCandidateSource[ + ert.ExploreRankerRequest, + ert.ExploreTweetRecommendation + ] = exploreRankerCandidateSource + + override val featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[ert.ExploreTweetRecommendation] + ] = Seq(ScoredTweetsPopularVideosResponseFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + ert.ExploreTweetRecommendation, + TweetCandidate + ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } + + override val preFilterFeatureHydrationPhase1: Seq[ + BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] + ] = Seq(tweetypieStaticEntitiesFeatureHydrator) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsCrMixerCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsTweetMixerCandidatePipelineConfig.scala similarity index 70% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsCrMixerCandidatePipelineConfig.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsTweetMixerCandidatePipelineConfig.scala index 1ec9353f8..df16e3f11 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsCrMixerCandidatePipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsTweetMixerCandidatePipelineConfig.scala @@ -1,16 +1,16 @@ package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline -import com.twitter.cr_mixer.{thriftscala => t} -import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieStaticEntitiesFeatureHydrator -import com.twitter.home_mixer.functional_component.filter.PredicateFeatureFilter -import com.twitter.home_mixer.functional_component.gate.MinCachedTweetsGate +import com.twitter.tweet_mixer.{thriftscala => t} import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TweetypieStaticEntitiesFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CrMixerSource -import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsCrMixerResponseFeatureTransformer +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidatePipeline +import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsTweetMixerResponseFeatureTransformer import com.twitter.home_mixer.util.CachedScoredTweetsHelper -import com.twitter.product_mixer.component_library.candidate_source.cr_mixer.CrMixerTweetRecommendationsCandidateSource +import com.twitter.product_mixer.component_library.candidate_source.tweet_mixer.TweetMixerCandidateSource +import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator @@ -24,55 +24,57 @@ import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineI import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.timelines.configapi.decider.DeciderParam + import javax.inject.Inject import javax.inject.Singleton /** - * Candidate Pipeline Config that fetches tweets from CrMixer. + * Candidate Pipeline Config that fetches tweets from TweetMixer. */ @Singleton -class ScoredTweetsCrMixerCandidatePipelineConfig @Inject() ( - crMixerTweetRecommendationsCandidateSource: CrMixerTweetRecommendationsCandidateSource, +class ScoredTweetsTweetMixerCandidatePipelineConfig @Inject() ( + tweetMixerCandidateSource: TweetMixerCandidateSource, tweetypieStaticEntitiesFeatureHydrator: TweetypieStaticEntitiesFeatureHydrator) extends CandidatePipelineConfig[ ScoredTweetsQuery, - t.CrMixerTweetRequest, - t.TweetRecommendation, + t.TweetMixerRequest, + t.TweetResult, TweetCandidate ] { override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ScoredTweetsCrMixer") + CandidatePipelineIdentifier("ScoredTweetsTweetMixer") val HasAuthorFilterId = "HasAuthor" override val enabledDeciderParam: Option[DeciderParam[Boolean]] = - Some(CrMixerSource.EnableCandidatePipelineParam) + Some(CandidatePipeline.EnableTweetMixerParam) override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( - MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) + MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam), ) - override val candidateSource: BaseCandidateSource[t.CrMixerTweetRequest, t.TweetRecommendation] = - crMixerTweetRecommendationsCandidateSource + override val candidateSource: BaseCandidateSource[t.TweetMixerRequest, t.TweetResult] = + tweetMixerCandidateSource - private val MaxTweetsToFetch = 500 + private val MaxTweetsToFetch = 400 override val queryTransformer: CandidatePipelineQueryTransformer[ ScoredTweetsQuery, - t.CrMixerTweetRequest + t.TweetMixerRequest ] = { query => val maxCount = (query.getQualityFactorCurrentValue(identifier) * MaxTweetsToFetch).toInt val excludedTweetIds = query.features.map( CachedScoredTweetsHelper.tweetImpressionsAndCachedScoredTweets(_, identifier)) - t.CrMixerTweetRequest( + t.TweetMixerRequest( clientContext = ClientContextMarshaller(query.clientContext), - product = t.Product.Home, - productContext = - Some(t.ProductContext.HomeContext(t.HomeContext(maxResults = Some(maxCount)))), - excludedTweetIds = excludedTweetIds + product = t.Product.HomeRecommendedTweets, + productContext = Some( + t.ProductContext.HomeRecommendedTweetsProductContext( + t.HomeRecommendedTweetsProductContext(excludedTweetIds = excludedTweetIds.map(_.toSet)))), + maxResults = Some(maxCount) ) } @@ -81,8 +83,8 @@ class ScoredTweetsCrMixerCandidatePipelineConfig @Inject() ( ] = Seq(tweetypieStaticEntitiesFeatureHydrator) override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[t.TweetRecommendation] - ] = Seq(ScoredTweetsCrMixerResponseFeatureTransformer) + CandidateFeatureTransformer[t.TweetResult] + ] = Seq(ScoredTweetsTweetMixerResponseFeatureTransformer) override val filters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq( PredicateFeatureFilter.fromPredicate( @@ -92,7 +94,7 @@ class ScoredTweetsCrMixerCandidatePipelineConfig @Inject() ( ) override val resultTransformer: CandidatePipelineResultsTransformer[ - t.TweetRecommendation, + t.TweetResult, TweetCandidate ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsUtegCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsUtegCandidatePipelineConfig.scala index b58ca784f..9178d0789 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsUtegCandidatePipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsUtegCandidatePipelineConfig.scala @@ -1,9 +1,9 @@ package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline -import com.twitter.home_mixer.functional_component.gate.MinCachedTweetsGate +import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.UtegSource +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidatePipeline import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerUtegQueryTransformer import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsUtegResponseFeatureTransformer import com.twitter.product_mixer.component_library.candidate_source.timeline_ranker.TimelineRankerUtegCandidateSource @@ -17,7 +17,6 @@ import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineI import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.timelineranker.{thriftscala => t} import com.twitter.timelines.configapi.decider.DeciderParam - import javax.inject.Inject import javax.inject.Singleton @@ -38,7 +37,7 @@ class ScoredTweetsUtegCandidatePipelineConfig @Inject() ( CandidatePipelineIdentifier("ScoredTweetsUteg") override val enabledDeciderParam: Option[DeciderParam[Boolean]] = - Some(UtegSource.EnableCandidatePipelineParam) + Some(CandidatePipeline.EnableUtegParam) override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/BUILD.bazel new file mode 100644 index 000000000..618c4a081 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/BUILD.bazel @@ -0,0 +1,21 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/earlybird", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", + "src/thrift/com/twitter/search:earlybird-scala", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdFrsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdFrsCandidatePipelineConfig.scala new file mode 100644 index 000000000..0d59bf3ca --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdFrsCandidatePipelineConfig.scala @@ -0,0 +1,69 @@ +package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.earlybird + +import com.twitter.finagle.thrift.ClientId +import com.twitter.home_mixer.functional_component.candidate_source.EarlybirdCandidateSource +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.FrsSeedUsersQueryFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate +import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets +import com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird.EarlybirdFrsQueryTransformer +import com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird.ScoredTweetsEarlybirdFrsResponseFeatureTransformer +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.search.earlybird.{thriftscala => eb} +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Candidate Pipeline Config that fetches tweets from the earlybird FRS Candidate Source + */ +@Singleton +class ScoredTweetsEarlybirdFrsCandidatePipelineConfig @Inject() ( + earlybirdCandidateSource: EarlybirdCandidateSource, + frsSeedUsersQueryFeatureHydrator: FrsSeedUsersQueryFeatureHydrator, + clientId: ClientId) + extends CandidatePipelineConfig[ + ScoredTweetsQuery, + eb.EarlybirdRequest, + eb.ThriftSearchResult, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier("ScoredTweetsEarlybirdFrs") + + override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( + MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) + ) + + override val queryFeatureHydration: Seq[ + BaseQueryFeatureHydrator[ScoredTweetsQuery, _] + ] = Seq(frsSeedUsersQueryFeatureHydrator) + + override val candidateSource: BaseCandidateSource[eb.EarlybirdRequest, eb.ThriftSearchResult] = + earlybirdCandidateSource + + override val queryTransformer: CandidatePipelineQueryTransformer[ + ScoredTweetsQuery, + eb.EarlybirdRequest + ] = EarlybirdFrsQueryTransformer(identifier, clientId = Some(clientId.name)) + + override val featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[eb.ThriftSearchResult] + ] = Seq(ScoredTweetsEarlybirdFrsResponseFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + eb.ThriftSearchResult, + TweetCandidate + ] = { sourceResult => TweetCandidate(id = sourceResult.id) } + + override def filters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq.empty +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig.scala new file mode 100644 index 000000000..acb26ce42 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig.scala @@ -0,0 +1,59 @@ +package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.earlybird + +import com.twitter.finagle.thrift.ClientId +import com.twitter.home_mixer.functional_component.candidate_source.EarlybirdCandidateSource +import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate +import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets +import com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird.EarlybirdInNetworkQueryTransformer +import com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird.ScoredTweetsEarlybirdInNetworkResponseFeatureTransformer +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.search.earlybird.{thriftscala => eb} +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Candidate Pipeline Config that fetches tweets from the earlybird InNetwork Candidate Source + */ +@Singleton +class ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig @Inject() ( + earlybirdCandidateSource: EarlybirdCandidateSource, + clientId: ClientId) + extends CandidatePipelineConfig[ + ScoredTweetsQuery, + eb.EarlybirdRequest, + eb.ThriftSearchResult, + TweetCandidate + ] { + + override val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier("ScoredTweetsEarlybirdInNetwork") + + override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( + MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) + ) + + override val candidateSource: BaseCandidateSource[eb.EarlybirdRequest, eb.ThriftSearchResult] = + earlybirdCandidateSource + + override val queryTransformer: CandidatePipelineQueryTransformer[ + ScoredTweetsQuery, + eb.EarlybirdRequest + ] = EarlybirdInNetworkQueryTransformer(identifier, clientId = Some(clientId.name)) + + override val featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[eb.ThriftSearchResult] + ] = Seq(ScoredTweetsEarlybirdInNetworkResponseFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + eb.ThriftSearchResult, + TweetCandidate + ] = { sourceResult => TweetCandidate(id = sourceResult.id) } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/BUILD.bazel index 58fe8621d..4461d8460 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/BUILD.bazel @@ -4,9 +4,10 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", + "src/thrift/com/twitter/timelineservice:thrift-scala", + "stitch/stitch-timelineservice", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/CachedScoredTweetsCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/CachedScoredTweetsCandidateSource.scala index 15a522efa..a5f556e7c 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/CachedScoredTweetsCandidateSource.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/CachedScoredTweetsCandidateSource.scala @@ -12,12 +12,12 @@ import javax.inject.Singleton @Singleton class CachedScoredTweetsCandidateSource @Inject() () - extends CandidateSource[PipelineQuery, hmt.CachedScoredTweet] { + extends CandidateSource[PipelineQuery, hmt.ScoredTweet] { override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("CachedScoredTweets") - override def apply(request: PipelineQuery): Stitch[Seq[hmt.CachedScoredTweet]] = { + override def apply(request: PipelineQuery): Stitch[Seq[hmt.ScoredTweet]] = { Stitch.value( request.features.map(CachedScoredTweetsHelper.unseenCachedScoredTweets).getOrElse(Seq.empty)) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/ListsCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/ListsCandidateSource.scala new file mode 100644 index 000000000..395ac7da8 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/ListsCandidateSource.scala @@ -0,0 +1,27 @@ +package com.twitter.home_mixer.product.scored_tweets.candidate_source + +import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource +import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier +import com.twitter.stitch.Stitch +import com.twitter.stitch.timelineservice.TimelineService +import com.twitter.timelineservice.{thriftscala => tls} + +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ListsCandidateSource @Inject() (timelineService: TimelineService) + extends CandidateSource[Seq[tls.TimelineQuery], tls.Tweet] { + + override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("Lists") + + override def apply(requests: Seq[tls.TimelineQuery]): Stitch[Seq[tls.Tweet]] = { + val timelines = Stitch.traverse(requests) { request => timelineService.getTimeline(request) } + + timelines.map { + _.flatMap { + _.entries.collect { case tls.TimelineEntry.Tweet(tweet) => tweet } + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/AncestorFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AncestorFeatureHydrator.scala similarity index 75% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/AncestorFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AncestorFeatureHydrator.scala index 8d1a0862d..9c5e8f3ab 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/AncestorFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AncestorFeatureHydrator.scala @@ -1,6 +1,7 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature +import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap @@ -8,9 +9,11 @@ import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.tweetconvosvc.tweet_ancestor.{thriftscala => ta} import com.twitter.tweetconvosvc.{thriftscala => tcs} +import com.twitter.util.Future import javax.inject.Inject import javax.inject.Singleton @@ -23,15 +26,16 @@ class AncestorFeatureHydrator @Inject() ( override val features: Set[Feature[_, _]] = Set(AncestorsFeature) + private val DefaultFeatureMap = FeatureMapBuilder().add(AncestorsFeature, Seq.empty).build() + override def apply( query: PipelineQuery, candidate: TweetCandidate, existingFeatures: FeatureMap - ): Stitch[FeatureMap] = { - val ancestorsRequest = tcs.GetAncestorsRequest(Seq(candidate.id)) - - Stitch.callFuture(conversationServiceClient.getAncestors(ancestorsRequest)).map { - getAncestorsResponse => + ): Stitch[FeatureMap] = OffloadFuturePools.offloadFuture { + if (existingFeatures.getOrElse(InReplyToTweetIdFeature, None).isDefined) { + val ancestorsRequest = tcs.GetAncestorsRequest(Seq(candidate.id)) + conversationServiceClient.getAncestors(ancestorsRequest).map { getAncestorsResponse => val ancestors = getAncestorsResponse.ancestors.headOption .collect { case tcs.TweetAncestorsResult.TweetAncestors(ancestorsResult) @@ -40,7 +44,8 @@ class AncestorFeatureHydrator @Inject() ( }.getOrElse(Seq.empty) FeatureMapBuilder().add(AncestorsFeature, ancestors).build() - } + } + } else Future.value(DefaultFeatureMap) } private def getTruncatedRootTweet( diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/AuthorFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AuthorFeatureHydrator.scala similarity index 68% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/AuthorFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AuthorFeatureHydrator.scala index a01269c2c..c8dd6a934 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/AuthorFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AuthorFeatureHydrator.scala @@ -1,28 +1,28 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.author_features.AuthorFeaturesAdapter -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.AuthorFeatureRepository +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.author_features.AuthorFeaturesAdapter +import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.repository.KeyValueResult +import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.servo.repository.KeyValueRepository +import com.twitter.servo.repository.KeyValueResult import com.twitter.stitch.Stitch import com.twitter.timelines.author_features.v1.{thriftjava => af} import com.twitter.util.Future import com.twitter.util.Try - import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @@ -51,34 +51,28 @@ class AuthorFeatureHydrator @Inject() ( override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - Stitch.callFuture { - val possiblyAuthorIds = extractKeys(candidates) - val authorIds = possiblyAuthorIds.flatten + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + val possiblyAuthorIds = extractKeys(candidates) + val authorIds = possiblyAuthorIds.flatten - val response: Future[KeyValueResult[Long, af.AuthorFeatures]] = - if (authorIds.isEmpty) { - Future.value(KeyValueResult.empty) - } else { - client(authorIds) - } + val response: Future[KeyValueResult[Long, af.AuthorFeatures]] = + if (authorIds.nonEmpty) client(authorIds) + else Future.value(KeyValueResult.empty) - response.map { result => - possiblyAuthorIds.map { possiblyAuthorId => - val value = observedGet(key = possiblyAuthorId, keyValueResult = result) - val transformedValue = postTransformer(value) + response.map { result => + possiblyAuthorIds.map { possiblyAuthorId => + val value = observedGet(key = possiblyAuthorId, keyValueResult = result) + val transformedValue = postTransformer(value) - FeatureMapBuilder() - .add(AuthorFeature, transformedValue) - .build() - } + FeatureMapBuilder().add(AuthorFeature, transformedValue).build() } } } private def postTransformer(authorFeatures: Try[Option[af.AuthorFeatures]]): Try[DataRecord] = { - authorFeatures.map { features => - AuthorFeaturesAdapter.adaptToDataRecords(features).asScala.head + authorFeatures.map { + _.map { features => AuthorFeaturesAdapter.adaptToDataRecords(features).asScala.head } + .getOrElse(new DataRecord()) } } @@ -86,10 +80,7 @@ class AuthorFeatureHydrator @Inject() ( candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[Option[Long]] = { candidates.map { candidate => - candidate.features - .getTry(AuthorIdFeature) - .toOption - .flatten + CandidatesUtil.getOriginalAuthorId(candidate.features) } } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AuthorIsCreatorFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AuthorIsCreatorFeatureHydrator.scala new file mode 100644 index 000000000..cc66983cc --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AuthorIsCreatorFeatureHydrator.scala @@ -0,0 +1,79 @@ +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.AuthorIsCreatorFeature +import com.twitter.home_mixer.util.MissingKeyException +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.strato.generated.client.audiencerewards.audienceRewardsService.GetSuperFollowEligibilityOnUserClientColumn +import com.twitter.util.Throw +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class AuthorIsCreatorFeatureHydrator @Inject() ( + getSuperFollowEligibilityOnUserClientColumn: GetSuperFollowEligibilityOnUserClientColumn, + statsReceiver: StatsReceiver) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("AuthorIsCreator") + + override val features: Set[Feature[_, _]] = + Set(AuthorIsCreatorFeature) + + private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val keyFoundCounter = scopedStatsReceiver.counter("key/found") + private val keyFailureCounter = scopedStatsReceiver.counter("key/failure") + + private val MissingKeyFeatureMap = FeatureMapBuilder() + .add(AuthorIsCreatorFeature, Throw(MissingKeyException)) + .build() + + private val DefaultFeatureMap = FeatureMapBuilder() + .add(AuthorIsCreatorFeature, false) + .build() + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = { + OffloadFuturePools.offloadStitch { + val authorIds = candidates.flatMap(_.features.getOrElse(AuthorIdFeature, None)).distinct + Stitch + .collect { + authorIds.map { authorId => + getSuperFollowEligibilityOnUserClientColumn.fetcher + .fetch(authorId) + .map { authorId -> _.v } + } + }.map { authorIdsToIsCreator => + val authorIdsToIsCreatorMap = authorIdsToIsCreator.toMap + candidates.map { candidate => + candidate.features.getOrElse(AuthorIdFeature, None) match { + case Some(authorId) => + authorIdsToIsCreatorMap.get(authorId) match { + case Some(response) => + keyFoundCounter.incr() + FeatureMapBuilder() + .add(AuthorIsCreatorFeature, response.getOrElse(false)).build() + case _ => + keyFailureCounter.incr() + DefaultFeatureMap + } + case _ => MissingKeyFeatureMap + } + } + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/BUILD.bazel index 504826eed..58a566de8 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/BUILD.bazel @@ -4,26 +4,67 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ + "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content", "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline/transformer", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/recommendations", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", + "representation-scorer/server/src/main/scala/com/twitter/representationscorer/common", "servo/repo/src/main/scala", "servo/util/src/main/scala", + "snowflake/src/main/scala/com/twitter/snowflake/id", + "src/java/com/twitter/ml/api/constant", + "src/scala/com/twitter/ml/api/util", + "src/scala/com/twitter/timelines/prediction/adapters/conversation_features", + "src/scala/com/twitter/timelines/prediction/adapters/real_graph", + "src/scala/com/twitter/timelines/prediction/adapters/realtime_interaction_graph", + "src/scala/com/twitter/timelines/prediction/adapters/twistly", + "src/scala/com/twitter/timelines/prediction/adapters/two_hop_features", + "src/scala/com/twitter/timelines/prediction/common/adapters", + "src/scala/com/twitter/timelines/prediction/common/util", + "src/scala/com/twitter/timelines/prediction/features/common", + "src/scala/com/twitter/timelines/prediction/features/realtime_interaction_graph", + "src/scala/com/twitter/timelines/prediction/features/time_features", + "src/thrift/com/twitter/ml/api:data-java", + "src/thrift/com/twitter/onboarding/relevance/features:features-java", + "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", + "src/thrift/com/twitter/search:earlybird-scala", "src/thrift/com/twitter/timelineranker:thrift-scala", - "stitch/stitch-core", - "timelineranker/common/src/main/scala/com/twitter/timelineranker/model", + "src/thrift/com/twitter/timelines/author_features:thrift-java", + "stitch/stitch-gizmoduck", + "stitch/stitch-socialgraph", + "stitch/stitch-tweetypie", + "strato/config/columns/audiencerewards/audienceRewardsService:getSuperFollowEligibility-strato-client", + "strato/config/columns/ml/featureStore:featureStore-strato-client", + "timelines/src/main/scala/com/twitter/timelines/clients/strato/topics", + "timelines/src/main/scala/com/twitter/timelines/clients/strato/twistly", "timelines/src/main/scala/com/twitter/timelines/common/model", "timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils", "timelines/src/main/scala/com/twitter/timelines/model/candidate", "timelines/src/main/scala/com/twitter/timelines/model/types", + "topic-social-proof/server/src/main/thrift:thrift-scala", + "topiclisting/topiclisting-core/src/main/scala/com/twitter/topiclisting", + "tweetconvosvc/thrift/src/main/thrift:thrift-scala", + "user_session_store/src/main/scala/com/twitter/user_session_store", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/CachedScoredTweetsQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/CachedScoredTweetsQueryFeatureHydrator.scala index 7801a9645..873f043be 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/CachedScoredTweetsQueryFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/CachedScoredTweetsQueryFeatureHydrator.scala @@ -11,7 +11,6 @@ import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIde import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.cache.TtlCache import com.twitter.stitch.Stitch -import com.twitter.timelines.model.UserId import com.twitter.util.Return import com.twitter.util.Throw import com.twitter.util.Time @@ -20,11 +19,11 @@ import javax.inject.Inject import javax.inject.Singleton /** - * Fetch scored Tweets from cache and exclude the seen ones + * Fetch scored Tweets from cache and exclude the expired ones */ @Singleton case class CachedScoredTweetsQueryFeatureHydrator @Inject() ( - scoredTweetsCache: TtlCache[UserId, hmt.CachedScoredTweets]) + scoredTweetsCache: TtlCache[Long, hmt.ScoredTweetsResponse]) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = @@ -39,7 +38,7 @@ case class CachedScoredTweetsQueryFeatureHydrator @Inject() ( Stitch.callFuture(scoredTweetsCache.get(Seq(userId))).map { keyValueResult => keyValueResult(userId) match { case Return(cachedCandidatesOpt) => - val cachedScoredTweets = cachedCandidatesOpt.map(_.tweets).getOrElse(Seq.empty) + val cachedScoredTweets = cachedCandidatesOpt.map(_.scoredTweets).getOrElse(Seq.empty) val nonExpiredTweets = cachedScoredTweets.filter { tweet => tweet.lastScoredTimestampMs.exists(Time.fromMilliseconds(_).untilNow < tweetScoreTtl) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/EarlybirdFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/EarlybirdFeatureHydrator.scala similarity index 68% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/EarlybirdFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/EarlybirdFeatureHydrator.scala index 639c3ac37..cc3c2aeeb 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/EarlybirdFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/EarlybirdFeatureHydrator.scala @@ -1,9 +1,10 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.earlybird.EarlybirdAdapter +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.earlybird.EarlybirdAdapter import com.twitter.home_mixer.model.HomeFeatures.DeviceLanguageFeature import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature +import com.twitter.home_mixer.model.HomeFeatures.EarlybirdSearchResultFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature import com.twitter.home_mixer.model.HomeFeatures.UserScreenNameFeature @@ -11,6 +12,7 @@ import com.twitter.home_mixer.param.HomeMixerInjectionNames.EarlybirdRepository import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.home_mixer.util.earlybird.EarlybirdResponseUtil import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure @@ -21,6 +23,7 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.search.earlybird.{thriftscala => eb} import com.twitter.servo.keyvalue.KeyValueResult import com.twitter.servo.repository.KeyValueRepository @@ -50,8 +53,12 @@ class EarlybirdFeatureHydrator @Inject() ( override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Earlybird") - override val features: Set[Feature[_, _]] = - Set(EarlybirdDataRecordFeature, EarlybirdFeature, TweetUrlsFeature) + override val features: Set[Feature[_, _]] = Set( + EarlybirdDataRecordFeature, + EarlybirdFeature, + EarlybirdSearchResultFeature, + TweetUrlsFeature + ) override val statScope: String = identifier.toString @@ -59,52 +66,66 @@ class EarlybirdFeatureHydrator @Inject() ( private val originalKeyFoundCounter = scopedStatsReceiver.counter("originalKey/found") private val originalKeyLossCounter = scopedStatsReceiver.counter("originalKey/loss") + private val ebSearchResultNotExistPredicate: CandidateWithFeatures[TweetCandidate] => Boolean = + candidate => candidate.features.getOrElse(EarlybirdSearchResultFeature, None).isEmpty private val ebFeaturesNotExistPredicate: CandidateWithFeatures[TweetCandidate] => Boolean = candidate => candidate.features.getOrElse(EarlybirdFeature, None).isEmpty override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val candidatesToHydrate = candidates.filter { candidate => - val isEmpty = ebFeaturesNotExistPredicate(candidate) + val isEmpty = + ebFeaturesNotExistPredicate(candidate) && ebSearchResultNotExistPredicate(candidate) if (isEmpty) originalKeyLossCounter.incr() else originalKeyFoundCounter.incr() isEmpty } - Stitch - .callFuture(client((candidatesToHydrate.map(_.candidate.id), query.getRequiredUserId))) - .map(handleResponse(query, candidates, _)) + + client((candidatesToHydrate.map(_.candidate.id), query.getRequiredUserId)) + .map(handleResponse(query, candidates, _, candidatesToHydrate)) } private def handleResponse( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]], - results: KeyValueResult[Long, eb.ThriftSearchResult] + results: KeyValueResult[Long, eb.ThriftSearchResult], + candidatesToHydrate: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[FeatureMap] = { val queryFeatureMap = query.features.getOrElse(FeatureMap.empty) val userLanguages = queryFeatureMap.getOrElse(UserLanguagesFeature, Seq.empty) val uiLanguageCode = queryFeatureMap.getOrElse(DeviceLanguageFeature, None) val screenName = queryFeatureMap.getOrElse(UserScreenNameFeature, None) + val followedUserIds = queryFeatureMap.getOrElse(SGSFollowedUsersFeature, Seq.empty).toSet - val searchResults = candidates - .filter(ebFeaturesNotExistPredicate).map { candidate => + val searchResults = candidatesToHydrate + .map { candidate => observedGet(Some(candidate.candidate.id), results) }.collect { case Return(Some(value)) => value } - val tweetIdToEbFeatures = EarlybirdResponseUtil.getOONTweetThriftFeaturesByTweetId( + val allSearchResults = searchResults ++ + candidates.filter(!ebSearchResultNotExistPredicate(_)).flatMap { candidate => + candidate.features + .getOrElse(EarlybirdSearchResultFeature, None) + } + val idToSearchResults = allSearchResults.map(sr => sr.id -> sr).toMap + val tweetIdToEbFeatures = EarlybirdResponseUtil.getTweetThriftFeaturesByTweetId( searcherUserId = query.getRequiredUserId, screenName = screenName, userLanguages = userLanguages, uiLanguageCode = uiLanguageCode, - searchResults = searchResults + followedUserIds = followedUserIds, + mutuallyFollowingUserIds = Set.empty, + searchResults = allSearchResults, + sourceTweetSearchResults = Seq.empty, ) candidates.map { candidate => - val hydratedEbFeatures = tweetIdToEbFeatures.get(candidate.candidate.id) + val transformedEbFeatures = tweetIdToEbFeatures.get(candidate.candidate.id) val earlybirdFeatures = - if (hydratedEbFeatures.nonEmpty) hydratedEbFeatures + if (transformedEbFeatures.nonEmpty) transformedEbFeatures else candidate.features.getOrElse(EarlybirdFeature, None) val candidateIsRetweet = candidate.features.getOrElse(IsRetweetFeature, false) @@ -122,6 +143,7 @@ class EarlybirdFeatureHydrator @Inject() ( FeatureMapBuilder() .add(EarlybirdFeature, earlybirdFeatures) .add(EarlybirdDataRecordFeature, earlybirdDataRecord) + .add(EarlybirdSearchResultFeature, idToSearchResults.get(candidate.candidate.id)) .add(TweetUrlsFeature, earlybirdFeatures.flatMap(_.urlsList).getOrElse(Seq.empty)) .build() } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_feature_hydrator/FrsSeedUsersQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/FrsSeedUsersQueryFeatureHydrator.scala similarity index 97% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_feature_hydrator/FrsSeedUsersQueryFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/FrsSeedUsersQueryFeatureHydrator.scala index 194146dac..2508067b0 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_feature_hydrator/FrsSeedUsersQueryFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/FrsSeedUsersQueryFeatureHydrator.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.product.scored_tweets.query_feature_hydrator +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.follow_recommendations.{thriftscala => frs} import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/GizmoduckAuthorFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/GizmoduckAuthorFeatureHydrator.scala new file mode 100644 index 000000000..7c96ef474 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/GizmoduckAuthorFeatureHydrator.scala @@ -0,0 +1,83 @@ +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator + +import com.twitter.conversions.DurationOps._ +import com.twitter.ads.entities.db.{thriftscala => ae} +import com.twitter.gizmoduck.{thriftscala => gt} +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature +import com.twitter.home_mixer.model.HomeFeatures.AuthorIsProtectedFeature +import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature +import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature +import com.twitter.home_mixer.model.HomeFeatures.IsSupportAccountReplyFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.snowflake.id.SnowflakeId +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class GizmoduckAuthorFeatureHydrator @Inject() (gizmoduck: gt.UserService.MethodPerEndpoint) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("GizmoduckAuthor") + + override val features: Set[Feature[_, _]] = + Set(AuthorIsBlueVerifiedFeature, AuthorIsProtectedFeature, IsSupportAccountReplyFeature) + + private val queryFields: Set[gt.QueryFields] = + Set(gt.QueryFields.AdvertiserAccount, gt.QueryFields.Profile, gt.QueryFields.Safety) + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + val authorIds = candidates.flatMap(_.features.getOrElse(AuthorIdFeature, None)) + + val response = gizmoduck.get( + userIds = authorIds.distinct, + queryFields = queryFields, + context = gt.LookupContext() + ) + + response.map { hydratedAuthors => + val userMetadataMap = hydratedAuthors + .collect { + case userResult if userResult.user.isDefined => + val user = userResult.user.get + val blueVerified = user.safety.flatMap(_.isBlueVerified).getOrElse(false) + val isProtected = user.safety.exists(_.isProtected) + (user.id, (blueVerified, isProtected)) + }.toMap.withDefaultValue((false, false)) + + candidates.map { candidate => + val authorId = candidate.features.get(AuthorIdFeature).get + val (isBlueVerified, isProtected) = userMetadataMap(authorId) + + // Some accounts run promotions on Twitter and send replies automatically. + // We assume that a reply that took more than one minute is not an auto-reply. + // If time difference doesn't exist, this means that one of the tweets was + // not snowflake and therefore much older, and therefore OK as an extended reply. + val timeDifference = candidate.features.getOrElse(InReplyToTweetIdFeature, None).map { + SnowflakeId.timeFromId(candidate.candidate.id) - SnowflakeId.timeFromId(_) + } + val isAutoReply = timeDifference.exists(_ < 1.minute) + + FeatureMapBuilder() + .add(AuthorIsBlueVerifiedFeature, isBlueVerified) + .add(AuthorIsProtectedFeature, isProtected) + .add(IsSupportAccountReplyFeature, isAutoReply) + .build() + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GraphTwoHopFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/GraphTwoHopFeatureHydrator.scala similarity index 76% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GraphTwoHopFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/GraphTwoHopFeatureHydrator.scala index 7a3ed69d2..03d70445e 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GraphTwoHopFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/GraphTwoHopFeatureHydrator.scala @@ -1,15 +1,13 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.graph_feature_service.{thriftscala => gfs} import com.twitter.home_mixer.model.HomeFeatures.FollowedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.IsExtendedReplyFeature +import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.GraphTwoHopRepository import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.home_mixer.util.ObservedKeyValueResultHandler -import com.twitter.home_mixer.util.ReplyRetweetUtil import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature @@ -21,6 +19,7 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.servo.repository.KeyValueRepository import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.adapters.two_hop_features.TwoHopFeaturesAdapter @@ -58,28 +57,19 @@ class GraphTwoHopFeatureHydrator @Inject() ( override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - // Apply filters to in network candidates for ExtendedReplyAncestors and retweets. - // ExtendedReplyAncestors should also be in candidates. No filter for oon. + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + // Apply filters to in network candidates for retweets only. val (inNetworkCandidates, oonCandidates) = candidates.partition { candidate => - candidate.features.getOrElse(InNetworkFeature, false) + candidate.features.getOrElse(FromInNetworkSourceFeature, false) } - val inNetworkReplyToAncestorTweet = - ReplyRetweetUtil.replyToAncestorTweetCandidatesMap(inNetworkCandidates) - - val inNetworkExtendedReplyAncestors = inNetworkCandidates - .filter(_.features.getOrElse(IsExtendedReplyFeature, false)).flatMap { inNetworkCandidate => - inNetworkReplyToAncestorTweet.get(inNetworkCandidate.candidate.id) - }.flatten - - val inNetworkCandidatesToHydrate = inNetworkExtendedReplyAncestors ++ + val inNetworkCandidatesToHydrate = inNetworkCandidates.filter(_.features.getOrElse(IsRetweetFeature, false)) val candidatesToHydrate = (inNetworkCandidatesToHydrate ++ oonCandidates) .flatMap(candidate => CandidatesUtil.getOriginalAuthorId(candidate.features)).distinct - val response = Stitch.callFuture(client((candidatesToHydrate, query.getRequiredUserId))) + val response = client((candidatesToHydrate, query.getRequiredUserId)) response.map { result => candidates.map { candidate => @@ -87,7 +77,9 @@ class GraphTwoHopFeatureHydrator @Inject() ( val value = observedGet(key = originalAuthorId, keyValueResult = result) val transformedValue = postTransformer(value) - val followedByUserIds = value.toOption.flatMap(getFollowedByUserIds(_)).getOrElse(Seq.empty) + val followedByUserIds = value.toOption + .flatMap(getFollowedByUserIds(_)) + .getOrElse(Seq.empty) FeatureMapBuilder() .add(GraphTwoHopFeature, transformedValue) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/IsExtendedReplyFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/IsExtendedReplyFeatureHydrator.scala new file mode 100644 index 000000000..fb114e55c --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/IsExtendedReplyFeatureHydrator.scala @@ -0,0 +1,45 @@ +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.IsExtendedReplyFeature +import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch + +object IsExtendedReplyFeatureHydrator + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("IsExtendedReply") + + override def features: Set[Feature[_, _]] = Set(IsExtendedReplyFeature) + + private val TrueFeatureMap = FeatureMapBuilder().add(IsExtendedReplyFeature, true).build() + private val FalseFeatureMap = FeatureMapBuilder().add(IsExtendedReplyFeature, false).build() + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { + val followedUsers = + query.features.map(_.get(SGSFollowedUsersFeature)).getOrElse(Seq.empty).toSet + + candidates.map { candidate => + val features = candidate.features + val isExtendedReply = features.getOrElse(InReplyToTweetIdFeature, None).nonEmpty && + !features.getOrElse(IsRetweetFeature, false) && + features.getOrElse(InReplyToUserIdFeature, None).exists(!followedUsers.contains(_)) + + if (isExtendedReply) TrueFeatureMap else FalseFeatureMap + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ListIdsQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ListIdsQueryFeatureHydrator.scala new file mode 100644 index 000000000..db60d8866 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ListIdsQueryFeatureHydrator.scala @@ -0,0 +1,52 @@ +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator + +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.socialgraph.{thriftscala => sg} +import com.twitter.stitch.Stitch +import com.twitter.stitch.socialgraph.SocialGraph +import javax.inject.Inject +import javax.inject.Singleton + +case object ListIdsFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] { + override val defaultValue: Seq[Long] = Seq.empty +} + +@Singleton +class ListIdsQueryFeatureHydrator @Inject() (socialGraph: SocialGraph) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ListIds") + + override val features: Set[Feature[_, _]] = Set(ListIdsFeature) + + private val MaxListsToFetch = 20 + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + val userId = query.getRequiredUserId + + val ownedSubscribedRequest = sg.IdsRequest( + relationships = Seq( + sg.SrcRelationship(userId, sg.RelationshipType.ListIsSubscriber, hasRelationship = true), + sg.SrcRelationship(userId, sg.RelationshipType.ListOwning, hasRelationship = true) + ), + pageRequest = Some(sg.PageRequest(selectAll = Some(false), count = Some(MaxListsToFetch))), + context = Some( + sg.LookupContext( + includeInactive = false, + performUnion = Some(true), + includeAll = Some(false) + ) + ) + ) + + socialGraph.ids(ownedSubscribedRequest).map { response => + FeatureMapBuilder().add(ListIdsFeature, response.ids).build() + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/MetricCenterUserCountingFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/MetricCenterUserCountingFeatureHydrator.scala similarity index 76% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/MetricCenterUserCountingFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/MetricCenterUserCountingFeatureHydrator.scala index de0b195e0..bc51aabb9 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/MetricCenterUserCountingFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/MetricCenterUserCountingFeatureHydrator.scala @@ -1,5 +1,4 @@ -package com.twitter.home_mixer -package functional_component.feature_hydrator +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature @@ -14,11 +13,11 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.servo.keyvalue.KeyValueResult import com.twitter.servo.repository.KeyValueRepository import com.twitter.stitch.Stitch import com.twitter.util.Future - import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @@ -45,25 +44,17 @@ class MetricCenterUserCountingFeatureHydrator @Inject() ( override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - Stitch.callFuture { - val possiblyAuthorIds = extractKeys(candidates) - val userIds = possiblyAuthorIds.flatten + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + val possiblyAuthorIds = extractKeys(candidates) + val userIds = possiblyAuthorIds.flatten - val response: Future[KeyValueResult[Long, rf.MCUserCountingFeatures]] = if (userIds.isEmpty) { - Future.value(KeyValueResult.empty) - } else { - client(userIds) - } + val response: Future[KeyValueResult[Long, rf.MCUserCountingFeatures]] = + if (userIds.isEmpty) Future.value(KeyValueResult.empty) else client(userIds) - response.map { result => - possiblyAuthorIds.map { possiblyAuthorId => - val value = observedGet(key = possiblyAuthorId, keyValueResult = result) - - FeatureMapBuilder() - .add(MetricCenterUserCountingFeature, value) - .build() - } + response.map { result => + possiblyAuthorIds.map { possiblyAuthorId => + val value = observedGet(key = possiblyAuthorId, keyValueResult = result) + FeatureMapBuilder().add(MetricCenterUserCountingFeature, value).build() } } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphQueryFeatureHydrator.scala similarity index 96% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphQueryFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphQueryFeatureHydrator.scala index 4eb735714..a48d3742e 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphQueryFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphQueryFeatureHydrator.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealGraphFeatureRepository import com.twitter.product_mixer.core.feature.Feature diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphViewerAuthorFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphViewerAuthorFeatureHydrator.scala similarity index 92% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphViewerAuthorFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphViewerAuthorFeatureHydrator.scala index 05b434ccf..6048213aa 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphViewerAuthorFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphViewerAuthorFeatureHydrator.scala @@ -1,8 +1,8 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.RealGraphViewerAuthorFeatureHydrator.getCombinedRealGraphFeatures import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RealGraphViewerAuthorFeatureHydrator.getCombinedRealGraphFeatures import com.twitter.home_mixer.util.MissingKeyException import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate @@ -14,6 +14,7 @@ import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.adapters.real_graph.RealGraphEdgeFeaturesCombineAdapter import com.twitter.timelines.prediction.adapters.real_graph.RealGraphFeaturesAdapter @@ -59,13 +60,13 @@ class RealGraphViewerAuthorFeatureHydrator @Inject() () query: PipelineQuery, candidate: TweetCandidate, existingFeatures: FeatureMap - ): Stitch[FeatureMap] = { + ): Stitch[FeatureMap] = OffloadFuturePools.offload { val viewerId = query.getRequiredUserId val realGraphFeatures = query.features .flatMap(_.getOrElse(RealGraphFeatures, None)) .getOrElse(Map.empty[Long, v1.RealGraphEdgeFeatures]) - val result: FeatureMap = existingFeatures.getOrElse(AuthorIdFeature, None) match { + existingFeatures.getOrElse(AuthorIdFeature, None) match { case Some(authorId) => val realGraphAuthorFeatures = getRealGraphViewerAuthorFeatures(viewerId, authorId, realGraphFeatures) @@ -90,7 +91,6 @@ class RealGraphViewerAuthorFeatureHydrator @Inject() () .build() case _ => MissingKeyFeatureMap } - Stitch(result) } private def getRealGraphViewerAuthorFeatures( diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphViewerRelatedUsersFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphViewerRelatedUsersFeatureHydrator.scala similarity index 86% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphViewerRelatedUsersFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphViewerRelatedUsersFeatureHydrator.scala index 45897ec05..47e2431cf 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphViewerRelatedUsersFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphViewerRelatedUsersFeatureHydrator.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature @@ -15,6 +15,7 @@ import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.adapters.real_graph.RealGraphEdgeFeaturesCombineAdapter import com.twitter.timelines.real_graph.v1.{thriftscala => v1} @@ -43,25 +44,23 @@ class RealGraphViewerRelatedUsersFeatureHydrator @Inject() () query: PipelineQuery, candidate: TweetCandidate, existingFeatures: FeatureMap - ): Stitch[FeatureMap] = { + ): Stitch[FeatureMap] = OffloadFuturePools.offload { val realGraphQueryFeatures = query.features .flatMap(_.getOrElse(RealGraphFeatures, None)) .getOrElse(Map.empty[Long, v1.RealGraphEdgeFeatures]) val allRelatedUserIds = getRelatedUserIds(existingFeatures) - val realGraphFeatures = - RealGraphViewerAuthorFeatureHydrator.getCombinedRealGraphFeatures( - allRelatedUserIds, - realGraphQueryFeatures) + val realGraphFeatures = RealGraphViewerAuthorFeatureHydrator.getCombinedRealGraphFeatures( + allRelatedUserIds, + realGraphQueryFeatures + ) val realGraphFeaturesDataRecord = RealGraphEdgeFeaturesCombineAdapter .adaptToDataRecords(Some(realGraphFeatures)).asScala.headOption .getOrElse(new DataRecord) - Stitch.value { - FeatureMapBuilder() - .add(RealGraphViewerRelatedUsersDataRecordFeature, realGraphFeaturesDataRecord) - .build() - } + FeatureMapBuilder() + .add(RealGraphViewerRelatedUsersDataRecordFeature, realGraphFeaturesDataRecord) + .build() } private def getRelatedUserIds(features: FeatureMap): Seq[Long] = { diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealTimeInteractionGraphEdgeFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealTimeInteractionGraphEdgeFeatureHydrator.scala similarity index 75% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealTimeInteractionGraphEdgeFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealTimeInteractionGraphEdgeFeatureHydrator.scala index 8f6435356..f690ff0e0 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealTimeInteractionGraphEdgeFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealTimeInteractionGraphEdgeFeatureHydrator.scala @@ -1,22 +1,22 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.adapters.realtime_interaction_graph.RealTimeInteractionGraphFeaturesAdapter import com.twitter.timelines.prediction.features.realtime_interaction_graph.RealTimeInteractionGraphEdgeFeatures import com.twitter.util.Time - import javax.inject.Inject import javax.inject.Singleton import scala.collection.JavaConverters._ @@ -31,8 +31,8 @@ object RealTimeInteractionGraphEdgeFeature class RealTimeInteractionGraphEdgeFeatureHydrator @Inject() () extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( - "RealTimeInteractionGraphEdge") + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("RealTimeInteractionGraphEdge") override val features: Set[Feature[_, _]] = Set(RealTimeInteractionGraphEdgeFeature) @@ -41,24 +41,21 @@ class RealTimeInteractionGraphEdgeFeatureHydrator @Inject() () override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { val userVertex = query.features.flatMap(_.getOrElse(RealTimeInteractionGraphUserVertexQueryFeature, None)) val realTimeInteractionGraphFeaturesMap = userVertex.map(RealTimeInteractionGraphEdgeFeatures(_, Time.now)) - Stitch.value { - candidates.map { candidate => - val feature = candidate.features.getOrElse(AuthorIdFeature, None).flatMap { authorId => - realTimeInteractionGraphFeaturesMap.flatMap(_.get(authorId)) - } - - FeatureMapBuilder() - .add( - RealTimeInteractionGraphEdgeFeature, - realTimeInteractionGraphFeaturesAdapter.adaptToDataRecords(feature).asScala.head) - .build() + candidates.map { candidate => + val feature = candidate.features.getOrElse(AuthorIdFeature, None).flatMap { authorId => + realTimeInteractionGraphFeaturesMap.flatMap(_.get(authorId)) } + + val dataRecordFeature = + realTimeInteractionGraphFeaturesAdapter.adaptToDataRecords(feature).asScala.head + + FeatureMapBuilder().add(RealTimeInteractionGraphEdgeFeature, dataRecordFeature).build() } } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealTimeInteractionGraphUserVertexQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealTimeInteractionGraphUserVertexQueryFeatureHydrator.scala similarity index 96% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealTimeInteractionGraphUserVertexQueryFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealTimeInteractionGraphUserVertexQueryFeatureHydrator.scala index 899f07630..60d82cc28 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealTimeInteractionGraphUserVertexQueryFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealTimeInteractionGraphUserVertexQueryFeatureHydrator.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver @@ -13,7 +13,6 @@ import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.cache.ReadCache import com.twitter.stitch.Stitch import com.twitter.wtf.real_time_interaction_graph.{thriftscala => ig} - import javax.inject.Inject import javax.inject.Singleton diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ReplyFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ReplyFeatureHydrator.scala similarity index 71% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ReplyFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ReplyFeatureHydrator.scala index 5833838f1..80d857a26 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ReplyFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ReplyFeatureHydrator.scala @@ -1,28 +1,56 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.ContentFeatures import com.twitter.home_mixer.model.HomeFeatures._ +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.content.InReplyToContentFeatureAdapter +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.earlybird.InReplyToEarlybirdAdapter import com.twitter.home_mixer.util.ReplyRetweetUtil +import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.search.common.features.thriftscala.ThriftTweetFeatures import com.twitter.snowflake.id.SnowflakeId import com.twitter.stitch.Stitch import com.twitter.timelines.conversation_features.v1.thriftscala.ConversationFeatures +import com.twitter.timelines.conversation_features.{thriftscala => cf} +import com.twitter.timelines.prediction.adapters.conversation_features.ConversationFeaturesAdapter import com.twitter.util.Duration import com.twitter.util.Time import javax.inject.Inject import javax.inject.Singleton +import scala.collection.JavaConverters._ object InReplyToTweetHydratedEarlybirdFeature extends Feature[TweetCandidate, Option[ThriftTweetFeatures]] +object ConversationDataRecordFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +object InReplyToEarlybirdDataRecordFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +object InReplyToTweetypieContentDataRecordFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + /** * The purpose of this hydrator is to * 1) hydrate simple features into replies and their ancestor tweets @@ -36,13 +64,19 @@ class ReplyFeatureHydrator @Inject() (statsReceiver: StatsReceiver) override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ReplyTweet") override val features: Set[Feature[_, _]] = Set( - ConversationFeature, - InReplyToTweetHydratedEarlybirdFeature + ConversationDataRecordFeature, + InReplyToTweetHydratedEarlybirdFeature, + InReplyToEarlybirdDataRecordFeature, + InReplyToTweetypieContentDataRecordFeature ) + private val defaulDataRecord: DataRecord = new DataRecord() + private val DefaultFeatureMap = FeatureMapBuilder() - .add(ConversationFeature, None) + .add(ConversationDataRecordFeature, defaulDataRecord) .add(InReplyToTweetHydratedEarlybirdFeature, None) + .add(InReplyToEarlybirdDataRecordFeature, defaulDataRecord) + .add(InReplyToTweetypieContentDataRecordFeature, defaulDataRecord) .build() private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) @@ -52,7 +86,7 @@ class ReplyFeatureHydrator @Inject() (statsReceiver: StatsReceiver) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { val replyToInReplyToTweetMap = ReplyRetweetUtil.replyTweetIdToInReplyToTweetMap(candidates) val candidatesWithRepliesHydrated = candidates.map { candidate => @@ -94,16 +128,28 @@ class ReplyFeatureHydrator @Inject() (statsReceiver: StatsReceiver) inReplyToTweetEarlyBirdFeature, updatedReplyConversationFeatures)) } - Stitch.value( - candidatesWithRepliesAndAncestorTweetsHydrated.map { - case (candidate, inReplyToTweetEarlyBirdFeature, updatedConversationFeatures) => - FeatureMapBuilder() - .add(ConversationFeature, updatedConversationFeatures) - .add(InReplyToTweetHydratedEarlybirdFeature, inReplyToTweetEarlyBirdFeature) - .build() - case _ => DefaultFeatureMap - } - ) + + candidatesWithRepliesAndAncestorTweetsHydrated.map { + case (candidate, inReplyToTweetEarlyBirdFeature, updatedConversationFeatures) => + val conversationDataRecordFeature = updatedConversationFeatures + .map(f => ConversationFeaturesAdapter.adaptToDataRecord(cf.ConversationFeatures.V1(f))) + .getOrElse(defaulDataRecord) + + val inReplyToEarlybirdDataRecord = + InReplyToEarlybirdAdapter + .adaptToDataRecords(inReplyToTweetEarlyBirdFeature).asScala.head + val inReplyToContentDataRecord = InReplyToContentFeatureAdapter + .adaptToDataRecords( + inReplyToTweetEarlyBirdFeature.map(ContentFeatures.fromThrift)).asScala.head + + FeatureMapBuilder() + .add(ConversationDataRecordFeature, conversationDataRecordFeature) + .add(InReplyToTweetHydratedEarlybirdFeature, inReplyToTweetEarlyBirdFeature) + .add(InReplyToEarlybirdDataRecordFeature, inReplyToEarlybirdDataRecord) + .add(InReplyToTweetypieContentDataRecordFeature, inReplyToContentDataRecord) + .build() + case _ => DefaultFeatureMap + } } private def hydratedReplyCandidate( diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RequestTimeQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RequestTimeQueryFeatureHydrator.scala new file mode 100644 index 000000000..fd9656d8c --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RequestTimeQueryFeatureHydrator.scala @@ -0,0 +1,122 @@ +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator + +import com.twitter.conversions.DurationOps._ +import com.twitter.home_mixer.model.HomeFeatures.FollowingLastNonPollingTimeFeature +import com.twitter.home_mixer.model.HomeFeatures.LastNonPollingTimeFeature +import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature +import com.twitter.ml.api.DataRecord +import com.twitter.ml.api.util.FDsl._ +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.snowflake.id.SnowflakeId +import com.twitter.stitch.Stitch +import com.twitter.timelines.prediction.features.time_features.AccountAgeInterval +import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.ACCOUNT_AGE_INTERVAL +import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.IS_12_MONTH_NEW_USER +import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.IS_30_DAY_NEW_USER +import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.TIME_BETWEEN_NON_POLLING_REQUESTS_AVG +import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.TIME_SINCE_LAST_NON_POLLING_REQUEST +import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.TIME_SINCE_VIEWER_ACCOUNT_CREATION_SECS +import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.USER_ID_IS_SNOWFLAKE_ID +import com.twitter.user_session_store.ReadRequest +import com.twitter.user_session_store.ReadWriteUserSessionStore +import com.twitter.user_session_store.UserSessionDataset +import com.twitter.user_session_store.UserSessionDataset.UserSessionDataset +import com.twitter.util.Time +import javax.inject.Inject +import javax.inject.Singleton + +object RequestTimeDataRecordFeature + extends DataRecordInAFeature[PipelineQuery] + with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +case class RequestTimeQueryFeatureHydrator @Inject() ( + userSessionStore: ReadWriteUserSessionStore) + extends QueryFeatureHydrator[PipelineQuery] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("RequestTime") + + override val features: Set[Feature[_, _]] = Set( + FollowingLastNonPollingTimeFeature, + LastNonPollingTimeFeature, + NonPollingTimesFeature, + RequestTimeDataRecordFeature + ) + + private val datasets: Set[UserSessionDataset] = Set(UserSessionDataset.NonPollingTimes) + + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { + userSessionStore + .read(ReadRequest(query.getRequiredUserId, datasets)) + .map { userSession => + val nonPollingTimestamps = userSession.flatMap(_.nonPollingTimestamps) + + val lastNonPollingTime = nonPollingTimestamps + .flatMap(_.nonPollingTimestampsMs.headOption) + .map(Time.fromMilliseconds) + + val followingLastNonPollingTime = nonPollingTimestamps + .flatMap(_.mostRecentHomeLatestNonPollingTimestampMs) + .map(Time.fromMilliseconds) + + val nonPollingTimes = nonPollingTimestamps + .map(_.nonPollingTimestampsMs) + .getOrElse(Seq.empty) + + val requestTimeDataRecord = getRequestTimeDataRecord(query, nonPollingTimes) + + FeatureMapBuilder() + .add(FollowingLastNonPollingTimeFeature, followingLastNonPollingTime) + .add(LastNonPollingTimeFeature, lastNonPollingTime) + .add(NonPollingTimesFeature, nonPollingTimes) + .add(RequestTimeDataRecordFeature, requestTimeDataRecord) + .build() + } + } + + def getRequestTimeDataRecord(query: PipelineQuery, nonPollingTimes: Seq[Long]): DataRecord = { + val requestTimeMs = query.queryTime.inMillis + val accountAge = SnowflakeId.timeFromIdOpt(query.getRequiredUserId) + val timeSinceAccountCreation = accountAge.map(query.queryTime.since) + val timeSinceEarliestNonPollingRequest = + nonPollingTimes.lastOption.map(requestTimeMs - _) + val timeSinceLastNonPollingRequest = + nonPollingTimes.headOption.map(requestTimeMs - _) + + new DataRecord() + .setFeatureValue(USER_ID_IS_SNOWFLAKE_ID, accountAge.isDefined) + .setFeatureValue( + IS_30_DAY_NEW_USER, + timeSinceAccountCreation.map(_ < 30.days).getOrElse(false) + ) + .setFeatureValue( + IS_12_MONTH_NEW_USER, + timeSinceAccountCreation.map(_ < 365.days).getOrElse(false) + ) + .setFeatureValueFromOption( + ACCOUNT_AGE_INTERVAL, + timeSinceAccountCreation.flatMap(AccountAgeInterval.fromDuration).map(_.id.toLong) + ) + .setFeatureValueFromOption( + TIME_SINCE_VIEWER_ACCOUNT_CREATION_SECS, + timeSinceAccountCreation.map(_.inSeconds.toDouble) + ) + .setFeatureValueFromOption( + TIME_BETWEEN_NON_POLLING_REQUESTS_AVG, + timeSinceEarliestNonPollingRequest.map(_.toDouble / math.max(1.0, nonPollingTimes.size)) + ) + .setFeatureValueFromOption( + TIME_SINCE_LAST_NON_POLLING_REQUEST, + timeSinceLastNonPollingRequest.map(_.toDouble) + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RetweetSourceTweetFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RetweetSourceTweetFeatureHydrator.scala similarity index 94% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RetweetSourceTweetFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RetweetSourceTweetFeatureHydrator.scala index 98385b130..33cbef884 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RetweetSourceTweetFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RetweetSourceTweetFeatureHydrator.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.product_mixer.component_library.candidate_source.timeline_ranker.TimelineRankerInNetworkSourceTweetsByTweetIdMapFeature @@ -22,8 +22,8 @@ object SourceTweetEarlybirdFeature extends Feature[TweetCandidate, Option[Thrift object RetweetSourceTweetFeatureHydrator extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( - "RetweetSourceTweet") + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("RetweetSourceTweet") override val features: Set[Feature[_, _]] = Set( SourceTweetEarlybirdFeature, diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SimClustersEngagementSimilarityFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SimClustersEngagementSimilarityFeatureHydrator.scala similarity index 85% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SimClustersEngagementSimilarityFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SimClustersEngagementSimilarityFeatureHydrator.scala index 0c04bce5a..f66966f00 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SimClustersEngagementSimilarityFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SimClustersEngagementSimilarityFeatureHydrator.scala @@ -1,6 +1,5 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator -import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate @@ -14,6 +13,7 @@ import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.timelines.clients.strato.twistly.SimClustersRecentEngagementSimilarityClient import com.twitter.timelines.configapi.decider.BooleanDeciderParam @@ -29,8 +29,7 @@ object SimClustersEngagementSimilarityFeature @Singleton class SimClustersEngagementSimilarityFeatureHydrator @Inject() ( - simClustersEngagementSimilarityClient: SimClustersRecentEngagementSimilarityClient, - statsReceiver: StatsReceiver) + simClustersEngagementSimilarityClient: SimClustersRecentEngagementSimilarityClient) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with Conditionally[PipelineQuery] { @@ -39,10 +38,6 @@ class SimClustersEngagementSimilarityFeatureHydrator @Inject() ( override val features: Set[Feature[_, _]] = Set(SimClustersEngagementSimilarityFeature) - private val scopedStatsReceiver = statsReceiver.scope(identifier.toString) - - private val hydratedCandidatesSizeStat = scopedStatsReceiver.stat("hydratedCandidatesSize") - private val simClustersRecentEngagementSimilarityFeaturesAdapter = new SimClustersRecentEngagementSimilarityFeaturesAdapter @@ -55,15 +50,14 @@ class SimClustersEngagementSimilarityFeatureHydrator @Inject() ( override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val tweetToCandidates = candidates.map(candidate => candidate.candidate.id -> candidate).toMap val tweetIds = tweetToCandidates.keySet.toSeq val userId = query.getRequiredUserId val userTweetEdges = tweetIds.map(tweetId => (userId, tweetId)) - val resultFuture = simClustersEngagementSimilarityClient + simClustersEngagementSimilarityClient .getSimClustersRecentEngagementSimilarityScores(userTweetEdges).map { simClustersRecentEngagementSimilarityScoresMap => - hydratedCandidatesSizeStat.add(simClustersRecentEngagementSimilarityScoresMap.size) candidates.map { candidate => val similarityFeatureOpt = simClustersRecentEngagementSimilarityScoresMap .get(userId -> candidate.candidate.id).flatten @@ -77,7 +71,5 @@ class SimClustersEngagementSimilarityFeatureHydrator @Inject() ( .build() } } - Stitch.callFuture(resultFuture) } - } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SimClustersUserTweetScoresHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SimClustersUserTweetScoresHydrator.scala new file mode 100644 index 000000000..c938f34c5 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SimClustersUserTweetScoresHydrator.scala @@ -0,0 +1,86 @@ +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator + +import com.twitter.dal.personal_data.{thriftjava => pd} +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.datarecord.DataRecordOptionalFeature +import com.twitter.product_mixer.core.feature.datarecord.DoubleDataRecordCompatible +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.strato.catalog.Fetch +import com.twitter.strato.generated.client.ml.featureStore.SimClustersUserInterestedInTweetEmbeddingDotProduct20M145K2020OnUserTweetEdgeClientColumn +import javax.inject.Inject +import javax.inject.Singleton + +object SimClustersUserInterestedInTweetEmbeddingDataRecordFeature + extends DataRecordOptionalFeature[TweetCandidate, Double] + with DoubleDataRecordCompatible { + override val featureName: String = + "user-tweet.recommendations.sim_clusters_scores.user_interested_in_tweet_embedding_dot_product_20m_145k_2020" + override val personalDataTypes: Set[pd.PersonalDataType] = + Set(pd.PersonalDataType.InferredInterests) +} + +@Singleton +class SimClustersUserTweetScoresHydrator @Inject() ( + simClustersColumn: SimClustersUserInterestedInTweetEmbeddingDotProduct20M145K2020OnUserTweetEdgeClientColumn, + statsReceiver: StatsReceiver) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("SimClustersUserTweetScores") + + override val features: Set[Feature[_, _]] = Set( + SimClustersUserInterestedInTweetEmbeddingDataRecordFeature) + + private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val keyFoundCounter = scopedStatsReceiver.counter("key/found") + private val keyLossCounter = scopedStatsReceiver.counter("key/loss") + private val keyFailureCounter = scopedStatsReceiver.counter("key/failure") + private val keySkipCounter = scopedStatsReceiver.counter("key/skip") + + private val DefaultFeatureMap = FeatureMapBuilder() + .add(SimClustersUserInterestedInTweetEmbeddingDataRecordFeature, None) + .build() + private val MinFavToHydrate = 9 + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + Stitch.run { + Stitch.collect { + candidates.map { candidate => + val ebFeatures = candidate.features.getOrElse(EarlybirdFeature, None) + val favCount = ebFeatures.flatMap(_.favCountV2).getOrElse(0) + + if (ebFeatures.isEmpty || favCount >= MinFavToHydrate) { + simClustersColumn.fetcher + .fetch((query.getRequiredUserId, candidate.candidate.id), Unit) + .map { + case Fetch.Result(response, _) => + if (response.nonEmpty) keyFoundCounter.incr() else keyLossCounter.incr() + FeatureMapBuilder() + .add(SimClustersUserInterestedInTweetEmbeddingDataRecordFeature, response) + .build() + case _ => + keyFailureCounter.incr() + DefaultFeatureMap + } + } else { + keySkipCounter.incr() + Stitch.value(DefaultFeatureMap) + } + } + } + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TSPInferredTopicFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TSPInferredTopicFeatureHydrator.scala new file mode 100644 index 000000000..f6047730f --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TSPInferredTopicFeatureHydrator.scala @@ -0,0 +1,148 @@ +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator + +import com.twitter.contentrecommender.{thriftscala => cr} +import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature +import com.twitter.home_mixer.model.HomeFeatures.TSPMetricTagFeature +import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature +import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.inferred_topic.InferredTopicAdapter +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType +import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContextFunctionalityType +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.timelines.clients.strato.topics.TopicSocialProofClient +import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => sid} +import com.twitter.topiclisting.TopicListingViewerContext +import com.twitter.tsp.{thriftscala => tsp} +import javax.inject.Inject +import javax.inject.Singleton +import scala.collection.JavaConverters._ + +object TSPInferredTopicFeature extends Feature[TweetCandidate, Map[Long, Double]] +object TSPInferredTopicDataRecordFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +class TSPInferredTopicFeatureHydrator @Inject() ( + topicSocialProofClient: TopicSocialProofClient) + extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TSPInferredTopic") + + override val features: Set[Feature[_, _]] = Set( + TSPInferredTopicFeature, + TSPInferredTopicDataRecordFeature, + TopicIdSocialContextFeature, + TopicContextFunctionalityTypeFeature + ) + + private val topK = 3 + + private val SourcesToSetSocialProof: Set[sid.CandidateTweetSourceId] = + Set(sid.CandidateTweetSourceId.Simcluster) + + private val DefaultFeatureMap = FeatureMapBuilder() + .add(TSPInferredTopicFeature, Map.empty[Long, Double]) + .add(TSPInferredTopicDataRecordFeature, new DataRecord()) + .add(TopicIdSocialContextFeature, None) + .add(TopicContextFunctionalityTypeFeature, None) + .build() + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + val tags = candidates.collect { + case candidate if candidate.features.getTry(TSPMetricTagFeature).isReturn => + candidate.candidate.id -> candidate.features + .getOrElse(TSPMetricTagFeature, Set.empty[tsp.MetricTag]) + }.toMap + + val topicSocialProofRequest = tsp.TopicSocialProofRequest( + userId = query.getRequiredUserId, + tweetIds = candidates.map(_.candidate.id).toSet, + displayLocation = cr.DisplayLocation.HomeTimeline, + topicListingSetting = tsp.TopicListingSetting.Followable, + context = TopicListingViewerContext.fromClientContext(query.clientContext).toThrift, + bypassModes = None, + // Only TweetMixer source has this data. Convert the TweetMixer metric tag to tsp metric tag. + tags = if (tags.isEmpty) None else Some(tags) + ) + + topicSocialProofClient + .getTopicTweetSocialProofResponse(topicSocialProofRequest) + .map { + case Some(response) => + handleResponse(response, candidates) + case _ => candidates.map { _ => DefaultFeatureMap } + } + } + + private def handleResponse( + response: tsp.TopicSocialProofResponse, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Seq[FeatureMap] = { + candidates.map { candidate => + val topicWithScores = response.socialProofs.getOrElse(candidate.candidate.id, Seq.empty) + if (topicWithScores.nonEmpty) { + val (socialProofId, socialProofFunctionalityType) = + if (candidate.features + .getOrElse(CandidateSourceIdFeature, None) + .exists(SourcesToSetSocialProof.contains)) { + getSocialProof(topicWithScores) + } else (None, None) + + val inferredTopicFeatures = + topicWithScores.sortBy(-_.score).take(topK).map(a => (a.topicId, a.score)).toMap + + val inferredTopicDataRecord = + InferredTopicAdapter.adaptToDataRecords(inferredTopicFeatures).asScala.head + + FeatureMapBuilder() + .add(TSPInferredTopicFeature, inferredTopicFeatures) + .add(TSPInferredTopicDataRecordFeature, inferredTopicDataRecord) + .add(TopicIdSocialContextFeature, socialProofId) + .add(TopicContextFunctionalityTypeFeature, socialProofFunctionalityType) + .build() + } else DefaultFeatureMap + } + } + + private def getSocialProof( + topicWithScores: Seq[tsp.TopicWithScore] + ): (Option[Long], Option[TopicContextFunctionalityType]) = { + val followingTopicId = topicWithScores.collectFirst { + case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.Following)) => topicId + } + + if (followingTopicId.nonEmpty) { + return (followingTopicId, Some(BasicTopicContextFunctionalityType)) + } + + val implicitFollowingId = topicWithScores.collectFirst { + case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.ImplicitFollow)) => + topicId + } + + if (implicitFollowingId.nonEmpty) { + return (implicitFollowingId, Some(RecommendationTopicContextFunctionalityType)) + } + + (None, None) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetMetaDataFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetMetaDataFeatureHydrator.scala similarity index 89% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetMetaDataFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetMetaDataFeatureHydrator.scala index 8c79c3874..1c237bf18 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetMetaDataFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetMetaDataFeatureHydrator.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature import com.twitter.home_mixer.util.CandidatesUtil @@ -15,6 +15,7 @@ import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures import java.lang.{Long => JLong} @@ -36,16 +37,10 @@ object TweetMetaDataFeatureHydrator query: PipelineQuery, candidate: TweetCandidate, existingFeatures: FeatureMap - ): Stitch[FeatureMap] = { + ): Stitch[FeatureMap] = OffloadFuturePools.offload { val richDataRecord = new RichDataRecord() - setFeatures(richDataRecord, candidate, existingFeatures) - - Stitch.value { - FeatureMapBuilder() - .add(TweetMetaDataDataRecord, richDataRecord.getRecord) - .build() - } + FeatureMapBuilder().add(TweetMetaDataDataRecord, richDataRecord.getRecord).build() } private def setFeatures( diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetTimeFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetTimeFeatureHydrator.scala new file mode 100644 index 000000000..947016826 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetTimeFeatureHydrator.scala @@ -0,0 +1,112 @@ +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator + +import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature +import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature +import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature +import com.twitter.ml.api.DataRecord +import com.twitter.ml.api.util.FDsl._ +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.snowflake.id.SnowflakeId +import com.twitter.stitch.Stitch +import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures._ +import com.twitter.util.Duration +import scala.collection.Searching._ + +object TweetTimeDataRecordFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +object TweetTimeFeatureHydrator extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] { + + override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetTime") + + override val features: Set[Feature[_, _]] = Set(TweetTimeDataRecordFeature) + + override def apply( + query: PipelineQuery, + candidate: TweetCandidate, + existingFeatures: FeatureMap + ): Stitch[FeatureMap] = { + val tweetFeatures = existingFeatures.getOrElse(EarlybirdFeature, None) + val timeSinceTweetCreation = SnowflakeId.timeFromIdOpt(candidate.id).map(query.queryTime.since) + val timeSinceTweetCreationMs = timeSinceTweetCreation.map(_.inMillis) + + val timeSinceSourceTweetCreationOpt = existingFeatures + .getOrElse(SourceTweetIdFeature, None) + .flatMap { sourceTweetId => + SnowflakeId.timeFromIdOpt(sourceTweetId).map(query.queryTime.since) + }.orElse(timeSinceTweetCreation) + + val lastFavSinceCreationHrs = + tweetFeatures.flatMap(_.lastFavSinceCreationHrs).map(_.toDouble) + val lastRetweetSinceCreationHrs = + tweetFeatures.flatMap(_.lastRetweetSinceCreationHrs).map(_.toDouble) + val lastReplySinceCreationHrs = + tweetFeatures.flatMap(_.lastReplySinceCreationHrs).map(_.toDouble) + val lastQuoteSinceCreationHrs = + tweetFeatures.flatMap(_.lastQuoteSinceCreationHrs).map(_.toDouble) + val timeSinceLastFavoriteHrs = + getTimeSinceLastEngagementHrs(lastFavSinceCreationHrs, timeSinceSourceTweetCreationOpt) + val timeSinceLastRetweetHrs = + getTimeSinceLastEngagementHrs(lastRetweetSinceCreationHrs, timeSinceSourceTweetCreationOpt) + val timeSinceLastReplyHrs = + getTimeSinceLastEngagementHrs(lastReplySinceCreationHrs, timeSinceSourceTweetCreationOpt) + val timeSinceLastQuoteHrs = + getTimeSinceLastEngagementHrs(lastQuoteSinceCreationHrs, timeSinceSourceTweetCreationOpt) + + val nonPollingTimestampsMs = query.features.get.getOrElse(NonPollingTimesFeature, Seq.empty) + val timeSinceLastNonPollingRequest = + nonPollingTimestampsMs.headOption.map(query.queryTime.inMillis - _) + + val nonPollingRequestsSinceTweetCreation = + if (nonPollingTimestampsMs.nonEmpty && timeSinceTweetCreationMs.isDefined) { + nonPollingTimestampsMs + .search(timeSinceTweetCreationMs.get)(Ordering[Long].reverse) + .insertionPoint + } else 0.0 + + val tweetAgeRatio = + if (timeSinceTweetCreationMs.exists(_ > 0.0) && timeSinceLastNonPollingRequest.isDefined) { + timeSinceLastNonPollingRequest.get / timeSinceTweetCreationMs.get.toDouble + } else 0.0 + + val dataRecord = new DataRecord() + .setFeatureValue(IS_TWEET_RECYCLED, false) + .setFeatureValue(TWEET_AGE_RATIO, tweetAgeRatio) + .setFeatureValueFromOption( + TIME_SINCE_TWEET_CREATION, + timeSinceTweetCreationMs.map(_.toDouble) + ) + .setFeatureValue( + NON_POLLING_REQUESTS_SINCE_TWEET_CREATION, + nonPollingRequestsSinceTweetCreation + ) + .setFeatureValueFromOption(LAST_FAVORITE_SINCE_CREATION_HRS, lastFavSinceCreationHrs) + .setFeatureValueFromOption(LAST_RETWEET_SINCE_CREATION_HRS, lastRetweetSinceCreationHrs) + .setFeatureValueFromOption(LAST_REPLY_SINCE_CREATION_HRS, lastReplySinceCreationHrs) + .setFeatureValueFromOption(LAST_QUOTE_SINCE_CREATION_HRS, lastQuoteSinceCreationHrs) + .setFeatureValueFromOption(TIME_SINCE_LAST_FAVORITE_HRS, timeSinceLastFavoriteHrs) + .setFeatureValueFromOption(TIME_SINCE_LAST_RETWEET_HRS, timeSinceLastRetweetHrs) + .setFeatureValueFromOption(TIME_SINCE_LAST_REPLY_HRS, timeSinceLastReplyHrs) + .setFeatureValueFromOption(TIME_SINCE_LAST_QUOTE_HRS, timeSinceLastQuoteHrs) + + Stitch.value(FeatureMapBuilder().add(TweetTimeDataRecordFeature, dataRecord).build()) + } + + private def getTimeSinceLastEngagementHrs( + lastEngagementTimeSinceCreationHrsOpt: Option[Double], + timeSinceTweetCreation: Option[Duration] + ): Option[Double] = lastEngagementTimeSinceCreationHrsOpt.flatMap { lastEngagementTimeHrs => + timeSinceTweetCreation.map(_.inHours - lastEngagementTimeHrs) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetypieContentFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieContentFeatureHydrator.scala similarity index 75% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetypieContentFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieContentFeatureHydrator.scala index 8ad3acc46..93039e8bc 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetypieContentFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieContentFeatureHydrator.scala @@ -1,12 +1,12 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.escherbird.{thriftscala => esb} import com.twitter.finagle.stats.Stat import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.content.ContentFeatureAdapter import com.twitter.home_mixer.model.HomeFeatures.MediaUnderstandingAnnotationIdsFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetypieContentRepository +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.content.ContentFeatureAdapter import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.home_mixer.util.tweetypie.content.FeatureExtractionHelper import com.twitter.ml.api.DataRecord @@ -64,45 +64,41 @@ class TweetypieContentFeatureHydrator @Inject() ( private val bulkPostTransformerLatencyStat = statsReceiver.scope(statScope).scope("bulkPostTransformer").stat("latency_ms") + private val DefaultDataRecord: DataRecord = new DataRecord() + override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - Stitch.callFuture { - val tweetIdsToHydrate = candidates.map(getCandidateOriginalTweetId).distinct + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + val tweetIdsToHydrate = candidates.map(getCandidateOriginalTweetId).distinct - val response: Future[KeyValueResult[Long, tp.Tweet]] = - Stat.timeFuture(bulkRequestLatencyStat)( - if (tweetIdsToHydrate.isEmpty) { - Future.value(KeyValueResult.empty) - } else { - client(tweetIdsToHydrate) - } - ) + val response: Future[KeyValueResult[Long, tp.Tweet]] = Stat.timeFuture(bulkRequestLatencyStat) { + if (tweetIdsToHydrate.isEmpty) Future.value(KeyValueResult.empty) + else client(tweetIdsToHydrate) + } - response.flatMap { result => - Stat.timeFuture(bulkPostTransformerLatencyStat) { - OffloadFuturePools - .parallelize[CandidateWithFeatures[TweetCandidate], Try[(Seq[Long], DataRecord)]]( - candidates, - parTransformer(result, _), - parallelism = 32, - default = Return((Seq.empty, new DataRecord)) - ).map { - _.map { - case Return(result) => - FeatureMapBuilder() - .add(MediaUnderstandingAnnotationIdsFeature, result._1) - .add(TweetypieContentDataRecordFeature, result._2) - .build() - case Throw(e) => - FeatureMapBuilder() - .add(MediaUnderstandingAnnotationIdsFeature, Throw(e)) - .add(TweetypieContentDataRecordFeature, Throw(e)) - .build() - } + response.flatMap { result => + Stat.timeFuture(bulkPostTransformerLatencyStat) { + OffloadFuturePools + .parallelize[CandidateWithFeatures[TweetCandidate], Try[(Seq[Long], DataRecord)]]( + candidates, + parTransformer(result, _), + parallelism = 32, + default = Return((Seq.empty, DefaultDataRecord)) + ).map { + _.map { + case Return(result) => + FeatureMapBuilder() + .add(MediaUnderstandingAnnotationIdsFeature, result._1) + .add(TweetypieContentDataRecordFeature, result._2) + .build() + case Throw(e) => + FeatureMapBuilder() + .add(MediaUnderstandingAnnotationIdsFeature, Throw(e)) + .add(TweetypieContentDataRecordFeature, Throw(e)) + .build() } - } + } } } } @@ -112,7 +108,6 @@ class TweetypieContentFeatureHydrator @Inject() ( candidate: CandidateWithFeatures[TweetCandidate] ): Try[(Seq[Long], DataRecord)] = { val originalTweetId = Some(getCandidateOriginalTweetId(candidate)) - val value = observedGet(key = originalTweetId, keyValueResult = result) Stat.time(postTransformerLatencyStat)(postTransformer(value)) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetypieStaticEntitiesFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieStaticEntitiesFeatureHydrator.scala similarity index 94% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetypieStaticEntitiesFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieStaticEntitiesFeatureHydrator.scala index 773f2144e..0ea9e8591 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetypieStaticEntitiesFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieStaticEntitiesFeatureHydrator.scala @@ -1,9 +1,10 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.google.inject.name.Named import com.twitter.conversions.DurationOps.RichDuration import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.HasImageFeature import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature @@ -13,7 +14,6 @@ import com.twitter.home_mixer.model.HomeFeatures.MentionScreenNameFeature import com.twitter.home_mixer.model.HomeFeatures.MentionUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SemanticAnnotationFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetypieStaticEntitiesCache @@ -47,6 +47,7 @@ class TweetypieStaticEntitiesFeatureHydrator @Inject() ( override val features: Set[Feature[_, _]] = Set( AuthorIdFeature, DirectedAtUserIdFeature, + ExclusiveConversationAuthorIdFeature, HasImageFeature, HasVideoFeature, InReplyToTweetIdFeature, @@ -56,7 +57,6 @@ class TweetypieStaticEntitiesFeatureHydrator @Inject() ( MentionUserIdFeature, QuotedTweetIdFeature, QuotedUserIdFeature, - SemanticAnnotationFeature, SourceTweetIdFeature, SourceUserIdFeature ) @@ -66,6 +66,7 @@ class TweetypieStaticEntitiesFeatureHydrator @Inject() ( private val DefaultFeatureMap = FeatureMapBuilder() .add(AuthorIdFeature, None) .add(DirectedAtUserIdFeature, None) + .add(ExclusiveConversationAuthorIdFeature, None) .add(HasImageFeature, false) .add(HasVideoFeature, false) .add(InReplyToTweetIdFeature, None) @@ -75,7 +76,6 @@ class TweetypieStaticEntitiesFeatureHydrator @Inject() ( .add(MentionUserIdFeature, Seq.empty) .add(QuotedTweetIdFeature, None) .add(QuotedUserIdFeature, None) - .add(SemanticAnnotationFeature, Seq.empty) .add(SourceTweetIdFeature, None) .add(SourceUserIdFeature, None) .build() @@ -114,12 +114,13 @@ class TweetypieStaticEntitiesFeatureHydrator @Inject() ( val mentions = tweet.mentions.getOrElse(Seq.empty) val share = coreData.flatMap(_.share) val reply = coreData.flatMap(_.reply) - val semanticAnnotations = - tweet.escherbirdEntityAnnotations.map(_.entityAnnotations).getOrElse(Seq.empty) FeatureMapBuilder() .add(AuthorIdFeature, coreData.map(_.userId)) .add(DirectedAtUserIdFeature, coreData.flatMap(_.directedAtUser.map(_.userId))) + .add( + ExclusiveConversationAuthorIdFeature, + tweet.exclusiveTweetControl.map(_.conversationAuthorId)) .add(HasImageFeature, TweetMediaFeaturesExtractor.hasImage(tweet)) .add(HasVideoFeature, TweetMediaFeaturesExtractor.hasVideo(tweet)) .add(InReplyToTweetIdFeature, reply.flatMap(_.inReplyToStatusId)) @@ -129,7 +130,6 @@ class TweetypieStaticEntitiesFeatureHydrator @Inject() ( .add(MentionUserIdFeature, mentions.flatMap(_.userId)) .add(QuotedTweetIdFeature, quotedTweet.map(_.tweetId)) .add(QuotedUserIdFeature, quotedTweet.map(_.userId)) - .add(SemanticAnnotationFeature, semanticAnnotations) .add(SourceTweetIdFeature, share.map(_.sourceStatusId)) .add(SourceUserIdFeature, share.map(_.sourceUserId)) .build() diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinAuthorFollow20220101FeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinAuthorFollowFeatureHydrator.scala similarity index 57% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinAuthorFollow20220101FeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinAuthorFollowFeatureHydrator.scala index 5d928a90b..2e531bfb2 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinAuthorFollow20220101FeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinAuthorFollowFeatureHydrator.scala @@ -1,9 +1,9 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinAuthorFollowEmbeddingsAdapter -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinAuthorFollow20200101FeatureRepository +import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinAuthorFollowFeatureRepository +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.twhin_embeddings.TwhinAuthorFollowEmbeddingsAdapter +import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.ml.api.DataRecord import com.twitter.ml.api.{thriftscala => ml} @@ -17,69 +17,61 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.servo.repository.KeyValueRepository import com.twitter.servo.repository.KeyValueResult import com.twitter.stitch.Stitch import com.twitter.util.Future import com.twitter.util.Try - import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton import scala.collection.JavaConverters._ -object TwhinAuthorFollow20220101Feature +object TwhinAuthorFollowFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton -class TwhinAuthorFollow20220101FeatureHydrator @Inject() ( - @Named(TwhinAuthorFollow20200101FeatureRepository) - client: KeyValueRepository[Seq[Long], Long, ml.Embedding], +class TwhinAuthorFollowFeatureHydrator @Inject() ( + @Named(TwhinAuthorFollowFeatureRepository) + client: KeyValueRepository[Seq[Long], Long, ml.FloatTensor], override val statsReceiver: StatsReceiver) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with ObservedKeyValueResultHandler { override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("TwhinAuthorFollow20220101") + FeatureHydratorIdentifier("TwhinAuthorFollow") - override val features: Set[Feature[_, _]] = Set(TwhinAuthorFollow20220101Feature) + override val features: Set[Feature[_, _]] = Set(TwhinAuthorFollowFeature) override val statScope: String = identifier.toString override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - Stitch.callFuture { - val possiblyAuthorIds = extractKeys(candidates) - val authorIds = possiblyAuthorIds.flatten + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + val possiblyAuthorIds = extractKeys(candidates) + val authorIds = possiblyAuthorIds.flatten - val response: Future[KeyValueResult[Long, ml.Embedding]] = - if (authorIds.isEmpty) { - Future.value(KeyValueResult.empty) - } else { - client(authorIds) - } + val response: Future[KeyValueResult[Long, ml.FloatTensor]] = + if (authorIds.isEmpty) Future.value(KeyValueResult.empty) else client(authorIds) - response.map { result => - possiblyAuthorIds.map { possiblyAuthorId => - val value = observedGet(key = possiblyAuthorId, keyValueResult = result) - val transformedValue = postTransformer(value) + response.map { result => + possiblyAuthorIds.map { possiblyAuthorId => + val value = observedGet(key = possiblyAuthorId, keyValueResult = result) + val transformedValue = postTransformer(value) - FeatureMapBuilder() - .add(TwhinAuthorFollow20220101Feature, transformedValue) - .build() - } + FeatureMapBuilder().add(TwhinAuthorFollowFeature, transformedValue).build() } } } - private def postTransformer(embedding: Try[Option[ml.Embedding]]): Try[DataRecord] = { - embedding.map { e => - TwhinAuthorFollowEmbeddingsAdapter.adaptToDataRecords(e).asScala.head + private def postTransformer(embedding: Try[Option[ml.FloatTensor]]): Try[DataRecord] = { + embedding.map { floatTensor => + TwhinAuthorFollowEmbeddingsAdapter.adaptToDataRecords(floatTensor).asScala.head } } @@ -87,10 +79,7 @@ class TwhinAuthorFollow20220101FeatureHydrator @Inject() ( candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[Option[Long]] = { candidates.map { candidate => - candidate.features - .getTry(AuthorIdFeature) - .toOption - .flatten + CandidatesUtil.getOriginalAuthorId(candidate.features) } } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinUserEngagementQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinUserEngagementQueryFeatureHydrator.scala similarity index 62% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinUserEngagementQueryFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinUserEngagementQueryFeatureHydrator.scala index bc602d90c..f43abc98f 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinUserEngagementQueryFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinUserEngagementQueryFeatureHydrator.scala @@ -1,17 +1,15 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinUserEngagementEmbeddingsAdapter import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinUserEngagementFeatureRepository +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.twhin_embeddings.TwhinUserEngagementEmbeddingsAdapter import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.RichDataRecord -import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions import com.twitter.ml.api.{thriftscala => ml} +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery @@ -22,6 +20,7 @@ import com.twitter.util.Throw import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton +import scala.collection.JavaConverters._ object TwhinUserEngagementFeature extends DataRecordInAFeature[PipelineQuery] @@ -48,33 +47,26 @@ class TwhinUserEngagementQueryFeatureHydrator @Inject() ( override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val userId = query.getRequiredUserId - Stitch.callFuture { - client(Seq(userId)).map { results => - val embedding: Option[ml.FloatTensor] = results(userId) match { - case Return(value) => - if (value.exists(_.floats.nonEmpty)) keyFoundCounter.incr() - else keyLossCounter.incr() - value - case Throw(_) => - keyFailureCounter.incr() - None - case _ => - None - } - val dataRecord = - new RichDataRecord(new DataRecord, TwhinUserEngagementEmbeddingsAdapter.getFeatureContext) - embedding.foreach { floatTensor => - dataRecord.setFeatureValue( - TwhinUserEngagementEmbeddingsAdapter.twhinEmbeddingsFeature, - ScalaToJavaDataRecordConversions.scalaTensor2Java( - ml.GeneralTensor.FloatTensor(floatTensor)) - ) - } - - FeatureMapBuilder() - .add(TwhinUserEngagementFeature, dataRecord.getRecord) - .build() + Stitch.callFuture(client(Seq(userId))).map { results => + val embedding: Option[ml.FloatTensor] = results(userId) match { + case Return(value) => + if (value.exists(_.floats.nonEmpty)) keyFoundCounter.incr() + else keyLossCounter.incr() + value + case Throw(_) => + keyFailureCounter.incr() + None + case _ => + None } + + val dataRecord = + TwhinUserEngagementEmbeddingsAdapter.adaptToDataRecords(embedding).asScala.head + + FeatureMapBuilder() + .add(TwhinUserEngagementFeature, dataRecord) + .build() } } + } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinUserFollowQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinUserFollowQueryFeatureHydrator.scala similarity index 62% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinUserFollowQueryFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinUserFollowQueryFeatureHydrator.scala index c0efba167..925043e1e 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinUserFollowQueryFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinUserFollowQueryFeatureHydrator.scala @@ -1,18 +1,15 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinUserFollowEmbeddingsAdapter import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinUserFollowFeatureRepository +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.twhin_embeddings.TwhinUserFollowEmbeddingsAdapter import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.RichDataRecord - -import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions import com.twitter.ml.api.{thriftscala => ml} +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery @@ -23,6 +20,7 @@ import com.twitter.util.Throw import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton +import scala.collection.JavaConverters._ object TwhinUserFollowFeature extends DataRecordInAFeature[PipelineQuery] @@ -49,32 +47,24 @@ class TwhinUserFollowQueryFeatureHydrator @Inject() ( override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val userId = query.getRequiredUserId - Stitch.callFuture( - client(Seq(userId)).map { results => - val embedding: Option[ml.FloatTensor] = results(userId) match { - case Return(value) => - if (value.exists(_.floats.nonEmpty)) keyFoundCounter.incr() - else keyLossCounter.incr() - value - case Throw(_) => - keyFailureCounter.incr() - None - case _ => - None - } - val dataRecord = - new RichDataRecord(new DataRecord, TwhinUserFollowEmbeddingsAdapter.getFeatureContext) - embedding.foreach { floatTensor => - dataRecord.setFeatureValue( - TwhinUserFollowEmbeddingsAdapter.twhinEmbeddingsFeature, - ScalaToJavaDataRecordConversions.scalaTensor2Java( - ml.GeneralTensor - .FloatTensor(floatTensor))) - } - FeatureMapBuilder() - .add(TwhinUserFollowFeature, dataRecord.getRecord) - .build() + Stitch.callFuture(client(Seq(userId))).map { results => + val embedding: Option[ml.FloatTensor] = results(userId) match { + case Return(value) => + if (value.exists(_.floats.nonEmpty)) keyFoundCounter.incr() + else keyLossCounter.incr() + value + case Throw(_) => + keyFailureCounter.incr() + None + case _ => + None } - ) + + val dataRecord = TwhinUserFollowEmbeddingsAdapter.adaptToDataRecords(embedding).asScala.head + + FeatureMapBuilder() + .add(TwhinUserFollowFeature, dataRecord) + .build() + } } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserFollowedTopicIdsFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserFollowedTopicIdsFeatureHydrator.scala similarity index 75% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserFollowedTopicIdsFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserFollowedTopicIdsFeatureHydrator.scala index 5aed770a1..90f618fd0 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserFollowedTopicIdsFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserFollowedTopicIdsFeatureHydrator.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature @@ -12,12 +12,12 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.servo.keyvalue.KeyValueResult import com.twitter.servo.repository.KeyValueRepository import com.twitter.stitch.Stitch import com.twitter.util.Future import com.twitter.util.Try - import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @@ -42,27 +42,19 @@ class UserFollowedTopicIdsFeatureHydrator @Inject() ( override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - Stitch.callFuture { - val possiblyAuthorIds = extractKeys(candidates) - val authorIds = possiblyAuthorIds.flatten + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { + val possiblyAuthorIds = extractKeys(candidates) + val authorIds = possiblyAuthorIds.flatten - val response: Future[KeyValueResult[Long, Seq[Long]]] = - if (authorIds.isEmpty) { - Future.value(KeyValueResult.empty) - } else { - client(authorIds) - } + val response: Future[KeyValueResult[Long, Seq[Long]]] = + if (authorIds.isEmpty) Future.value(KeyValueResult.empty) else client(authorIds) - response.map { result => - possiblyAuthorIds.map { possiblyAuthorId => - val value = observedGet(key = possiblyAuthorId, keyValueResult = result) - val transformedValue = postTransformer(value) + response.map { result => + possiblyAuthorIds.map { possiblyAuthorId => + val value = observedGet(key = possiblyAuthorId, keyValueResult = result) + val transformedValue = postTransformer(value) - FeatureMapBuilder() - .add(UserFollowedTopicIdsFeature, transformedValue) - .build() - } + FeatureMapBuilder().add(UserFollowedTopicIdsFeature, transformedValue).build() } } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserLanguagesFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserLanguagesFeatureHydrator.scala similarity index 58% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserLanguagesFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserLanguagesFeatureHydrator.scala index ad97f3a23..e2431c43e 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserLanguagesFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserLanguagesFeatureHydrator.scala @@ -1,6 +1,8 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator -import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserLanguagesStore +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserLanguagesRepository +import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder @@ -8,8 +10,8 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Quer import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.search.common.constants.{thriftscala => scc} +import com.twitter.servo.repository.KeyValueRepository import com.twitter.stitch.Stitch -import com.twitter.storehaus.ReadableStore import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @@ -18,17 +20,26 @@ object UserLanguagesFeature extends Feature[PipelineQuery, Seq[scc.ThriftLanguag @Singleton case class UserLanguagesFeatureHydrator @Inject() ( - @Named(UserLanguagesStore) store: ReadableStore[Long, Seq[scc.ThriftLanguage]]) - extends QueryFeatureHydrator[PipelineQuery] { + @Named(UserLanguagesRepository) client: KeyValueRepository[Seq[Long], Long, Seq[ + scc.ThriftLanguage + ]], + statsReceiver: StatsReceiver) + extends QueryFeatureHydrator[PipelineQuery] + with ObservedKeyValueResultHandler { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserLanguages") override val features: Set[Feature[_, _]] = Set(UserLanguagesFeature) + override val statScope: String = identifier.toString + override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - Stitch.callFuture(store.get(query.getRequiredUserId)).map { languages => + val key = query.getRequiredUserId + Stitch.callFuture(client(Seq(key))).map { result => + val feature = + observedGet(key = Some(key), keyValueResult = result).map(_.getOrElse(Seq.empty)) FeatureMapBuilder() - .add(UserLanguagesFeature, languages.getOrElse(Seq.empty)) + .add(UserLanguagesFeature, feature) .build() } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserStateQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserStateQueryFeatureHydrator.scala similarity index 96% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserStateQueryFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserStateQueryFeatureHydrator.scala index 29532c8e1..5a61f31fb 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserStateQueryFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserStateQueryFeatureHydrator.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature import com.twitter.home_mixer.service.HomeMixerAlertConfig @@ -9,13 +9,12 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Quer import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch -import com.twitter.timelines.user_health.{thriftscala => uh} import com.twitter.timelines.user_health.v1.{thriftscala => uhv1} +import com.twitter.timelines.user_health.{thriftscala => uh} import com.twitter.user_session_store.ReadOnlyUserSessionStore import com.twitter.user_session_store.ReadRequest import com.twitter.user_session_store.UserSessionDataset import com.twitter.user_session_store.UserSessionDataset.UserSessionDataset - import javax.inject.Inject import javax.inject.Singleton diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UtegFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UtegFeatureHydrator.scala similarity index 77% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UtegFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UtegFeatureHydrator.scala index 389a50d03..96b9657e7 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UtegFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UtegFeatureHydrator.scala @@ -1,9 +1,13 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator +import com.twitter.home_mixer.model.HomeFeatures.FavoritedByCountFeature import com.twitter.home_mixer.model.HomeFeatures.FavoritedByUserIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature +import com.twitter.home_mixer.model.HomeFeatures.RepliedByCountFeature import com.twitter.home_mixer.model.HomeFeatures.RepliedByEngagerIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.RetweetedByCountFeature import com.twitter.home_mixer.model.HomeFeatures.RetweetedByEngagerIdsFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.UtegSocialProofRepository @@ -16,12 +20,12 @@ import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.recos.recos_common.{thriftscala => rc} import com.twitter.recos.user_tweet_entity_graph.{thriftscala => uteg} import com.twitter.servo.keyvalue.KeyValueResult import com.twitter.servo.repository.KeyValueRepository import com.twitter.stitch.Stitch - import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @@ -40,7 +44,10 @@ class UtegFeatureHydrator @Inject() ( override val features: Set[Feature[_, _]] = Set( FavoritedByUserIdsFeature, RetweetedByEngagerIdsFeature, - RepliedByEngagerIdsFeature + RepliedByEngagerIdsFeature, + FavoritedByCountFeature, + RetweetedByCountFeature, + RepliedByCountFeature ) override def onlyIf(query: PipelineQuery): Boolean = query.features @@ -49,7 +56,7 @@ class UtegFeatureHydrator @Inject() ( override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val seedUserWeights = query.features.map(_.get(RealGraphInNetworkScoresFeature)).get val sourceTweetIds = candidates.flatMap(_.features.getOrElse(SourceTweetIdFeature, None)) @@ -59,9 +66,7 @@ class UtegFeatureHydrator @Inject() ( val utegQuery = (tweetIdsToSend, (query.getRequiredUserId, seedUserWeights)) - Stitch - .callFuture(client(utegQuery)) - .map(handleResponse(candidates, _)) + client(utegQuery).map(handleResponse(candidates, _)) } private def handleResponse( @@ -69,6 +74,7 @@ class UtegFeatureHydrator @Inject() ( results: KeyValueResult[Long, uteg.TweetRecommendation], ): Seq[FeatureMap] = { candidates.map { candidate => + val inNetwork = candidate.features.getOrElse(FromInNetworkSourceFeature, false) val candidateProof = results(candidate.candidate.id).toOption.flatten val sourceProof = candidate.features .getOrElse(SourceTweetIdFeature, None).flatMap(results(_).toOption.flatten) @@ -78,10 +84,18 @@ class UtegFeatureHydrator @Inject() ( val retweetedBy = proofs.flatMap(_.get(rc.SocialProofType.Retweet)).flatten val repliedBy = proofs.flatMap(_.get(rc.SocialProofType.Reply)).flatten + val (favoritedByCount, retweetedByCount, repliedByCount) = + if (!inNetwork) { + (favoritedBy.size.toDouble, retweetedBy.size.toDouble, repliedBy.size.toDouble) + } else { (0.0, 0.0, 0.0) } + FeatureMapBuilder() .add(FavoritedByUserIdsFeature, favoritedBy) .add(RetweetedByEngagerIdsFeature, retweetedBy) .add(RepliedByEngagerIdsFeature, repliedBy) + .add(FavoritedByCountFeature, favoritedByCount) + .add(RetweetedByCountFeature, retweetedByCount) + .add(RepliedByCountFeature, repliedByCount) .build() } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features/AuthorFeaturesAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features/AuthorFeaturesAdapter.scala new file mode 100644 index 000000000..cbaa3f18c --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features/AuthorFeaturesAdapter.scala @@ -0,0 +1,92 @@ +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.author_features + +import com.twitter.home_mixer.util.DataRecordUtil +import com.twitter.ml.api.DataRecord +import com.twitter.ml.api.Feature +import com.twitter.ml.api.FeatureContext +import com.twitter.ml.api.util.CompactDataRecordConverter +import com.twitter.ml.api.util.FDsl._ +import com.twitter.timelines.author_features.v1.{thriftjava => af} +import com.twitter.timelines.prediction.common.adapters.TimelinesAdapterBase +import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig +import com.twitter.timelines.prediction.features.user_health.UserHealthFeatures +import scala.collection.JavaConverters._ + +object AuthorFeaturesAdapter extends TimelinesAdapterBase[af.AuthorFeatures] { + + private val Prefix = "original_author.timelines.original_author_aggregates." + + private val typedAggregateGroups = + TimelinesAggregationConfig.originalAuthorAggregatesV1.buildTypedAggregateGroups() + + private val aggregateFeaturesRenameMap: Map[Feature[_], Feature[_]] = + typedAggregateGroups.map(_.outputFeaturesToRenamedOutputFeatures(Prefix)).reduce(_ ++ _) + + private val prefixedOriginalAuthorAggregateFeatures = + typedAggregateGroups.flatMap(_.allOutputFeatures).map { feature => + aggregateFeaturesRenameMap.getOrElse(feature, feature) + } + + private val authorFeatures = prefixedOriginalAuthorAggregateFeatures ++ Seq( + UserHealthFeatures.AuthorState, + UserHealthFeatures.NumAuthorFollowers, + UserHealthFeatures.NumAuthorConnectDays, + UserHealthFeatures.NumAuthorConnect + ) + + private val aggregateFeatureContext: FeatureContext = + new FeatureContext(typedAggregateGroups.flatMap(_.allOutputFeatures).asJava) + + private lazy val prefixedAggregateFeatureContext: FeatureContext = + new FeatureContext(prefixedOriginalAuthorAggregateFeatures.asJava) + + override val getFeatureContext: FeatureContext = new FeatureContext(authorFeatures: _*) + + override val commonFeatures: Set[Feature[_]] = Set.empty + + private val compactDataRecordConverter = new CompactDataRecordConverter() + + override def adaptToDataRecords( + authorFeatures: af.AuthorFeatures + ): java.util.List[DataRecord] = { + val dataRecord = + if (authorFeatures.aggregates != null) { + val originalAuthorAggregatesDataRecord = + compactDataRecordConverter.compactDataRecordToDataRecord(authorFeatures.aggregates) + + DataRecordUtil.applyRename( + originalAuthorAggregatesDataRecord, + aggregateFeatureContext, + prefixedAggregateFeatureContext, + aggregateFeaturesRenameMap) + } else new DataRecord + + if (authorFeatures.user_health != null) { + val userHealth = authorFeatures.user_health + + if (userHealth.user_state != null) { + dataRecord.setFeatureValue( + UserHealthFeatures.AuthorState, + userHealth.user_state.getValue.toLong + ) + } + + dataRecord.setFeatureValue( + UserHealthFeatures.NumAuthorFollowers, + userHealth.num_followers.toDouble + ) + + dataRecord.setFeatureValue( + UserHealthFeatures.NumAuthorConnectDays, + userHealth.num_connect_days.toDouble + ) + + dataRecord.setFeatureValue( + UserHealthFeatures.NumAuthorConnect, + userHealth.num_connect.toDouble + ) + } + + List(dataRecord).asJava + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features/BUILD.bazel similarity index 90% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features/BUILD.bazel rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features/BUILD.bazel index ad8e1cd82..7a00a9b26 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features/BUILD.bazel @@ -4,6 +4,7 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "src/java/com/twitter/ml/api:api-base", "src/java/com/twitter/ml/api/util", "src/scala/com/twitter/ml/api/util", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/BUILD.bazel similarity index 89% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/BUILD.bazel rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/BUILD.bazel index bf9d7e2b8..6a9393121 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/BUILD.bazel @@ -11,6 +11,7 @@ scala_library( "src/scala/com/twitter/timelines/prediction/common/adapters", "src/scala/com/twitter/timelines/prediction/common/adapters:base", "src/scala/com/twitter/timelines/prediction/features/common", + "src/scala/com/twitter/timelines/prediction/features/conversation_features", "src/thrift/com/twitter/ml/api:data-java", "src/thrift/com/twitter/ml/api:data-scala", ], diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/ContentFeatureAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/ContentFeatureAdapter.scala similarity index 94% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/ContentFeatureAdapter.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/ContentFeatureAdapter.scala index 93cb6036d..2d73ece45 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/ContentFeatureAdapter.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/ContentFeatureAdapter.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.content +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.content import com.twitter.home_mixer.model.ContentFeatures import com.twitter.ml.api.Feature @@ -8,11 +8,14 @@ import com.twitter.ml.api.util.DataRecordConverters._ import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase import com.twitter.timelines.prediction.common.adapters.TweetLengthType import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures +import com.twitter.timelines.prediction.features.conversation_features.ConversationFeatures import scala.collection.JavaConverters._ object ContentFeatureAdapter extends TimelinesMutatingAdapterBase[Option[ContentFeatures]] { override val getFeatureContext: FeatureContext = new FeatureContext( + ConversationFeatures.IS_SELF_THREAD_TWEET, + ConversationFeatures.IS_LEAF_IN_SELF_THREAD, TimelinesSharedFeatures.ASPECT_RATIO_DEN, TimelinesSharedFeatures.ASPECT_RATIO_NUM, TimelinesSharedFeatures.BIT_RATE, @@ -79,6 +82,16 @@ object ContentFeatureAdapter extends TimelinesMutatingAdapterBase[Option[Content ): Unit = { if (contentFeatures.nonEmpty) { val features = contentFeatures.get + // Conversation Features + richDataRecord.setFeatureValueFromOption( + ConversationFeatures.IS_SELF_THREAD_TWEET, + Some(features.selfThreadMetadata.nonEmpty) + ) + richDataRecord.setFeatureValueFromOption( + ConversationFeatures.IS_LEAF_IN_SELF_THREAD, + features.selfThreadMetadata.map(_.isLeaf) + ) + // Media Features richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.ASPECT_RATIO_DEN, diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/InReplyToContentFeatureAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/InReplyToContentFeatureAdapter.scala new file mode 100644 index 000000000..fa27ddac0 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/InReplyToContentFeatureAdapter.scala @@ -0,0 +1,75 @@ +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.content + +import com.twitter.home_mixer.model.ContentFeatures +import com.twitter.ml.api.Feature +import com.twitter.ml.api.FeatureContext +import com.twitter.ml.api.RichDataRecord +import com.twitter.ml.api.util.DataRecordConverters.RichDataRecordWrapper +import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase +import com.twitter.timelines.prediction.features.common.InReplyToTweetTimelinesSharedFeatures + +object InReplyToContentFeatureAdapter + extends TimelinesMutatingAdapterBase[Option[ContentFeatures]] { + + override val getFeatureContext: FeatureContext = new FeatureContext( + // Media Features + InReplyToTweetTimelinesSharedFeatures.ASPECT_RATIO_DEN, + InReplyToTweetTimelinesSharedFeatures.ASPECT_RATIO_NUM, + InReplyToTweetTimelinesSharedFeatures.HEIGHT_1, + InReplyToTweetTimelinesSharedFeatures.HEIGHT_2, + InReplyToTweetTimelinesSharedFeatures.VIDEO_DURATION, + // TextFeatures + InReplyToTweetTimelinesSharedFeatures.NUM_CAPS, + InReplyToTweetTimelinesSharedFeatures.TWEET_LENGTH, + InReplyToTweetTimelinesSharedFeatures.HAS_QUESTION, + ) + + override val commonFeatures: Set[Feature[_]] = Set.empty + + override def setFeatures( + contentFeatures: Option[ContentFeatures], + richDataRecord: RichDataRecord + ): Unit = { + if (contentFeatures.nonEmpty) { + val features = contentFeatures.get + richDataRecord.setFeatureValueFromOption( + InReplyToTweetTimelinesSharedFeatures.ASPECT_RATIO_DEN, + features.aspectRatioNum.map(_.toDouble) + ) + + richDataRecord.setFeatureValueFromOption( + InReplyToTweetTimelinesSharedFeatures.ASPECT_RATIO_NUM, + features.aspectRatioNum.map(_.toDouble) + ) + + richDataRecord.setFeatureValueFromOption( + InReplyToTweetTimelinesSharedFeatures.HEIGHT_1, + features.heights.flatMap(_.lift(0)).map(_.toDouble) + ) + richDataRecord.setFeatureValueFromOption( + InReplyToTweetTimelinesSharedFeatures.HEIGHT_2, + features.heights.flatMap(_.lift(1)).map(_.toDouble) + ) + + richDataRecord.setFeatureValueFromOption( + InReplyToTweetTimelinesSharedFeatures.VIDEO_DURATION, + features.videoDurationMs.map(_.toDouble) + ) + + richDataRecord.setFeatureValueFromOption( + InReplyToTweetTimelinesSharedFeatures.NUM_CAPS, + Some(features.numCaps.toDouble) + ) + + richDataRecord.setFeatureValueFromOption( + InReplyToTweetTimelinesSharedFeatures.TWEET_LENGTH, + Some(features.length.toDouble) + ) + + richDataRecord.setFeatureValueFromOption( + InReplyToTweetTimelinesSharedFeatures.HAS_QUESTION, + Some(features.hasQuestion) + ) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/earlybird/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/BUILD.bazel similarity index 100% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/earlybird/BUILD.bazel rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/BUILD.bazel diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/earlybird/EarlybirdAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/EarlybirdAdapter.scala similarity index 98% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/earlybird/EarlybirdAdapter.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/EarlybirdAdapter.scala index 833733c16..5a0207e8a 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/earlybird/EarlybirdAdapter.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/EarlybirdAdapter.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.earlybird +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.earlybird import com.twitter.ml.api.Feature import com.twitter.ml.api.FeatureContext @@ -305,6 +305,10 @@ object EarlybirdAdapter extends TimelinesMutatingAdapterBase[Option[sc.ThriftTwe richDataRecord.setFeatureValueFromOption( RecapFeatures.REPLY_COUNT_V2, features.replyCountV2.map(_.toDouble)) + richDataRecord.setFeatureValueFromOption( + RecapFeatures.MENTIONED_SCREEN_NAMES, + features.mentionsList.map(_.toSet.asJava) + ) val urls = features.urlsList.getOrElse(Seq.empty) richDataRecord.setFeatureValue( RecapFeatures.URL_DOMAINS, diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/InReplyToEarlybirdAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/InReplyToEarlybirdAdapter.scala new file mode 100644 index 000000000..b6a7b0ad3 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/InReplyToEarlybirdAdapter.scala @@ -0,0 +1,206 @@ +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.earlybird + +import com.twitter.ml.api.Feature +import com.twitter.ml.api.FeatureContext +import com.twitter.ml.api.RichDataRecord +import com.twitter.ml.api.util.DataRecordConverters.RichDataRecordWrapper +import com.twitter.search.common.features.{thriftscala => sc} +import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase +import com.twitter.timelines.prediction.features.common.InReplyToTweetTimelinesSharedFeatures +import com.twitter.timelines.prediction.features.recap.InReplyToRecapFeatures +import java.lang.{Boolean => JBoolean} +import java.lang.{Double => JDouble} + +object InReplyToEarlybirdAdapter + extends TimelinesMutatingAdapterBase[Option[sc.ThriftTweetFeatures]] { + + override val getFeatureContext: FeatureContext = new FeatureContext( + // TextFeatures + InReplyToTweetTimelinesSharedFeatures.WEIGHTED_FAV_COUNT, + InReplyToTweetTimelinesSharedFeatures.WEIGHTED_RETWEET_COUNT, + InReplyToTweetTimelinesSharedFeatures.WEIGHTED_REPLY_COUNT, + InReplyToTweetTimelinesSharedFeatures.WEIGHTED_QUOTE_COUNT, + InReplyToTweetTimelinesSharedFeatures.DECAYED_FAVORITE_COUNT, + InReplyToTweetTimelinesSharedFeatures.DECAYED_RETWEET_COUNT, + InReplyToTweetTimelinesSharedFeatures.DECAYED_REPLY_COUNT, + InReplyToTweetTimelinesSharedFeatures.DECAYED_QUOTE_COUNT, + InReplyToTweetTimelinesSharedFeatures.QUOTE_COUNT, + InReplyToTweetTimelinesSharedFeatures.HAS_QUOTE, + InReplyToTweetTimelinesSharedFeatures.EARLYBIRD_SCORE, + InReplyToRecapFeatures.PREV_USER_TWEET_ENGAGEMENT, + InReplyToRecapFeatures.IS_SENSITIVE, + InReplyToRecapFeatures.IS_AUTHOR_NEW, + InReplyToRecapFeatures.NUM_MENTIONS, + InReplyToRecapFeatures.HAS_MENTION, + InReplyToRecapFeatures.HAS_HASHTAG, + InReplyToRecapFeatures.IS_AUTHOR_NSFW, + InReplyToRecapFeatures.IS_AUTHOR_SPAM, + InReplyToRecapFeatures.IS_AUTHOR_BOT, + InReplyToRecapFeatures.FROM_MUTUAL_FOLLOW, + InReplyToRecapFeatures.USER_REP, + InReplyToRecapFeatures.FROM_VERIFIED_ACCOUNT, + InReplyToRecapFeatures.HAS_IMAGE, + InReplyToRecapFeatures.HAS_NEWS, + InReplyToRecapFeatures.HAS_VIDEO, + InReplyToRecapFeatures.HAS_VISIBLE_LINK, + InReplyToRecapFeatures.IS_OFFENSIVE, + InReplyToRecapFeatures.IS_REPLY, + InReplyToRecapFeatures.BIDIRECTIONAL_REPLY_COUNT, + InReplyToRecapFeatures.UNIDIRECTIONAL_REPLY_COUNT, + InReplyToRecapFeatures.BIDIRECTIONAL_RETWEET_COUNT, + InReplyToRecapFeatures.UNIDIRECTIONAL_RETWEET_COUNT, + InReplyToRecapFeatures.BIDIRECTIONAL_FAV_COUNT, + InReplyToRecapFeatures.UNIDIRECTIONAL_FAV_COUNT, + InReplyToRecapFeatures.CONVERSATIONAL_COUNT, + InReplyToRecapFeatures.REPLY_COUNT, + InReplyToRecapFeatures.RETWEET_COUNT, + InReplyToRecapFeatures.FAV_COUNT, + InReplyToRecapFeatures.TEXT_SCORE, + InReplyToRecapFeatures.FAV_COUNT_V2, + InReplyToRecapFeatures.RETWEET_COUNT_V2, + InReplyToRecapFeatures.REPLY_COUNT_V2) + + override val commonFeatures: Set[Feature[_]] = Set.empty + + override def setFeatures( + ebFeatures: Option[sc.ThriftTweetFeatures], + richDataRecord: RichDataRecord + ): Unit = { + if (ebFeatures.nonEmpty) { + val features = ebFeatures.get + + richDataRecord.setFeatureValueFromOption( + InReplyToTweetTimelinesSharedFeatures.WEIGHTED_FAV_COUNT, + features.weightedFavoriteCount.map(_.toDouble) + ) + + richDataRecord.setFeatureValueFromOption( + InReplyToTweetTimelinesSharedFeatures.WEIGHTED_RETWEET_COUNT, + features.weightedRetweetCount.map(_.toDouble) + ) + + richDataRecord.setFeatureValueFromOption( + InReplyToTweetTimelinesSharedFeatures.WEIGHTED_REPLY_COUNT, + features.weightedReplyCount.map(_.toDouble) + ) + + richDataRecord.setFeatureValueFromOption( + InReplyToTweetTimelinesSharedFeatures.WEIGHTED_QUOTE_COUNT, + features.weightedQuoteCount.map(_.toDouble) + ) + + richDataRecord.setFeatureValueFromOption( + InReplyToTweetTimelinesSharedFeatures.DECAYED_FAVORITE_COUNT, + features.decayedFavoriteCount.map(_.toDouble) + ) + + richDataRecord.setFeatureValueFromOption( + InReplyToTweetTimelinesSharedFeatures.DECAYED_RETWEET_COUNT, + features.decayedRetweetCount.map(_.toDouble) + ) + + richDataRecord.setFeatureValueFromOption( + InReplyToTweetTimelinesSharedFeatures.DECAYED_REPLY_COUNT, + features.decayedReplyCount.map(_.toDouble) + ) + + richDataRecord.setFeatureValueFromOption( + InReplyToTweetTimelinesSharedFeatures.DECAYED_QUOTE_COUNT, + features.decayedQuoteCount.map(_.toDouble) + ) + + richDataRecord.setFeatureValueFromOption( + InReplyToTweetTimelinesSharedFeatures.QUOTE_COUNT, + features.quoteCount.map(_.toDouble) + ) + + richDataRecord.setFeatureValueFromOption( + InReplyToTweetTimelinesSharedFeatures.HAS_QUOTE, + features.hasQuote + ) + + if (features.earlybirdScore > 0) + richDataRecord.setFeatureValue[JDouble]( + InReplyToTweetTimelinesSharedFeatures.EARLYBIRD_SCORE, + features.earlybirdScore + ) + + richDataRecord.setFeatureValue[JDouble]( + InReplyToRecapFeatures.PREV_USER_TWEET_ENGAGEMENT, + features.prevUserTweetEngagement.toDouble + ) + + richDataRecord + .setFeatureValue[JBoolean](InReplyToRecapFeatures.IS_SENSITIVE, features.isSensitiveContent) + richDataRecord + .setFeatureValue[JBoolean](InReplyToRecapFeatures.IS_AUTHOR_NEW, features.isAuthorNew) + richDataRecord.setFeatureValue[JDouble]( + InReplyToRecapFeatures.NUM_MENTIONS, + features.numMentions.toDouble) + richDataRecord + .setFeatureValue[JBoolean](InReplyToRecapFeatures.HAS_MENTION, (features.numMentions > 0)) + richDataRecord + .setFeatureValue[JBoolean](InReplyToRecapFeatures.HAS_HASHTAG, (features.numHashtags > 0)) + richDataRecord + .setFeatureValue[JBoolean](InReplyToRecapFeatures.IS_AUTHOR_NSFW, features.isAuthorNSFW) + richDataRecord + .setFeatureValue[JBoolean](InReplyToRecapFeatures.IS_AUTHOR_SPAM, features.isAuthorSpam) + richDataRecord + .setFeatureValue[JBoolean](InReplyToRecapFeatures.IS_AUTHOR_BOT, features.isAuthorBot) + richDataRecord.setFeatureValue[JBoolean]( + InReplyToRecapFeatures.FROM_MUTUAL_FOLLOW, + features.fromMutualFollow) + richDataRecord.setFeatureValue[JDouble](InReplyToRecapFeatures.USER_REP, features.userRep) + richDataRecord.setFeatureValue[JBoolean]( + InReplyToRecapFeatures.FROM_VERIFIED_ACCOUNT, + features.fromVerifiedAccount) + richDataRecord.setFeatureValue[JBoolean](InReplyToRecapFeatures.HAS_IMAGE, features.hasImage) + richDataRecord.setFeatureValue[JBoolean](InReplyToRecapFeatures.HAS_NEWS, features.hasNews) + richDataRecord.setFeatureValue[JBoolean](InReplyToRecapFeatures.HAS_VIDEO, features.hasVideo) + richDataRecord + .setFeatureValue[JBoolean](InReplyToRecapFeatures.HAS_VISIBLE_LINK, features.hasVisibleLink) + richDataRecord + .setFeatureValue[JBoolean](InReplyToRecapFeatures.IS_OFFENSIVE, features.isOffensive) + richDataRecord.setFeatureValue[JBoolean](InReplyToRecapFeatures.IS_REPLY, features.isReply) + richDataRecord.setFeatureValue[JDouble]( + InReplyToRecapFeatures.BIDIRECTIONAL_REPLY_COUNT, + features.bidirectionalReplyCount) + richDataRecord.setFeatureValue[JDouble]( + InReplyToRecapFeatures.UNIDIRECTIONAL_REPLY_COUNT, + features.unidirectionalReplyCount) + richDataRecord.setFeatureValue[JDouble]( + InReplyToRecapFeatures.BIDIRECTIONAL_RETWEET_COUNT, + features.bidirectionalRetweetCount) + richDataRecord.setFeatureValue[JDouble]( + InReplyToRecapFeatures.UNIDIRECTIONAL_RETWEET_COUNT, + features.unidirectionalRetweetCount) + richDataRecord.setFeatureValue[JDouble]( + InReplyToRecapFeatures.BIDIRECTIONAL_FAV_COUNT, + features.bidirectionalFavCount) + richDataRecord.setFeatureValue[JDouble]( + InReplyToRecapFeatures.UNIDIRECTIONAL_FAV_COUNT, + features.unidirectionalFavCount) + richDataRecord.setFeatureValue[JDouble]( + InReplyToRecapFeatures.CONVERSATIONAL_COUNT, + features.conversationCount) + richDataRecord + .setFeatureValue[JDouble](InReplyToRecapFeatures.REPLY_COUNT, features.replyCount.toDouble) + richDataRecord.setFeatureValue[JDouble]( + InReplyToRecapFeatures.RETWEET_COUNT, + features.retweetCount.toDouble) + richDataRecord + .setFeatureValue[JDouble](InReplyToRecapFeatures.FAV_COUNT, features.favCount.toDouble) + richDataRecord.setFeatureValue[JDouble](InReplyToRecapFeatures.TEXT_SCORE, features.textScore) + richDataRecord.setFeatureValueFromOption( + InReplyToRecapFeatures.FAV_COUNT_V2, + features.favCountV2.map(_.toDouble)) + richDataRecord.setFeatureValueFromOption( + InReplyToRecapFeatures.RETWEET_COUNT_V2, + features.retweetCountV2.map(_.toDouble) + ) + richDataRecord.setFeatureValueFromOption( + InReplyToRecapFeatures.REPLY_COUNT_V2, + features.replyCountV2.map(_.toDouble)) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/inferred_topic/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic/BUILD.bazel similarity index 100% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/inferred_topic/BUILD.bazel rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic/BUILD.bazel diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/inferred_topic/InferredTopicAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic/InferredTopicAdapter.scala similarity index 90% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/inferred_topic/InferredTopicAdapter.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic/InferredTopicAdapter.scala index 62125ea5a..1439cc2ad 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/inferred_topic/InferredTopicAdapter.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic/InferredTopicAdapter.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.inferred_topic +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.inferred_topic import com.twitter.ml.api.Feature import com.twitter.ml.api.FeatureContext diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/non_ml_features/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/BUILD.bazel similarity index 100% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/non_ml_features/BUILD.bazel rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/BUILD.bazel diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/non_ml_features/NonMLCandidateFeaturesAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/NonMLCandidateFeaturesAdapter.scala similarity index 94% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/non_ml_features/NonMLCandidateFeaturesAdapter.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/NonMLCandidateFeaturesAdapter.scala index 24e0edb9b..6c4b79eee 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/non_ml_features/NonMLCandidateFeaturesAdapter.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/NonMLCandidateFeaturesAdapter.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.non_ml_features +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.non_ml_features import com.twitter.ml.api.constant.SharedFeatures import com.twitter.ml.api.Feature diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/non_ml_features/NonMLCommonFeaturesAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/NonMLCommonFeaturesAdapter.scala similarity index 94% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/non_ml_features/NonMLCommonFeaturesAdapter.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/NonMLCommonFeaturesAdapter.scala index 612c4900c..a2777538c 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/non_ml_features/NonMLCommonFeaturesAdapter.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/NonMLCommonFeaturesAdapter.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.non_ml_features +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.non_ml_features import com.twitter.ml.api.constant.SharedFeatures import com.twitter.ml.api.Feature diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/offline_aggregates/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/BUILD.bazel similarity index 100% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/offline_aggregates/BUILD.bazel rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/BUILD.bazel diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/offline_aggregates/PassThroughAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/PassThroughAdapter.scala similarity index 79% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/offline_aggregates/PassThroughAdapter.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/PassThroughAdapter.scala index cd5ab020f..dce0a81db 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/offline_aggregates/PassThroughAdapter.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/PassThroughAdapter.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.offline_aggregates +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.offline_aggregates import com.twitter.ml.api.DataRecord import com.twitter.ml.api.IRecordOneToOneAdapter diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/offline_aggregates/SparseAggregatesToDenseAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/SparseAggregatesToDenseAdapter.scala similarity index 87% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/offline_aggregates/SparseAggregatesToDenseAdapter.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/SparseAggregatesToDenseAdapter.scala index 816b35069..4c54f78d8 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/offline_aggregates/SparseAggregatesToDenseAdapter.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/SparseAggregatesToDenseAdapter.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.offline_aggregates +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.offline_aggregates import com.twitter.ml.api.DataRecord import com.twitter.ml.api.FeatureContext diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings/BUILD.bazel similarity index 100% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings/BUILD.bazel rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings/BUILD.bazel diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings/TwhinEmbeddingsAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings/TwhinEmbeddingsAdapter.scala similarity index 63% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings/TwhinEmbeddingsAdapter.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings/TwhinEmbeddingsAdapter.scala index 99c4ba8dd..c2830e462 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings/TwhinEmbeddingsAdapter.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings/TwhinEmbeddingsAdapter.scala @@ -1,17 +1,14 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.twhin_embeddings -import com.twitter.ml.api.util.BufferToIterators.RichFloatBuffer -import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions import com.twitter.ml.api.DataType import com.twitter.ml.api.Feature import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.RichDataRecord +import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions import com.twitter.ml.api.{thriftscala => ml} import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase -import java.nio.ByteOrder - -sealed trait TwhinEmbeddingsAdapter extends TimelinesMutatingAdapterBase[Option[ml.Embedding]] { +sealed trait TwhinEmbeddingsAdapter extends TimelinesMutatingAdapterBase[Option[ml.FloatTensor]] { def twhinEmbeddingsFeature: Feature.Tensor override def getFeatureContext: FeatureContext = new FeatureContext( @@ -19,42 +16,32 @@ sealed trait TwhinEmbeddingsAdapter extends TimelinesMutatingAdapterBase[Option[ ) override def setFeatures( - embedding: Option[ml.Embedding], + embedding: Option[ml.FloatTensor], richDataRecord: RichDataRecord ): Unit = { - embedding.foreach { embedding => - val floatTensor = embedding.tensor map { tensor => - ml.FloatTensor( - tensor.content - .order(ByteOrder.LITTLE_ENDIAN) - .asFloatBuffer - .iterator.toList - .map(_.toDouble)) - } - - floatTensor.foreach { v => - richDataRecord.setFeatureValue( - twhinEmbeddingsFeature, - ScalaToJavaDataRecordConversions.scalaTensor2Java(ml.GeneralTensor.FloatTensor(v)) - ) - } + embedding.foreach { floatTensor => + richDataRecord.setFeatureValue( + twhinEmbeddingsFeature, + ScalaToJavaDataRecordConversions.scalaTensor2Java( + ml.GeneralTensor + .FloatTensor(floatTensor))) } } } object TwhinEmbeddingsFeatures { val TwhinAuthorFollowEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( - "original_author.timelines.twhin_author_follow_embeddings.twhin_author_follow_embeddings", + "original_author.twhin.tw_hi_n.author_follow_as_float_tensor", DataType.FLOAT ) val TwhinUserEngagementEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( - "user.timelines.twhin_user_engagement_embeddings.twhin_user_engagement_embeddings", + "user.twhin.tw_hi_n.user_engagement_as_float_tensor", DataType.FLOAT ) val TwhinUserFollowEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( - "user.timelines.twhin_user_follow_embeddings.twhin_user_follow_embeddings", + "user.twhin.tw_hi_n.user_follow_as_float_tensor", DataType.FLOAT ) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/AggregateFeatureInfo.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/AggregateFeatureInfo.scala similarity index 94% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/AggregateFeatureInfo.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/AggregateFeatureInfo.scala index a1186e2f4..00dee1209 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/AggregateFeatureInfo.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/AggregateFeatureInfo.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates import com.twitter.ml.api.FeatureContext import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/AggregateFeaturesToDecodeWithMetadata.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/AggregateFeaturesToDecodeWithMetadata.scala similarity index 96% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/AggregateFeaturesToDecodeWithMetadata.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/AggregateFeaturesToDecodeWithMetadata.scala index cbda35437..5fb599240 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/AggregateFeaturesToDecodeWithMetadata.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/AggregateFeaturesToDecodeWithMetadata.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.timelinemixer.injection.repository.uss.VersionedAggregateFeaturesDecoder diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BUILD.bazel similarity index 90% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/BUILD.bazel rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BUILD.bazel index d9c353df1..10f696409 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BUILD.bazel @@ -6,14 +6,16 @@ scala_library( dependencies = [ "finatra/inject/inject-core/src/main/scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/offline_aggregates", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", "servo/repo/src/main/scala", "src/java/com/twitter/ml/api:api-base", "src/scala/com/twitter/timelines/prediction/adapters/request_context", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/BaseAggregateQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BaseAggregateQueryFeatureHydrator.scala similarity index 97% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/BaseAggregateQueryFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BaseAggregateQueryFeatureHydrator.scala index b58f23d2e..c81ed5054 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/BaseAggregateQueryFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BaseAggregateQueryFeatureHydrator.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/BaseEdgeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BaseEdgeAggregateFeatureHydrator.scala similarity index 88% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/BaseEdgeAggregateFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BaseEdgeAggregateFeatureHydrator.scala index fde5ab909..43cfa967b 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/BaseEdgeAggregateFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BaseEdgeAggregateFeatureHydrator.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates import com.twitter.ml.api.DataRecord import com.twitter.ml.api.FeatureContext @@ -12,6 +12,7 @@ import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateType.AggregateType @@ -44,8 +45,7 @@ trait BaseEdgeAggregateFeatureHydrator override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { val featureMapBuilders: Seq[FeatureMapBuilder] = for (_ <- candidates) yield FeatureMapBuilder() @@ -56,7 +56,7 @@ trait BaseEdgeAggregateFeatureHydrator } } - Stitch.value(featureMapBuilders.map(_.build())) + featureMapBuilders.map(_.build()) } private def hydrateAggregateFeature( @@ -74,17 +74,17 @@ trait BaseEdgeAggregateFeatureHydrator .getOrElse(AggregateFeaturesToDecodeWithMetadata.empty) // Decode the DenseCompactDataRecords into DataRecords for each required secondary id. - val decoded: Map[Long, DataRecord] = - Utils.selectAndTransform( - secondaryIds.flatten.distinct, - featuresToDecodeWithMetadata.toDataRecord, - extractMapFn(featuresToDecodeWithMetadata)) + val decoded: Map[Long, DataRecord] = Utils.selectAndTransform( + secondaryIds.flatten.distinct, + featuresToDecodeWithMetadata.toDataRecord, + extractMapFn(featuresToDecodeWithMetadata) + ) // Remove unnecessary features in-place. This is safe because the underlying DataRecords // are unique and have just been generated in the previous step. decoded.values.foreach(Utils.filterDataRecord(_, featureContext)) - // Put features into the FeatureMapBuilder's + // Put features into the FeatureMapBuilders secondaryIds.map { ids => val dataRecords = ids.flatMap(decoded.get) feature.adapter.adaptToDataRecord(dataRecords) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/EdgeAggregateFeatures.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/EdgeAggregateFeatures.scala similarity index 79% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/EdgeAggregateFeatures.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/EdgeAggregateFeatures.scala index d637b3b3d..f769bb980 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/EdgeAggregateFeatures.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/EdgeAggregateFeatures.scala @@ -1,11 +1,11 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates -import com.twitter.home_mixer.functional_component.feature_hydrator.TSPInferredTopicFeature -import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.offline_aggregates.PassThroughAdapter -import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.offline_aggregates.SparseAggregatesToDenseAdapter +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.offline_aggregates.PassThroughAdapter +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.offline_aggregates.SparseAggregatesToDenseAdapter import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.MentionUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.MentionScreenNameFeature import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TSPInferredTopicFeature import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateType import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig @@ -28,8 +28,7 @@ object EdgeAggregateFeatures { object UserOriginalAuthorAggregateFeature extends BaseEdgeAggregateFeature( - aggregateGroups = Set( - TimelinesAggregationConfig.userOriginalAuthorReciprocalEngagementAggregates), + aggregateGroups = Set(TimelinesAggregationConfig.userOriginalAuthorAggregatesV1), aggregateType = AggregateType.UserOriginalAuthor, extractMapFn = _.userOriginalAuthorAggregates, adapter = PassThroughAdapter, @@ -57,13 +56,25 @@ object EdgeAggregateFeatures { extractMapFn = _.userMentionAggregates, adapter = new SparseAggregatesToDenseAdapter(CombineCountPolicies.MentionCountsPolicy), getSecondaryKeysFn = candidate => - candidate.features.getOrElse(MentionUserIdFeature, Seq.empty) + candidate.features.getOrElse(MentionScreenNameFeature, Seq.empty).map(_.hashCode.toLong) ) object UserInferredTopicAggregateFeature extends BaseEdgeAggregateFeature( aggregateGroups = Set( TimelinesAggregationConfig.userInferredTopicAggregates, + ), + aggregateType = AggregateType.UserInferredTopic, + extractMapFn = _.userInferredTopicAggregates, + adapter = new SparseAggregatesToDenseAdapter( + CombineCountPolicies.UserInferredTopicCountsPolicy), + getSecondaryKeysFn = candidate => + candidate.features.getOrElse(TSPInferredTopicFeature, Map.empty[Long, Double]).keys.toSeq + ) + + object UserInferredTopicAggregateV2Feature + extends BaseEdgeAggregateFeature( + aggregateGroups = Set( TimelinesAggregationConfig.userInferredTopicAggregatesV2 ), aggregateType = AggregateType.UserInferredTopic, diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/PartAAggregateQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/PartAAggregateQueryFeatureHydrator.scala similarity index 94% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/PartAAggregateQueryFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/PartAAggregateQueryFeatureHydrator.scala index aa75e2c05..f18cceaca 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/PartAAggregateQueryFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/PartAAggregateQueryFeatureHydrator.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelineAggregateMetadataRepository import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelineAggregatePartARepository diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/PartBAggregateQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/PartBAggregateQueryFeatureHydrator.scala similarity index 96% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/PartBAggregateQueryFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/PartBAggregateQueryFeatureHydrator.scala index 4a00c7ca4..fa0336165 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/PartBAggregateQueryFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/PartBAggregateQueryFeatureHydrator.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelineAggregateMetadataRepository import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelineAggregatePartBRepository @@ -58,7 +58,7 @@ class PartBAggregateQueryFeatureHydrator @Inject() ( aggregateGroups = Set( TimelinesAggregationConfig.userAggregatesV2, TimelinesAggregationConfig.userAggregatesV5Continuous, - TimelinesAggregationConfig.userReciprocalEngagementAggregates, + TimelinesAggregationConfig.userAggregatesV6, TimelinesAggregationConfig.twitterWideUserAggregates, ), aggregateType = AggregateType.User @@ -104,7 +104,7 @@ class PartBAggregateQueryFeatureHydrator @Inject() ( Option(featuresWithMetadata.userRequestHourAggregates.get(hourOfDay)) .map(featuresWithMetadata.toDataRecord) val userRequestDowAggregatesDr = - Option(featuresWithMetadata.userRequestHourAggregates.get(dayOfWeek)) + Option(featuresWithMetadata.userRequestDowAggregates.get(dayOfWeek)) .map(featuresWithMetadata.toDataRecord) dropUnknownFeatures(userAggregatesDr, userAggregateFeatureInfo.featureContext) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/Phase1EdgeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Phase1EdgeAggregateFeatureHydrator.scala similarity index 56% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/Phase1EdgeAggregateFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Phase1EdgeAggregateFeatureHydrator.scala index c8c912b63..d9df51d5c 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/Phase1EdgeAggregateFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Phase1EdgeAggregateFeatureHydrator.scala @@ -1,6 +1,6 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates -import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures._ +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.EdgeAggregateFeatures._ import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import javax.inject.Inject import javax.inject.Singleton @@ -11,10 +11,9 @@ class Phase1EdgeAggregateFeatureHydrator @Inject() extends BaseEdgeAggregateFeat override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Phase1EdgeAggregate") - override val aggregateFeatures: Set[BaseEdgeAggregateFeature] = - Set( - UserAuthorAggregateFeature, - UserOriginalAuthorAggregateFeature, - UserMentionAggregateFeature - ) + override val aggregateFeatures: Set[BaseEdgeAggregateFeature] = Set( + UserAuthorAggregateFeature, + UserOriginalAuthorAggregateFeature, + UserMentionAggregateFeature + ) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Phase2EdgeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Phase2EdgeAggregateFeatureHydrator.scala new file mode 100644 index 000000000..cfbb0b2c9 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Phase2EdgeAggregateFeatureHydrator.scala @@ -0,0 +1,28 @@ +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates + +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserEngagerAggregateFeature +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserEngagerGoodClickAggregateFeature +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserInferredTopicAggregateFeature +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserInferredTopicAggregateV2Feature +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserMediaUnderstandingAnnotationAggregateFeature +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserTopicAggregateFeature +import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class Phase2EdgeAggregateFeatureHydrator @Inject() extends BaseEdgeAggregateFeatureHydrator { + + override val identifier: FeatureHydratorIdentifier = + FeatureHydratorIdentifier("Phase2EdgeAggregate") + + override val aggregateFeatures: Set[BaseEdgeAggregateFeature] = + Set( + UserEngagerAggregateFeature, + UserEngagerGoodClickAggregateFeature, + UserInferredTopicAggregateFeature, + UserInferredTopicAggregateV2Feature, + UserTopicAggregateFeature, + UserMediaUnderstandingAnnotationAggregateFeature + ) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/Utils.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Utils.scala similarity index 93% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/Utils.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Utils.scala index 45d205888..6ae1a0f4b 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/Utils.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Utils.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates import com.twitter.ml.api.DataRecord import com.twitter.ml.api.FeatureContext diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BUILD.bazel similarity index 93% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/BUILD.bazel rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BUILD.bazel index 93f042fca..1954dd77d 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BUILD.bazel @@ -10,6 +10,7 @@ scala_library( "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", "servo/repo/src/main/scala", "src/java/com/twitter/ml/api:api-base", "src/java/com/twitter/ml/api/constant", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateBulkCandidateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateBulkCandidateFeatureHydrator.scala similarity index 83% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateBulkCandidateFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateBulkCandidateFeatureHydrator.scala index d892a072a..41b565ded 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateBulkCandidateFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateBulkCandidateFeatureHydrator.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature @@ -8,6 +8,7 @@ import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch trait BaseRealTimeAggregateBulkCandidateFeatureHydrator[K] @@ -28,13 +29,11 @@ trait BaseRealTimeAggregateBulkCandidateFeatureHydrator[K] override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { + ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val possiblyKeys = keysFromQueryAndCandidates(query, candidates) fetchAndConstructDataRecords(possiblyKeys).map { dataRecords => dataRecords.map { dataRecord => - FeatureMapBuilder() - .add(outputFeature, dataRecord) - .build() + FeatureMapBuilder().add(outputFeature, dataRecord).build() } } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateQueryFeatureHydrator.scala similarity index 84% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateQueryFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateQueryFeatureHydrator.scala index 772e0bb92..8f5b17d64 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateQueryFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateQueryFeatureHydrator.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature @@ -6,6 +6,7 @@ import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch trait BaseRealTimeAggregateQueryFeatureHydrator[K] @@ -24,7 +25,7 @@ trait BaseRealTimeAggregateQueryFeatureHydrator[K] override def hydrate( query: PipelineQuery - ): Stitch[FeatureMap] = { + ): Stitch[FeatureMap] = OffloadFuturePools.offloadFuture { val possiblyKeys = keysFromQueryAndCandidates(query) fetchAndConstructDataRecords(Seq(possiblyKeys)).map { dataRecords => FeatureMapBuilder() diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/BaseRealtimeAggregateHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealtimeAggregateHydrator.scala similarity index 68% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/BaseRealtimeAggregateHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealtimeAggregateHydrator.scala index 004312f2f..f97820be0 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/BaseRealtimeAggregateHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealtimeAggregateHydrator.scala @@ -1,21 +1,22 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates -import com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates.BaseRealtimeAggregateHydrator._ +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates.BaseRealtimeAggregateHydrator._ +import com.twitter.home_mixer.util.DataRecordUtil import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.ml.api.DataRecord import com.twitter.ml.api.DataRecordMerger import com.twitter.ml.api.FeatureContext +import com.twitter.ml.api.constant.SharedFeatures import com.twitter.ml.api.util.SRichDataRecord import com.twitter.ml.api.{Feature => MLApiFeature} import com.twitter.servo.cache.ReadCache import com.twitter.servo.keyvalue.KeyValueResult -import com.twitter.stitch.Stitch import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup import com.twitter.util.Future import com.twitter.util.Time import com.twitter.util.Try -import scala.collection.JavaConverters._ import java.lang.{Double => JDouble} +import scala.collection.JavaConverters._ trait BaseRealtimeAggregateHydrator[K] extends ObservedKeyValueResultHandler { @@ -29,7 +30,9 @@ trait BaseRealtimeAggregateHydrator[K] extends ObservedKeyValueResultHandler { private lazy val featureContexts: Seq[FeatureContext] = typedAggregateGroupsList.map { typedAggregateGroups => - new FeatureContext(typedAggregateGroups.flatMap(_.allOutputFeatures).asJava) + new FeatureContext( + (SharedFeatures.TIMESTAMP +: typedAggregateGroups.flatMap(_.allOutputFeatures)).asJava + ) } private lazy val aggregateFeaturesRenameMap: Map[MLApiFeature[_], MLApiFeature[_]] = { @@ -72,7 +75,7 @@ trait BaseRealtimeAggregateHydrator[K] extends ObservedKeyValueResultHandler { featureContexts.zip(renamedFeatureContexts).zip(decays).foreach { case ((featureContext, renamedFeatureContext), decay) => val decayedDr = applyDecay(dr, featureContext, decay) - val renamedDr = applyRename( + val renamedDr = DataRecordUtil.applyRename( dataRecord = decayedDr, featureContext, renamedFeatureContext, @@ -80,33 +83,36 @@ trait BaseRealtimeAggregateHydrator[K] extends ObservedKeyValueResultHandler { drMerger.merge(newDr, renamedDr) } newDr - case _ => - new DataRecord + case _ => new DataRecord } } - def fetchAndConstructDataRecords(possiblyKeys: Seq[Option[K]]): Stitch[Seq[Try[DataRecord]]] = { - Stitch.callFuture { - val keys = possiblyKeys.flatten + def fetchAndConstructDataRecords(possiblyKeys: Seq[Option[K]]): Future[Seq[Try[DataRecord]]] = { + val keys = possiblyKeys.flatten - val response: Future[KeyValueResult[K, DataRecord]] = - if (keys.isEmpty) { - Future.value(KeyValueResult.empty) - } else { - client.get(keys) - } + val response: Future[KeyValueResult[K, DataRecord]] = + if (keys.isEmpty) Future.value(KeyValueResult.empty) + else { + val batchResponses = keys + .grouped(RequestBatchSize) + .map(keyGroup => client.get(keyGroup)) + .toSeq - response.map { result => - possiblyKeys.map { possiblyKey => - val value = observedGet(key = possiblyKey, keyValueResult = result) - postTransformer(value) - } + Future.collect(batchResponses).map(_.reduce(_ ++ _)) + } + + response.map { result => + possiblyKeys.map { possiblyKey => + val value = observedGet(key = possiblyKey, keyValueResult = result) + postTransformer(value) } } } } object BaseRealtimeAggregateHydrator { + private val RequestBatchSize = 5 + type TimeDecay = scala.Function2[com.twitter.ml.api.DataRecord, scala.Long, scala.Unit] private def applyDecay( @@ -129,26 +135,4 @@ object BaseRealtimeAggregateHydrator { decay(resultDr, time) resultDr } - - private def applyRename( - dataRecord: DataRecord, - featureContext: FeatureContext, - renamedFeatureContext: FeatureContext, - featureRenamingMap: Map[MLApiFeature[_], MLApiFeature[_]] - ): DataRecord = { - val richFullDr = new SRichDataRecord(dataRecord, featureContext) - val richNewDr = new SRichDataRecord(new DataRecord, renamedFeatureContext) - val featureIterator = featureContext.iterator() - featureIterator.forEachRemaining { feature => - if (richFullDr.hasFeature(feature)) { - val renamedFeature = featureRenamingMap.getOrElse(feature, feature) - - val typedFeature = feature.asInstanceOf[MLApiFeature[JDouble]] - val typedRenamedFeature = renamedFeature.asInstanceOf[MLApiFeature[JDouble]] - - richNewDr.setFeatureValue(typedRenamedFeature, richFullDr.getFeatureValue(typedFeature)) - } - } - richNewDr.getRecord - } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator.scala similarity index 96% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator.scala index 57463dd79..990fe1ca4 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/RealTimeAggregateTimeDecay.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/RealTimeAggregateTimeDecay.scala similarity index 95% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/RealTimeAggregateTimeDecay.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/RealTimeAggregateTimeDecay.scala index 110a82635..dbecd12c6 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/RealTimeAggregateTimeDecay.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/RealTimeAggregateTimeDecay.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates import com.twitter.ml.api.DataRecord import com.twitter.ml.api.constant.SharedFeatures.TIMESTAMP diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TopicCountryEngagementRealTimeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TopicCountryEngagementRealTimeAggregateFeatureHydrator.scala similarity index 96% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TopicCountryEngagementRealTimeAggregateFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TopicCountryEngagementRealTimeAggregateFeatureHydrator.scala index d8ffad311..13b2a0d2d 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TopicCountryEngagementRealTimeAggregateFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TopicCountryEngagementRealTimeAggregateFeatureHydrator.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TopicEngagementRealTimeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TopicEngagementRealTimeAggregateFeatureHydrator.scala similarity index 79% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TopicEngagementRealTimeAggregateFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TopicEngagementRealTimeAggregateFeatureHydrator.scala index 6d71f45b3..12a2e1c47 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TopicEngagementRealTimeAggregateFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TopicEngagementRealTimeAggregateFeatureHydrator.scala @@ -1,6 +1,5 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates -import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.TopicEngagementCache @@ -15,6 +14,7 @@ import com.twitter.servo.cache.ReadCache import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ import javax.inject.Inject +import javax.inject.Named import javax.inject.Singleton object TopicEngagementRealTimeAggregateFeature @@ -36,7 +36,14 @@ class TopicEngagementRealTimeAggregateFeatureHydrator @Inject() ( TopicEngagementRealTimeAggregateFeature override val aggregateGroups: Seq[AggregateGroup] = Seq( - topicEngagementRealTimeAggregatesProd + topicEngagementRealTimeAggregatesProd, + topicEngagement24HourRealTimeAggregatesProd, + topicShareEngagementsRealTimeAggregates + ) + + override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( + topicEngagement24HourRealTimeAggregatesProd -> "topic.timelines.topic_engagement_24_hour_real_time_aggregates.", + topicShareEngagementsRealTimeAggregates -> "topic.timelines.topic_share_engagements_real_time_aggregates." ) override def keysFromQueryAndCandidates( diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TweetCountryEngagementRealTimeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TweetCountryEngagementRealTimeAggregateFeatureHydrator.scala similarity index 80% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TweetCountryEngagementRealTimeAggregateFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TweetCountryEngagementRealTimeAggregateFeatureHydrator.scala index 4656b1945..cd9e34a3e 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TweetCountryEngagementRealTimeAggregateFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TweetCountryEngagementRealTimeAggregateFeatureHydrator.scala @@ -1,8 +1,9 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetCountryEngagementCache +import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure @@ -35,11 +36,13 @@ class TweetCountryEngagementRealTimeAggregateFeatureHydrator @Inject() ( TweetCountryEngagementRealTimeAggregateFeature override val aggregateGroups: Seq[AggregateGroup] = Seq( - tweetCountryRealTimeAggregates + tweetCountryRealTimeAggregates, + tweetCountryPrivateEngagementsRealTimeAggregates ) override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( - tweetCountryRealTimeAggregates -> "tweet-country_code.timelines.tweet_country_engagement_real_time_aggregates." + tweetCountryRealTimeAggregates -> "tweet-country_code.timelines.tweet_country_engagement_real_time_aggregates.", + tweetCountryPrivateEngagementsRealTimeAggregates -> "tweet-country_code.timelines.tweet_country_private_engagement_real_time_aggregates." ) override def keysFromQueryAndCandidates( @@ -48,8 +51,8 @@ class TweetCountryEngagementRealTimeAggregateFeatureHydrator @Inject() ( ): Seq[Option[(Long, String)]] = { val countryCode = query.clientContext.countryCode candidates.map { candidate => - val tweetId = candidate.candidate.id - countryCode.map((tweetId, _)) + val originalTweetId = CandidatesUtil.getOriginalTweetId(candidate) + countryCode.map((originalTweetId, _)) } } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TweetEngagementRealTimeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TweetEngagementRealTimeAggregateFeatureHydrator.scala similarity index 69% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TweetEngagementRealTimeAggregateFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TweetEngagementRealTimeAggregateFeatureHydrator.scala index 5a607dec2..99bae79d9 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TweetEngagementRealTimeAggregateFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TweetEngagementRealTimeAggregateFeatureHydrator.scala @@ -1,8 +1,9 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetEngagementCache +import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure @@ -36,7 +37,18 @@ class TweetEngagementRealTimeAggregateFeatureHydrator @Inject() ( override val aggregateGroups: Seq[AggregateGroup] = Seq( tweetEngagement30MinuteCountsProd, - tweetEngagementTotalCountsProd + tweetEngagementTotalCountsProd, + tweetEngagementUserStateRealTimeAggregatesProd, + tweetNegativeEngagementUserStateRealTimeAggregates, + tweetNegativeEngagement6HourCounts, + tweetNegativeEngagementTotalCounts, + tweetShareEngagementsRealTimeAggregates, + tweetBCEDwellEngagementsRealTimeAggregates + ) + + override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( + tweetShareEngagementsRealTimeAggregates -> "original_tweet.timelines.tweet_share_engagements_real_time_aggregates.", + tweetBCEDwellEngagementsRealTimeAggregates -> "original_tweet.timelines.tweet_bce_dwell_engagements_real_time_aggregates." ) override def keysFromQueryAndCandidates( @@ -44,6 +56,6 @@ class TweetEngagementRealTimeAggregateFeatureHydrator @Inject() ( candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[Option[Long]] = { candidates - .map(candidate => Some(candidate.candidate.id)) + .map(candidate => Some(CandidatesUtil.getOriginalTweetId(candidate))) } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TwitterListEngagementRealTimeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TwitterListEngagementRealTimeAggregateFeatureHydrator.scala similarity index 96% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TwitterListEngagementRealTimeAggregateFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TwitterListEngagementRealTimeAggregateFeatureHydrator.scala index e6d7b86f1..b5f3af2a9 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TwitterListEngagementRealTimeAggregateFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TwitterListEngagementRealTimeAggregateFeatureHydrator.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/UserAuthorEngagementRealTimeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/UserAuthorEngagementRealTimeAggregateFeatureHydrator.scala similarity index 90% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/UserAuthorEngagementRealTimeAggregateFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/UserAuthorEngagementRealTimeAggregateFeatureHydrator.scala index 9671c7bf8..2c13fc0f6 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/UserAuthorEngagementRealTimeAggregateFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/UserAuthorEngagementRealTimeAggregateFeatureHydrator.scala @@ -1,9 +1,9 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserAuthorEngagementCache +import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure @@ -51,10 +51,8 @@ class UserAuthorEngagementRealTimeAggregateFeatureHydrator @Inject() ( ): Seq[Option[(Long, Long)]] = { val userId = query.getRequiredUserId candidates.map { candidate => - candidate.features - .getTry(AuthorIdFeature) - .toOption - .flatten + CandidatesUtil + .getOriginalAuthorId(candidate.features) .map((userId, _)) } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/UserEngagementRealTimeAggregatesFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/UserEngagementRealTimeAggregatesFeatureHydrator.scala similarity index 92% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/UserEngagementRealTimeAggregatesFeatureHydrator.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/UserEngagementRealTimeAggregatesFeatureHydrator.scala index 7526f75df..cc05b52aa 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/UserEngagementRealTimeAggregatesFeatureHydrator.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/UserEngagementRealTimeAggregatesFeatureHydrator.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates +package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver @@ -43,7 +43,7 @@ class UserEngagementRealTimeAggregatesFeatureHydrator @Inject() ( ) override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( - userEngagementRealTimeAggregatesProd -> "user.timelines.user_share_engagements_real_time_aggregates.", + userShareEngagementsRealTimeAggregates -> "user.timelines.user_share_engagements_real_time_aggregates.", userBCEDwellEngagementsRealTimeAggregates -> "user.timelines.user_bce_dwell_engagements_real_time_aggregates.", userEngagement48HourRealTimeAggregatesProd -> "user.timelines.user_engagement_48_hour_real_time_aggregates.", userNegativeEngagementAuthorUserState72HourRealTimeAggregates -> "user.timelines.user_negative_engagement_author_user_state_72_hour_real_time_aggregates.", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/BUILD.bazel new file mode 100644 index 000000000..07a2e7cb3 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/BUILD.bazel @@ -0,0 +1,14 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter", + "stitch/stitch-core", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/DuplicateConversationTweetsFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/DuplicateConversationTweetsFilter.scala new file mode 100644 index 000000000..adc11d255 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/DuplicateConversationTweetsFilter.scala @@ -0,0 +1,37 @@ +package com.twitter.home_mixer.product.scored_tweets.filter + +import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.filter.FilterResult +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * Remove any candidate that is in the ancestor list of any reply, including retweets of ancestors. + * + * E.g. if B replied to A and D was a retweet of A, we would prefer to drop D since otherwise + * we may end up serving the same tweet twice in the timeline (e.g. serving both A->B and D). + */ +object DuplicateConversationTweetsFilter extends Filter[PipelineQuery, TweetCandidate] { + + override val identifier: FilterIdentifier = FilterIdentifier("DuplicateConversationTweets") + + override def apply( + query: PipelineQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[FilterResult[TweetCandidate]] = { + val allAncestors = candidates + .flatMap(_.features.getOrElse(AncestorsFeature, Seq.empty)) + .map(_.tweetId).toSet + + val (kept, removed) = candidates.partition { candidate => + !allAncestors.contains(CandidatesUtil.getOriginalTweetId(candidate)) + } + + Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/OutOfNetworkCompetitorFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/OutOfNetworkCompetitorFilter.scala similarity index 96% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/OutOfNetworkCompetitorFilter.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/OutOfNetworkCompetitorFilter.scala index 5bbf35185..3c6024268 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/OutOfNetworkCompetitorFilter.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/OutOfNetworkCompetitorFilter.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.filter +package com.twitter.home_mixer.product.scored_tweets.filter import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/OutOfNetworkCompetitorURLFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/OutOfNetworkCompetitorURLFilter.scala similarity index 90% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/OutOfNetworkCompetitorURLFilter.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/OutOfNetworkCompetitorURLFilter.scala index f7163bee3..00e2bb200 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/OutOfNetworkCompetitorURLFilter.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/OutOfNetworkCompetitorURLFilter.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.filter +package com.twitter.home_mixer.product.scored_tweets.filter import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature @@ -32,6 +32,7 @@ object OutOfNetworkCompetitorURLFilter extends Filter[PipelineQuery, TweetCandid ): Boolean = { !candidate.features.getOrElse(InNetworkFeature, true) && !candidate.features.getOrElse(IsRetweetFeature, false) && - candidate.features.get(TweetUrlsFeature).toSet.intersect(competitorUrls).nonEmpty + candidate.features + .getOrElse(TweetUrlsFeature, Seq.empty).toSet.intersect(competitorUrls).nonEmpty } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetSourceTweetRemovingFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/RetweetSourceTweetRemovingFilter.scala similarity index 96% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetSourceTweetRemovingFilter.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/RetweetSourceTweetRemovingFilter.scala index 6dc88f02c..5900756d3 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetSourceTweetRemovingFilter.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/RetweetSourceTweetRemovingFilter.scala @@ -1,4 +1,4 @@ -package com.twitter.home_mixer.functional_component.filter +package com.twitter.home_mixer.product.scored_tweets.filter import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/SocialContextFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/ScoredTweetsSocialContextFilter.scala similarity index 65% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/SocialContextFilter.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/ScoredTweetsSocialContextFilter.scala index 5002137c4..fef427a6d 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/SocialContextFilter.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/ScoredTweetsSocialContextFilter.scala @@ -1,12 +1,6 @@ -package com.twitter.home_mixer.functional_component.filter +package com.twitter.home_mixer.product.scored_tweets.filter -import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature -import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature +import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.filter.Filter @@ -15,10 +9,18 @@ import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch +import com.twitter.timelineservice.suggests.{thriftscala => st} -object SocialContextFilter extends Filter[PipelineQuery, TweetCandidate] { +object ScoredTweetsSocialContextFilter extends Filter[PipelineQuery, TweetCandidate] { - override val identifier: FilterIdentifier = FilterIdentifier("SocialContext") + override val identifier: FilterIdentifier = FilterIdentifier("ScoredTweetsSocialContext") + + // Tweets from candidate sources which don't need generic like/follow/topic proof + private val AllowedSources: Set[st.SuggestType] = Set( + st.SuggestType.RankedListTweet, + st.SuggestType.RecommendedTrendTweet, + st.SuggestType.MediaTweet + ) override def apply( query: PipelineQuery, @@ -27,10 +29,11 @@ object SocialContextFilter extends Filter[PipelineQuery, TweetCandidate] { val validTweetIds = candidates .filter { candidate => candidate.features.getOrElse(InNetworkFeature, true) || + candidate.features.getOrElse(SuggestTypeFeature, None).exists(AllowedSources.contains) || + candidate.features.getOrElse(InReplyToTweetIdFeature, None).isDefined || hasLikedBySocialContext(candidate.features) || hasFollowedBySocialContext(candidate.features) || - hasTopicSocialContext(candidate.features) || - candidate.features.getOrElse(ConversationModuleFocalTweetIdFeature, None).isDefined + hasTopicSocialContext(candidate.features) }.map(_.candidate.id).toSet val (kept, removed) = @@ -51,7 +54,8 @@ object SocialContextFilter extends Filter[PipelineQuery, TweetCandidate] { private def hasFollowedBySocialContext(candidateFeatures: FeatureMap): Boolean = candidateFeatures.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty).nonEmpty - private def hasTopicSocialContext(candidateFeatures: FeatureMap): Boolean = + private def hasTopicSocialContext(candidateFeatures: FeatureMap): Boolean = { candidateFeatures.getOrElse(TopicIdSocialContextFeature, None).isDefined && - candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None).isDefined + candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None).isDefined + } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/BUILD.bazel new file mode 100644 index 000000000..6d3fd9b1c --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/BUILD.bazel @@ -0,0 +1,12 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", + "home-mixer/thrift/src/main/thrift:thrift-scala", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/MinCachedTweetsGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/MinCachedTweetsGate.scala similarity index 89% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/MinCachedTweetsGate.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/MinCachedTweetsGate.scala index a160a5b5d..bdeb33a92 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/MinCachedTweetsGate.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/MinCachedTweetsGate.scala @@ -1,6 +1,6 @@ -package com.twitter.home_mixer.functional_component.gate +package com.twitter.home_mixer.product.scored_tweets.gate -import com.twitter.home_mixer.functional_component.gate.MinCachedTweetsGate.identifierSuffix +import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate.identifierSuffix import com.twitter.home_mixer.util.CachedScoredTweetsHelper import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/MinTimeSinceLastRequestGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/MinTimeSinceLastRequestGate.scala new file mode 100644 index 000000000..050f2ab67 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/MinTimeSinceLastRequestGate.scala @@ -0,0 +1,27 @@ +package com.twitter.home_mixer.product.scored_tweets.gate + +import com.twitter.conversions.DurationOps._ +import com.twitter.home_mixer.model.HomeFeatures.LastNonPollingTimeFeature +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.stitch.Stitch + +/** + * Gate continues if the amount of time passed since the previous request is greater + * than the configured amount or if the previous request time in not available + */ +object MinTimeSinceLastRequestGate extends Gate[PipelineQuery] { + + override val identifier: GateIdentifier = GateIdentifier("TimeSinceLastRequest") + + private val MinTimeSinceLastRequest = 24.hours + + override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = Stitch.value { + query.features.exists { features => + features + .getOrElse(LastNonPollingTimeFeature, None) + .forall(lnpt => (query.queryTime - lnpt) > MinTimeSinceLastRequest) + } + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/BUILD.bazel index db158c23f..4347c1093 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/BUILD.bazel @@ -4,14 +4,10 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tweet_tlx", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/premarshaller", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/ScoredTweetsResponseDomainMarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/ScoredTweetsResponseDomainMarshaller.scala index 3ba4b201b..796970bec 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/ScoredTweetsResponseDomainMarshaller.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/ScoredTweetsResponseDomainMarshaller.scala @@ -1,17 +1,10 @@ package com.twitter.home_mixer.product.scored_tweets.marshaller -import com.twitter.home_mixer.model.HomeFeatures._ -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweet import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.metadata.TopicContextFunctionalityTypeMarshaller import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller import com.twitter.product_mixer.core.model.common.identifier.DomainMarshallerIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails /** * Creates a domain model of the Scored Tweets product response from the set of candidates selected @@ -25,40 +18,5 @@ object ScoredTweetsResponseDomainMarshaller override def apply( query: ScoredTweetsQuery, selections: Seq[CandidateWithDetails] - ): ScoredTweetsResponse = ScoredTweetsResponse( - scoredTweets = selections.collect { - case ItemCandidateWithDetails(candidate: TweetCandidate, _, features) => - Seq(mkScoredTweet(candidate.id, features)) - case ModuleCandidateWithDetails(candidates, _, _) => - candidates.map { candidate => mkScoredTweet(candidate.candidateIdLong, candidate.features) } - }.flatten - ) - - private def mkScoredTweet(tweetId: Long, features: FeatureMap): ScoredTweet = { - val topicFunctionalityType = features - .getOrElse(TopicContextFunctionalityTypeFeature, None) - .map(TopicContextFunctionalityTypeMarshaller(_)) - - ScoredTweet( - tweetId = tweetId, - authorId = features.get(AuthorIdFeature).get, - score = features.get(ScoreFeature), - suggestType = features.get(SuggestTypeFeature).get, - sourceTweetId = features.getOrElse(SourceTweetIdFeature, None), - sourceUserId = features.getOrElse(SourceUserIdFeature, None), - quotedTweetId = features.getOrElse(QuotedTweetIdFeature, None), - quotedUserId = features.getOrElse(QuotedUserIdFeature, None), - inReplyToTweetId = features.getOrElse(InReplyToTweetIdFeature, None), - inReplyToUserId = features.getOrElse(InReplyToUserIdFeature, None), - directedAtUserId = features.getOrElse(DirectedAtUserIdFeature, None), - inNetwork = Some(features.getOrElse(InNetworkFeature, false)), - favoritedByUserIds = Some(features.getOrElse(FavoritedByUserIdsFeature, Seq.empty)), - followedByUserIds = Some(features.getOrElse(FollowedByUserIdsFeature, Seq.empty)), - topicId = features.getOrElse(TopicIdSocialContextFeature, None), - topicFunctionalityType = topicFunctionalityType, - ancestors = Some(features.getOrElse(AncestorsFeature, Seq.empty)), - isReadFromCache = Some(features.getOrElse(IsReadFromCacheFeature, false)), - streamToKafka = Some(features.getOrElse(StreamToKafkaFeature, false)) - ) - } + ): ScoredTweetsResponse = ScoredTweetsResponse(scoredTweets = selections) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/ScoredTweetsResponseTransportMarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/ScoredTweetsResponseTransportMarshaller.scala index 355f076fe..27486768b 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/ScoredTweetsResponseTransportMarshaller.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/ScoredTweetsResponseTransportMarshaller.scala @@ -1,8 +1,11 @@ package com.twitter.home_mixer.product.scored_tweets.marshaller +import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse import com.twitter.home_mixer.{thriftscala => t} +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller +import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.metadata.TopicContextFunctionalityTypeMarshaller import com.twitter.product_mixer.core.model.common.identifier.TransportMarshallerIdentifier /** @@ -16,28 +19,52 @@ object ScoredTweetsResponseTransportMarshaller override def apply(input: ScoredTweetsResponse): t.ScoredTweetsResponse = { val scoredTweets = input.scoredTweets.map { tweet => - t.ScoredTweet( - tweetId = tweet.tweetId, - authorId = tweet.authorId, - score = tweet.score, - suggestType = Some(tweet.suggestType), - sourceTweetId = tweet.sourceTweetId, - sourceUserId = tweet.sourceUserId, - quotedTweetId = tweet.quotedTweetId, - quotedUserId = tweet.quotedUserId, - inReplyToTweetId = tweet.inReplyToTweetId, - inReplyToUserId = tweet.inReplyToUserId, - directedAtUserId = tweet.directedAtUserId, - inNetwork = tweet.inNetwork, - favoritedByUserIds = tweet.favoritedByUserIds, - followedByUserIds = tweet.followedByUserIds, - topicId = tweet.topicId, - topicFunctionalityType = tweet.topicFunctionalityType, - ancestors = tweet.ancestors, - isReadFromCache = tweet.isReadFromCache, - streamToKafka = tweet.streamToKafka - ) + mkScoredTweet(tweet.candidateIdLong, tweet.features) } t.ScoredTweetsResponse(scoredTweets) } + + private def mkScoredTweet(tweetId: Long, features: FeatureMap): t.ScoredTweet = { + val topicFunctionalityType = features + .getOrElse(TopicContextFunctionalityTypeFeature, None) + .map(TopicContextFunctionalityTypeMarshaller(_)) + + t.ScoredTweet( + tweetId = tweetId, + authorId = features.get(AuthorIdFeature).get, + score = features.get(ScoreFeature), + suggestType = features.get(SuggestTypeFeature), + sourceTweetId = features.getOrElse(SourceTweetIdFeature, None), + sourceUserId = features.getOrElse(SourceUserIdFeature, None), + quotedTweetId = features.getOrElse(QuotedTweetIdFeature, None), + quotedUserId = features.getOrElse(QuotedUserIdFeature, None), + inReplyToTweetId = features.getOrElse(InReplyToTweetIdFeature, None), + inReplyToUserId = features.getOrElse(InReplyToUserIdFeature, None), + directedAtUserId = features.getOrElse(DirectedAtUserIdFeature, None), + inNetwork = Some(features.getOrElse(InNetworkFeature, true)), + sgsValidLikedByUserIds = Some(features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty)), + sgsValidFollowedByUserIds = + Some(features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty)), + topicId = features.getOrElse(TopicIdSocialContextFeature, None), + topicFunctionalityType = topicFunctionalityType, + ancestors = Some(features.getOrElse(AncestorsFeature, Seq.empty)), + isReadFromCache = Some(features.getOrElse(IsReadFromCacheFeature, false)), + streamToKafka = Some(features.getOrElse(StreamToKafkaFeature, false)), + exclusiveConversationAuthorId = + features.getOrElse(ExclusiveConversationAuthorIdFeature, None), + authorMetadata = Some( + t.AuthorMetadata( + blueVerified = features.getOrElse(AuthorIsBlueVerifiedFeature, false), + goldVerified = features.getOrElse(AuthorIsGoldVerifiedFeature, false), + grayVerified = features.getOrElse(AuthorIsGrayVerifiedFeature, false), + legacyVerified = features.getOrElse(AuthorIsLegacyVerifiedFeature, false), + creator = features.getOrElse(AuthorIsCreatorFeature, false) + )), + lastScoredTimestampMs = None, + candidatePipelineIdentifier = None, + tweetUrls = None, + perspectiveFilteredLikedByUserIds = + Some(features.getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Seq.empty)), + ) + } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/BUILD.bazel index 8fd8325cb..6414453db 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/BUILD.bazel @@ -7,17 +7,11 @@ scala_library( "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor", - "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", ], exports = [ "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/ScoredTweetsResponse.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/ScoredTweetsResponse.scala index 81cdc2211..e9bd7cd61 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/ScoredTweetsResponse.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/ScoredTweetsResponse.scala @@ -1,29 +1,6 @@ package com.twitter.home_mixer.product.scored_tweets.model +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.timelineservice.suggests.{thriftscala => st} -import com.twitter.tweetconvosvc.tweet_ancestor.{thriftscala => ta} -import com.twitter.timelines.render.{thriftscala => urt} -case class ScoredTweet( - tweetId: Long, - authorId: Long, - score: Option[Double], - suggestType: st.SuggestType, - sourceTweetId: Option[Long], - sourceUserId: Option[Long], - quotedTweetId: Option[Long], - quotedUserId: Option[Long], - inReplyToTweetId: Option[Long], - inReplyToUserId: Option[Long], - directedAtUserId: Option[Long], - inNetwork: Option[Boolean], - favoritedByUserIds: Option[Seq[Long]], - followedByUserIds: Option[Seq[Long]], - topicId: Option[Long], - topicFunctionalityType: Option[urt.TopicContextFunctionalityType], - ancestors: Option[Seq[ta.TweetAncestor]], - isReadFromCache: Option[Boolean], - streamToKafka: Option[Boolean]) - -case class ScoredTweetsResponse(scoredTweets: Seq[ScoredTweet]) extends HasMarshalling +case class ScoredTweetsResponse(scoredTweets: Seq[CandidateWithDetails]) extends HasMarshalling diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/BUILD.bazel index d402ecc18..065841642 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/BUILD.bazel @@ -4,12 +4,8 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/timelineranker", - "util/util-core/src/main/scala/com/twitter/conversions", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/ScoredTweetsParam.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/ScoredTweetsParam.scala index 261be593a..9720b9344 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/ScoredTweetsParam.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/ScoredTweetsParam.scala @@ -12,38 +12,93 @@ import com.twitter.util.Duration object ScoredTweetsParam { val SupportedClientFSName = "scored_tweets_supported_client" - object CrMixerSource { - object EnableCandidatePipelineParam - extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsCrMixerCandidatePipeline) - } - - object FrsTweetSource { - object EnableCandidatePipelineParam - extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsFrsCandidatePipeline) - } - - object InNetworkSource { - object EnableCandidatePipelineParam + object CandidatePipeline { + object EnableInNetworkParam extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsInNetworkCandidatePipeline) + + object EnableTweetMixerParam + extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsTweetMixerCandidatePipeline) + + object EnableUtegParam + extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsUtegCandidatePipeline) + + object EnableFrsParam + extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsFrsCandidatePipeline) + + object EnableListsParam + extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsListsCandidatePipeline) + + object EnablePopularVideosParam + extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsPopularVideosCandidatePipeline) + + object EnableBackfillParam + extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsBackfillCandidatePipeline) } + object EnableBackfillCandidatePipelineParam + extends FSParam[Boolean]( + name = "scored_tweets_enable_backfill_candidate_pipeline", + default = true + ) + object QualityFactor { - object MaxTweetsToScoreParam + object InNetworkMaxTweetsToScoreParam extends FSBoundedParam[Int]( - name = "scored_tweets_quality_factor_max_tweets_to_score", - default = 1100, + name = "scored_tweets_quality_factor_earlybird_max_tweets_to_score", + default = 500, min = 0, max = 10000 ) - object CrMixerMaxTweetsToScoreParam + object UtegMaxTweetsToScoreParam extends FSBoundedParam[Int]( - name = "scored_tweets_quality_factor_cr_mixer_max_tweets_to_score", + name = "scored_tweets_quality_factor_uteg_max_tweets_to_score", + default = 500, + min = 0, + max = 10000 + ) + + object FrsMaxTweetsToScoreParam + extends FSBoundedParam[Int]( + name = "scored_tweets_quality_factor_frs_max_tweets_to_score", + default = 500, + min = 0, + max = 10000 + ) + + object TweetMixerMaxTweetsToScoreParam + extends FSBoundedParam[Int]( + name = "scored_tweets_quality_factor_tweet_mixer_max_tweets_to_score", + default = 500, + min = 0, + max = 10000 + ) + + object ListsMaxTweetsToScoreParam + extends FSBoundedParam[Int]( + name = "scored_tweets_quality_factor_lists_max_tweets_to_score", + default = 500, + min = 0, + max = 100 + ) + + object PopularVideosMaxTweetsToScoreParam + extends FSBoundedParam[Int]( + name = "scored_tweets_quality_factor_popular_videos_max_tweets_to_score", + default = 40, + min = 0, + max = 10000 + ) + + object BackfillMaxTweetsToScoreParam + extends FSBoundedParam[Int]( + name = "scored_tweets_quality_factor_backfill_max_tweets_to_score", default = 500, min = 0, max = 10000 ) } + object ServerMaxResultsParam extends FSBoundedParam[Int]( name = "scored_tweets_server_max_results", @@ -51,10 +106,22 @@ object ScoredTweetsParam { min = 1, max = 500 ) - object UtegSource { - object EnableCandidatePipelineParam - extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsUtegCandidatePipeline) - } + + object MaxInNetworkResultsParam + extends FSBoundedParam[Int]( + name = "scored_tweets_max_in_network_results", + default = 60, + min = 1, + max = 500 + ) + + object MaxOutOfNetworkResultsParam + extends FSBoundedParam[Int]( + name = "scored_tweets_max_out_of_network_results", + default = 60, + min = 1, + max = 500 + ) object CachedScoredTweets { object TTLParam @@ -147,6 +214,46 @@ object ScoredTweetsParam { max = 1000000.0 ) + object TweetDetailDwellParam + extends FSBoundedParam[Double]( + name = "scored_tweets_model_weight_tweet_detail_dwell", + default = 0.0, + min = 0.0, + max = 100.0 + ) + + object ProfileDwelledParam + extends FSBoundedParam[Double]( + name = "scored_tweets_model_weight_profile_dwelled", + default = 0.0, + min = 0.0, + max = 100.0 + ) + + object BookmarkParam + extends FSBoundedParam[Double]( + name = "scored_tweets_model_weight_bookmark", + default = 0.0, + min = 0.0, + max = 100.0 + ) + + object ShareParam + extends FSBoundedParam[Double]( + name = "scored_tweets_model_weight_share", + default = 0.0, + min = 0.0, + max = 100.0 + ) + + object ShareMenuClickParam + extends FSBoundedParam[Double]( + name = "scored_tweets_model_weight_share_menu_click", + default = 0.0, + min = 0.0, + max = 100.0 + ) + object NegativeFeedbackV2Param extends FSBoundedParam[Double]( name = "scored_tweets_model_weight_negative_feedback_v2", @@ -162,6 +269,22 @@ object ScoredTweetsParam { min = -20000.0, max = 0.0 ) + + object WeakNegativeFeedbackParam + extends FSBoundedParam[Double]( + name = "scored_tweets_model_weight_weak_negative_feedback", + default = 0.0, + min = -1000.0, + max = 0.0 + ) + + object StrongNegativeFeedbackParam + extends FSBoundedParam[Double]( + name = "scored_tweets_model_weight_strong_negative_feedback", + default = 0.0, + min = -1000.0, + max = 0.0 + ) } } @@ -173,4 +296,66 @@ object ScoredTweetsParam { object CompetitorURLSeqParam extends FSParam[Seq[String]](name = "scored_tweets_competitor_url_list", default = Seq.empty) + + object BlueVerifiedAuthorInNetworkMultiplierParam + extends FSBoundedParam[Double]( + name = "scored_tweets_blue_verified_author_in_network_multiplier", + default = 4.0, + min = 0.0, + max = 100.0 + ) + + object BlueVerifiedAuthorOutOfNetworkMultiplierParam + extends FSBoundedParam[Double]( + name = "scored_tweets_blue_verified_author_out_of_network_multiplier", + default = 2.0, + min = 0.0, + max = 100.0 + ) + + object CreatorInNetworkMultiplierParam + extends FSBoundedParam[Double]( + name = "scored_tweets_creator_in_network_multiplier", + default = 1.1, + min = 0.0, + max = 100.0 + ) + + object CreatorOutOfNetworkMultiplierParam + extends FSBoundedParam[Double]( + name = "scored_tweets_creator_out_of_network_multiplier", + default = 1.3, + min = 0.0, + max = 100.0 + ) + + object OutOfNetworkScaleFactorParam + extends FSBoundedParam[Double]( + name = "scored_tweets_out_of_network_scale_factor", + default = 1.0, + min = 0.0, + max = 100.0 + ) + + object EnableScribeScoredCandidatesParam + extends FSParam[Boolean](name = "scored_tweets_enable_scribing", default = false) + + object EarlybirdTensorflowModel { + + object InNetworkParam + extends FSParam[String]( + name = "scored_tweets_in_network_earlybird_tensorflow_model", + default = "timelines_recap_replica") + + object FrsParam + extends FSParam[String]( + name = "scored_tweets_frs_earlybird_tensorflow_model", + default = "timelines_rectweet_replica") + + object UtegParam + extends FSParam[String]( + name = "scored_tweets_uteg_earlybird_tensorflow_model", + default = "timelines_rectweet_replica") + } + } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/ScoredTweetsParamConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/ScoredTweetsParamConfig.scala index ae82db20f..10b9de49d 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/ScoredTweetsParamConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/ScoredTweetsParamConfig.scala @@ -4,7 +4,6 @@ import com.twitter.home_mixer.param.decider.DeciderKey import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam._ import com.twitter.product_mixer.core.product.ProductParamConfig import com.twitter.servo.decider.DeciderKeyName - import javax.inject.Inject import javax.inject.Singleton @@ -14,17 +13,32 @@ class ScoredTweetsParamConfig @Inject() () extends ProductParamConfig { override val supportedClientFSName: String = SupportedClientFSName override val booleanDeciderOverrides = Seq( - CrMixerSource.EnableCandidatePipelineParam, - FrsTweetSource.EnableCandidatePipelineParam, - InNetworkSource.EnableCandidatePipelineParam, - UtegSource.EnableCandidatePipelineParam, + CandidatePipeline.EnableBackfillParam, + CandidatePipeline.EnableTweetMixerParam, + CandidatePipeline.EnableFrsParam, + CandidatePipeline.EnableInNetworkParam, + CandidatePipeline.EnableListsParam, + CandidatePipeline.EnablePopularVideosParam, + CandidatePipeline.EnableUtegParam, ScoredTweetsParam.EnableSimClustersSimilarityFeatureHydrationDeciderParam ) + override val booleanFSOverrides = Seq( + EnableBackfillCandidatePipelineParam, + EnableScribeScoredCandidatesParam + ) + override val boundedIntFSOverrides = Seq( CachedScoredTweets.MinCachedTweetsParam, - QualityFactor.CrMixerMaxTweetsToScoreParam, - QualityFactor.MaxTweetsToScoreParam, + MaxInNetworkResultsParam, + MaxOutOfNetworkResultsParam, + QualityFactor.BackfillMaxTweetsToScoreParam, + QualityFactor.TweetMixerMaxTweetsToScoreParam, + QualityFactor.FrsMaxTweetsToScoreParam, + QualityFactor.InNetworkMaxTweetsToScoreParam, + QualityFactor.ListsMaxTweetsToScoreParam, + QualityFactor.PopularVideosMaxTweetsToScoreParam, + QualityFactor.UtegMaxTweetsToScoreParam, ServerMaxResultsParam ) @@ -33,10 +47,19 @@ class ScoredTweetsParamConfig @Inject() () extends ProductParamConfig { ) override val stringFSOverrides = Seq( - Scoring.HomeModelParam + Scoring.HomeModelParam, + EarlybirdTensorflowModel.InNetworkParam, + EarlybirdTensorflowModel.FrsParam, + EarlybirdTensorflowModel.UtegParam ) override val boundedDoubleFSOverrides = Seq( + BlueVerifiedAuthorInNetworkMultiplierParam, + BlueVerifiedAuthorOutOfNetworkMultiplierParam, + CreatorInNetworkMultiplierParam, + CreatorOutOfNetworkMultiplierParam, + OutOfNetworkScaleFactorParam, + // Model Weights Scoring.ModelWeights.FavParam, Scoring.ModelWeights.ReplyParam, Scoring.ModelWeights.RetweetParam, @@ -47,10 +70,17 @@ class ScoredTweetsParamConfig @Inject() () extends ProductParamConfig { Scoring.ModelWeights.VideoPlayback50Param, Scoring.ModelWeights.ReportParam, Scoring.ModelWeights.NegativeFeedbackV2Param, + Scoring.ModelWeights.TweetDetailDwellParam, + Scoring.ModelWeights.ProfileDwelledParam, + Scoring.ModelWeights.BookmarkParam, + Scoring.ModelWeights.ShareParam, + Scoring.ModelWeights.ShareMenuClickParam, + Scoring.ModelWeights.StrongNegativeFeedbackParam, + Scoring.ModelWeights.WeakNegativeFeedbackParam ) override val longSetFSOverrides = Seq( - CompetitorSetParam, + CompetitorSetParam ) override val stringSeqFSOverrides = Seq( diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/BUILD.bazel index a7db4784b..15fc94f47 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/BUILD.bazel @@ -5,14 +5,13 @@ scala_library( dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_feature_hydrator", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "snowflake/src/main/scala/com/twitter/snowflake/id", "src/thrift/com/twitter/timelineranker:thrift-scala", - "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/timelineranker", "timelineranker/common/src/main/scala/com/twitter/timelineranker/model", "timelines:util", "timelines/src/main/scala/com/twitter/timelines/common/model", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerFrsQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerFrsQueryTransformer.scala index e141d1ee7..2592ec82e 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerFrsQueryTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerFrsQueryTransformer.scala @@ -1,8 +1,11 @@ package com.twitter.home_mixer.product.scored_tweets.query_transformer import com.twitter.conversions.DurationOps._ +import com.twitter.core_workflows.user_model.{thriftscala => um} +import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.home_mixer.product.scored_tweets.query_feature_hydrator.FrsSeedUserIdsFeature +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.FrsSeedUserIdsFeature +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerFrsQueryTransformer._ import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier @@ -11,33 +14,51 @@ import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus import com.twitter.timelineranker.{thriftscala => t} import com.twitter.timelines.common.model.TweetKindOption import com.twitter.timelines.model.candidate.CandidateTweetSourceId -import com.twitter.util.Duration object TimelineRankerFrsQueryTransformer { - private val SinceDuration = 24.hours + private val DefaultSinceDuration = 24.hours + private val ExpandedSinceDuration = 48.hours private val MaxTweetsToFetch = 100 private val tweetKindOptions: TweetKindOption.ValueSet = TweetKindOption(includeOriginalTweetsAndQuotes = true) + + private val UserStatesForExtendedSinceDuration: Set[um.UserState] = Set( + um.UserState.Light, + um.UserState.MediumNonTweeter, + um.UserState.MediumTweeter, + um.UserState.NearZero, + um.UserState.New, + um.UserState.VeryLight + ) } case class TimelineRankerFrsQueryTransformer[ Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext ]( override val candidatePipelineIdentifier: CandidatePipelineIdentifier, - override val maxTweetsToFetch: Int = MaxTweetsToFetch, - override val sinceDuration: Duration = SinceDuration) + override val maxTweetsToFetch: Int = MaxTweetsToFetch) extends CandidatePipelineQueryTransformer[Query, t.RecapQuery] with TimelineRankerQueryTransformer[Query] { override val candidateTweetSourceId = CandidateTweetSourceId.FrsTweet - override val skipVeryRecentTweets = false override val options = tweetKindOptions + override def getTensorflowModel(query: Query): Option[String] = { + Some(query.params(ScoredTweetsParam.EarlybirdTensorflowModel.FrsParam)) + } + override def seedAuthorIds(query: Query): Option[Seq[Long]] = { query.features.flatMap(_.getOrElse(FrsSeedUserIdsFeature, None)) } - override def transform(input: Query): t.RecapQuery = - buildTimelineRankerQuery(input).toThriftRecapQuery + override def transform(input: Query): t.RecapQuery = { + val userState = input.features.get.getOrElse(UserStateFeature, None) + + val sinceDuration = + if (userState.exists(UserStatesForExtendedSinceDuration.contains)) ExpandedSinceDuration + else DefaultSinceDuration + + buildTimelineRankerQuery(input, sinceDuration).toThriftRecapQuery + } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerInNetworkQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerInNetworkQueryTransformer.scala index bec8f74f2..4514dc2c4 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerInNetworkQueryTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerInNetworkQueryTransformer.scala @@ -1,7 +1,10 @@ package com.twitter.home_mixer.product.scored_tweets.query_transformer import com.twitter.conversions.DurationOps._ +import com.twitter.core_workflows.user_model.{thriftscala => um} +import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature import com.twitter.home_mixer.model.request.HasDeviceContext +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerInNetworkQueryTransformer._ import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier @@ -10,11 +13,11 @@ import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus import com.twitter.timelineranker.{thriftscala => t} import com.twitter.timelines.common.model.TweetKindOption import com.twitter.timelines.model.candidate.CandidateTweetSourceId -import com.twitter.util.Duration object TimelineRankerInNetworkQueryTransformer { - private val SinceDuration = 24.hours - private val MaxTweetsToFetch = 500 + private val DefaultSinceDuration = 24.hours + private val ExpandedSinceDuration = 48.hours + private val MaxTweetsToFetch = 600 private val tweetKindOptions: TweetKindOption.ValueSet = TweetKindOption( includeReplies = true, @@ -22,21 +25,39 @@ object TimelineRankerInNetworkQueryTransformer { includeOriginalTweetsAndQuotes = true, includeExtendedReplies = true ) + + private val UserStatesForExtendedSinceDuration: Set[um.UserState] = Set( + um.UserState.Light, + um.UserState.MediumNonTweeter, + um.UserState.MediumTweeter, + um.UserState.NearZero, + um.UserState.New, + um.UserState.VeryLight + ) } case class TimelineRankerInNetworkQueryTransformer[ Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext ]( override val candidatePipelineIdentifier: CandidatePipelineIdentifier, - override val maxTweetsToFetch: Int = MaxTweetsToFetch, - override val sinceDuration: Duration = SinceDuration) + override val maxTweetsToFetch: Int = MaxTweetsToFetch) extends CandidatePipelineQueryTransformer[Query, t.RecapQuery] with TimelineRankerQueryTransformer[Query] { override val candidateTweetSourceId = CandidateTweetSourceId.RecycledTweet - override val skipVeryRecentTweets = false override val options = tweetKindOptions - override def transform(input: Query): t.RecapQuery = - buildTimelineRankerQuery(input).toThriftRecapQuery + override def getTensorflowModel(query: Query): Option[String] = { + Some(query.params(ScoredTweetsParam.EarlybirdTensorflowModel.InNetworkParam)) + } + + override def transform(input: Query): t.RecapQuery = { + val userState = input.features.get.getOrElse(UserStateFeature, None) + + val sinceDuration = + if (userState.exists(UserStatesForExtendedSinceDuration.contains)) ExpandedSinceDuration + else DefaultSinceDuration + + buildTimelineRankerQuery(input, sinceDuration).toThriftRecapQuery + } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerQueryTransformer.scala index d5b29974b..e187a0aa0 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerQueryTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerQueryTransformer.scala @@ -4,11 +4,10 @@ import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerQueryTransformer._ import com.twitter.home_mixer.util.CachedScoredTweetsHelper +import com.twitter.home_mixer.util.earlybird.EarlybirdRequestUtil import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus -import com.twitter.timelinemixer.clients.timelineranker.EarlybirdScoringModels -import com.twitter.timelinemixer.clients.timelineranker.EarlybirdScoringModelsId import com.twitter.timelineranker.{model => tlr} import com.twitter.timelines.common.model.TweetKindOption import com.twitter.timelines.earlybird.common.options.EarlybirdOptions @@ -37,24 +36,22 @@ object TimelineRankerQueryTransformer { /** * Maximum number of results TLR should retrieve from each earlybird shard. */ - private val EarlybirdMaxResults = 200 + private val EarlybirdMaxResults = 300 } trait TimelineRankerQueryTransformer[ Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext] { def maxTweetsToFetch: Int - def sinceDuration: Duration def options: TweetKindOption.ValueSet = TweetKindOption.Default def candidateTweetSourceId: CandidateTweetSourceId.Value - def skipVeryRecentTweets: Boolean def utegLikedByTweetsOptions(query: Query): Option[tlr.UtegLikedByTweetsOptions] = None def seedAuthorIds(query: Query): Option[Seq[Long]] = None def candidatePipelineIdentifier: CandidatePipelineIdentifier def earlybirdModels: Seq[EarlybirdScoringModelConfig] = - EarlybirdScoringModels.fromEnum(EarlybirdScoringModelsId.UnifiedEngagementProd) - def tensorflowModel: Option[String] = None + EarlybirdRequestUtil.EarlybirdScoringModels.UnifiedEngagementProd + def getTensorflowModel(query: Query): Option[String] = None - def buildTimelineRankerQuery(query: Query): tlr.RecapQuery = { + def buildTimelineRankerQuery(query: Query, sinceDuration: Duration): tlr.RecapQuery = { val sinceTime: Time = sinceDuration.ago val untilTime: Time = Time.now @@ -81,12 +78,14 @@ trait TimelineRankerQueryTransformer[ val deviceContext = query.deviceContext.map(_.toTimelineServiceDeviceContext(query.clientContext)) + val tensorflowModel = getTensorflowModel(query) + val earlyBirdOptions = EarlybirdOptions( maxNumHitsPerShard = EarlybirdMaxHits, maxNumResultsPerShard = EarlybirdMaxResults, models = earlybirdModels, authorScoreMap = authorScoreMap, - skipVeryRecentTweets = skipVeryRecentTweets, + skipVeryRecentTweets = true, tensorflowModel = tensorflowModel ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerUtegQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerUtegQueryTransformer.scala index 03a63acf9..ea051f331 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerUtegQueryTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerUtegQueryTransformer.scala @@ -3,46 +3,46 @@ package com.twitter.home_mixer.product.scored_tweets.query_transformer import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature import com.twitter.home_mixer.model.request.HasDeviceContext +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerUtegQueryTransformer._ +import com.twitter.home_mixer.util.earlybird.EarlybirdRequestUtil import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus -import com.twitter.timelinemixer.clients.timelineranker.EarlybirdScoringModels -import com.twitter.timelinemixer.clients.timelineranker.EarlybirdScoringModelsId import com.twitter.timelineranker.{model => tlr} import com.twitter.timelineranker.{thriftscala => t} import com.twitter.timelines.common.model.TweetKindOption +import com.twitter.timelines.earlybird.common.options.EarlybirdScoringModelConfig import com.twitter.timelines.model.UserId import com.twitter.timelines.model.candidate.CandidateTweetSourceId -import com.twitter.util.Duration object TimelineRankerUtegQueryTransformer { private val SinceDuration = 24.hours - private val MaxTweetsToFetch = 500 + private val MaxTweetsToFetch = 300 private val MaxUtegCandidates = 800 - private val TensorflowModel = "timelines_rectweet_replica" + private val tweetKindOptions = + TweetKindOption(includeOriginalTweetsAndQuotes = true, includeReplies = true) - private val tweetKindOptions = TweetKindOption(includeReplies = true) - - def utegEarlybirdModels = - EarlybirdScoringModels.fromEnum(EarlybirdScoringModelsId.UnifiedEngagementRectweet) + def utegEarlybirdModels: Seq[EarlybirdScoringModelConfig] = + EarlybirdRequestUtil.EarlybirdScoringModels.UnifiedEngagementRectweet } case class TimelineRankerUtegQueryTransformer[ Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext ]( override val candidatePipelineIdentifier: CandidatePipelineIdentifier, - override val maxTweetsToFetch: Int = MaxTweetsToFetch, - override val sinceDuration: Duration = SinceDuration) + override val maxTweetsToFetch: Int = MaxTweetsToFetch) extends CandidatePipelineQueryTransformer[Query, t.UtegLikedByTweetsQuery] with TimelineRankerQueryTransformer[Query] { override val candidateTweetSourceId = CandidateTweetSourceId.RecommendedTweet - override val skipVeryRecentTweets = true + override val options = tweetKindOptions override val earlybirdModels = utegEarlybirdModels - override val tensorflowModel = Some(TensorflowModel) + override def getTensorflowModel(query: Query): Option[String] = { + Some(query.params(ScoredTweetsParam.EarlybirdTensorflowModel.UtegParam)) + } override def utegLikedByTweetsOptions(input: Query): Option[tlr.UtegLikedByTweetsOptions] = Some( tlr.UtegLikedByTweetsOptions( @@ -55,5 +55,5 @@ case class TimelineRankerUtegQueryTransformer[ ) override def transform(input: Query): t.UtegLikedByTweetsQuery = - buildTimelineRankerQuery(input).toThriftUtegLikedByTweetsQuery + buildTimelineRankerQuery(input, SinceDuration).toThriftUtegLikedByTweetsQuery } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/BUILD.bazel new file mode 100644 index 000000000..a337a328b --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/BUILD.bazel @@ -0,0 +1,23 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", + "src/thrift/com/twitter/search:earlybird-scala", + "timelines:util", + "timelines/src/main/scala/com/twitter/timelines/clients/relevance_search", + "timelines/src/main/scala/com/twitter/timelines/common/model", + "timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils", + "timelines/src/main/scala/com/twitter/timelines/model/candidate", + "timelines/src/main/scala/com/twitter/timelines/model/types", + "timelines/src/main/scala/com/twitter/timelines/util/stats", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdFrsQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdFrsQueryTransformer.scala new file mode 100644 index 000000000..9b2ac341e --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdFrsQueryTransformer.scala @@ -0,0 +1,41 @@ +package com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird + +import com.twitter.conversions.DurationOps._ +import com.twitter.home_mixer.model.request.HasDeviceContext +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.FrsSeedUserIdsFeature +import com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird.EarlybirdFrsQueryTransformer._ +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus +import com.twitter.search.earlybird.{thriftscala => eb} +import com.twitter.timelines.common.model.TweetKindOption + +object EarlybirdFrsQueryTransformer { + private val SinceDuration = 24.hours + private val MaxTweetsToFetch = 100 + private val TensorflowModel = Some("timelines_rectweet_replica") + + private val TweetKindOptions: TweetKindOption.ValueSet = + TweetKindOption(includeOriginalTweetsAndQuotes = true) +} + +case class EarlybirdFrsQueryTransformer[ + Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext +]( + candidatePipelineIdentifier: CandidatePipelineIdentifier, + override val clientId: Option[String]) + extends CandidatePipelineQueryTransformer[Query, eb.EarlybirdRequest] + with EarlybirdQueryTransformer[Query] { + + override val tweetKindOptions: TweetKindOption.ValueSet = TweetKindOptions + override val maxTweetsToFetch: Int = MaxTweetsToFetch + override val tensorflowModel: Option[String] = TensorflowModel + + override def transform(query: Query): eb.EarlybirdRequest = { + val seedUserIds = query.features + .flatMap(_.getOrElse(FrsSeedUserIdsFeature, None)) + .getOrElse(Seq.empty).toSet + buildEarlybirdQuery(query, SinceDuration, seedUserIds) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdInNetworkQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdInNetworkQueryTransformer.scala new file mode 100644 index 000000000..5c5464b8b --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdInNetworkQueryTransformer.scala @@ -0,0 +1,68 @@ +package com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird + +import com.twitter.conversions.DurationOps._ +import com.twitter.core_workflows.user_model.{thriftscala => um} +import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature +import com.twitter.home_mixer.model.request.HasDeviceContext +import com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird.EarlybirdInNetworkQueryTransformer._ +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus +import com.twitter.search.earlybird.{thriftscala => eb} +import com.twitter.timelines.common.model.TweetKindOption + +object EarlybirdInNetworkQueryTransformer { + private val DefaultSinceDuration = 24.hours + private val ExpandedSinceDuration = 48.hours + private val MaxTweetsToFetch = 600 + private val TensorflowModel = Some("timelines_recap_replica") + + private val TweetKindOptions: TweetKindOption.ValueSet = TweetKindOption( + includeReplies = true, + includeRetweets = true, + includeOriginalTweetsAndQuotes = true, + includeExtendedReplies = true + ) + + private val UserStatesForExtendedSinceDuration: Set[um.UserState] = Set( + um.UserState.Light, + um.UserState.MediumNonTweeter, + um.UserState.MediumTweeter, + um.UserState.NearZero, + um.UserState.New, + um.UserState.VeryLight + ) +} + +case class EarlybirdInNetworkQueryTransformer[ + Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext +]( + candidatePipelineIdentifier: CandidatePipelineIdentifier, + override val clientId: Option[String]) + extends CandidatePipelineQueryTransformer[Query, eb.EarlybirdRequest] + with EarlybirdQueryTransformer[Query] { + + override val tweetKindOptions: TweetKindOption.ValueSet = TweetKindOptions + override val maxTweetsToFetch: Int = MaxTweetsToFetch + override val tensorflowModel: Option[String] = TensorflowModel + + override def transform(query: Query): eb.EarlybirdRequest = { + + val userState = query.features.get.getOrElse(UserStateFeature, None) + + val sinceDuration = + if (userState.exists(UserStatesForExtendedSinceDuration.contains)) ExpandedSinceDuration + else DefaultSinceDuration + + val followedUserIds = + query.features + .map( + _.getOrElse( + SGSFollowedUsersFeature, + Seq.empty)).toSeq.flatten.toSet + query.getRequiredUserId + + buildEarlybirdQuery(query, sinceDuration, followedUserIds) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdQueryTransformer.scala new file mode 100644 index 000000000..0e51c54ea --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdQueryTransformer.scala @@ -0,0 +1,70 @@ +package com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird + +import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature +import com.twitter.home_mixer.model.request.HasDeviceContext +import com.twitter.home_mixer.util.CachedScoredTweetsHelper +import com.twitter.home_mixer.util.earlybird.EarlybirdRequestUtil +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus +import com.twitter.search.earlybird.{thriftscala => eb} +import com.twitter.timelines.clients.relevance_search.SearchClient.TweetTypes +import com.twitter.timelines.common.model.TweetKindOption +import com.twitter.timelines.util.SnowflakeSortIndexHelper +import com.twitter.util.Duration +import com.twitter.util.Time + +trait EarlybirdQueryTransformer[ + Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext] { + + def candidatePipelineIdentifier: CandidatePipelineIdentifier + def clientId: Option[String] = None + def maxTweetsToFetch: Int = 100 + def tweetKindOptions: TweetKindOption.ValueSet + def tensorflowModel: Option[String] = None + + private val EarlybirdMaxExcludedTweets = 1500 + + def buildEarlybirdQuery( + query: Query, + sinceDuration: Duration, + followedUserIds: Set[Long] = Set.empty + ): eb.EarlybirdRequest = { + val sinceTime: Time = sinceDuration.ago + val untilTime: Time = Time.now + + val fromTweetIdExclusive = SnowflakeSortIndexHelper.timestampToFakeId(sinceTime) + val toTweetIdExclusive = SnowflakeSortIndexHelper.timestampToFakeId(untilTime) + + val excludedTweetIds = query.features.map { featureMap => + CachedScoredTweetsHelper.tweetImpressionsAndCachedScoredTweetsInRange( + featureMap, + candidatePipelineIdentifier, + EarlybirdMaxExcludedTweets, + sinceTime, + untilTime) + } + + val maxCount = + (query.getQualityFactorCurrentValue(candidatePipelineIdentifier) * maxTweetsToFetch).toInt + + val authorScoreMap = query.features + .map(_.getOrElse(RealGraphInNetworkScoresFeature, Map.empty[Long, Double])) + .getOrElse(Map.empty) + + EarlybirdRequestUtil.getTweetsRequest( + userId = Some(query.getRequiredUserId), + clientId = clientId, + skipVeryRecentTweets = true, + followedUserIds = followedUserIds, + retweetsMutedUserIds = Set.empty, + beforeTweetIdExclusive = Some(toTweetIdExclusive), + afterTweetIdExclusive = Some(fromTweetIdExclusive), + excludedTweetIds = excludedTweetIds.map(_.toSet), + maxCount = maxCount, + tweetTypes = TweetTypes.fromTweetKindOption(tweetKindOptions), + authorScoreMap = Some(authorScoreMap), + tensorflowModel = tensorflowModel + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/BUILD.bazel index 32f150c38..30774b91f 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/BUILD.bazel @@ -3,16 +3,14 @@ scala_library( compiler_option_sets = ["fatal_warnings"], strict_deps = True, dependencies = [ - "cr-mixer/thrift/src/main/thrift:thrift-scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", + "explore/explore-ranker/thrift/src/main/thrift:thrift-scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", "src/thrift/com/twitter/timelineranker:thrift-scala", "topic-social-proof/server/src/main/thrift:thrift-scala", + "tweet-mixer/thrift/src/main/thrift:thrift-scala", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/CachedScoredTweetsResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/CachedScoredTweetsResponseFeatureTransformer.scala index 9822a5331..0fbe7a438 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/CachedScoredTweetsResponseFeatureTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/CachedScoredTweetsResponseFeatureTransformer.scala @@ -4,21 +4,28 @@ import com.twitter.home_mixer.marshaller.timelines.TopicContextFunctionalityType import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature +import com.twitter.home_mixer.model.HomeFeatures.AuthorIsCreatorFeature +import com.twitter.home_mixer.model.HomeFeatures.AuthorIsGoldVerifiedFeature +import com.twitter.home_mixer.model.HomeFeatures.AuthorIsGrayVerifiedFeature +import com.twitter.home_mixer.model.HomeFeatures.AuthorIsLegacyVerifiedFeature import com.twitter.home_mixer.model.HomeFeatures.CachedCandidatePipelineIdentifierFeature import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.FavoritedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.FollowedByUserIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsReadFromCacheFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.LastScoredTimestampMsFeature +import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.StreamToKafkaFeature import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature @@ -32,7 +39,7 @@ import com.twitter.product_mixer.core.functional_component.transformer.Candidate import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier object CachedScoredTweetsResponseFeatureTransformer - extends CandidateFeatureTransformer[hmt.CachedScoredTweet] { + extends CandidateFeatureTransformer[hmt.ScoredTweet] { override val identifier: TransformerIdentifier = TransformerIdentifier("CachedScoredTweetsResponse") @@ -41,21 +48,28 @@ object CachedScoredTweetsResponseFeatureTransformer AncestorsFeature, AuthorIdFeature, AuthorIsBlueVerifiedFeature, + AuthorIsCreatorFeature, + AuthorIsGoldVerifiedFeature, + AuthorIsGrayVerifiedFeature, + AuthorIsLegacyVerifiedFeature, CachedCandidatePipelineIdentifierFeature, DirectedAtUserIdFeature, - FavoritedByUserIdsFeature, - FollowedByUserIdsFeature, + ExclusiveConversationAuthorIdFeature, InNetworkFeature, InReplyToTweetIdFeature, InReplyToUserIdFeature, IsReadFromCacheFeature, IsRetweetFeature, LastScoredTimestampMsFeature, + PerspectiveFilteredLikedByUserIdsFeature, QuotedTweetIdFeature, QuotedUserIdFeature, + SGSValidFollowedByUserIdsFeature, + SGSValidLikedByUserIdsFeature, ScoreFeature, SourceTweetIdFeature, SourceUserIdFeature, + StreamToKafkaFeature, SuggestTypeFeature, TopicContextFunctionalityTypeFeature, TopicIdSocialContextFeature, @@ -63,32 +77,43 @@ object CachedScoredTweetsResponseFeatureTransformer WeightedModelScoreFeature ) - override def transform(candidate: hmt.CachedScoredTweet): FeatureMap = + override def transform(candidate: hmt.ScoredTweet): FeatureMap = FeatureMapBuilder() .add(AncestorsFeature, candidate.ancestors.getOrElse(Seq.empty)) - .add(AuthorIdFeature, candidate.userId) - .add(AuthorIsBlueVerifiedFeature, candidate.authorIsBlueVerified.getOrElse(false)) + .add(AuthorIdFeature, Some(candidate.authorId)) + .add(AuthorIsBlueVerifiedFeature, candidate.authorMetadata.exists(_.blueVerified)) + .add(AuthorIsGoldVerifiedFeature, candidate.authorMetadata.exists(_.goldVerified)) + .add(AuthorIsGrayVerifiedFeature, candidate.authorMetadata.exists(_.grayVerified)) + .add(AuthorIsLegacyVerifiedFeature, candidate.authorMetadata.exists(_.legacyVerified)) + .add(AuthorIsCreatorFeature, candidate.authorMetadata.exists(_.creator)) .add(CachedCandidatePipelineIdentifierFeature, candidate.candidatePipelineIdentifier) .add(DirectedAtUserIdFeature, candidate.directedAtUserId) - .add(FavoritedByUserIdsFeature, candidate.favoritedByUserIds.getOrElse(Seq.empty)) - .add(FollowedByUserIdsFeature, candidate.followedByUserIds.getOrElse(Seq.empty)) - .add(InNetworkFeature, candidate.isInNetwork.getOrElse(false)) + .add(ExclusiveConversationAuthorIdFeature, candidate.exclusiveConversationAuthorId) + .add(InNetworkFeature, candidate.inNetwork.getOrElse(true)) .add(InReplyToTweetIdFeature, candidate.inReplyToTweetId) .add(InReplyToUserIdFeature, candidate.inReplyToUserId) .add(IsReadFromCacheFeature, true) - .add(IsRetweetFeature, candidate.isRetweet.getOrElse(false)) + .add(IsRetweetFeature, candidate.sourceTweetId.isDefined) .add(LastScoredTimestampMsFeature, candidate.lastScoredTimestampMs) + .add( + PerspectiveFilteredLikedByUserIdsFeature, + candidate.perspectiveFilteredLikedByUserIds.getOrElse(Seq.empty)) .add(QuotedTweetIdFeature, candidate.quotedTweetId) .add(QuotedUserIdFeature, candidate.quotedUserId) .add(ScoreFeature, candidate.score) + .add(SGSValidLikedByUserIdsFeature, candidate.sgsValidLikedByUserIds.getOrElse(Seq.empty)) + .add( + SGSValidFollowedByUserIdsFeature, + candidate.sgsValidFollowedByUserIds.getOrElse(Seq.empty)) .add(SourceTweetIdFeature, candidate.sourceTweetId) .add(SourceUserIdFeature, candidate.sourceUserId) + .add(StreamToKafkaFeature, false) .add(SuggestTypeFeature, candidate.suggestType) .add( TopicContextFunctionalityTypeFeature, candidate.topicFunctionalityType.map(TopicContextFunctionalityTypeUnmarshaller(_))) .add(TopicIdSocialContextFeature, candidate.topicId) - .add(TweetUrlsFeature, candidate.urlsList.getOrElse(Seq.empty)) + .add(TweetUrlsFeature, candidate.tweetUrls.getOrElse(Seq.empty)) .add(WeightedModelScoreFeature, candidate.score) .build() } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsBackfillResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsBackfillResponseFeatureTransformer.scala new file mode 100644 index 000000000..806c0d11d --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsBackfillResponseFeatureTransformer.scala @@ -0,0 +1,30 @@ +package com.twitter.home_mixer.product.scored_tweets.response_transformer + +import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature +import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature +import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts} +import com.twitter.timelineservice.suggests.{thriftscala => st} + +object ScoredTweetsBackfillResponseFeatureTransformer extends CandidateFeatureTransformer[Long] { + + override val identifier: TransformerIdentifier = + TransformerIdentifier("ScoredTweetsBackfillResponse") + + override val features: Set[Feature[_, _]] = Set( + CandidateSourceIdFeature, + FromInNetworkSourceFeature, + SuggestTypeFeature + ) + + override def transform(candidate: Long): FeatureMap = FeatureMapBuilder() + .add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.BackfillOrganicTweet)) + .add(FromInNetworkSourceFeature, true) + .add(SuggestTypeFeature, Some(st.SuggestType.RankedOrganicTweet)) + .build() +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsInNetworkResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsInNetworkResponseFeatureTransformer.scala index ada45d3e1..d7d23bc98 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsInNetworkResponseFeatureTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsInNetworkResponseFeatureTransformer.scala @@ -26,7 +26,7 @@ object ScoredTweetsInNetworkResponseFeatureTransformer val features = FeatureMapBuilder() .add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.RecycledTweet)) .add(FromInNetworkSourceFeature, true) - .add(SuggestTypeFeature, Some(st.SuggestType.RecycledTweetInline)) + .add(SuggestTypeFeature, Some(st.SuggestType.RankedTimelineTweet)) .build() baseFeatures ++ features diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsListsResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsListsResponseFeatureTransformer.scala new file mode 100644 index 000000000..6a48e84c8 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsListsResponseFeatureTransformer.scala @@ -0,0 +1,45 @@ +package com.twitter.home_mixer.product.scored_tweets.response_transformer + +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature +import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature +import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature +import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.timelineservice.{thriftscala => t} +import com.twitter.timelineservice.suggests.{thriftscala => st} +import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts} + +object ScoredTweetsListsResponseFeatureTransformer extends CandidateFeatureTransformer[t.Tweet] { + + override val identifier: TransformerIdentifier = + TransformerIdentifier("ScoredTweetsListsResponse") + + override val features: Set[Feature[_, _]] = Set( + AuthorIdFeature, + CandidateSourceIdFeature, + FromInNetworkSourceFeature, + IsRetweetFeature, + SuggestTypeFeature, + SourceTweetIdFeature, + SourceUserIdFeature, + ) + + override def transform(candidate: t.Tweet): FeatureMap = { + FeatureMapBuilder() + .add(AuthorIdFeature, candidate.userId) + .add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.ListTweet)) + .add(FromInNetworkSourceFeature, false) + .add(IsRetweetFeature, candidate.sourceStatusId.isDefined) + .add(SuggestTypeFeature, Some(st.SuggestType.RankedListTweet)) + .add(SourceTweetIdFeature, candidate.sourceStatusId) + .add(SourceUserIdFeature, candidate.sourceUserId) + .build() + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsCrMixerResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsPopularVideosResponseFeatureTransformer.scala similarity index 51% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsCrMixerResponseFeatureTransformer.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsPopularVideosResponseFeatureTransformer.scala index 96aef36b6..6657e6f5c 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsCrMixerResponseFeatureTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsPopularVideosResponseFeatureTransformer.scala @@ -1,13 +1,13 @@ package com.twitter.home_mixer.product.scored_tweets.response_transformer -import com.twitter.cr_mixer.{thriftscala => crm} +import com.twitter.explore_ranker.{thriftscala => ert} import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature +import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature import com.twitter.home_mixer.model.HomeFeatures.IsRandomTweetFeature import com.twitter.home_mixer.model.HomeFeatures.StreamToKafkaFeature import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.model.HomeFeatures.TSPMetricTagFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder @@ -15,51 +15,32 @@ import com.twitter.product_mixer.core.functional_component.transformer.Candidate import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts} import com.twitter.timelineservice.suggests.{thriftscala => st} -import com.twitter.tsp.{thriftscala => tsp} -object ScoredTweetsCrMixerResponseFeatureTransformer - extends CandidateFeatureTransformer[crm.TweetRecommendation] { +object ScoredTweetsPopularVideosResponseFeatureTransformer + extends CandidateFeatureTransformer[ert.ExploreTweetRecommendation] { override val identifier: TransformerIdentifier = - TransformerIdentifier("ScoredTweetsCrMixerResponse") + TransformerIdentifier("ScoredTweetsPopularVideosResponse") override val features: Set[Feature[_, _]] = Set( AuthorIdFeature, CandidateSourceIdFeature, FromInNetworkSourceFeature, + HasVideoFeature, IsRandomTweetFeature, StreamToKafkaFeature, - SuggestTypeFeature, - TSPMetricTagFeature + SuggestTypeFeature ) - override def transform(candidate: crm.TweetRecommendation): FeatureMap = { - val crMixerMetricTags = candidate.metricTags.getOrElse(Seq.empty) - val tspMetricTag = crMixerMetricTags - .map(CrMixerMetricTagToTspMetricTag) - .filter(_.nonEmpty).map(_.get).toSet - + override def transform(candidate: ert.ExploreTweetRecommendation): FeatureMap = { FeatureMapBuilder() .add(AuthorIdFeature, candidate.authorId) - .add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.Simcluster)) + .add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.MediaTweet)) .add(FromInNetworkSourceFeature, false) + .add(HasVideoFeature, candidate.mediaType.contains(ert.MediaType.Video)) .add(IsRandomTweetFeature, false) .add(StreamToKafkaFeature, true) - .add(SuggestTypeFeature, Some(st.SuggestType.ScTweet)) - .add(TSPMetricTagFeature, tspMetricTag) + .add(SuggestTypeFeature, Some(st.SuggestType.MediaTweet)) .build() } - - private def CrMixerMetricTagToTspMetricTag( - crMixerMetricTag: crm.MetricTag - ): Option[tsp.MetricTag] = crMixerMetricTag match { - case crm.MetricTag.TweetFavorite => Some(tsp.MetricTag.TweetFavorite) - case crm.MetricTag.Retweet => Some(tsp.MetricTag.Retweet) - case crm.MetricTag.UserFollow => Some(tsp.MetricTag.UserFollow) - case crm.MetricTag.PushOpenOrNtabClick => Some(tsp.MetricTag.PushOpenOrNtabClick) - case crm.MetricTag.UserInterestedIn => Some(tsp.MetricTag.UserInterestedIn) - case crm.MetricTag.HomeTweetClick => Some(tsp.MetricTag.HomeTweetClick) - case crm.MetricTag.HomeVideoView => Some(tsp.MetricTag.HomeVideoView) - case _ => None - } } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsTweetMixerResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsTweetMixerResponseFeatureTransformer.scala new file mode 100644 index 000000000..4312b5104 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsTweetMixerResponseFeatureTransformer.scala @@ -0,0 +1,52 @@ +package com.twitter.home_mixer.product.scored_tweets.response_transformer + +import com.twitter.tweet_mixer.{thriftscala => tmt} +import com.twitter.home_mixer.model.HomeFeatures._ +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts} +import com.twitter.timelineservice.suggests.{thriftscala => st} +import com.twitter.tsp.{thriftscala => tsp} + +object ScoredTweetsTweetMixerResponseFeatureTransformer + extends CandidateFeatureTransformer[tmt.TweetResult] { + + override val identifier: TransformerIdentifier = + TransformerIdentifier("ScoredTweetsTweetMixerResponse") + + override val features: Set[Feature[_, _]] = Set( + CandidateSourceIdFeature, + FromInNetworkSourceFeature, + IsRandomTweetFeature, + StreamToKafkaFeature, + SuggestTypeFeature, + TSPMetricTagFeature + ) + + override def transform(candidate: tmt.TweetResult): FeatureMap = { + val tweetMixerMetricTags = candidate.metricTags.getOrElse(Seq.empty) + val tspMetricTag = tweetMixerMetricTags + .map(TweetMixerMetricTagToTspMetricTag) + .filter(_.nonEmpty).map(_.get).toSet + + FeatureMapBuilder() + .add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.Simcluster)) + .add(FromInNetworkSourceFeature, false) + .add(IsRandomTweetFeature, false) + .add(StreamToKafkaFeature, true) + .add(SuggestTypeFeature, Some(st.SuggestType.ScTweet)) + .add(TSPMetricTagFeature, tspMetricTag) + .build() + } + + private def TweetMixerMetricTagToTspMetricTag( + tweetMixerMetricTag: tmt.MetricTag + ): Option[tsp.MetricTag] = tweetMixerMetricTag match { + case tmt.MetricTag.TweetFavorite => Some(tsp.MetricTag.TweetFavorite) + case tmt.MetricTag.Retweet => Some(tsp.MetricTag.Retweet) + case _ => None + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/TimelineRankerResponseTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/TimelineRankerResponseTransformer.scala index e431ef6f4..a261b2fc2 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/TimelineRankerResponseTransformer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/TimelineRankerResponseTransformer.scala @@ -5,6 +5,7 @@ import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature import com.twitter.home_mixer.model.HomeFeatures.EarlybirdScoreFeature +import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature import com.twitter.home_mixer.model.HomeFeatures.HasImageFeature import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature @@ -16,7 +17,6 @@ import com.twitter.home_mixer.model.HomeFeatures.MentionScreenNameFeature import com.twitter.home_mixer.model.HomeFeatures.MentionUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SemanticAnnotationFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.StreamToKafkaFeature @@ -36,6 +36,7 @@ object TimelineRankerResponseTransformer { DirectedAtUserIdFeature, EarlybirdFeature, EarlybirdScoreFeature, + ExclusiveConversationAuthorIdFeature, FromInNetworkSourceFeature, HasImageFeature, HasVideoFeature, @@ -45,7 +46,6 @@ object TimelineRankerResponseTransformer { IsRetweetFeature, MentionScreenNameFeature, MentionUserIdFeature, - SemanticAnnotationFeature, StreamToKafkaFeature, QuotedTweetIdFeature, QuotedUserIdFeature, @@ -57,19 +57,20 @@ object TimelineRankerResponseTransformer { def transform(candidate: tlr.CandidateTweet): FeatureMap = { val tweet = candidate.tweet - val quotedTweet = tweet.flatMap(_.quotedTweet) + val quotedTweet = tweet.filter(_.quotedTweet.exists(_.tweetId != 0)).flatMap(_.quotedTweet) val mentions = tweet.flatMap(_.mentions).getOrElse(Seq.empty) val coreData = tweet.flatMap(_.coreData) val share = coreData.flatMap(_.share) val reply = coreData.flatMap(_.reply) - val semanticAnnotations = - tweet.flatMap(_.escherbirdEntityAnnotations.map(_.entityAnnotations)).getOrElse(Seq.empty) FeatureMapBuilder() .add(AuthorIdFeature, coreData.map(_.userId)) .add(DirectedAtUserIdFeature, coreData.flatMap(_.directedAtUser.map(_.userId))) .add(EarlybirdFeature, candidate.features) .add(EarlybirdScoreFeature, candidate.features.map(_.earlybirdScore)) + .add( + ExclusiveConversationAuthorIdFeature, + tweet.flatMap(_.exclusiveTweetControl.map(_.conversationAuthorId))) .add(FromInNetworkSourceFeature, false) .add(HasImageFeature, tweet.exists(TweetMediaFeaturesExtractor.hasImage)) .add(HasVideoFeature, tweet.exists(TweetMediaFeaturesExtractor.hasVideo)) @@ -79,7 +80,6 @@ object TimelineRankerResponseTransformer { .add(IsRetweetFeature, share.isDefined) .add(MentionScreenNameFeature, mentions.map(_.screenName)) .add(MentionUserIdFeature, mentions.flatMap(_.userId)) - .add(SemanticAnnotationFeature, semanticAnnotations) .add(StreamToKafkaFeature, true) .add(QuotedTweetIdFeature, quotedTweet.map(_.tweetId)) .add(QuotedUserIdFeature, quotedTweet.map(_.userId)) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/BUILD.bazel new file mode 100644 index 000000000..81cf19e9b --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/BUILD.bazel @@ -0,0 +1,13 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content", + "home-mixer/thrift/src/main/thrift:thrift-scala", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", + "src/thrift/com/twitter/search:earlybird-scala", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/EarlybirdResponseTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/EarlybirdResponseTransformer.scala new file mode 100644 index 000000000..f0b1b59b1 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/EarlybirdResponseTransformer.scala @@ -0,0 +1,92 @@ +package com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird + +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature +import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.EarlybirdScoreFeature +import com.twitter.home_mixer.model.HomeFeatures.EarlybirdSearchResultFeature +import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature +import com.twitter.home_mixer.model.HomeFeatures.HasImageFeature +import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature +import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.IsRandomTweetFeature +import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature +import com.twitter.home_mixer.model.HomeFeatures.MentionScreenNameFeature +import com.twitter.home_mixer.model.HomeFeatures.MentionUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.StreamToKafkaFeature +import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature +import com.twitter.home_mixer.util.tweetypie.content.TweetMediaFeaturesExtractor +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.search.earlybird.{thriftscala => eb} + +object EarlybirdResponseTransformer { + + val features: Set[Feature[_, _]] = Set( + AuthorIdFeature, + CandidateSourceIdFeature, + DirectedAtUserIdFeature, + EarlybirdScoreFeature, + EarlybirdSearchResultFeature, + ExclusiveConversationAuthorIdFeature, + FromInNetworkSourceFeature, + HasImageFeature, + HasVideoFeature, + InReplyToTweetIdFeature, + InReplyToUserIdFeature, + IsRandomTweetFeature, + IsRetweetFeature, + MentionScreenNameFeature, + MentionUserIdFeature, + StreamToKafkaFeature, + QuotedTweetIdFeature, + QuotedUserIdFeature, + SourceTweetIdFeature, + SourceUserIdFeature, + SuggestTypeFeature, + TweetUrlsFeature + ) + + def transform(candidate: eb.ThriftSearchResult): FeatureMap = { + val tweet = candidate.tweetypieTweet + val quotedTweet = tweet.flatMap(_.quotedTweet) + val mentions = tweet.flatMap(_.mentions).getOrElse(Seq.empty) + val coreData = tweet.flatMap(_.coreData) + val share = coreData.flatMap(_.share) + val reply = coreData.flatMap(_.reply) + FeatureMapBuilder() + .add(AuthorIdFeature, coreData.map(_.userId)) + .add(DirectedAtUserIdFeature, coreData.flatMap(_.directedAtUser.map(_.userId))) + .add(EarlybirdSearchResultFeature, Some(candidate)) + .add(EarlybirdScoreFeature, candidate.metadata.flatMap(_.score)) + .add( + ExclusiveConversationAuthorIdFeature, + tweet.flatMap(_.exclusiveTweetControl.map(_.conversationAuthorId))) + .add(FromInNetworkSourceFeature, false) + .add(HasImageFeature, tweet.exists(TweetMediaFeaturesExtractor.hasImage)) + .add(HasVideoFeature, tweet.exists(TweetMediaFeaturesExtractor.hasVideo)) + .add(InReplyToTweetIdFeature, reply.flatMap(_.inReplyToStatusId)) + .add(InReplyToUserIdFeature, reply.map(_.inReplyToUserId)) + .add(IsRandomTweetFeature, candidate.tweetFeatures.exists(_.isRandomTweet.getOrElse(false))) + .add(IsRetweetFeature, share.isDefined) + .add(MentionScreenNameFeature, mentions.map(_.screenName)) + .add(MentionUserIdFeature, mentions.flatMap(_.userId)) + .add(StreamToKafkaFeature, true) + .add(QuotedTweetIdFeature, quotedTweet.map(_.tweetId)) + .add(QuotedUserIdFeature, quotedTweet.map(_.userId)) + .add(SourceTweetIdFeature, share.map(_.sourceStatusId)) + .add(SourceUserIdFeature, share.map(_.sourceUserId)) + .add( + TweetUrlsFeature, + candidate.metadata.flatMap(_.tweetUrls.map(_.map(_.originalUrl))).getOrElse(Seq.empty)) + .build() + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsEarlybirdFrsResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsEarlybirdFrsResponseFeatureTransformer.scala new file mode 100644 index 000000000..bb9ea8bee --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsEarlybirdFrsResponseFeatureTransformer.scala @@ -0,0 +1,33 @@ +package com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird + +import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature +import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.search.earlybird.{thriftscala => eb} +import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts} +import com.twitter.timelineservice.suggests.{thriftscala => st} + +object ScoredTweetsEarlybirdFrsResponseFeatureTransformer + extends CandidateFeatureTransformer[eb.ThriftSearchResult] { + + override val identifier: TransformerIdentifier = + TransformerIdentifier("ScoredTweetsEarlybirdFrsResponse") + + override val features: Set[Feature[_, _]] = EarlybirdResponseTransformer.features + + override def transform(candidate: eb.ThriftSearchResult): FeatureMap = { + + val baseFeatures = EarlybirdResponseTransformer.transform(candidate) + + val features = FeatureMapBuilder() + .add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.FrsTweet)) + .add(SuggestTypeFeature, Some(st.SuggestType.FrsTweet)) + .build() + + baseFeatures ++ features + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsEarlybirdInNetworkResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsEarlybirdInNetworkResponseFeatureTransformer.scala new file mode 100644 index 000000000..6b6a9d003 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsEarlybirdInNetworkResponseFeatureTransformer.scala @@ -0,0 +1,32 @@ +package com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird + +import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature +import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.search.earlybird.{thriftscala => eb} +import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts} +import com.twitter.timelineservice.suggests.{thriftscala => st} + +object ScoredTweetsEarlybirdInNetworkResponseFeatureTransformer + extends CandidateFeatureTransformer[eb.ThriftSearchResult] { + override val identifier: TransformerIdentifier = + TransformerIdentifier("ScoredTweetsEarlybirdInNetworkResponse") + + override val features: Set[Feature[_, _]] = EarlybirdResponseTransformer.features + + override def transform(candidate: eb.ThriftSearchResult): FeatureMap = { + + val baseFeatures = EarlybirdResponseTransformer.transform(candidate) + + val features = FeatureMapBuilder() + .add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.RecycledTweet)) + .add(SuggestTypeFeature, Some(st.SuggestType.RecycledTweet)) + .build() + + baseFeatures ++ features + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/BUILD.bazel index 6fcaddf0f..91ff12ee1 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/BUILD.bazel @@ -3,20 +3,18 @@ scala_library( compiler_option_sets = ["fatal_warnings"], strict_deps = True, dependencies = [ - "3rdparty/jvm/io/opil:tensorflow-serving-client", - "cortex-deepbird/thrift/src/main/thrift:thrift-java", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/module", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", "src/scala/com/twitter/timelines/prediction/features/recap", - "src/thrift/com/twitter/timelinescorer:thrift-scala", + "timelineservice/common:model", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/DiversityDiscountProvider.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/DiversityDiscountProvider.scala index aa292e89a..62dea8748 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/DiversityDiscountProvider.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/DiversityDiscountProvider.scala @@ -1,20 +1,53 @@ package com.twitter.home_mixer.product.scored_tweets.scorer import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.model.common.CandidateWithFeatures trait DiversityDiscountProvider { + /** + * Fetch the ID of the entity to diversify + */ def entityId(candidate: CandidateWithFeatures[TweetCandidate]): Option[Long] /** - * Compute the discounted score for the position - * @param score the previous score for the candidate - * @param position zero-based position for the candidate for the given entity - * @return the discounted score for the candidate + * Compute discount factor for each candidate based on position (zero-based) + * relative to other candidates associated with the same entity */ - def discount(score: Double, position: Int): Double + def discount(position: Int): Double + + /** + * Return candidate IDs sorted by score in descending order + */ + def sort(candidates: Seq[CandidateWithFeatures[TweetCandidate]]): Seq[Long] = candidates + .map { candidate => + (candidate.candidate.id, candidate.features.getOrElse(ScoreFeature, None).getOrElse(0.0)) + } + .sortBy(_._2)(Ordering.Double.reverse) + .map(_._1) + + /** + * Group by the specified entity ID (e.g. authors, likers, followers) + * Sort each group by score in descending order + * Determine the discount factor based on the position of each candidate + */ + def apply( + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Map[Long, Double] = candidates + .groupBy(entityId) + .flatMap { + case (entityIdOpt, entityCandidates) => + val sortedCandidateIds = sort(entityCandidates) + + if (entityIdOpt.isDefined) { + sortedCandidateIds.zipWithIndex.map { + case (candidateId, index) => + candidateId -> discount(index) + } + } else sortedCandidateIds.map(_ -> 1.0) + } } object AuthorDiversityDiscountProvider extends DiversityDiscountProvider { @@ -25,6 +58,6 @@ object AuthorDiversityDiscountProvider extends DiversityDiscountProvider { candidate.features.getOrElse(AuthorIdFeature, None) // Provides an exponential decay based discount by position (with a floor) - override def discount(score: Double, position: Int): Double = - score * ((1 - Floor) * Math.pow(Decay, position) + Floor) + override def discount(position: Int): Double = + (1 - Floor) * Math.pow(Decay, position) + Floor } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/DiversityScorer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/DiversityScorer.scala deleted file mode 100644 index 758467717..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/DiversityScorer.scala +++ /dev/null @@ -1,57 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.scorer - -import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier -import com.twitter.stitch.Stitch - -/** - * Discounts scores of each consecutive tweet (ordered by score desc) from the - * same entity (e.g. author, engager, topic) based on the discount factor provided - */ - -case class DiversityScorer(diversityDiscountProvider: DiversityDiscountProvider) - extends Scorer[ScoredTweetsQuery, TweetCandidate] { - - override val identifier: ScorerIdentifier = ScorerIdentifier("Diversity") - - override val features: Set[Feature[_, _]] = Set(ScoreFeature) - - override def apply( - query: ScoredTweetsQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - val candidateIdScoreMap = candidates - .groupBy(diversityDiscountProvider.entityId) - .flatMap { - case (entityIdOpt, entityCandidates) => - val candidateScores = entityCandidates - .map { candidate => - val score = candidate.features.getOrElse(ScoreFeature, None).getOrElse(0.0) - (candidate.candidate.id, score) - }.sortBy(_._2)(Ordering.Double.reverse) - - if (entityIdOpt.isDefined) { - candidateScores.zipWithIndex.map { - case ((candidateId, score), index) => - candidateId -> diversityDiscountProvider.discount(score, index) - } - } else candidateScores - } - - Stitch.value { - candidates.map { candidate => - val score = candidateIdScoreMap.getOrElse(candidate.candidate.id, 0.0) - FeatureMapBuilder() - .add(ScoreFeature, Some(score)) - .build() - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/HeuristicScorer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/HeuristicScorer.scala new file mode 100644 index 000000000..765523b53 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/HeuristicScorer.scala @@ -0,0 +1,46 @@ +package com.twitter.home_mixer.product.scored_tweets.scorer + +import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature +import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.scorer.Scorer +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier +import com.twitter.stitch.Stitch + +/** + * Apply various heuristics to the model score + */ +object HeuristicScorer extends Scorer[ScoredTweetsQuery, TweetCandidate] { + + override val identifier: ScorerIdentifier = ScorerIdentifier("Heuristic") + + override val features: Set[Feature[_, _]] = Set(ScoreFeature) + + override def apply( + query: ScoredTweetsQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = { + val rescorers = Seq( + RescoreOutOfNetwork, + RescoreReplies, + RescoreBlueVerified, + RescoreCreators, + RescoreMTLNormalization, + RescoreAuthorDiversity(AuthorDiversityDiscountProvider(candidates)), + RescoreFeedbackFatigue(query) + ) + + val updatedScores = candidates.map { candidate => + val score = candidate.features.getOrElse(ScoreFeature, None) + val scaleFactor = rescorers.map(_(query, candidate)).product + val updatedScore = score.map(_ * scaleFactor) + FeatureMapBuilder().add(ScoreFeature, updatedScore).build() + } + + Stitch.value(updatedScores) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/HomeNaviModelDataRecordScorer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/HomeNaviModelDataRecordScorer.scala deleted file mode 100644 index 23bf07da6..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/HomeNaviModelDataRecordScorer.scala +++ /dev/null @@ -1,233 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.scorer - -import com.twitter.dal.personal_data.{thriftjava => pd} -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.Scoring.ModelWeights -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.BaseDataRecordFeature -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.datarecord.DataRecordOptionalFeature -import com.twitter.product_mixer.core.feature.datarecord.DoubleDataRecordCompatible -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.datarecord.AllFeatures -import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordConverter -import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordExtractor -import com.twitter.product_mixer.core.feature.featuremap.datarecord.FeaturesScope -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.pipeline_failure.IllegalStateFailure -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch -import com.twitter.timelines.clients.predictionservice.PredictionServiceGRPCClient -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.prediction.features.recap.RecapFeatures -import com.twitter.util.Future -import com.twitter.util.Return - -object CommonFeaturesDataRecordFeature - extends DataRecordInAFeature[PipelineQuery] - with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -object CandidateFeaturesDataRecordFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -case class HomeNaviModelDataRecordScorer[ - Query <: PipelineQuery, - Candidate <: UniversalNoun[Any], - CandidateFeatures <: BaseDataRecordFeature[Candidate, _], - ResultFeatures <: BaseDataRecordFeature[Candidate, _] -]( - override val identifier: ScorerIdentifier, - modelClient: PredictionServiceGRPCClient, - candidateFeatures: FeaturesScope[CandidateFeatures], - resultFeatures: Set[ResultFeatures], - statsReceiver: StatsReceiver) - extends Scorer[Query, Candidate] { - - require(resultFeatures.nonEmpty, "Result features cannot be empty") - - override val features: Set[Feature[_, _]] = - resultFeatures.asInstanceOf[ - Set[Feature[_, _]]] + CommonFeaturesDataRecordFeature + CandidateFeaturesDataRecordFeature - - private val queryDataRecordAdapter = new DataRecordConverter(AllFeatures()) - private val candidatesDataRecordAdapter = new DataRecordConverter(candidateFeatures) - private val resultDataRecordExtractor = new DataRecordExtractor(resultFeatures) - - private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) - private val failuresStat = scopedStatsReceiver.stat("failures") - private val responsesStat = scopedStatsReceiver.stat("responses") - private val invalidResponsesSizeCounter = scopedStatsReceiver.counter("invalidResponsesSize") - private val candidatesDataRecordAdapterLatencyStat = - scopedStatsReceiver.scope("candidatesDataRecordAdapter").stat("latency_ms") - - private val DataRecordConstructionParallelism = 32 - - override def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[Candidate]] - ): Stitch[Seq[FeatureMap]] = { - val commonRecord = query.features.map(queryDataRecordAdapter.toDataRecord) - val candidateRecords: Future[Seq[DataRecord]] = - Stat.time(candidatesDataRecordAdapterLatencyStat) { - OffloadFuturePools.parallelize[FeatureMap, DataRecord]( - candidates.map(_.features), - candidatesDataRecordAdapter.toDataRecord(_), - DataRecordConstructionParallelism, - new DataRecord - ) - } - - Stitch.callFuture { - candidateRecords.flatMap { records => - val predictionResponses = - modelClient.getPredictions( - records = records, - commonFeatures = commonRecord, - modelId = Some("Home") - ) - - predictionResponses.map { responses => - failuresStat.add(responses.count(_.isThrow)) - responsesStat.add(responses.size) - - if (responses.size == candidates.size) { - val predictedScoreFeatureMaps = responses.map { - case Return(dataRecord) => - resultDataRecordExtractor.fromDataRecord(dataRecord) - case _ => - resultDataRecordExtractor.fromDataRecord(new DataRecord()) - } - - // add Data Record to feature map, which will be used for logging in later stage - predictedScoreFeatureMaps.zip(records).map { - case (predictedScoreFeatureMap, candidateRecord) => - predictedScoreFeatureMap + - (key = CandidateFeaturesDataRecordFeature, value = candidateRecord) + - (key = CommonFeaturesDataRecordFeature, value = - commonRecord.getOrElse(new DataRecord())) - } - } else { - invalidResponsesSizeCounter.incr() - throw PipelineFailure(IllegalStateFailure, "Result Size mismatched candidates size") - } - } - } - } - } -} - -/** - * Features for results returned by Navi user-tweet prediction models. - */ -object HomeNaviModelDataRecordScorer { - val RequestBatchSize = 32 - - sealed trait PredictedScoreFeature - extends DataRecordOptionalFeature[TweetCandidate, Double] - with DoubleDataRecordCompatible { - def statName: String - - def modelWeightParam: FSBoundedParam[Double] - } - - object PredictedFavoriteScoreFeature extends PredictedScoreFeature { - override val featureName: String = RecapFeatures.PREDICTED_IS_FAVORITED.getFeatureName - override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty - override val statName = "fav" - override val modelWeightParam = ModelWeights.FavParam - } - - object PredictedReplyScoreFeature extends PredictedScoreFeature { - override val featureName: String = RecapFeatures.PREDICTED_IS_REPLIED.getFeatureName - override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty - override val statName = "reply" - override val modelWeightParam = ModelWeights.ReplyParam - } - - object PredictedRetweetScoreFeature extends PredictedScoreFeature { - override val featureName: String = RecapFeatures.PREDICTED_IS_RETWEETED.getFeatureName - override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty - override val statName = "retweet" - override val modelWeightParam = ModelWeights.RetweetParam - } - - object PredictedReplyEngagedByAuthorScoreFeature extends PredictedScoreFeature { - override val featureName: String = - RecapFeatures.PREDICTED_IS_REPLIED_REPLY_ENGAGED_BY_AUTHOR.getFeatureName - override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty - override val statName = "reply_engaged_by_author" - override val modelWeightParam = ModelWeights.ReplyEngagedByAuthorParam - } - - object PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature extends PredictedScoreFeature { - override val featureName: String = RecapFeatures.PREDICTED_IS_GOOD_CLICKED_V1.getFeatureName - override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty - override val statName = "good_click_convo_desc_favorited_or_replied" - override val modelWeightParam = ModelWeights.GoodClickParam - } - - object PredictedGoodClickConvoDescUamGt2ScoreFeature extends PredictedScoreFeature { - override val featureName: String = RecapFeatures.PREDICTED_IS_GOOD_CLICKED_V2.getFeatureName - override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty - override val statName = "good_click_convo_desc_uam_gt_2" - override val modelWeightParam = ModelWeights.GoodClickV2Param - } - - object PredictedNegativeFeedbackV2ScoreFeature extends PredictedScoreFeature { - override val featureName: String = - RecapFeatures.PREDICTED_IS_NEGATIVE_FEEDBACK_V2.getFeatureName - override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty - override val statName = "negative_feedback_v2" - override val modelWeightParam = ModelWeights.NegativeFeedbackV2Param - } - - object PredictedGoodProfileClickScoreFeature extends PredictedScoreFeature { - override val featureName: String = - RecapFeatures.PREDICTED_IS_PROFILE_CLICKED_AND_PROFILE_ENGAGED.getFeatureName - override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty - override val statName = "good_profile_click" - override val modelWeightParam = ModelWeights.GoodProfileClickParam - } - - object PredictedReportedScoreFeature extends PredictedScoreFeature { - override val featureName: String = - RecapFeatures.PREDICTED_IS_REPORT_TWEET_CLICKED.getFeatureName - override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty - override val statName = "reported" - override val modelWeightParam = ModelWeights.ReportParam - } - - object PredictedVideoPlayback50ScoreFeature extends PredictedScoreFeature { - override val featureName: String = RecapFeatures.PREDICTED_IS_VIDEO_PLAYBACK_50.getFeatureName - override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty - override val statName = "video_playback_50" - override val modelWeightParam = ModelWeights.VideoPlayback50Param - } - - val PredictedScoreFeatures: Seq[PredictedScoreFeature] = Seq( - PredictedFavoriteScoreFeature, - PredictedReplyScoreFeature, - PredictedRetweetScoreFeature, - PredictedReplyEngagedByAuthorScoreFeature, - PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature, - PredictedGoodClickConvoDescUamGt2ScoreFeature, - PredictedNegativeFeedbackV2ScoreFeature, - PredictedGoodProfileClickScoreFeature, - PredictedReportedScoreFeature, - PredictedVideoPlayback50ScoreFeature, - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/NaviModelScorer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/NaviModelScorer.scala new file mode 100644 index 000000000..cbde87f73 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/NaviModelScorer.scala @@ -0,0 +1,179 @@ +package com.twitter.home_mixer.product.scored_tweets.scorer + +import com.twitter.finagle.stats.Stat +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature +import com.twitter.home_mixer.model.HomeFeatures.WeightedModelScoreFeature +import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery +import com.twitter.home_mixer.product.scored_tweets.scorer.PredictedScoreFeature.PredictedScoreFeatures +import com.twitter.ml.api.DataRecord +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure +import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.datarecord.AllFeatures +import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordConverter +import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordExtractor +import com.twitter.product_mixer.core.functional_component.scorer.Scorer +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.product_mixer.core.pipeline.pipeline_failure.IllegalStateFailure +import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure +import com.twitter.product_mixer.core.util.OffloadFuturePools +import com.twitter.stitch.Stitch +import com.twitter.timelines.clients.predictionservice.PredictionGRPCService +import com.twitter.timelines.clients.predictionservice.PredictionServiceGRPCClient +import com.twitter.util.Future +import com.twitter.util.Return +import javax.inject.Inject +import javax.inject.Singleton + +object CommonFeaturesDataRecordFeature + extends DataRecordInAFeature[PipelineQuery] + with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +object CandidateFeaturesDataRecordFeature + extends DataRecordInAFeature[TweetCandidate] + with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { + override def defaultValue: DataRecord = new DataRecord() +} + +@Singleton +case class NaviModelScorer @Inject() ( + predictionGRPCService: PredictionGRPCService, + statsReceiver: StatsReceiver) + extends Scorer[ScoredTweetsQuery, TweetCandidate] { + + override val identifier: ScorerIdentifier = ScorerIdentifier("NaviModel") + + override val features: Set[Feature[_, _]] = Set( + CommonFeaturesDataRecordFeature, + CandidateFeaturesDataRecordFeature, + WeightedModelScoreFeature, + ScoreFeature + ) ++ PredictedScoreFeatures.asInstanceOf[Set[Feature[_, _]]] + + private val queryDataRecordAdapter = new DataRecordConverter(AllFeatures()) + private val candidatesDataRecordAdapter = new DataRecordConverter(AllFeatures()) + private val resultDataRecordExtractor = new DataRecordExtractor(PredictedScoreFeatures) + + private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) + private val failuresStat = scopedStatsReceiver.stat("failures") + private val responsesStat = scopedStatsReceiver.stat("responses") + private val invalidResponsesCounter = scopedStatsReceiver.counter("invalidResponses") + private val candidatesDataRecordAdapterLatencyStat = + scopedStatsReceiver.scope("candidatesDataRecordAdapter").stat("latency_ms") + + private val StatsReadabilityMultiplier = 1000 + private val Epsilon = 0.001 + private val PredictedScoreStatName = f"predictedScore${StatsReadabilityMultiplier}x" + private val MissingScoreStatName = "missingScore" + private val scoreStat = scopedStatsReceiver.stat(f"score${StatsReadabilityMultiplier}x") + + private val RequestBatchSize = 64 + private val DataRecordConstructionParallelism = 32 + private val ModelId = "Home" + + private val modelClient = new PredictionServiceGRPCClient( + service = predictionGRPCService, + statsReceiver = statsReceiver, + requestBatchSize = RequestBatchSize, + useCompact = false + ) + + override def apply( + query: ScoredTweetsQuery, + candidates: Seq[CandidateWithFeatures[TweetCandidate]] + ): Stitch[Seq[FeatureMap]] = { + val commonRecord = query.features.map(queryDataRecordAdapter.toDataRecord) + val candidateRecords: Future[Seq[DataRecord]] = + Stat.time(candidatesDataRecordAdapterLatencyStat) { + OffloadFuturePools.parallelize[FeatureMap, DataRecord]( + inputSeq = candidates.map(_.features), + transformer = candidatesDataRecordAdapter.toDataRecord(_), + parallelism = DataRecordConstructionParallelism, + default = new DataRecord + ) + } + + val scoreFeatureMaps = candidateRecords.flatMap { records => + val predictionResponses = + modelClient.getPredictions(records, commonRecord, modelId = Some(ModelId)) + + predictionResponses.map { responses => + failuresStat.add(responses.count(_.isThrow)) + responsesStat.add(responses.size) + + if (responses.size == candidates.size) { + val predictedScoreFeatureMaps = responses.map { + case Return(dataRecord) => resultDataRecordExtractor.fromDataRecord(dataRecord) + case _ => resultDataRecordExtractor.fromDataRecord(new DataRecord()) + } + + // Add Data Record to candidate Feature Map for logging in later stages + predictedScoreFeatureMaps.zip(records).map { + case (predictedScoreFeatureMap, candidateRecord) => + val weightedModelScore = computeWeightedModelScore(query, predictedScoreFeatureMap) + scoreStat.add((weightedModelScore * StatsReadabilityMultiplier).toFloat) + + predictedScoreFeatureMap + + (CandidateFeaturesDataRecordFeature, candidateRecord) + + (CommonFeaturesDataRecordFeature, commonRecord.getOrElse(new DataRecord())) + + (ScoreFeature, Some(weightedModelScore)) + + (WeightedModelScoreFeature, Some(weightedModelScore)) + } + } else { + invalidResponsesCounter.incr() + throw PipelineFailure(IllegalStateFailure, "Result size mismatched candidates size") + } + } + } + + Stitch.callFuture(scoreFeatureMaps) + } + + /** + * Compute the weighted sum of predicted scores of all engagements + * Convert negative score to positive, if needed + */ + private def computeWeightedModelScore( + query: PipelineQuery, + features: FeatureMap + ): Double = { + val weightedScoreAndModelWeightSeq = PredictedScoreFeatures.toSeq.map { predictedScoreFeature => + val predictedScoreOpt = predictedScoreFeature.extractScore(features) + + predictedScoreOpt match { + case Some(predictedScore) => + scopedStatsReceiver + .stat(predictedScoreFeature.statName, PredictedScoreStatName) + .add((predictedScore * StatsReadabilityMultiplier).toFloat) + case None => + scopedStatsReceiver.counter(predictedScoreFeature.statName, MissingScoreStatName).incr() + } + + val weight = query.params(predictedScoreFeature.modelWeightParam) + val weightedScore = predictedScoreOpt.getOrElse(0.0) * weight + (weightedScore, weight) + } + + val (weightedScores, modelWeights) = weightedScoreAndModelWeightSeq.unzip + val combinedScoreSum = weightedScores.sum + + val positiveModelWeightsSum = modelWeights.filter(_ > 0.0).sum + val negativeModelWeightsSum = modelWeights.filter(_ < 0).sum.abs + val modelWeightsSum = positiveModelWeightsSum + negativeModelWeightsSum + + val weightedScoresSum = + if (modelWeightsSum == 0) combinedScoreSum.max(0.0) + else if (combinedScoreSum < 0) + (combinedScoreSum + negativeModelWeightsSum) / modelWeightsSum * Epsilon + else combinedScoreSum + Epsilon + + weightedScoresSum + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/PredictedScoreFeature.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/PredictedScoreFeature.scala new file mode 100644 index 000000000..f24223a99 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/PredictedScoreFeature.scala @@ -0,0 +1,166 @@ +package com.twitter.home_mixer.product.scored_tweets.scorer + +import com.twitter.dal.personal_data.{thriftjava => pd} +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.Scoring.ModelWeights +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.datarecord.DataRecordOptionalFeature +import com.twitter.product_mixer.core.feature.datarecord.DoubleDataRecordCompatible +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.timelines.prediction.features.recap.RecapFeatures + +sealed trait PredictedScoreFeature + extends DataRecordOptionalFeature[TweetCandidate, Double] + with DoubleDataRecordCompatible { + + override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty + def statName: String + def modelWeightParam: FSBoundedParam[Double] + def extractScore: FeatureMap => Option[Double] = _.getOrElse(this, None) +} + +object PredictedFavoriteScoreFeature extends PredictedScoreFeature { + override val featureName: String = RecapFeatures.PREDICTED_IS_FAVORITED.getFeatureName + override val statName = "fav" + override val modelWeightParam = ModelWeights.FavParam +} + +object PredictedReplyScoreFeature extends PredictedScoreFeature { + override val featureName: String = RecapFeatures.PREDICTED_IS_REPLIED.getFeatureName + override val statName = "reply" + override val modelWeightParam = ModelWeights.ReplyParam +} + +object PredictedRetweetScoreFeature extends PredictedScoreFeature { + override val featureName: String = RecapFeatures.PREDICTED_IS_RETWEETED.getFeatureName + override val statName = "retweet" + override val modelWeightParam = ModelWeights.RetweetParam +} + +object PredictedReplyEngagedByAuthorScoreFeature extends PredictedScoreFeature { + override val featureName: String = + RecapFeatures.PREDICTED_IS_REPLIED_REPLY_ENGAGED_BY_AUTHOR.getFeatureName + override val statName = "reply_engaged_by_author" + override val modelWeightParam = ModelWeights.ReplyEngagedByAuthorParam +} + +object PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature extends PredictedScoreFeature { + override val featureName: String = RecapFeatures.PREDICTED_IS_GOOD_CLICKED_V1.getFeatureName + override val statName = "good_click_convo_desc_favorited_or_replied" + override val modelWeightParam = ModelWeights.GoodClickParam + + override def extractScore: FeatureMap => Option[Double] = { featureMap => + val goodClickV1Opt = featureMap.getOrElse(this, None) + val goodClickV2Opt = featureMap.getOrElse(PredictedGoodClickConvoDescUamGt2ScoreFeature, None) + + (goodClickV1Opt, goodClickV2Opt) match { + case (Some(v1Score), Some(v2Score)) => Some(Math.max(v1Score, v2Score)) + case _ => goodClickV1Opt.orElse(goodClickV2Opt) + } + } +} + +object PredictedGoodClickConvoDescUamGt2ScoreFeature extends PredictedScoreFeature { + override val featureName: String = RecapFeatures.PREDICTED_IS_GOOD_CLICKED_V2.getFeatureName + override val statName = "good_click_convo_desc_uam_gt_2" + override val modelWeightParam = ModelWeights.GoodClickV2Param +} + +object PredictedGoodProfileClickScoreFeature extends PredictedScoreFeature { + override val featureName: String = + RecapFeatures.PREDICTED_IS_PROFILE_CLICKED_AND_PROFILE_ENGAGED.getFeatureName + override val statName = "good_profile_click" + override val modelWeightParam = ModelWeights.GoodProfileClickParam +} + +object PredictedVideoPlayback50ScoreFeature extends PredictedScoreFeature { + override val featureName: String = RecapFeatures.PREDICTED_IS_VIDEO_PLAYBACK_50.getFeatureName + override val statName = "video_playback_50" + override val modelWeightParam = ModelWeights.VideoPlayback50Param +} + +object PredictedTweetDetailDwellScoreFeature extends PredictedScoreFeature { + override val featureName: String = + RecapFeatures.PREDICTED_IS_TWEET_DETAIL_DWELLED_15_SEC.getFeatureName + override val statName = "tweet_detail_dwell" + override val modelWeightParam = ModelWeights.TweetDetailDwellParam +} + +object PredictedProfileDwelledScoreFeature extends PredictedScoreFeature { + override val featureName: String = + RecapFeatures.PREDICTED_IS_PROFILE_DWELLED_20_SEC.getFeatureName + override val statName = "profile_dwell" + override val modelWeightParam = ModelWeights.ProfileDwelledParam +} + +object PredictedBookmarkScoreFeature extends PredictedScoreFeature { + override val featureName: String = RecapFeatures.PREDICTED_IS_BOOKMARKED.getFeatureName + override val statName = "bookmark" + override val modelWeightParam = ModelWeights.BookmarkParam +} + +object PredictedShareScoreFeature extends PredictedScoreFeature { + override val featureName: String = + RecapFeatures.PREDICTED_IS_SHARED.getFeatureName + override val statName = "share" + override val modelWeightParam = ModelWeights.ShareParam +} + +object PredictedShareMenuClickScoreFeature extends PredictedScoreFeature { + override val featureName: String = + RecapFeatures.PREDICTED_IS_SHARE_MENU_CLICKED.getFeatureName + override val statName = "share_menu_click" + override val modelWeightParam = ModelWeights.ShareMenuClickParam +} + +// Negative Engagements +object PredictedNegativeFeedbackV2ScoreFeature extends PredictedScoreFeature { + override val featureName: String = + RecapFeatures.PREDICTED_IS_NEGATIVE_FEEDBACK_V2.getFeatureName + override val statName = "negative_feedback_v2" + override val modelWeightParam = ModelWeights.NegativeFeedbackV2Param +} + +object PredictedReportedScoreFeature extends PredictedScoreFeature { + override val featureName: String = + RecapFeatures.PREDICTED_IS_REPORT_TWEET_CLICKED.getFeatureName + override val statName = "reported" + override val modelWeightParam = ModelWeights.ReportParam +} + +object PredictedStrongNegativeFeedbackScoreFeature extends PredictedScoreFeature { + override val featureName: String = + RecapFeatures.PREDICTED_IS_STRONG_NEGATIVE_FEEDBACK.getFeatureName + override val statName = "strong_negative_feedback" + override val modelWeightParam = ModelWeights.StrongNegativeFeedbackParam +} + +object PredictedWeakNegativeFeedbackScoreFeature extends PredictedScoreFeature { + override val featureName: String = + RecapFeatures.PREDICTED_IS_WEAK_NEGATIVE_FEEDBACK.getFeatureName + override val statName = "weak_negative_feedback" + override val modelWeightParam = ModelWeights.WeakNegativeFeedbackParam +} + +object PredictedScoreFeature { + val PredictedScoreFeatures: Set[PredictedScoreFeature] = Set( + PredictedFavoriteScoreFeature, + PredictedReplyScoreFeature, + PredictedRetweetScoreFeature, + PredictedReplyEngagedByAuthorScoreFeature, + PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature, + PredictedGoodClickConvoDescUamGt2ScoreFeature, + PredictedGoodProfileClickScoreFeature, + PredictedVideoPlayback50ScoreFeature, + PredictedTweetDetailDwellScoreFeature, + PredictedProfileDwelledScoreFeature, + PredictedBookmarkScoreFeature, + PredictedShareScoreFeature, + PredictedShareMenuClickScoreFeature, + // Negative Engagements + PredictedNegativeFeedbackV2ScoreFeature, + PredictedReportedScoreFeature, + PredictedStrongNegativeFeedbackScoreFeature, + PredictedWeakNegativeFeedbackScoreFeature, + ) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/RescoringFactorProvider.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/RescoringFactorProvider.scala new file mode 100644 index 000000000..d9538b66d --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/RescoringFactorProvider.scala @@ -0,0 +1,180 @@ +package com.twitter.home_mixer.product.scored_tweets.scorer + +import com.twitter.home_mixer.functional_component.scorer.FeedbackFatigueScorer +import com.twitter.home_mixer.model.HomeFeatures +import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature +import com.twitter.home_mixer.model.HomeFeatures.AuthorIsCreatorFeature +import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature +import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature +import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.BlueVerifiedAuthorInNetworkMultiplierParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.BlueVerifiedAuthorOutOfNetworkMultiplierParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CreatorInNetworkMultiplierParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CreatorOutOfNetworkMultiplierParam +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.OutOfNetworkScaleFactorParam +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.model.common.CandidateWithFeatures +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.timelineservice.{thriftscala => tls} + +trait RescoringFactorProvider { + + def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean + + def factor( + query: PipelineQuery, + candidate: CandidateWithFeatures[TweetCandidate] + ): Double + + def apply( + query: PipelineQuery, + candidate: CandidateWithFeatures[TweetCandidate], + ): Double = if (selector(candidate)) factor(query, candidate) else 1.0 +} + +/** + * Re-scoring multiplier to apply to authors who are eligible subscription content creators + */ +object RescoreCreators extends RescoringFactorProvider { + + def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = + candidate.features.getOrElse(AuthorIsCreatorFeature, false) && + CandidatesUtil.isOriginalTweet(candidate) + + def factor( + query: PipelineQuery, + candidate: CandidateWithFeatures[TweetCandidate] + ): Double = + if (candidate.features.getOrElse(InNetworkFeature, false)) + query.params(CreatorInNetworkMultiplierParam) + else query.params(CreatorOutOfNetworkMultiplierParam) +} + +/** + * Re-scoring multiplier to apply to authors who are verified by Twitter Blue + */ +object RescoreBlueVerified extends RescoringFactorProvider { + + def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = + candidate.features.getOrElse(AuthorIsBlueVerifiedFeature, false) && + CandidatesUtil.isOriginalTweet(candidate) + + def factor( + query: PipelineQuery, + candidate: CandidateWithFeatures[TweetCandidate] + ): Double = + if (candidate.features.getOrElse(InNetworkFeature, false)) + query.params(BlueVerifiedAuthorInNetworkMultiplierParam) + else query.params(BlueVerifiedAuthorOutOfNetworkMultiplierParam) +} + +/** + * Re-scoring multiplier to apply to out-of-network tweets + */ +object RescoreOutOfNetwork extends RescoringFactorProvider { + + def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = + !candidate.features.getOrElse(InNetworkFeature, false) + + def factor( + query: PipelineQuery, + candidate: CandidateWithFeatures[TweetCandidate] + ): Double = query.params(OutOfNetworkScaleFactorParam) +} + +/** + * Re-scoring multiplier to apply to reply candidates + */ +object RescoreReplies extends RescoringFactorProvider { + + private val ScaleFactor = 0.75 + + def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = + candidate.features.getOrElse(InReplyToTweetIdFeature, None).isDefined + + def factor( + query: PipelineQuery, + candidate: CandidateWithFeatures[TweetCandidate] + ): Double = ScaleFactor +} + +/** + * Re-scoring multiplier to calibrate multi-tasks learning model prediction + */ +object RescoreMTLNormalization extends RescoringFactorProvider { + + private val ScaleFactor = 1.0 + + def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = { + candidate.features.contains(HomeFeatures.FocalTweetAuthorIdFeature) + } + + def factor( + query: PipelineQuery, + candidate: CandidateWithFeatures[TweetCandidate] + ): Double = ScaleFactor +} + +/** + * Re-scoring multiplier to apply to multiple tweets from the same author + */ +case class RescoreAuthorDiversity(diversityDiscounts: Map[Long, Double]) + extends RescoringFactorProvider { + + def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = + diversityDiscounts.contains(candidate.candidate.id) + + def factor( + query: PipelineQuery, + candidate: CandidateWithFeatures[TweetCandidate] + ): Double = diversityDiscounts(candidate.candidate.id) +} + +case class RescoreFeedbackFatigue(query: PipelineQuery) extends RescoringFactorProvider { + + def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = true + + private val feedbackEntriesByEngagementType = + query.features + .getOrElse(FeatureMap.empty).getOrElse(FeedbackHistoryFeature, Seq.empty) + .filter { entry => + val timeSinceFeedback = query.queryTime.minus(entry.timestamp) + timeSinceFeedback < FeedbackFatigueScorer.DurationForFiltering + FeedbackFatigueScorer.DurationForDiscounting && + entry.feedbackType == tls.FeedbackType.SeeFewer + }.groupBy(_.engagementType) + + private val authorsToDiscount = + FeedbackFatigueScorer.getUserDiscounts( + query.queryTime, + feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Tweet, Seq.empty)) + + private val likersToDiscount = + FeedbackFatigueScorer.getUserDiscounts( + query.queryTime, + feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Like, Seq.empty)) + + private val followersToDiscount = + FeedbackFatigueScorer.getUserDiscounts( + query.queryTime, + feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Follow, Seq.empty)) + + private val retweetersToDiscount = + FeedbackFatigueScorer.getUserDiscounts( + query.queryTime, + feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Retweet, Seq.empty)) + + def factor( + query: PipelineQuery, + candidate: CandidateWithFeatures[TweetCandidate] + ): Double = { + FeedbackFatigueScorer.getScoreMultiplier( + candidate, + authorsToDiscount, + likersToDiscount, + followersToDiscount, + retweetersToDiscount + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/WeightedScoresSumScorer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/WeightedScoresSumScorer.scala deleted file mode 100644 index 9f937448f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/WeightedScoresSumScorer.scala +++ /dev/null @@ -1,91 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.scorer - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature -import com.twitter.home_mixer.model.HomeFeatures.WeightedModelScoreFeature -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class WeightedScoresSumScorer @Inject() (statsReceiver: StatsReceiver) - extends Scorer[ScoredTweetsQuery, TweetCandidate] { - - override val identifier: ScorerIdentifier = ScorerIdentifier("WeightedScoresSum") - - override val features: Set[Feature[_, _]] = Set(WeightedModelScoreFeature, ScoreFeature) - - private val StatsReadabilityMultiplier = 1000 - private val Epsilon = 0.001 - private val PredictedScoreStatName = f"predicted_score_${StatsReadabilityMultiplier}x" - private val MissingScoreStatName = "missing_score" - - private val scopedStatsProvider = statsReceiver.scope(getClass.getSimpleName) - private val scoreStat = scopedStatsProvider.stat(f"score_${StatsReadabilityMultiplier}x") - - override def apply( - query: ScoredTweetsQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - val features = candidates.map { candidate => - val score = weightedModelScore(query, candidate.features) - scoreStat.add((score * StatsReadabilityMultiplier).toFloat) - FeatureMapBuilder() - .add(WeightedModelScoreFeature, Some(score)) - .add(ScoreFeature, Some(score)) - .build() - } - - Stitch.value(features) - } - - /** - * (1) compute weighted sum of predicted scores of all engagements - * (2) convert negative score to positive score if needed - */ - private def weightedModelScore( - query: PipelineQuery, - features: FeatureMap - ): Double = { - val weightedScoreAndModelWeightSeq: Seq[(Double, Double)] = - HomeNaviModelDataRecordScorer.PredictedScoreFeatures.map { scoreFeature => - val predictedScoreOpt = features.getOrElse(scoreFeature, None) - - predictedScoreOpt match { - case Some(predictedScore) => - scopedStatsProvider - .stat(scoreFeature.statName, PredictedScoreStatName) - .add((predictedScore * StatsReadabilityMultiplier).toFloat) - case None => - scopedStatsProvider.counter(scoreFeature.statName, MissingScoreStatName).incr() - } - - val weight = query.params(scoreFeature.modelWeightParam) - (predictedScoreOpt.getOrElse(0.0) * weight, weight) - } - - val (weightedScores, modelWeights) = weightedScoreAndModelWeightSeq.unzip - val combinedScoreSum = weightedScores.sum - - val positiveModelWeightsSum = modelWeights.filter(_ > 0.0).sum - val negativeModelWeightsSum = modelWeights.filter(_ < 0).sum.abs - val modelWeightsSum = positiveModelWeightsSum + negativeModelWeightsSum - - val weightedScoresSum = - if (modelWeightsSum == 0) combinedScoreSum.max(0.0) - else if (combinedScoreSum < 0) - (combinedScoreSum + negativeModelWeightsSum) / modelWeightsSum * Epsilon - else combinedScoreSum + Epsilon - - weightedScoresSum - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/BUILD.bazel index fcd1bbda8..a7fccd8ff 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/BUILD.bazel @@ -4,30 +4,22 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/module", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tweet_tlx", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsDiversityScoringPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsDiversityScoringPipelineConfig.scala deleted file mode 100644 index 3a202f594..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsDiversityScoringPipelineConfig.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.scoring_pipeline - -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.scorer.AuthorDiversityDiscountProvider -import com.twitter.home_mixer.product.scored_tweets.scorer.DiversityScorer -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.component_library.selector.InsertAppendResults -import com.twitter.product_mixer.core.functional_component.common.AllPipelines -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier -import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipelineConfig - -object ScoredTweetsDiversityScoringPipelineConfig - extends ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate] { - - override val identifier: ScoringPipelineIdentifier = - ScoringPipelineIdentifier("ScoredTweetsDiversity") - - override val selectors: Seq[Selector[ScoredTweetsQuery]] = - Seq(InsertAppendResults(AllPipelines)) - - override val scorers: Seq[Scorer[ScoredTweetsQuery, TweetCandidate]] = - Seq(DiversityScorer(AuthorDiversityDiscountProvider)) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsRescoreOONScoringPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsHeuristicScoringPipelineConfig.scala similarity index 73% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsRescoreOONScoringPipelineConfig.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsHeuristicScoringPipelineConfig.scala index 71d53d2ec..aedfc15b5 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsRescoreOONScoringPipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsHeuristicScoringPipelineConfig.scala @@ -1,7 +1,7 @@ package com.twitter.home_mixer.product.scored_tweets.scoring_pipeline -import com.twitter.home_mixer.functional_component.scorer.OONTweetScalingScorer import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery +import com.twitter.home_mixer.product.scored_tweets.scorer.HeuristicScorer import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.selector.InsertAppendResults import com.twitter.product_mixer.core.functional_component.common.AllPipelines @@ -10,14 +10,14 @@ import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipelineConfig -object ScoredTweetsRescoreOONScoringPipelineConfig +object ScoredTweetsHeuristicScoringPipelineConfig extends ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate] { override val identifier: ScoringPipelineIdentifier = - ScoringPipelineIdentifier("ScoredTweetsRescoreOON") + ScoringPipelineIdentifier("ScoredTweetsHeuristic") - override val selectors: Seq[Selector[ScoredTweetsQuery]] = - Seq(InsertAppendResults(AllPipelines)) + override val selectors: Seq[Selector[ScoredTweetsQuery]] = Seq(InsertAppendResults(AllPipelines)) - override val scorers: Seq[Scorer[ScoredTweetsQuery, TweetCandidate]] = Seq(OONTweetScalingScorer) + override val scorers: Seq[Scorer[ScoredTweetsQuery, TweetCandidate]] = + Seq(HeuristicScorer) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsScoringPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsModelScoringPipelineConfig.scala similarity index 54% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsScoringPipelineConfig.scala rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsModelScoringPipelineConfig.scala index 6d2b35c19..ab1b49a83 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsScoringPipelineConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsModelScoringPipelineConfig.scala @@ -1,79 +1,110 @@ package com.twitter.home_mixer.product.scored_tweets.scoring_pipeline -import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.functional_component.feature_hydrator._ -import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.Phase1EdgeAggregateFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.Phase2EdgeAggregateFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates._ import com.twitter.home_mixer.model.HomeFeatures.EarlybirdScoreFeature import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.CachedScoredTweetsCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsCrMixerCandidatePipelineConfig +import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsBackfillCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsFrsCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsInNetworkCandidatePipelineConfig +import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsListsCandidatePipelineConfig +import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsPopularVideosCandidatePipelineConfig +import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsTweetMixerCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsUtegCandidatePipelineConfig +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.AncestorFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.AuthorFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.AuthorIsCreatorFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.EarlybirdFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.GizmoduckAuthorFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.GraphTwoHopFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.MetricCenterUserCountingFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RealGraphViewerAuthorFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RealGraphViewerRelatedUsersFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RealTimeInteractionGraphEdgeFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.SimClustersEngagementSimilarityFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.SimClustersUserTweetScoresHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TSPInferredTopicFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TweetMetaDataFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TweetTimeFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TweetypieContentFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TwhinAuthorFollowFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.UtegFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.Phase1EdgeAggregateFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.Phase2EdgeAggregateFeatureHydrator +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates._ import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.QualityFactor -import com.twitter.home_mixer.product.scored_tweets.scorer.HomeNaviModelDataRecordScorer +import com.twitter.home_mixer.product.scored_tweets.scorer.NaviModelScorer +import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.selector.DropMaxCandidates import com.twitter.product_mixer.component_library.selector.InsertAppendResults import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates -import com.twitter.product_mixer.core.feature.featuremap.datarecord.AllFeatures import com.twitter.product_mixer.core.functional_component.common.AllExceptPipelines +import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.gate.BaseGate import com.twitter.product_mixer.core.functional_component.scorer.Scorer import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure import com.twitter.product_mixer.core.pipeline.pipeline_failure.UnexpectedCandidateResult import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipelineConfig -import com.twitter.timelines.clients.predictionservice.PredictionGRPCService -import com.twitter.timelines.clients.predictionservice.PredictionServiceGRPCClient +import com.twitter.timelines.configapi.Param + import javax.inject.Inject import javax.inject.Singleton @Singleton -class ScoredTweetsScoringPipelineConfig @Inject() ( +class ScoredTweetsModelScoringPipelineConfig @Inject() ( + // candidate sources scoredTweetsInNetworkCandidatePipelineConfig: ScoredTweetsInNetworkCandidatePipelineConfig, scoredTweetsUtegCandidatePipelineConfig: ScoredTweetsUtegCandidatePipelineConfig, - scoredTweetsCrMixerCandidatePipelineConfig: ScoredTweetsCrMixerCandidatePipelineConfig, + scoredTweetsTweetMixerCandidatePipelineConfig: ScoredTweetsTweetMixerCandidatePipelineConfig, scoredTweetsFrsCandidatePipelineConfig: ScoredTweetsFrsCandidatePipelineConfig, - predictionGRPCService: PredictionGRPCService, + scoredTweetsListsCandidatePipelineConfig: ScoredTweetsListsCandidatePipelineConfig, + scoredTweetsPopularVideosCandidatePipelineConfig: ScoredTweetsPopularVideosCandidatePipelineConfig, + scoredTweetsBackfillCandidatePipelineConfig: ScoredTweetsBackfillCandidatePipelineConfig, + // feature hydrators ancestorFeatureHydrator: AncestorFeatureHydrator, authorFeatureHydrator: AuthorFeatureHydrator, + authorIsCreatorFeatureHydrator: AuthorIsCreatorFeatureHydrator, earlybirdFeatureHydrator: EarlybirdFeatureHydrator, - metricCenterUserCountingFeatureHydrator: MetricCenterUserCountingFeatureHydrator, - tweetypieContentFeatureHydrator: TweetypieContentFeatureHydrator, - gizmoduckAuthorSafetyFeatureHydrator: GizmoduckAuthorSafetyFeatureHydrator, + gizmoduckAuthorSafetyFeatureHydrator: GizmoduckAuthorFeatureHydrator, graphTwoHopFeatureHydrator: GraphTwoHopFeatureHydrator, - socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator, - twhinAuthorFollow20220101FeatureHydrator: TwhinAuthorFollow20220101FeatureHydrator, - userFollowedTopicIdsFeatureHydrator: UserFollowedTopicIdsFeatureHydrator, - utegFeatureHydrator: UtegFeatureHydrator, + metricCenterUserCountingFeatureHydrator: MetricCenterUserCountingFeatureHydrator, + perspectiveFilteredSocialContextFeatureHydrator: PerspectiveFilteredSocialContextFeatureHydrator, realGraphViewerAuthorFeatureHydrator: RealGraphViewerAuthorFeatureHydrator, realGraphViewerRelatedUsersFeatureHydrator: RealGraphViewerRelatedUsersFeatureHydrator, realTimeInteractionGraphEdgeFeatureHydrator: RealTimeInteractionGraphEdgeFeatureHydrator, + sgsValidSocialContextFeatureHydrator: SGSValidSocialContextFeatureHydrator, + simClustersEngagementSimilarityFeatureHydrator: SimClustersEngagementSimilarityFeatureHydrator, + simClustersUserTweetScoresHydrator: SimClustersUserTweetScoresHydrator, + tspInferredTopicFeatureHydrator: TSPInferredTopicFeatureHydrator, + tweetypieContentFeatureHydrator: TweetypieContentFeatureHydrator, + twhinAuthorFollowFeatureHydrator: TwhinAuthorFollowFeatureHydrator, + utegFeatureHydrator: UtegFeatureHydrator, + // real time aggregate feature hydrators engagementsReceivedByAuthorRealTimeAggregateFeatureHydrator: EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator, topicCountryEngagementRealTimeAggregateFeatureHydrator: TopicCountryEngagementRealTimeAggregateFeatureHydrator, topicEngagementRealTimeAggregateFeatureHydrator: TopicEngagementRealTimeAggregateFeatureHydrator, - tspInferredTopicFeatureHydrator: TSPInferredTopicFeatureHydrator, tweetCountryEngagementRealTimeAggregateFeatureHydrator: TweetCountryEngagementRealTimeAggregateFeatureHydrator, tweetEngagementRealTimeAggregateFeatureHydrator: TweetEngagementRealTimeAggregateFeatureHydrator, twitterListEngagementRealTimeAggregateFeatureHydrator: TwitterListEngagementRealTimeAggregateFeatureHydrator, userAuthorEngagementRealTimeAggregateFeatureHydrator: UserAuthorEngagementRealTimeAggregateFeatureHydrator, - simClustersEngagementSimilarityFeatureHydrator: SimClustersEngagementSimilarityFeatureHydrator, + // offline aggregate feature hydrators phase1EdgeAggregateFeatureHydrator: Phase1EdgeAggregateFeatureHydrator, phase2EdgeAggregateFeatureHydrator: Phase2EdgeAggregateFeatureHydrator, - statsReceiver: StatsReceiver) + // model + naviModelScorer: NaviModelScorer) extends ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate] { - override val identifier: ScoringPipelineIdentifier = ScoringPipelineIdentifier("ScoredTweets") + override val identifier: ScoringPipelineIdentifier = + ScoringPipelineIdentifier("ScoredTweetsModel") private val nonCachedScoringPipelineScope = AllExceptPipelines( pipelinesToExclude = Set(CachedScoredTweetsCandidatePipelineConfig.Identifier) @@ -96,19 +127,51 @@ class ScoredTweetsScoringPipelineConfig @Inject() ( case _ => throw PipelineFailure(UnexpectedCandidateResult, "Invalid candidate type") } + private def qualityFactorDropMaxCandidates( + pipelineIdentifier: CandidatePipelineIdentifier, + qualityFactorParam: Param[Int] + ): DropMaxCandidates[ScoredTweetsQuery] = { + new DropMaxCandidates( + pipelineScope = SpecificPipelines(pipelineIdentifier), + maxSelector = (query, _, _) => + (query.getQualityFactorCurrentValue(identifier) * + query.params(qualityFactorParam)).toInt + ) + } + override val selectors: Seq[Selector[ScoredTweetsQuery]] = Seq( UpdateSortCandidates(SpecificPipelines(earlybirdScorePipelineScope), earlybirdScoreOrdering), - new DropMaxCandidates( - pipelineScope = SpecificPipelines(earlybirdScorePipelineScope), - maxSelector = (query, _, _) => - (query.getQualityFactorCurrentValue(identifier) * - query.params(QualityFactor.MaxTweetsToScoreParam)).toInt + UpdateSortCandidates( + SpecificPipeline(scoredTweetsBackfillCandidatePipelineConfig.identifier), + CandidatesUtil.reverseChronTweetsOrdering ), - new DropMaxCandidates( - pipelineScope = SpecificPipelines(scoredTweetsCrMixerCandidatePipelineConfig.identifier), - maxSelector = (query, _, _) => - (query.getQualityFactorCurrentValue(identifier) * - query.params(QualityFactor.CrMixerMaxTweetsToScoreParam)).toInt + qualityFactorDropMaxCandidates( + scoredTweetsInNetworkCandidatePipelineConfig.identifier, + QualityFactor.InNetworkMaxTweetsToScoreParam + ), + qualityFactorDropMaxCandidates( + scoredTweetsUtegCandidatePipelineConfig.identifier, + QualityFactor.UtegMaxTweetsToScoreParam + ), + qualityFactorDropMaxCandidates( + scoredTweetsFrsCandidatePipelineConfig.identifier, + QualityFactor.FrsMaxTweetsToScoreParam + ), + qualityFactorDropMaxCandidates( + scoredTweetsTweetMixerCandidatePipelineConfig.identifier, + QualityFactor.TweetMixerMaxTweetsToScoreParam + ), + qualityFactorDropMaxCandidates( + scoredTweetsListsCandidatePipelineConfig.identifier, + QualityFactor.ListsMaxTweetsToScoreParam + ), + qualityFactorDropMaxCandidates( + scoredTweetsPopularVideosCandidatePipelineConfig.identifier, + QualityFactor.PopularVideosMaxTweetsToScoreParam + ), + qualityFactorDropMaxCandidates( + scoredTweetsBackfillCandidatePipelineConfig.identifier, + QualityFactor.BackfillMaxTweetsToScoreParam ), // Select candidates for Heavy Ranker Feature Hydration and Scoring InsertAppendResults(nonCachedScoringPipelineScope) @@ -117,24 +180,25 @@ class ScoredTweetsScoringPipelineConfig @Inject() ( override val preScoringFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] ] = Seq( + TweetMetaDataFeatureHydrator, ancestorFeatureHydrator, authorFeatureHydrator, + authorIsCreatorFeatureHydrator, earlybirdFeatureHydrator, gizmoduckAuthorSafetyFeatureHydrator, graphTwoHopFeatureHydrator, metricCenterUserCountingFeatureHydrator, - socialGraphServiceFeatureHydrator, - TweetMetaDataFeatureHydrator, - tweetypieContentFeatureHydrator, - twhinAuthorFollow20220101FeatureHydrator, - userFollowedTopicIdsFeatureHydrator, - utegFeatureHydrator, realTimeInteractionGraphEdgeFeatureHydrator, realGraphViewerAuthorFeatureHydrator, + simClustersEngagementSimilarityFeatureHydrator, + simClustersUserTweetScoresHydrator, + InNetworkFeatureHydrator, + tspInferredTopicFeatureHydrator, + tweetypieContentFeatureHydrator, + twhinAuthorFollowFeatureHydrator, + utegFeatureHydrator, // real time aggregates engagementsReceivedByAuthorRealTimeAggregateFeatureHydrator, - simClustersEngagementSimilarityFeatureHydrator, - tspInferredTopicFeatureHydrator, tweetCountryEngagementRealTimeAggregateFeatureHydrator, tweetEngagementRealTimeAggregateFeatureHydrator, twitterListEngagementRealTimeAggregateFeatureHydrator, @@ -146,29 +210,14 @@ class ScoredTweetsScoringPipelineConfig @Inject() ( override val preScoringFeatureHydrationPhase2: Seq[ BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] ] = Seq( + perspectiveFilteredSocialContextFeatureHydrator, + phase2EdgeAggregateFeatureHydrator, realGraphViewerRelatedUsersFeatureHydrator, - TimeFeaturesHydrator, + sgsValidSocialContextFeatureHydrator, + TweetTimeFeatureHydrator, topicCountryEngagementRealTimeAggregateFeatureHydrator, - topicEngagementRealTimeAggregateFeatureHydrator, - phase2EdgeAggregateFeatureHydrator + topicEngagementRealTimeAggregateFeatureHydrator ) - private val homeNaviModelDataRecordScorer: Scorer[ScoredTweetsQuery, TweetCandidate] = { - val modelClient = new PredictionServiceGRPCClient( - service = predictionGRPCService, - statsReceiver = statsReceiver, - requestBatchSize = HomeNaviModelDataRecordScorer.RequestBatchSize, - useCompact = false - ) - HomeNaviModelDataRecordScorer( - identifier = ScorerIdentifier("HomeNaviModel"), - modelClient = modelClient, - candidateFeatures = AllFeatures(), - resultFeatures = HomeNaviModelDataRecordScorer.PredictedScoreFeatures.toSet, - statsReceiver = statsReceiver - ) - } - - override val scorers: Seq[Scorer[ScoredTweetsQuery, TweetCandidate]] = - Seq(homeNaviModelDataRecordScorer) + override val scorers: Seq[Scorer[ScoredTweetsQuery, TweetCandidate]] = Seq(naviModelScorer) } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsRescoreVerifiedAuthorScoringPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsRescoreVerifiedAuthorScoringPipelineConfig.scala deleted file mode 100644 index d07e5c965..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsRescoreVerifiedAuthorScoringPipelineConfig.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.scoring_pipeline - -import com.twitter.home_mixer.functional_component.scorer.VerifiedAuthorScalingScorer -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.component_library.selector.InsertAppendResults -import com.twitter.product_mixer.core.functional_component.common.AllPipelines -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier -import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipelineConfig - -object ScoredTweetsRescoreVerifiedAuthorScoringPipelineConfig - extends ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate] { - - override val identifier: ScoringPipelineIdentifier = - ScoringPipelineIdentifier("ScoredTweetsRescoreVerifiedAuthor") - - override val selectors: Seq[Selector[ScoredTweetsQuery]] = - Seq(InsertAppendResults(AllPipelines)) - - override val scorers: Seq[Scorer[ScoredTweetsQuery, TweetCandidate]] = - Seq(VerifiedAuthorScalingScorer) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsWeightedScoresSumScoringPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsWeightedScoresSumScoringPipelineConfig.scala deleted file mode 100644 index 459bc4d44..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsWeightedScoresSumScoringPipelineConfig.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.scoring_pipeline - -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.CachedScoredTweetsCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.scorer.WeightedScoresSumScorer -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.component_library.selector.InsertAppendResults -import com.twitter.product_mixer.core.functional_component.common.AllExceptPipelines -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier -import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipelineConfig -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ScoredTweetsWeightedScoresSumScoringPipelineConfig @Inject() ( - weightedScoresSumScorer: WeightedScoresSumScorer) - extends ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate] { - - override val identifier: ScoringPipelineIdentifier = - ScoringPipelineIdentifier("ScoredTweetsWeightedScoresSum") - - override val selectors: Seq[Selector[ScoredTweetsQuery]] = Seq( - InsertAppendResults( - AllExceptPipelines(pipelinesToExclude = - Set(CachedScoredTweetsCandidatePipelineConfig.Identifier)) - ) - ) - - override val scorers: Seq[Scorer[ScoredTweetsQuery, TweetCandidate]] = Seq( - weightedScoresSumScorer - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/selector/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/selector/BUILD.bazel new file mode 100644 index 000000000..c5dbb187f --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/selector/BUILD.bazel @@ -0,0 +1,12 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/selector/KeepBestOutOfNetworkCandidatePerAuthorPerSuggestType.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/selector/KeepBestOutOfNetworkCandidatePerAuthorPerSuggestType.scala new file mode 100644 index 000000000..c485c4c2a --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/selector/KeepBestOutOfNetworkCandidatePerAuthorPerSuggestType.scala @@ -0,0 +1,39 @@ +package com.twitter.home_mixer.product.scored_tweets.selector + +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature +import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature +import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature +import com.twitter.product_mixer.core.functional_component.common.CandidateScope +import com.twitter.product_mixer.core.functional_component.selector.Selector +import com.twitter.product_mixer.core.functional_component.selector.SelectorResult +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.product_mixer.core.pipeline.PipelineQuery + +case class KeepBestOutOfNetworkCandidatePerAuthorPerSuggestType( + override val pipelineScope: CandidateScope) + extends Selector[PipelineQuery] { + + override def apply( + query: PipelineQuery, + remainingCandidates: Seq[CandidateWithDetails], + result: Seq[CandidateWithDetails] + ): SelectorResult = { + val (selectedCandidates, otherCandidates) = + remainingCandidates.partition(candidate => + pipelineScope.contains(candidate) && !candidate.features.getOrElse(InNetworkFeature, true)) + + val filteredCandidates = selectedCandidates + .groupBy { candidate => + ( + candidate.features.getOrElse(AuthorIdFeature, None), + candidate.features.getOrElse(SuggestTypeFeature, None) + ) + } + .values.map(_.maxBy(_.features.getOrElse(ScoreFeature, None))) + .toSeq + + val updatedCandidates = otherCandidates ++ filteredCandidates + SelectorResult(remainingCandidates = updatedCandidates, result = result) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/BUILD.bazel index f6c404ec6..2147ee217 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/BUILD.bazel @@ -4,29 +4,27 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", "finagle/finagle-mysql/src/main/scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/non_ml_features", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tweet_tlx", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", "servo/repo/src/main/scala", "servo/util/src/main/scala", "src/scala/com/twitter/timelines/prediction/common/adapters", "src/thrift/com/twitter/timelines/suggests/common:data_record_metadata-scala", "src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java", + "src/thrift/com/twitter/timelines/timeline_logging:thrift-scala", "timelines/ml:pldr-client", "timelines/ml:pldr-conversion", "timelines/ml/cont_train/common/domain/src/main/scala/com/twitter/timelines/ml/cont_train/common/domain/non_scalding", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CachedScoredTweetsSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CachedScoredTweetsSideEffect.scala index abe076c54..3d66ff54a 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CachedScoredTweetsSideEffect.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CachedScoredTweetsSideEffect.scala @@ -3,17 +3,22 @@ package com.twitter.home_mixer.product.scored_tweets.side_effect import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature +import com.twitter.home_mixer.model.HomeFeatures.AuthorIsCreatorFeature +import com.twitter.home_mixer.model.HomeFeatures.AuthorIsGoldVerifiedFeature +import com.twitter.home_mixer.model.HomeFeatures.AuthorIsGrayVerifiedFeature +import com.twitter.home_mixer.model.HomeFeatures.AuthorIsLegacyVerifiedFeature import com.twitter.home_mixer.model.HomeFeatures.CachedCandidatePipelineIdentifierFeature import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.FavoritedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.FollowedByUserIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.LastScoredTimestampMsFeature +import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature @@ -33,14 +38,12 @@ import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDet import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.cache.TtlCache import com.twitter.stitch.Stitch -import com.twitter.util.Time - import javax.inject.Inject import javax.inject.Singleton @Singleton class CachedScoredTweetsSideEffect @Inject() ( - scoredTweetsCache: TtlCache[Long, hmt.CachedScoredTweets]) + scoredTweetsCache: TtlCache[Long, hmt.ScoredTweetsResponse]) extends PipelineResultSideEffect[PipelineQuery, ScoredTweetsResponse] { override val identifier: SideEffectIdentifier = SideEffectIdentifier("CachedScoredTweets") @@ -48,71 +51,79 @@ class CachedScoredTweetsSideEffect @Inject() ( private val MaxTweetsToCache = 1000 def buildCachedScoredTweets( + query: PipelineQuery, candidates: Seq[CandidateWithDetails] - ): hmt.CachedScoredTweets = { + ): hmt.ScoredTweetsResponse = { val tweets = candidates.map { candidate => - val favoritedByUserIds = candidate.features.getOrElse(FavoritedByUserIdsFeature, Seq.empty) - val followedByUserIds = candidate.features.getOrElse(FollowedByUserIdsFeature, Seq.empty) + val sgsValidLikedByUserIds = + candidate.features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty) + val sgsValidFollowedByUserIds = + candidate.features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty) + val perspectiveFilteredLikedByUserIds = + candidate.features.getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Seq.empty) val ancestors = candidate.features.getOrElse(AncestorsFeature, Seq.empty) - val urlsList = candidate.features.getOrElse(TweetUrlsFeature, Seq.empty) - hmt.CachedScoredTweet( + hmt.ScoredTweet( tweetId = candidate.candidateIdLong, + authorId = candidate.features.get(AuthorIdFeature).get, // Cache the model score instead of the final score because rescoring is per-request score = candidate.features.getOrElse(WeightedModelScoreFeature, None), - lastScoredTimestampMs = Some( - candidate.features - .getOrElse(LastScoredTimestampMsFeature, None) - .getOrElse(Time.now.inMilliseconds)), - candidatePipelineIdentifier = Some( - candidate.features - .getOrElse(CachedCandidatePipelineIdentifierFeature, None) - .getOrElse(candidate.source.name)), - userId = candidate.features.getOrElse(AuthorIdFeature, None), + suggestType = candidate.features.getOrElse(SuggestTypeFeature, None), sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None), sourceUserId = candidate.features.getOrElse(SourceUserIdFeature, None), - isRetweet = Some(candidate.features.getOrElse(IsRetweetFeature, false)), - isInNetwork = Some(candidate.features.getOrElse(InNetworkFeature, false)), - suggestType = candidate.features.getOrElse(SuggestTypeFeature, None), quotedTweetId = candidate.features.getOrElse(QuotedTweetIdFeature, None), quotedUserId = candidate.features.getOrElse(QuotedUserIdFeature, None), inReplyToTweetId = candidate.features.getOrElse(InReplyToTweetIdFeature, None), inReplyToUserId = candidate.features.getOrElse(InReplyToUserIdFeature, None), directedAtUserId = candidate.features.getOrElse(DirectedAtUserIdFeature, None), - favoritedByUserIds = if (favoritedByUserIds.nonEmpty) Some(favoritedByUserIds) else None, - followedByUserIds = if (followedByUserIds.nonEmpty) Some(followedByUserIds) else None, + inNetwork = Some(candidate.features.getOrElse(InNetworkFeature, true)), + sgsValidLikedByUserIds = Some(sgsValidLikedByUserIds), + sgsValidFollowedByUserIds = Some(sgsValidFollowedByUserIds), topicId = candidate.features.getOrElse(TopicIdSocialContextFeature, None), topicFunctionalityType = candidate.features .getOrElse(TopicContextFunctionalityTypeFeature, None).map( TopicContextFunctionalityTypeMarshaller(_)), ancestors = if (ancestors.nonEmpty) Some(ancestors) else None, - urlsList = if (urlsList.nonEmpty) Some(urlsList) else None, - authorIsBlueVerified = - Some(candidate.features.getOrElse(AuthorIsBlueVerifiedFeature, false)) + isReadFromCache = Some(true), + streamToKafka = Some(false), + exclusiveConversationAuthorId = candidate.features + .getOrElse(ExclusiveConversationAuthorIdFeature, None), + authorMetadata = Some( + hmt.AuthorMetadata( + blueVerified = candidate.features.getOrElse(AuthorIsBlueVerifiedFeature, false), + goldVerified = candidate.features.getOrElse(AuthorIsGoldVerifiedFeature, false), + grayVerified = candidate.features.getOrElse(AuthorIsGrayVerifiedFeature, false), + legacyVerified = candidate.features.getOrElse(AuthorIsLegacyVerifiedFeature, false), + creator = candidate.features.getOrElse(AuthorIsCreatorFeature, false) + )), + lastScoredTimestampMs = candidate.features + .getOrElse(LastScoredTimestampMsFeature, Some(query.queryTime.inMilliseconds)), + candidatePipelineIdentifier = candidate.features + .getOrElse(CachedCandidatePipelineIdentifierFeature, Some(candidate.source.name)), + tweetUrls = Some(candidate.features.getOrElse(TweetUrlsFeature, Seq.empty)), + perspectiveFilteredLikedByUserIds = Some(perspectiveFilteredLikedByUserIds) ) } - hmt.CachedScoredTweets(tweets = tweets) + hmt.ScoredTweetsResponse(tweets) } final override def apply( inputs: PipelineResultSideEffect.Inputs[PipelineQuery, ScoredTweetsResponse] ): Stitch[Unit] = { - val candidates = (inputs.selectedCandidates ++ inputs.remainingCandidates).filter { candidate => - val score = candidate.features.getOrElse(ScoreFeature, None) - score.exists(_ > 0.0) - } + val candidates = + (inputs.selectedCandidates ++ inputs.remainingCandidates ++ inputs.droppedCandidates) + .filter(_.features.getOrElse(ScoreFeature, None).exists(_ > 0.0)) val truncatedCandidates = if (candidates.size > MaxTweetsToCache) candidates - .sortBy(-_.features.getOrElse(ScoreFeature, None).getOrElse(0.0)) - .take(MaxTweetsToCache) + .sortBy(-_.features.getOrElse(ScoreFeature, None).getOrElse(0.0)).take(MaxTweetsToCache) else candidates if (truncatedCandidates.nonEmpty) { val ttl = inputs.query.params(CachedScoredTweets.TTLParam) - val scoredTweets = buildCachedScoredTweets(truncatedCandidates) + val scoredTweets = buildCachedScoredTweets(inputs.query, truncatedCandidates) Stitch.callFuture(scoredTweetsCache.set(inputs.query.getRequiredUserId, scoredTweets, ttl)) } else Stitch.Unit } diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScribeScoredCandidatesSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScribeScoredCandidatesSideEffect.scala new file mode 100644 index 000000000..ef7e3b41a --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScribeScoredCandidatesSideEffect.scala @@ -0,0 +1,126 @@ +package com.twitter.home_mixer.product.scored_tweets.side_effect + +import com.twitter.finagle.tracing.Trace +import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.EarlybirdScoreFeature +import com.twitter.home_mixer.model.HomeFeatures.FavoritedByUserIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.FollowedByUserIdsFeature +import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature +import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature +import com.twitter.home_mixer.model.HomeFeatures.RequestJoinIdFeature +import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature +import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature +import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeScoredCandidatesFlag +import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery +import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse +import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableScribeScoredCandidatesParam +import com.twitter.inject.annotations.Flag +import com.twitter.logpipeline.client.common.EventPublisher +import com.twitter.product_mixer.component_library.side_effect.ScribeLogEventSideEffect +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect +import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidatePipelines +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails +import com.twitter.timelines.timeline_logging.{thriftscala => t} +import javax.inject.Inject +import javax.inject.Singleton +import com.twitter.util.logging.Logging + +/** + * Side effect that logs scored candidates from scoring pipelines + */ +@Singleton +class ScribeScoredCandidatesSideEffect @Inject() ( + @Flag(ScribeScoredCandidatesFlag) enableScribeScoredCandidates: Boolean, + eventBusPublisher: EventPublisher[t.ScoredCandidate]) + extends ScribeLogEventSideEffect[ + t.ScoredCandidate, + ScoredTweetsQuery, + ScoredTweetsResponse + ] + with PipelineResultSideEffect.Conditionally[ + ScoredTweetsQuery, + ScoredTweetsResponse + ] + with Logging { + + override val identifier: SideEffectIdentifier = + SideEffectIdentifier("ScribeScoredCandidates") + + override def onlyIf( + query: ScoredTweetsQuery, + selectedCandidates: Seq[CandidateWithDetails], + remainingCandidates: Seq[CandidateWithDetails], + droppedCandidates: Seq[CandidateWithDetails], + response: ScoredTweetsResponse + ): Boolean = enableScribeScoredCandidates && query.params(EnableScribeScoredCandidatesParam) + + /** + * Build the log events from query, selections and response + * + * @param query PipelineQuery + * @param selectedCandidates Result after Selectors are executed + * @param remainingCandidates Candidates which were not selected + * @param droppedCandidates Candidates dropped during selection + * @param response Result after Unmarshalling + * + * @return LogEvent in thrift + */ + override def buildLogEvents( + query: ScoredTweetsQuery, + selectedCandidates: Seq[CandidateWithDetails], + remainingCandidates: Seq[CandidateWithDetails], + droppedCandidates: Seq[CandidateWithDetails], + response: ScoredTweetsResponse + ): Seq[t.ScoredCandidate] = { + val returned = (selectedCandidates ++ remainingCandidates).map(toThrift(_, query, false)) + val dropped = droppedCandidates.map(toThrift(_, query, true)) + returned ++ dropped + } + + private def toThrift( + candidate: CandidateWithDetails, + query: ScoredTweetsQuery, + isDropped: Boolean + ): t.ScoredCandidate = { + t.ScoredCandidate( + tweetId = candidate.candidateIdLong, + viewerId = query.getOptionalUserId, + authorId = candidate.features.getOrElse(AuthorIdFeature, None), + traceId = Some(Trace.id.traceId.toLong), + requestJoinId = query.features.flatMap(_.getOrElse(RequestJoinIdFeature, None)), + score = candidate.features.getOrElse(ScoreFeature, None), + suggestType = candidate.features.getOrElse(SuggestTypeFeature, None).map(_.name), + isInNetwork = candidate.features.getTry(FromInNetworkSourceFeature).toOption, + inReplyToTweetId = candidate.features.getOrElse(InReplyToTweetIdFeature, None), + inReplyToUserId = candidate.features.getOrElse(InReplyToUserIdFeature, None), + quotedTweetId = candidate.features.getOrElse(QuotedTweetIdFeature, None), + quotedUserId = candidate.features.getOrElse(QuotedUserIdFeature, None), + directedAtUserId = candidate.features.getOrElse(DirectedAtUserIdFeature, None), + favoritedByUserIds = convertSeqFeature(candidate, FavoritedByUserIdsFeature), + followedByUserIds = convertSeqFeature(candidate, FollowedByUserIdsFeature), + ancestors = convertSeqFeature(candidate, AncestorsFeature), + requestTimeMs = Some(query.queryTime.inMilliseconds), + candidatePipelineIdentifier = + candidate.features.getTry(CandidatePipelines).toOption.map(_.head.name), + earlybirdScore = candidate.features.getOrElse(EarlybirdScoreFeature, None), + isDropped = Some(isDropped) + ) + } + + private def convertSeqFeature[T]( + candidateWithDetails: CandidateWithDetails, + feature: Feature[_, Seq[T]] + ): Option[Seq[T]] = + Option( + candidateWithDetails.features + .getOrElse(feature, Seq.empty)).filter(_.nonEmpty) + + override val logPipelinePublisher: EventPublisher[t.ScoredCandidate] = eventBusPublisher +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScribeServedCommonFeaturesAndCandidateFeaturesSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScribeServedCommonFeaturesAndCandidateFeaturesSideEffect.scala index 2599832cc..c93adee27 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScribeServedCommonFeaturesAndCandidateFeaturesSideEffect.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScribeServedCommonFeaturesAndCandidateFeaturesSideEffect.scala @@ -5,21 +5,22 @@ import com.twitter.finagle.mysql.Client import com.twitter.finagle.mysql.Transactions import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.util.DefaultTimer -import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.non_ml_features.NonMLCandidateFeatures -import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.non_ml_features.NonMLCandidateFeaturesAdapter -import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.non_ml_features.NonMLCommonFeatures -import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.non_ml_features.NonMLCommonFeaturesAdapter import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.param.HomeMixerFlagName.DataRecordMetadataStoreConfigsYmlFlag +import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeServedCommonFeaturesAndCandidateFeaturesFlag import com.twitter.home_mixer.param.HomeMixerInjectionNames.CandidateFeaturesScribeEventPublisher import com.twitter.home_mixer.param.HomeMixerInjectionNames.CommonFeaturesScribeEventPublisher import com.twitter.home_mixer.param.HomeMixerInjectionNames.MinimumFeaturesScribeEventPublisher +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.non_ml_features.NonMLCandidateFeatures +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.non_ml_features.NonMLCandidateFeaturesAdapter +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.non_ml_features.NonMLCommonFeatures +import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.non_ml_features.NonMLCommonFeaturesAdapter import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse import com.twitter.home_mixer.product.scored_tweets.scorer.CandidateFeaturesDataRecordFeature import com.twitter.home_mixer.product.scored_tweets.scorer.CommonFeaturesDataRecordFeature -import com.twitter.home_mixer.product.scored_tweets.scorer.HomeNaviModelDataRecordScorer.PredictedScoreFeatures +import com.twitter.home_mixer.product.scored_tweets.scorer.PredictedScoreFeature.PredictedScoreFeatures import com.twitter.home_mixer.util.CandidatesUtil.getOriginalAuthorId import com.twitter.inject.annotations.Flag import com.twitter.logpipeline.client.common.EventPublisher @@ -28,6 +29,7 @@ import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordCo import com.twitter.product_mixer.core.feature.featuremap.datarecord.SpecificFeatures import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier +import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.stitch.Stitch import com.twitter.timelines.ml.cont_train.common.domain.non_scalding.CandidateAndCommonFeaturesStreamingUtils import com.twitter.timelines.ml.pldr.client.MysqlClientUtils @@ -51,6 +53,7 @@ import scala.collection.JavaConverters._ @Singleton class ScribeServedCommonFeaturesAndCandidateFeaturesSideEffect @Inject() ( @Flag(DataRecordMetadataStoreConfigsYmlFlag) dataRecordMetadataStoreConfigsYml: String, + @Flag(ScribeServedCommonFeaturesAndCandidateFeaturesFlag) enableScribeServedCommonFeaturesAndCandidateFeatures: Boolean, @Named(CommonFeaturesScribeEventPublisher) commonFeaturesScribeEventPublisher: EventPublisher[ pldr.PolyDataRecord ], @@ -62,27 +65,28 @@ class ScribeServedCommonFeaturesAndCandidateFeaturesSideEffect @Inject() ( ], statsReceiver: StatsReceiver, ) extends PipelineResultSideEffect[ScoredTweetsQuery, ScoredTweetsResponse] + with PipelineResultSideEffect.Conditionally[ScoredTweetsQuery, ScoredTweetsResponse] with Logging { - override val identifier: SideEffectIdentifier = SideEffectIdentifier( - "ScribeServedCommonFeaturesAndCandidateFeatures") + override val identifier: SideEffectIdentifier = + SideEffectIdentifier("ScribeServedCommonFeaturesAndCandidateFeatures") private val drMerger = new DataRecordMerger - private val postScoringCandidateFeatures = SpecificFeatures(PredictedScoreFeatures.toSet) - private val postScoringCandidateFeaturesDataRecordAdapter = new DataRecordConverter( - postScoringCandidateFeatures) + private val postScoringCandidateFeatures = SpecificFeatures(PredictedScoreFeatures) + private val postScoringCandidateFeaturesDataRecordAdapter = + new DataRecordConverter(postScoringCandidateFeatures) private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val metadataFetchFailedCounter = scopedStatsReceiver.counter("metadataFetchFailed") private val commonFeaturesScribeCounter = scopedStatsReceiver.counter("commonFeaturesScribe") - private val commonFeaturesPLDROptionObserver = OptionObserver( - scopedStatsReceiver.scope("commonFeaturesPLDR")) + private val commonFeaturesPLDROptionObserver = + OptionObserver(scopedStatsReceiver.scope("commonFeaturesPLDR")) private val candidateFeaturesScribeCounter = scopedStatsReceiver.counter("candidateFeaturesScribe") - private val candidateFeaturesPLDROptionObserver = OptionObserver( - scopedStatsReceiver.scope("candidateFeaturesPLDR")) - private val minimumFeaturesPLDROptionObserver = OptionObserver( - scopedStatsReceiver.scope("minimumFeaturesPLDR")) + private val candidateFeaturesPLDROptionObserver = + OptionObserver(scopedStatsReceiver.scope("candidateFeaturesPLDR")) + private val minimumFeaturesPLDROptionObserver = + OptionObserver(scopedStatsReceiver.scope("minimumFeaturesPLDR")) private val minimumFeaturesScribeCounter = scopedStatsReceiver.counter("minimumFeaturesScribe") @@ -115,6 +119,14 @@ class ScribeServedCommonFeaturesAndCandidateFeaturesSideEffect @Inject() ( ) } + override def onlyIf( + query: ScoredTweetsQuery, + selectedCandidates: Seq[CandidateWithDetails], + remainingCandidates: Seq[CandidateWithDetails], + droppedCandidates: Seq[CandidateWithDetails], + response: ScoredTweetsResponse + ): Boolean = enableScribeServedCommonFeaturesAndCandidateFeatures + override def apply( inputs: PipelineResultSideEffect.Inputs[ScoredTweetsQuery, ScoredTweetsResponse] ): Stitch[Unit] = { diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/BUILD.bazel new file mode 100644 index 000000000..c1b5032db --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/BUILD.bazel @@ -0,0 +1,82 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "3rdparty/jvm/javax/inject:javax.inject", + "ads-injection/lib/src/main/scala/com/twitter/goldfinch/api", + "finagle/finagle-memcached/src/main/scala", + "finatra/inject/inject-core/src/main/scala", + "finatra/inject/inject-core/src/main/scala/com/twitter/inject", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/model", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/async", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/param_gated", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/urt", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", + "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/item", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", + "src/java/com/twitter/search/common/schema/base", + "src/java/com/twitter/search/common/schema/earlybird", + "src/java/com/twitter/search/common/util/lang", + "src/java/com/twitter/search/queryparser/query:core-query-nodes", + "src/java/com/twitter/search/queryparser/query/search:search-query-nodes", + "src/thrift/com/twitter/search:earlybird-scala", + "src/thrift/com/twitter/search/common:constants-java", + "src/thrift/com/twitter/tweetypie:service-scala", + "stitch/stitch-gizmoduck", + "stitch/stitch-tweetypie", + "stringcenter/client", + "stringcenter/client/src/main/java", + "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan", + "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", + "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", + "timelines/src/main/scala/com/twitter/timelines/clients/relevance_search", + "timelines/src/main/scala/com/twitter/timelines/injection/scribe", + ], + exports = [ + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", + "src/thrift/com/twitter/timelines/render:thrift-scala", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedEarlybirdCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedEarlybirdCandidatePipelineConfig.scala new file mode 100644 index 000000000..c331f955b --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedEarlybirdCandidatePipelineConfig.scala @@ -0,0 +1,64 @@ +package com.twitter.home_mixer.product.subscribed + +import com.google.inject.Inject +import com.twitter.home_mixer.functional_component.candidate_source.EarlybirdCandidateSource +import com.twitter.home_mixer.product.subscribed.model.SubscribedQuery +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSSubscribedUsersFeature +import com.twitter.product_mixer.component_library.filter.TweetVisibilityFilter +import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource +import com.twitter.product_mixer.core.functional_component.filter.Filter +import com.twitter.product_mixer.core.functional_component.gate.Gate +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.search.earlybird.{thriftscala => t} +import com.twitter.spam.rtf.thriftscala.SafetyLevel.TimelineHomeSubscribed +import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient} +import com.twitter.tweetypie.thriftscala.TweetVisibilityPolicy + +class SubscribedEarlybirdCandidatePipelineConfig @Inject() ( + earlybirdCandidateSource: EarlybirdCandidateSource, + tweetyPieStitchClient: TweetypieStitchClient, + subscribedEarlybirdQueryTransformer: SubscribedEarlybirdQueryTransformer) + extends CandidatePipelineConfig[ + SubscribedQuery, + t.EarlybirdRequest, + t.ThriftSearchResult, + TweetCandidate + ] { + override val identifier: CandidatePipelineIdentifier = + CandidatePipelineIdentifier("SubscribedEarlybird") + + override val candidateSource: BaseCandidateSource[t.EarlybirdRequest, t.ThriftSearchResult] = + earlybirdCandidateSource + + override val gates: Seq[Gate[SubscribedQuery]] = Seq( + NonEmptySeqFeatureGate(SGSSubscribedUsersFeature) + ) + + override def filters: Seq[Filter[SubscribedQuery, TweetCandidate]] = Seq( + new TweetVisibilityFilter( + tweetyPieStitchClient, + TweetVisibilityPolicy.UserVisible, + TimelineHomeSubscribed + ) + ) + + override val queryTransformer: CandidatePipelineQueryTransformer[ + SubscribedQuery, + t.EarlybirdRequest + ] = subscribedEarlybirdQueryTransformer + + override val featuresFromCandidateSourceTransformers: Seq[ + CandidateFeatureTransformer[t.ThriftSearchResult] + ] = Seq(SubscribedEarlybirdResponseFeatureTransformer) + + override val resultTransformer: CandidatePipelineResultsTransformer[ + t.ThriftSearchResult, + TweetCandidate + ] = { sourceResult => TweetCandidate(id = sourceResult.id) } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedEarlybirdQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedEarlybirdQueryTransformer.scala new file mode 100644 index 000000000..6e0d57c13 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedEarlybirdQueryTransformer.scala @@ -0,0 +1,67 @@ +package com.twitter.home_mixer.product.subscribed + +import com.twitter.finagle.thrift.ClientId +import com.twitter.finagle.tracing.Trace +import com.twitter.home_mixer.product.subscribed.model.SubscribedQuery +import com.twitter.home_mixer.product.subscribed.param.SubscribedParam.ServerMaxResultsParam +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSSubscribedUsersFeature +import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer +import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.BottomCursor +import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor +import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor +import com.twitter.product_mixer.core.pipeline.pipeline_failure.MalformedCursor +import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure +import com.twitter.search.common.schema.earlybird.EarlybirdFieldConstants.EarlybirdFieldConstant +import com.twitter.search.earlybird.{thriftscala => t} +import com.twitter.search.queryparser.query.Conjunction +import com.twitter.search.queryparser.query.search.SearchOperator +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +case class SubscribedEarlybirdQueryTransformer @Inject() (clientId: ClientId) + extends CandidatePipelineQueryTransformer[SubscribedQuery, t.EarlybirdRequest] { + + override def transform(query: SubscribedQuery): t.EarlybirdRequest = { + val subscribedUserIds = + query.features.map(_.get(SGSSubscribedUsersFeature)).getOrElse(Seq.empty) + + val subscribedUsersQuery = new SearchOperator.Builder() + .setType(SearchOperator.Type.FILTER) + .addOperand(EarlybirdFieldConstant.EXCLUSIVE_FILTER_TERM) + .build() + + val searchQuery = query.pipelineCursor + .map { cursor => + val sinceIdQuery = + (id: Long) => new SearchOperator(SearchOperator.Type.SINCE_ID, id.toString) + val maxIdQuery = // max ID is inclusive, so subtract 1 + (id: Long) => new SearchOperator(SearchOperator.Type.MAX_ID, (id - 1).toString) + + (cursor.cursorType, cursor.id, cursor.gapBoundaryId) match { + case (Some(TopCursor), Some(sinceId), _) => + new Conjunction(sinceIdQuery(sinceId), subscribedUsersQuery) + case (Some(BottomCursor), Some(maxId), _) => + new Conjunction(maxIdQuery(maxId), subscribedUsersQuery) + case (Some(GapCursor), Some(maxId), Some(sinceId)) => + new Conjunction(sinceIdQuery(sinceId), maxIdQuery(maxId), subscribedUsersQuery) + case (Some(GapCursor), _, _) => + throw PipelineFailure(MalformedCursor, "Invalid cursor " + cursor.toString) + case _ => subscribedUsersQuery + } + }.getOrElse(subscribedUsersQuery) + + t.EarlybirdRequest( + searchQuery = t.ThriftSearchQuery( + serializedQuery = Some(searchQuery.serialize), + fromUserIDFilter64 = Some(subscribedUserIds), + numResults = query.requestedMaxResults.getOrElse(query.params(ServerMaxResultsParam)), + rankingMode = t.ThriftSearchRankingMode.Recency, + ), + getOlderResults = Some(true), // needed for archive access to older tweets + clientRequestID = Some(s"${Trace.id.traceId}"), + numResultsToReturnAtRoot = Some(query.params(ServerMaxResultsParam)), + clientId = Some(clientId.name), + ) + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedEarlybirdResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedEarlybirdResponseFeatureTransformer.scala new file mode 100644 index 000000000..a8136102c --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedEarlybirdResponseFeatureTransformer.scala @@ -0,0 +1,38 @@ +package com.twitter.home_mixer.product.subscribed + +import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature +import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature +import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature +import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature +import com.twitter.product_mixer.core.feature.Feature +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder +import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer +import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier +import com.twitter.search.earlybird.{thriftscala => t} + +object SubscribedEarlybirdResponseFeatureTransformer + extends CandidateFeatureTransformer[t.ThriftSearchResult] { + + override val identifier: TransformerIdentifier = + TransformerIdentifier("SubscribedEarlybirdResponse") + + override val features: Set[Feature[_, _]] = Set( + AuthorIdFeature, + InReplyToTweetIdFeature, + IsRetweetFeature, + SourceTweetIdFeature, + SourceUserIdFeature, + ) + + override def transform(candidate: t.ThriftSearchResult): FeatureMap = FeatureMapBuilder() + .add(AuthorIdFeature, candidate.tweetypieTweet.flatMap(_.coreData.map(_.userId))) + .add( + InReplyToTweetIdFeature, + candidate.tweetypieTweet.flatMap(_.coreData.flatMap(_.reply.flatMap(_.inReplyToStatusId)))) + .add(IsRetweetFeature, candidate.metadata.exists(_.isRetweet.contains(true))) + .add(SourceTweetIdFeature, candidate.sourceTweetypieTweet.map(_.id)) + .add(SourceUserIdFeature, candidate.sourceTweetypieTweet.flatMap(_.coreData.map(_.userId))) + .build() +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedMixerPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedMixerPipelineConfig.scala new file mode 100644 index 000000000..5b1f74b37 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedMixerPipelineConfig.scala @@ -0,0 +1,221 @@ +package com.twitter.home_mixer.product.subscribed + +import com.twitter.clientapp.{thriftscala => ca} +import com.twitter.home_mixer.candidate_pipeline.ConversationServiceCandidatePipelineConfigBuilder +import com.twitter.home_mixer.candidate_pipeline.EditedTweetsCandidatePipelineConfig +import com.twitter.home_mixer.candidate_pipeline.NewTweetsPillCandidatePipelineConfig +import com.twitter.home_mixer.functional_component.decorator.HomeConversationServiceCandidateDecorator +import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder +import com.twitter.home_mixer.functional_component.feature_hydrator._ +import com.twitter.home_mixer.functional_component.selector.UpdateHomeClientEventDetails +import com.twitter.home_mixer.functional_component.selector.UpdateNewTweetsPillDecoration +import com.twitter.home_mixer.functional_component.side_effect._ +import com.twitter.home_mixer.model.GapIncludeInstruction +import com.twitter.home_mixer.param.HomeGlobalParams.MaxNumberReplaceInstructionsParam +import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeClientEventsFlag +import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings +import com.twitter.home_mixer.product.subscribed.model.SubscribedQuery +import com.twitter.home_mixer.product.subscribed.param.SubscribedParam.ServerMaxResultsParam +import com.twitter.home_mixer.util.CandidatesUtil +import com.twitter.inject.annotations.Flag +import com.twitter.logpipeline.client.common.EventPublisher +import com.twitter.product_mixer.component_library.feature_hydrator.query.async.AsyncQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweetsQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator +import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSSubscribedUsersQueryFeatureHydrator +import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate +import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate +import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.AddEntriesWithReplaceAndShowAlertInstructionBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedBottomCursorBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedGapCursorBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedTopCursorBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceAllEntries +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceEntryInstructionBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowAlertInstructionBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.StaticTimelineScribeConfigBuilder +import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtMetadataBuilder +import com.twitter.product_mixer.component_library.selector.DropMaxCandidates +import com.twitter.product_mixer.component_library.selector.InsertAppendResults +import com.twitter.product_mixer.component_library.selector.SelectConditionally +import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates +import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline +import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines +import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator +import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller +import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.UrtTransportMarshaller +import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller +import com.twitter.product_mixer.core.functional_component.selector.Selector +import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect +import com.twitter.product_mixer.core.model.common.UniversalNoun +import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier +import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier +import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline +import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule +import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig +import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem +import com.twitter.product_mixer.core.pipeline.FailOpenPolicy +import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig +import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig +import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig +import com.twitter.product_mixer.core.product.guice.scope.ProductScoped +import com.twitter.stringcenter.client.StringCenter +import com.twitter.timelines.render.{thriftscala => urt} +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +@Singleton +class SubscribedMixerPipelineConfig @Inject() ( + subscribedEarlybirdCandidatePipelineConfig: SubscribedEarlybirdCandidatePipelineConfig, + conversationServiceCandidatePipelineConfigBuilder: ConversationServiceCandidatePipelineConfigBuilder[ + SubscribedQuery + ], + homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder, + editedTweetsCandidatePipelineConfig: EditedTweetsCandidatePipelineConfig, + newTweetsPillCandidatePipelineConfig: NewTweetsPillCandidatePipelineConfig[SubscribedQuery], + dismissInfoQueryFeatureHydrator: DismissInfoQueryFeatureHydrator, + gizmoduckUserQueryFeatureHydrator: GizmoduckUserQueryFeatureHydrator, + requestQueryFeatureHydrator: RequestQueryFeatureHydrator[SubscribedQuery], + sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, + sgsSubscribedUsersQueryFeatureHydrator: SGSSubscribedUsersQueryFeatureHydrator, + manhattanTweetImpressionsQueryFeatureHydrator: TweetImpressionsQueryFeatureHydrator[ + SubscribedQuery + ], + memcacheTweetImpressionsQueryFeatureHydrator: ImpressedTweetsQueryFeatureHydrator, + publishClientSentImpressionsEventBusSideEffect: PublishClientSentImpressionsEventBusSideEffect, + publishClientSentImpressionsManhattanSideEffect: PublishClientSentImpressionsManhattanSideEffect, + homeTimelineServedCandidatesSideEffect: HomeScribeServedCandidatesSideEffect, + clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent], + externalStrings: HomeMixerExternalStrings, + @ProductScoped stringCenterProvider: Provider[StringCenter], + urtTransportMarshaller: UrtTransportMarshaller, + @Flag(ScribeClientEventsFlag) enableScribeClientEvents: Boolean) + extends MixerPipelineConfig[SubscribedQuery, Timeline, urt.TimelineResponse] { + + override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("Subscribed") + + private val dependentCandidatesStep = MixerPipelineConfig.dependentCandidatePipelinesStep + private val resultSelectorsStep = MixerPipelineConfig.resultSelectorsStep + + override val fetchQueryFeatures: Seq[QueryFeatureHydrator[SubscribedQuery]] = Seq( + requestQueryFeatureHydrator, + sgsFollowedUsersQueryFeatureHydrator, + sgsSubscribedUsersQueryFeatureHydrator, + AsyncQueryFeatureHydrator(dependentCandidatesStep, dismissInfoQueryFeatureHydrator), + AsyncQueryFeatureHydrator(dependentCandidatesStep, gizmoduckUserQueryFeatureHydrator), + AsyncQueryFeatureHydrator(resultSelectorsStep, manhattanTweetImpressionsQueryFeatureHydrator), + AsyncQueryFeatureHydrator(resultSelectorsStep, memcacheTweetImpressionsQueryFeatureHydrator) + ) + + private val earlybirdCandidatePipelineScope = + SpecificPipeline(subscribedEarlybirdCandidatePipelineConfig.identifier) + + private val conversationServiceCandidatePipelineConfig = + conversationServiceCandidatePipelineConfigBuilder.build( + Seq(NonEmptyCandidatesGate(earlybirdCandidatePipelineScope)), + HomeConversationServiceCandidateDecorator(homeFeedbackActionInfoBuilder) + ) + + override val candidatePipelines: Seq[CandidatePipelineConfig[SubscribedQuery, _, _, _]] = + Seq(subscribedEarlybirdCandidatePipelineConfig) + + override val dependentCandidatePipelines: Seq[ + DependentCandidatePipelineConfig[SubscribedQuery, _, _, _] + ] = Seq( + conversationServiceCandidatePipelineConfig, + editedTweetsCandidatePipelineConfig, + newTweetsPillCandidatePipelineConfig + ) + + override val failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = Map( + editedTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + newTweetsPillCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, + ) + + override val resultSelectors: Seq[Selector[SubscribedQuery]] = Seq( + UpdateSortCandidates( + ordering = CandidatesUtil.reverseChronTweetsOrdering, + candidatePipeline = conversationServiceCandidatePipelineConfig.identifier + ), + DropMaxCandidates( + candidatePipeline = editedTweetsCandidatePipelineConfig.identifier, + maxSelectionsParam = MaxNumberReplaceInstructionsParam + ), + DropMaxCandidates( + candidatePipeline = conversationServiceCandidatePipelineConfig.identifier, + maxSelectionsParam = ServerMaxResultsParam + ), + InsertAppendResults(candidatePipeline = conversationServiceCandidatePipelineConfig.identifier), + // This selector must come after the tweets are inserted into the results + UpdateNewTweetsPillDecoration( + pipelineScope = SpecificPipelines( + conversationServiceCandidatePipelineConfig.identifier, + newTweetsPillCandidatePipelineConfig.identifier + ), + stringCenter = stringCenterProvider.get(), + seeNewTweetsString = externalStrings.seeNewTweetsString, + tweetedString = externalStrings.tweetedString + ), + InsertAppendResults(candidatePipeline = editedTweetsCandidatePipelineConfig.identifier), + SelectConditionally( + selector = + InsertAppendResults(candidatePipeline = newTweetsPillCandidatePipelineConfig.identifier), + includeSelector = (_, _, results) => CandidatesUtil.containsType[TweetCandidate](results) + ), + UpdateHomeClientEventDetails( + candidatePipelines = Set(conversationServiceCandidatePipelineConfig.identifier) + ), + ) + + private val homeScribeClientEventSideEffect = HomeScribeClientEventSideEffect( + enableScribeClientEvents = enableScribeClientEvents, + logPipelinePublisher = clientEventsScribeEventPublisher, + injectedTweetsCandidatePipelineIdentifiers = + Seq(conversationServiceCandidatePipelineConfig.identifier), + ) + + override val resultSideEffects: Seq[PipelineResultSideEffect[SubscribedQuery, Timeline]] = Seq( + homeScribeClientEventSideEffect, + homeTimelineServedCandidatesSideEffect, + publishClientSentImpressionsEventBusSideEffect, + publishClientSentImpressionsManhattanSideEffect + ) + + override val domainMarshaller: DomainMarshaller[SubscribedQuery, Timeline] = { + val instructionBuilders = Seq( + ReplaceEntryInstructionBuilder(ReplaceAllEntries), + AddEntriesWithReplaceAndShowAlertInstructionBuilder(), + ShowAlertInstructionBuilder(), + ) + + val idSelector: PartialFunction[UniversalNoun[_], Long] = { + // exclude ads while determining tweet cursor values + case item: TweetItem if item.promotedMetadata.isEmpty => item.id + case module: TimelineModule + if module.items.headOption.exists(_.item.isInstanceOf[TweetItem]) => + module.items.last.item match { case item: TweetItem => item.id } + } + + val topCursorBuilder = OrderedTopCursorBuilder(idSelector) + val bottomCursorBuilder = + OrderedBottomCursorBuilder(idSelector, GapIncludeInstruction.inverse()) + val gapCursorBuilder = OrderedGapCursorBuilder(idSelector, GapIncludeInstruction) + + val metadataBuilder = UrtMetadataBuilder( + title = None, + scribeConfigBuilder = Some( + StaticTimelineScribeConfigBuilder( + TimelineScribeConfig(page = Some("subscribed"), section = None, entityToken = None))) + ) + + UrtDomainMarshaller( + instructionBuilders = instructionBuilders, + metadataBuilder = Some(metadataBuilder), + cursorBuilders = Seq(topCursorBuilder, bottomCursorBuilder, gapCursorBuilder) + ) + } + + override val transportMarshaller: TransportMarshaller[Timeline, urt.TimelineResponse] = + urtTransportMarshaller +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedProductPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedProductPipelineConfig.scala new file mode 100644 index 000000000..0d524391a --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedProductPipelineConfig.scala @@ -0,0 +1,123 @@ +package com.twitter.home_mixer.product.subscribed + +import com.twitter.conversions.DurationOps._ +import com.twitter.home_mixer.marshaller.timelines.ChronologicalCursorUnmarshaller +import com.twitter.home_mixer.model.request.HomeMixerRequest +import com.twitter.home_mixer.model.request.SubscribedProduct +import com.twitter.home_mixer.model.request.SubscribedProductContext +import com.twitter.home_mixer.product.subscribed.model.SubscribedQuery +import com.twitter.home_mixer.product.subscribed.param.SubscribedParam.ServerMaxResultsParam +import com.twitter.home_mixer.service.HomeMixerAccessPolicy.DefaultHomeMixerAccessPolicy +import com.twitter.home_mixer.service.HomeMixerAlertConfig.DefaultNotificationGroup +import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor +import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer +import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy +import com.twitter.product_mixer.core.functional_component.common.alert.Alert +import com.twitter.product_mixer.core.functional_component.common.alert.LatencyAlert +import com.twitter.product_mixer.core.functional_component.common.alert.P99 +import com.twitter.product_mixer.core.functional_component.common.alert.SuccessRateAlert +import com.twitter.product_mixer.core.functional_component.common.alert.ThroughputAlert +import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfAbove +import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfBelow +import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfLatencyAbove +import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier +import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier +import com.twitter.product_mixer.core.model.marshalling.request.Product +import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor +import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor +import com.twitter.product_mixer.core.pipeline.PipelineConfig +import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest +import com.twitter.product_mixer.core.pipeline.pipeline_failure.MalformedCursor +import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure +import com.twitter.product_mixer.core.pipeline.product.ProductPipelineConfig +import com.twitter.product_mixer.core.product.ProductParamConfig +import com.twitter.product_mixer.core.util.SortIndexBuilder +import com.twitter.timelines.configapi.Params +import com.twitter.timelines.render.{thriftscala => urt} +import com.twitter.timelines.util.RequestCursorSerializer +import com.twitter.util.Time +import com.twitter.util.Try +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class SubscribedProductPipelineConfig @Inject() ( + subscribedMixerPipelineConfig: SubscribedMixerPipelineConfig, + subscribedParamConfig: param.SubscribedParamConfig) + extends ProductPipelineConfig[HomeMixerRequest, SubscribedQuery, urt.TimelineResponse] { + + override val identifier: ProductPipelineIdentifier = ProductPipelineIdentifier("Subscribed") + + override val product: Product = SubscribedProduct + override val paramConfig: ProductParamConfig = subscribedParamConfig + + override def pipelineQueryTransformer( + request: HomeMixerRequest, + params: Params + ): SubscribedQuery = { + val context = request.productContext match { + case Some(context: SubscribedProductContext) => context + case _ => throw PipelineFailure(BadRequest, "SubscribedProductContext not found") + } + + val debugOptions = request.debugParams.flatMap(_.debugOptions) + + /** + * Unlike other clients, newly created tweets on Android have the sort index set to the current + * time instead of the top sort index + 1, so these tweets get stuck at the top of the timeline + * if subsequent timeline responses use the sort index from the previous response instead of + * the current time. + */ + val pipelineCursor = request.serializedRequestCursor.flatMap { cursor => + Try(UrtCursorSerializer.deserializeOrderedCursor(cursor)) + .getOrElse(ChronologicalCursorUnmarshaller(RequestCursorSerializer.deserialize(cursor))) + .map { + case UrtOrderedCursor(_, id, Some(GapCursor), gapBoundaryId) + if id.isEmpty || gapBoundaryId.isEmpty => + throw PipelineFailure(MalformedCursor, "Gap Cursor bounds not defined") + case topCursor @ UrtOrderedCursor(_, _, Some(TopCursor), _) => + val queryTime = debugOptions.flatMap(_.requestTimeOverride).getOrElse(Time.now) + topCursor.copy(initialSortIndex = SortIndexBuilder.timeToId(queryTime)) + case cursor => cursor + } + } + + SubscribedQuery( + params = params, + clientContext = request.clientContext, + features = None, + pipelineCursor = pipelineCursor, + requestedMaxResults = Some(params(ServerMaxResultsParam)), + debugOptions = debugOptions, + deviceContext = context.deviceContext, + seenTweetIds = context.seenTweetIds + ) + } + + override val pipelines: Seq[PipelineConfig] = Seq(subscribedMixerPipelineConfig) + + override def pipelineSelector( + query: SubscribedQuery + ): ComponentIdentifier = subscribedMixerPipelineConfig.identifier + + override val alerts: Seq[Alert] = Seq( + SuccessRateAlert( + notificationGroup = DefaultNotificationGroup, + warnPredicate = TriggerIfBelow(99.9, 20, 30), + criticalPredicate = TriggerIfBelow(99.9, 30, 30), + ), + LatencyAlert( + notificationGroup = DefaultNotificationGroup, + percentile = P99, + warnPredicate = TriggerIfLatencyAbove(1100.millis, 15, 30), + criticalPredicate = TriggerIfLatencyAbove(1200.millis, 15, 30) + ), + ThroughputAlert( + notificationGroup = DefaultNotificationGroup, + warnPredicate = TriggerIfAbove(18000), + criticalPredicate = TriggerIfAbove(20000) + ) + ) + + override val debugAccessPolicies: Set[AccessPolicy] = DefaultHomeMixerAccessPolicy +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_feature_hydrator/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/model/BUILD.bazel similarity index 57% rename from home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_feature_hydrator/BUILD.bazel rename to home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/model/BUILD.bazel index 154f84007..b821a8f8d 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_feature_hydrator/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/model/BUILD.bazel @@ -2,17 +2,19 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/recommendations", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline/transformer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", + "stringcenter/client", + "stringcenter/client/src/main/java", + ], + exports = [ + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "src/thrift/com/twitter/timelineranker:thrift-scala", - "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/timelineranker", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/model/SubscribedQuery.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/model/SubscribedQuery.scala new file mode 100644 index 000000000..2085ef54f --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/model/SubscribedQuery.scala @@ -0,0 +1,31 @@ +package com.twitter.home_mixer.product.subscribed.model + +import com.twitter.home_mixer.model.request.DeviceContext +import com.twitter.home_mixer.model.request.HasDeviceContext +import com.twitter.home_mixer.model.request.HasSeenTweetIds +import com.twitter.home_mixer.model.request.SubscribedProduct +import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor +import com.twitter.product_mixer.core.feature.featuremap.FeatureMap +import com.twitter.product_mixer.core.model.marshalling.request._ +import com.twitter.product_mixer.core.pipeline.HasPipelineCursor +import com.twitter.product_mixer.core.pipeline.PipelineQuery +import com.twitter.timelines.configapi.Params + +case class SubscribedQuery( + override val params: Params, + override val clientContext: ClientContext, + override val pipelineCursor: Option[UrtOrderedCursor], + override val requestedMaxResults: Option[Int], + override val debugOptions: Option[DebugOptions], + override val features: Option[FeatureMap], + override val deviceContext: Option[DeviceContext], + override val seenTweetIds: Option[Seq[Long]]) + extends PipelineQuery + with HasPipelineCursor[UrtOrderedCursor] + with HasDeviceContext + with HasSeenTweetIds { + override val product: Product = SubscribedProduct + + override def withFeatureMap(features: FeatureMap): SubscribedQuery = + copy(features = Some(features)) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/BUILD.bazel new file mode 100644 index 000000000..a56e3a1fd --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/BUILD.bazel @@ -0,0 +1,14 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = ["bazel-compatible"], + dependencies = [ + "3rdparty/jvm/javax/inject:javax.inject", + "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", + "home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider", + "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", + "util/util-core/src/main/scala/com/twitter/conversions", + ], +) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/SubscribedParam.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/SubscribedParam.scala new file mode 100644 index 000000000..9e4ade43a --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/SubscribedParam.scala @@ -0,0 +1,15 @@ +package com.twitter.home_mixer.product.subscribed.param + +import com.twitter.timelines.configapi.FSBoundedParam + +object SubscribedParam { + val SupportedClientFSName = "subscribed_supported_client" + + object ServerMaxResultsParam + extends FSBoundedParam[Int]( + name = "subscribed_server_max_results", + default = 100, + min = 1, + max = 500 + ) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/SubscribedParamConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/SubscribedParamConfig.scala new file mode 100644 index 000000000..58ce7ec35 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/SubscribedParamConfig.scala @@ -0,0 +1,18 @@ +package com.twitter.home_mixer.product.subscribed.param + +import com.twitter.home_mixer.param.decider.DeciderKey +import com.twitter.home_mixer.product.subscribed.param.SubscribedParam._ +import com.twitter.product_mixer.core.product.ProductParamConfig +import com.twitter.servo.decider.DeciderKeyName +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class SubscribedParamConfig @Inject() () extends ProductParamConfig { + override val enabledDeciderKey: DeciderKeyName = DeciderKey.EnableSubscribedProduct + override val supportedClientFSName: String = SupportedClientFSName + + override val boundedIntFSOverrides = Seq( + ServerMaxResultsParam + ) +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/HomeMixerAlertConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/HomeMixerAlertConfig.scala index 93c361516..597fd4d36 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/HomeMixerAlertConfig.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/HomeMixerAlertConfig.scala @@ -26,7 +26,8 @@ object HomeMixerAlertConfig { object BusinessHours { val DefaultNotificationGroup: NotificationGroup = NotificationGroup( warn = Destination(emails = Seq("")), - critical = Destination(emails = Seq("")) + critical = Destination(emails = + Seq("")) ) def defaultEmptyResponseRateAlert(warnThreshold: Double = 50, criticalThreshold: Double = 80) = diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/BUILD.bazel index 89535dbd3..c4855d9e7 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/BUILD.bazel @@ -6,12 +6,8 @@ scala_library( dependencies = [ "3rdparty/jvm/com/twitter/bijection:scrooge", "3rdparty/jvm/com/twitter/storehaus:core", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "src/thrift/com/twitter/service/metastore/gen:thrift-java", - "src/thrift/com/twitter/service/metastore/gen:thrift-scala", "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", "stitch/stitch-core", "storage/clients/manhattan/client/src/main/scala", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/repository/uss", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/UserLanguagesStore.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/UserLanguagesStore.scala deleted file mode 100644 index 3f78ddb87..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/UserLanguagesStore.scala +++ /dev/null @@ -1,47 +0,0 @@ -package com.twitter.home_mixer.store - -import com.twitter.bijection.Injection -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.store.ManhattanUserLanguagesKVDescriptor._ -import com.twitter.home_mixer.util.LanguageUtil -import com.twitter.search.common.constants.{thriftscala => scc} -import com.twitter.service.metastore.gen.{thriftscala => smg} -import com.twitter.stitch.Stitch -import com.twitter.storage.client.manhattan.bijections.Bijections -import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpoint -import com.twitter.storage.client.manhattan.kv.ManhattanValue -import com.twitter.storage.client.manhattan.kv.impl.ReadOnlyKeyDescriptor -import com.twitter.storage.client.manhattan.kv.impl.ValueDescriptor -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -object ManhattanUserLanguagesKVDescriptor { - val userLanguagesDatasetName = "languages" - val keyInjection = Injection.connect[Long, Array[Byte]].andThen(Bijections.BytesInjection) - val keyDescriptor = ReadOnlyKeyDescriptor(keyInjection) - val valueDescriptor = ValueDescriptor(Bijections.BinaryScalaInjection(smg.UserLanguages)) - val userLanguagesDatasetKey = keyDescriptor.withDataset(userLanguagesDatasetName) -} - -class UserLanguagesStore( - manhattanKVEndpoint: ManhattanKVEndpoint, - statsReceiver: StatsReceiver) - extends ReadableStore[Long, Seq[scc.ThriftLanguage]] { - - private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) - private val keyFoundCounter = scopedStatsReceiver.counter("key/found") - private val keyLossCounter = scopedStatsReceiver.counter("key/loss") - - override def get(viewerId: Long): Future[Option[Seq[scc.ThriftLanguage]]] = - Stitch - .run( - manhattanKVEndpoint.get(key = userLanguagesDatasetKey.withPkey(viewerId), valueDescriptor)) - .map { - case Some(mhResponse: ManhattanValue[smg.UserLanguages]) => - keyFoundCounter.incr() - Some(LanguageUtil.computeLanguages(mhResponse.contents)) - case _ => - keyLossCounter.incr() - None - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/BUILD.bazel index 2330dd541..8e04fa11f 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/BUILD.bazel @@ -11,8 +11,11 @@ scala_library( "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", "servo/repo/src/main/scala", "snowflake/src/main/scala/com/twitter/snowflake/id", + "src/java/com/twitter/ml/api:api-base", + "src/java/com/twitter/ml/api/util", "src/java/com/twitter/search/common/util/lang", "src/scala/com/twitter/ml/api/util", + "src/thrift/com/twitter/ml/api:data-java", "src/thrift/com/twitter/search/common:constants-java", "src/thrift/com/twitter/search/common:constants-scala", "src/thrift/com/twitter/service/metastore/gen:thrift-java", diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/CachedScoredTweetsHelper.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/CachedScoredTweetsHelper.scala index 740fd03ec..e0fbdd76f 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/CachedScoredTweetsHelper.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/CachedScoredTweetsHelper.scala @@ -32,6 +32,7 @@ object CachedScoredTweetsHelper { untilTime: Time ): Seq[Long] = tweetImpressionsAndCachedScoredTweets(features, candidatePipelineIdentifier) + .filter { tweetId => SnowflakeId.isSnowflakeId(tweetId) } .filter { tweetId => val creationTime = SnowflakeId.timeFromId(tweetId) sinceTime <= creationTime && untilTime >= creationTime @@ -39,7 +40,7 @@ object CachedScoredTweetsHelper { def unseenCachedScoredTweets( features: FeatureMap - ): Seq[hmt.CachedScoredTweet] = { + ): Seq[hmt.ScoredTweet] = { val seenTweetIds = TweetImpressionsHelper.tweetImpressions(features) features diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/CandidatesUtil.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/CandidatesUtil.scala index 4bf265fd1..06fe5646c 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/CandidatesUtil.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/CandidatesUtil.scala @@ -3,6 +3,7 @@ package com.twitter.home_mixer.util import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.FavoritedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.HasImageFeature +import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.MediaUnderstandingAnnotationIdsFeature import com.twitter.home_mixer.model.HomeFeatures.RepliedByEngagerIdsFeature @@ -21,7 +22,6 @@ import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateW import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure import com.twitter.product_mixer.core.pipeline.pipeline_failure.UnexpectedCandidateResult - import scala.reflect.ClassTag object CandidatesUtil { @@ -52,11 +52,22 @@ object CandidatesUtil { case _ => false } + def getOriginalTweetId(candidate: CandidateWithFeatures[TweetCandidate]): Long = { + if (candidate.features.getOrElse(IsRetweetFeature, false)) + candidate.features.getOrElse(SourceTweetIdFeature, None).getOrElse(candidate.candidate.id) + else + candidate.candidate.id + } + def getOriginalAuthorId(candidateFeatures: FeatureMap): Option[Long] = if (candidateFeatures.getOrElse(IsRetweetFeature, false)) candidateFeatures.getOrElse(SourceUserIdFeature, None) else candidateFeatures.getOrElse(AuthorIdFeature, None) + def isOriginalTweet(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = + !candidate.features.getOrElse(IsRetweetFeature, false) && + candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty + def getEngagerUserIds( candidateFeatures: FeatureMap ): Seq[Long] = { diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/DataRecordUtil.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/DataRecordUtil.scala new file mode 100644 index 000000000..b972b8158 --- /dev/null +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/DataRecordUtil.scala @@ -0,0 +1,31 @@ +package com.twitter.home_mixer.util + +import com.twitter.ml.api.DataRecord +import com.twitter.ml.api.FeatureContext +import com.twitter.ml.api.util.SRichDataRecord +import com.twitter.ml.api.Feature +import java.lang.{Double => JDouble} + +object DataRecordUtil { + def applyRename( + dataRecord: DataRecord, + featureContext: FeatureContext, + renamedFeatureContext: FeatureContext, + featureRenamingMap: Map[Feature[_], Feature[_]] + ): DataRecord = { + val richFullDr = new SRichDataRecord(dataRecord, featureContext) + val richNewDr = new SRichDataRecord(new DataRecord, renamedFeatureContext) + val featureIterator = featureContext.iterator() + featureIterator.forEachRemaining { feature => + if (richFullDr.hasFeature(feature)) { + val renamedFeature = featureRenamingMap.getOrElse(feature, feature) + + val typedFeature = feature.asInstanceOf[Feature[JDouble]] + val typedRenamedFeature = renamedFeature.asInstanceOf[Feature[JDouble]] + + richNewDr.setFeatureValue(typedRenamedFeature, richFullDr.getFeatureValue(typedFeature)) + } + } + richNewDr.getRecord + } +} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/LanguageUtil.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/LanguageUtil.scala index 3969ef64d..23c77c27f 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/LanguageUtil.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/LanguageUtil.scala @@ -6,7 +6,7 @@ import com.twitter.service.metastore.gen.{thriftscala => smg} object LanguageUtil { - private val DafaultMinProducedLanguageRatio = 0.05 + private val DefaultMinProducedLanguageRatio = 0.05 private val DefaultMinConsumedLanguageConfidence = 0.8 /** @@ -17,7 +17,7 @@ object LanguageUtil { */ def computeLanguages( userLanguages: smg.UserLanguages, - minProducedLanguageRatio: Double = DafaultMinProducedLanguageRatio, + minProducedLanguageRatio: Double = DefaultMinProducedLanguageRatio, minConsumedLanguageConfidence: Double = DefaultMinConsumedLanguageConfidence ): Seq[scc.ThriftLanguage] = { val languageConfidenceMap = computeLanguageConfidenceMap( @@ -25,7 +25,7 @@ object LanguageUtil { minProducedLanguageRatio, minConsumedLanguageConfidence ) - languageConfidenceMap.toSeq.sortWith(_._2 > _._2).map(_._1) // _1 = language, _2 = score + languageConfidenceMap.toSeq.sortBy(-_._2).map(_._1) // _1 = language, _2 = score } /** diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/BUILD.bazel index 0ca1f89aa..2b5722179 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/BUILD.bazel +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/BUILD.bazel @@ -14,5 +14,10 @@ scala_library( "src/thrift/com/twitter/search/common:constants-scala", "src/thrift/com/twitter/search/common:query-scala", "src/thrift/com/twitter/search/common:ranking-scala", + "timelines/src/main/scala/com/twitter/timelines/clients/relevance_search", + "timelines/src/main/scala/com/twitter/timelines/earlybird/common/options", + "timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils", + "timelines/src/main/scala/com/twitter/timelines/model/types", + "timelines/src/main/scala/com/twitter/timelines/util/stats", ], ) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/EarlybirdRequestUtil.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/EarlybirdRequestUtil.scala index 19df02a20..4c482b251 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/EarlybirdRequestUtil.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/EarlybirdRequestUtil.scala @@ -2,41 +2,165 @@ package com.twitter.home_mixer.util.earlybird import com.twitter.conversions.DurationOps._ import com.twitter.search.common.query.thriftjava.{thriftscala => scq} +import com.twitter.search.common.ranking.{thriftscala => scr} import com.twitter.search.earlybird.{thriftscala => eb} +import com.twitter.timelines.clients.relevance_search.SearchClient.TweetFeatures +import com.twitter.timelines.clients.relevance_search.SearchClient.TweetTypes +import com.twitter.timelines.clients.relevance_search.SearchQueryBuilder +import com.twitter.timelines.clients.relevance_search.SearchQueryBuilder.QueryWithNamedDisjunctions +import com.twitter.timelines.earlybird.common.options.EarlybirdScoringModelConfig +import com.twitter.timelines.earlybird.common.utils.SearchOperator import com.twitter.util.Duration object EarlybirdRequestUtil { - // If no EarlybirdOptions.maxNumHitsPerShard is set then default to this value. val DefaultMaxHitsToProcess = 1000 val DefaultSearchProcessingTimeout: Duration = 200.milliseconds - val DefaultMaxNumResultsPerShard = 100 - val DeafultCollectorParams = scq.CollectorParams( - // numResultsToReturn defines how many results each EB shard will return to search root - numResultsToReturn = DefaultMaxNumResultsPerShard, - // terminationParams.maxHitsToProcess is used for early terminating per shard results fetching. - terminationParams = Some( - scq.CollectorTerminationParams( - maxHitsToProcess = Some(DefaultMaxHitsToProcess), - timeoutMs = DefaultSearchProcessingTimeout.inMilliseconds.toInt - )) - ) + val DefaultHydrationMaxNumResultsPerShard = 1000 + val DefaultQueryMaxNumResultsPerShard = 300 + val DefaultHydrationCollectorParams = mkCollectorParams(DefaultHydrationMaxNumResultsPerShard) - def getTweetsEBFeaturesRequest( + private val queryBuilder = new SearchQueryBuilder + + object EarlybirdScoringModels { + val UnifiedEngagementProd: Seq[EarlybirdScoringModelConfig] = Seq( + EarlybirdScoringModelConfig("timelines_unified_engagement_prod.schema_based", 1.0) + ) + + val UnifiedEngagementRectweet: Seq[EarlybirdScoringModelConfig] = Seq( + EarlybirdScoringModelConfig("timelines_unified_engagement_rectweet.schema_based", 1.0) + ) + } + + private[earlybird] def mkCollectorParams(numResultsToReturn: Int): scq.CollectorParams = { + scq.CollectorParams( + // numResultsToReturn defines how many results each EB shard will return to search root + numResultsToReturn = numResultsToReturn, + // terminationParams.maxHitsToProcess is used for early terminating per shard results fetching. + terminationParams = Some( + scq.CollectorTerminationParams( + maxHitsToProcess = Some(DefaultMaxHitsToProcess), + timeoutMs = DefaultSearchProcessingTimeout.inMilliseconds.toInt + )) + ) + } + + private def getRankingParams( + authorScoreMap: Option[Map[Long, Double]], + tensorflowModel: Option[String], + ebModels: Seq[EarlybirdScoringModelConfig] + ): Option[scr.ThriftRankingParams] = { + if (tensorflowModel.nonEmpty) { + Some( + scr.ThriftRankingParams( + `type` = Some(scr.ThriftScoringFunctionType.TensorflowBased), + selectedTensorflowModel = tensorflowModel, + minScore = -1.0e100, + applyBoosts = false, + authorSpecificScoreAdjustments = authorScoreMap + ) + ) + } else if (ebModels.nonEmpty) { + Some( + scr.ThriftRankingParams( + `type` = Some(scr.ThriftScoringFunctionType.ModelBased), + selectedModels = Some(ebModels.map(m => m.name -> m.weight).toMap), + applyBoosts = false, + minScore = -1.0e100, + authorSpecificScoreAdjustments = authorScoreMap + ) + ) + } else None + } + + def getTweetsRequest( + userId: Option[Long], + clientId: Option[String], + skipVeryRecentTweets: Boolean, + followedUserIds: Set[Long], + retweetsMutedUserIds: Set[Long], + beforeTweetIdExclusive: Option[Long], + afterTweetIdExclusive: Option[Long], + excludedTweetIds: Option[Set[Long]] = None, + maxCount: Int, + tweetTypes: TweetTypes.ValueSet, + authorScoreMap: Option[Map[Long, Double]] = None, + tensorflowModel: Option[String] = None, + ebModels: Seq[EarlybirdScoringModelConfig] = Seq.empty, + queryMaxNumResultsPerShard: Int = DefaultQueryMaxNumResultsPerShard + ): eb.EarlybirdRequest = { + + val QueryWithNamedDisjunctions(query, namedDisjunctionMap) = queryBuilder.create( + followedUserIds, + retweetsMutedUserIds, + beforeTweetIdExclusive, + afterTweetIdExclusive, + semanticCoreIds = None, + languages = None, + tweetTypes = tweetTypes, + searchOperator = SearchOperator.Exclude, + tweetFeatures = TweetFeatures.All, + excludedTweetIds = excludedTweetIds.getOrElse(Set.empty), + enableExcludeSourceTweetIdsQuery = false + ) + val ebRankingParams = getRankingParams(authorScoreMap, tensorflowModel, ebModels) + val relOptions = RelevanceSearchUtil.RelevanceOptions.copy( + rankingParams = ebRankingParams + ) + + val followedUserIdsSeq = followedUserIds.toSeq + val namedDisjunctionMapOpt = + if (namedDisjunctionMap.isEmpty) None + else Some(namedDisjunctionMap.mapValues(_.toSeq)) + + val thriftQuery = eb.ThriftSearchQuery( + serializedQuery = Some(query.serialize), + fromUserIDFilter64 = Some(followedUserIdsSeq), + numResults = maxCount, + collectConversationId = true, + rankingMode = eb.ThriftSearchRankingMode.Relevance, + relevanceOptions = Some(relOptions), + collectorParams = Some(mkCollectorParams(queryMaxNumResultsPerShard)), + facetFieldNames = Some(RelevanceSearchUtil.FacetsToFetch), + resultMetadataOptions = Some(RelevanceSearchUtil.MetadataOptions), + searcherId = userId, + searchStatusIds = None, + namedDisjunctionMap = namedDisjunctionMapOpt + ) + + eb.EarlybirdRequest( + searchQuery = thriftQuery, + clientId = clientId, + getOlderResults = Some(false), + followedUserIds = Some(followedUserIdsSeq), + getProtectedTweetsOnly = Some(false), + timeoutMs = DefaultSearchProcessingTimeout.inMilliseconds.toInt, + skipVeryRecentTweets = skipVeryRecentTweets, + numResultsToReturnAtRoot = Some(maxCount) + ) + } + + def getTweetsFeaturesRequest( userId: Option[Long], tweetIds: Option[Seq[Long]], clientId: Option[String], - getTweetsFromArchiveIndex: Boolean = false, getOnlyProtectedTweets: Boolean = false, + authorScoreMap: Option[Map[Long, Double]] = None, + tensorflowModel: Option[String] = None, + ebModels: Seq[EarlybirdScoringModelConfig] = Seq.empty ): eb.EarlybirdRequest = { val candidateSize = tweetIds.getOrElse(Seq.empty).size + val ebRankingParams = getRankingParams(authorScoreMap, tensorflowModel, ebModels) + val relOptions = RelevanceSearchUtil.RelevanceOptions.copy( + rankingParams = ebRankingParams + ) val thriftQuery = eb.ThriftSearchQuery( numResults = candidateSize, collectConversationId = true, rankingMode = eb.ThriftSearchRankingMode.Relevance, - relevanceOptions = Some(RelevanceSearchUtil.RelevanceOptions), - collectorParams = Some(DeafultCollectorParams), + relevanceOptions = Some(relOptions), + collectorParams = Some(DefaultHydrationCollectorParams), facetFieldNames = Some(RelevanceSearchUtil.FacetsToFetch), resultMetadataOptions = Some(RelevanceSearchUtil.MetadataOptions), searcherId = userId, @@ -46,7 +170,7 @@ object EarlybirdRequestUtil { eb.EarlybirdRequest( searchQuery = thriftQuery, clientId = clientId, - getOlderResults = Some(getTweetsFromArchiveIndex), + getOlderResults = Some(false), getProtectedTweetsOnly = Some(getOnlyProtectedTweets), timeoutMs = DefaultSearchProcessingTimeout.inMilliseconds.toInt, skipVeryRecentTweets = true, diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/EarlybirdResponseUtil.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/EarlybirdResponseUtil.scala index fb52e9689..06e0dd708 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/EarlybirdResponseUtil.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/EarlybirdResponseUtil.scala @@ -6,6 +6,7 @@ import com.twitter.search.common.schema.earlybird.EarlybirdFieldConstants.Earlyb import com.twitter.search.common.schema.earlybird.EarlybirdFieldConstants.EarlybirdFieldConstant._ import com.twitter.search.common.util.lang.ThriftLanguageUtil import com.twitter.search.earlybird.{thriftscala => eb} +import com.twitter.timelines.earlybird.common.utils.InNetworkEngagement object EarlybirdResponseUtil { @@ -53,6 +54,13 @@ object EarlybirdResponseUtil { facetLabels.filter(_.fieldName == facetName).map(_.label.filterNot(charsToRemove)) } + private def isUserMentioned( + screenName: Option[String], + mentions: Seq[String], + mentionsInSourceTweet: Seq[String] + ): Boolean = + isUserMentioned(screenName, mentions) || isUserMentioned(screenName, mentionsInSourceTweet) + private def isUserMentioned( screenName: Option[String], mentions: Seq[String] @@ -121,44 +129,64 @@ object EarlybirdResponseUtil { None } - def getOONTweetThriftFeaturesByTweetId( + def getTweetThriftFeaturesByTweetId( searcherUserId: Long, screenName: Option[String], userLanguages: Seq[scc.ThriftLanguage], uiLanguageCode: Option[String] = None, + followedUserIds: Set[Long], + mutuallyFollowingUserIds: Set[Long], searchResults: Seq[eb.ThriftSearchResult], + sourceTweetSearchResults: Seq[eb.ThriftSearchResult], ): Map[Long, sc.ThriftTweetFeatures] = { + val allSearchResults = searchResults ++ sourceTweetSearchResults + val sourceTweetSearchResultById = + sourceTweetSearchResults.map(result => (result.id -> result)).toMap + val inNetworkEngagement = + InNetworkEngagement(followedUserIds.toSeq, mutuallyFollowingUserIds, allSearchResults) searchResults.map { searchResult => - val features = getOONThriftTweetFeaturesFromSearchResult( + val features = getThriftTweetFeaturesFromSearchResult( searcherUserId, screenName, userLanguages, getLanguage(uiLanguageCode), getTweetCountByAuthorId(searchResults), + followedUserIds, + mutuallyFollowingUserIds, + sourceTweetSearchResultById, + inNetworkEngagement, searchResult ) (searchResult.id -> features) }.toMap } - private[earlybird] def getOONThriftTweetFeaturesFromSearchResult( + private[earlybird] def getThriftTweetFeaturesFromSearchResult( searcherUserId: Long, screenName: Option[String], userLanguages: Seq[scc.ThriftLanguage], uiLanguage: Option[scc.ThriftLanguage], tweetCountByAuthorId: Map[Long, Int], - searchResult: eb.ThriftSearchResult + followedUserIds: Set[Long], + mutuallyFollowingUserIds: Set[Long], + sourceTweetSearchResultById: Map[Long, eb.ThriftSearchResult], + inNetworkEngagement: InNetworkEngagement, + searchResult: eb.ThriftSearchResult, ): sc.ThriftTweetFeatures = { val applyFeatures = (applyUserIndependentFeatures( searchResult )(_)).andThen( - applyOONUserDependentFeatures( + applyUserDependentFeatures( searcherUserId, screenName, userLanguages, uiLanguage, tweetCountByAuthorId, + followedUserIds, + mutuallyFollowingUserIds, + sourceTweetSearchResultById, + inNetworkEngagement, searchResult )(_) ) @@ -317,21 +345,19 @@ object EarlybirdResponseUtil { } .getOrElse(thriftTweetFeatures) - if (result.tweetSource.contains(eb.ThriftTweetSource.RealtimeProtectedCluster)) { - features.copy(isProtected = true) - } else { - features - } + features } - // Omitting inNetwork features e.g source tweet features and follow graph. - // Can be expanded to include InNetwork in the future. - def applyOONUserDependentFeatures( + private def applyUserDependentFeatures( searcherUserId: Long, screenName: Option[String], userLanguages: Seq[scc.ThriftLanguage], uiLanguage: Option[scc.ThriftLanguage], tweetCountByAuthorId: Map[Long, Int], + followedUserIds: Set[Long], + mutuallyFollowingUserIds: Set[Long], + sourceTweetSearchResultById: Map[Long, eb.ThriftSearchResult], + inNetworkEngagement: InNetworkEngagement, result: eb.ThriftSearchResult )( thriftTweetFeatures: sc.ThriftTweetFeatures @@ -339,21 +365,30 @@ object EarlybirdResponseUtil { result.metadata .map { metadata => val isRetweet = metadata.isRetweet.getOrElse(false) + val sourceTweet = + if (isRetweet) sourceTweetSearchResultById.get(metadata.sharedStatusId) + else None + val mentionsInSourceTweet = sourceTweet.map(getMentions).getOrElse(Seq.empty) + val isReply = metadata.isReply.getOrElse(false) val replyToSearcher = isReply && (metadata.referencedTweetAuthorId == searcherUserId) val replyOther = isReply && !replyToSearcher val retweetOther = isRetweet && (metadata.referencedTweetAuthorId != searcherUserId) val tweetLanguage = metadata.language.getOrElse(scc.ThriftLanguage.Unknown) + val referencedTweetAuthorId = + if (metadata.referencedTweetAuthorId > 0) Some(metadata.referencedTweetAuthorId) else None + val inReplyToUserId = if (!isRetweet) referencedTweetAuthorId else None + thriftTweetFeatures.copy( // Info about the Tweet. fromSearcher = metadata.fromUserId == searcherUserId, - probablyFromFollowedAuthor = false, - fromMutualFollow = false, + probablyFromFollowedAuthor = followedUserIds.contains(metadata.fromUserId), + fromMutualFollow = mutuallyFollowingUserIds.contains(metadata.fromUserId), replySearcher = replyToSearcher, replyOther = replyOther, retweetOther = retweetOther, - mentionSearcher = isUserMentioned(screenName, getMentions(result)), + mentionSearcher = isUserMentioned(screenName, getMentions(result), mentionsInSourceTweet), // Info about Tweet content/media. matchesSearcherMainLang = isUsersMainLanguage(tweetLanguage, userLanguages), matchesSearcherLangs = isUsersLanguage(tweetLanguage, userLanguages), @@ -362,6 +397,13 @@ object EarlybirdResponseUtil { prevUserTweetEngagement = metadata.extraMetadata.flatMap(_.prevUserTweetEngagement).getOrElse(DefaultCount), tweetCountFromUserInSnapshot = tweetCountByAuthorId(metadata.fromUserId), + bidirectionalReplyCount = inNetworkEngagement.biDirectionalReplyCounts(result.id), + unidirectionalReplyCount = inNetworkEngagement.uniDirectionalReplyCounts(result.id), + bidirectionalRetweetCount = inNetworkEngagement.biDirectionalRetweetCounts(result.id), + unidirectionalRetweetCount = inNetworkEngagement.uniDirectionalRetweetCounts(result.id), + conversationCount = inNetworkEngagement.descendantReplyCounts(result.id), + directedAtUserIdIsInFirstDegree = + if (isReply) inReplyToUserId.map(followedUserIds.contains) else None, ) } .getOrElse(thriftTweetFeatures) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/RelevanceSearchUtil.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/RelevanceSearchUtil.scala index 0de4546a6..40b727141 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/RelevanceSearchUtil.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/RelevanceSearchUtil.scala @@ -1,7 +1,6 @@ package com.twitter.home_mixer.util.earlybird import com.twitter.search.common.schema.earlybird.EarlybirdFieldConstants.EarlybirdFieldConstant -import com.twitter.search.common.ranking.{thriftscala => scr} import com.twitter.search.earlybird.{thriftscala => eb} object RelevanceSearchUtil { @@ -10,16 +9,6 @@ object RelevanceSearchUtil { val Hashtags: String = EarlybirdFieldConstant.HASHTAGS_FACET val FacetsToFetch: Seq[String] = Seq(Mentions, Hashtags) - private val RankingParams: scr.ThriftRankingParams = { - scr.ThriftRankingParams( - `type` = Some(scr.ThriftScoringFunctionType.TensorflowBased), - selectedTensorflowModel = Some("timelines_rectweet_replica"), - minScore = -1.0e100, - selectedModels = Some(Map("home_mixer_unified_engagement_prod" -> 1.0)), - applyBoosts = false, - ) - } - val MetadataOptions: eb.ThriftSearchResultMetadataOptions = { eb.ThriftSearchResultMetadataOptions( getTweetUrls = true, @@ -39,7 +28,7 @@ object RelevanceSearchUtil { eb.ThriftSearchRelevanceOptions( proximityScoring = true, maxConsecutiveSameUser = Some(2), - rankingParams = Some(RankingParams), + rankingParams = None, maxHitsToProcess = Some(500), maxUserBlendCount = Some(3), proximityPhraseWeight = 9.0, diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/RequestFields.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/RequestFields.scala index 9d048005e..e20aead94 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/RequestFields.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/RequestFields.scala @@ -44,7 +44,8 @@ object RequestFields { tp.TweetInclude.TweetFieldId(tp.Tweet.QuotedTweetField.id), tp.TweetInclude.TweetFieldId(tp.Tweet.CommunitiesField.id), // Field required for determining if a Tweet was created via News Camera. - tp.TweetInclude.TweetFieldId(tp.Tweet.ComposerSourceField.id) + tp.TweetInclude.TweetFieldId(tp.Tweet.ComposerSourceField.id), + tp.TweetInclude.TweetFieldId(tp.Tweet.LanguageField.id) ) val TweetStaticEntitiesFields: Set[tp.TweetInclude] =