diff --git a/home-mixer/BUILD.bazel b/home-mixer/BUILD.bazel deleted file mode 100644 index ab7b358e7..000000000 --- a/home-mixer/BUILD.bazel +++ /dev/null @@ -1,30 +0,0 @@ -jvm_binary( - name = "bin", - basename = "home-mixer", - main = "com.twitter.home_mixer.HomeMixerServerMain", - runtime_platform = "java11", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/ch/qos/logback:logback-classic", - "finagle/finagle-zipkin-scribe/src/main/scala", - "finatra/inject/inject-logback/src/main/scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer", - "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", - "twitter-server-internal/src/main/scala", - "twitter-server/logback-classic/src/main/scala", - ], -) - -# Aurora Workflows build phase convention requires a jvm_app named with home-mixer-app -jvm_app( - name = "home-mixer-app", - archive = "zip", - binary = ":bin", - bundles = [ - bundle( - fileset = ["config/**/*"], - owning_target = "home-mixer/config:files", - ), - ], - tags = ["bazel-compatible"], -) diff --git a/home-mixer/README.md b/home-mixer/README.md deleted file mode 100644 index 20861d7a0..000000000 --- a/home-mixer/README.md +++ /dev/null @@ -1,101 +0,0 @@ -Home Mixer -========== - -Home Mixer is the main service used to construct and serve Twitter's Home Timelines. It currently -powers: -- For you - best Tweets from people you follow + recommended out-of-network content -- Following - reverse chronological Tweets from people you follow -- Lists - reverse chronological Tweets from List members - -Home Mixer is built on Product Mixer, our custom Scala framework that facilitates building -feeds of content. - -## Overview - -The For You recommendation algorithm in Home Mixer involves the following stages: - -- Candidate Generation - fetch Tweets from various Candidate Sources. For example: - - Earlybird Search Index - - User Tweet Entity Graph - - Cr Mixer - - Follow Recommendations Service -- Feature Hydration - - Fetch the ~6000 features needed for ranking -- Scoring and Ranking using ML model -- Filters and Heuristics. For example: - - Author Diversity - - Content Balance (In network vs Out of Network) - - Feedback fatigue - - Deduplication / previously seen Tweets removal - - Visibility Filtering (blocked, muted authors/tweets, NSFW settings) -- Mixing - integrate Tweets with non-Tweet content - - Ads - - Who-to-follow modules - - Prompts -- Product Features and Serving - - Conversation Modules for replies - - Social Context - - Timeline Navigation - - Edited Tweets - - Feedback options - - Pagination and cursoring - - Observability and logging - - Client instructions and content marshalling - -## Pipeline Structure - -### General - -Product Mixer services like Home Mixer are structured around Pipelines that split the execution -into transparent and structured steps. - -Requests first go to Product Pipelines, which are used to select which Mixer Pipeline or -Recommendation Pipeline to run for a given request. Each Mixer or Recommendation -Pipeline may run multiple Candidate Pipelines to fetch candidates to include in the response. - -Mixer Pipelines combine the results of multiple heterogeneous Candidate Pipelines together -(e.g. ads, tweets, users) while Recommendation Pipelines are used to score (via Scoring Pipelines) -and rank the results of homogenous Candidate Pipelines so that the top ranked ones can be returned. -These pipelines also marshall candidates into a domain object and then into a transport object -to return to the caller. - -Candidate Pipelines fetch candidates from underlying Candidate Sources and perform some basic -operations on the Candidates, such as filtering out unwanted candidates, applying decorations, -and hydrating features. - -The sections below describe the high level pipeline structure (non-exhaustive) for the main Home -Timeline tabs powered by Home Mixer. - -### For You - -- ForYouProductPipelineConfig - - ForYouScoredTweetsMixerPipelineConfig (main orchestration layer - mixes Tweets with ads and users) - - ForYouScoredTweetsCandidatePipelineConfig (fetch Tweets) - - ScoredTweetsRecommendationPipelineConfig (main Tweet recommendation layer) - - Fetch Tweet Candidates - - ScoredTweetsInNetworkCandidatePipelineConfig - - ScoredTweetsTweetMixerCandidatePipelineConfig - - ScoredTweetsUtegCandidatePipelineConfig - - ScoredTweetsFrsCandidatePipelineConfig - - Feature Hydration and Scoring - - ScoredTweetsScoringPipelineConfig - - ForYouConversationServiceCandidatePipelineConfig (backup reverse chron pipeline in case Scored Tweets fails) - - ForYouAdsCandidatePipelineConfig (fetch ads) - - ForYouWhoToFollowCandidatePipelineConfig (fetch users to recommend) - -### Following - -- FollowingProductPipelineConfig - - FollowingMixerPipelineConfig - - FollowingEarlybirdCandidatePipelineConfig (fetch tweets from Search Index) - - ConversationServiceCandidatePipelineConfig (fetch ancestors for conversation modules) - - FollowingAdsCandidatePipelineConfig (fetch ads) - - FollowingWhoToFollowCandidatePipelineConfig (fetch users to recommend) - -### Lists - -- ListTweetsProductPipelineConfig - - ListTweetsMixerPipelineConfig - - ListTweetsTimelineServiceCandidatePipelineConfig (fetch tweets from timeline service) - - ConversationServiceCandidatePipelineConfig (fetch ancestors for conversation modules) - - ListTweetsAdsCandidatePipelineConfig (fetch ads) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/BUILD.bazel deleted file mode 100644 index 103e079da..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/BUILD.bazel +++ /dev/null @@ -1,51 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finagle/finagle-core/src/main", - "finagle/finagle-http/src/main/scala", - "finagle/finagle-thriftmux/src/main/scala", - "finatra-internal/mtls-http/src/main/scala", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/http-core/src/main/java/com/twitter/finatra/http", - "finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations", - "finatra/inject/inject-app/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-server/src/main/scala", - "finatra/inject/inject-utils/src/main/scala", - "home-mixer/server/src/main/resources", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/controller", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/federated", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/module", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/module", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter", - "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", - "src/thrift/com/twitter/timelines/render:thrift-scala", - "strato/config/columns/auth-context:auth-context-strato-client", - "strato/config/columns/gizmoduck:gizmoduck-strato-client", - "strato/src/main/scala/com/twitter/strato/fed", - "strato/src/main/scala/com/twitter/strato/fed/server", - "stringcenter/client", - "stringcenter/client/src/main/java", - "stringcenter/client/src/main/scala/com/twitter/stringcenter/client", - "thrift-web-forms/src/main/scala/com/twitter/thriftwebforms/view", - "timelines/src/main/scala/com/twitter/timelines/config", - "timelines/src/main/scala/com/twitter/timelines/features/app", - "twitter-server-internal", - "twitter-server/server/src/main/scala", - "util/util-app/src/main/scala", - "util/util-core:scala", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerHttpServerWarmupHandler.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerHttpServerWarmupHandler.scala deleted file mode 100644 index e27133b23..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerHttpServerWarmupHandler.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.home_mixer - -import com.twitter.finatra.http.routing.HttpWarmup -import com.twitter.finatra.httpclient.RequestBuilder._ -import com.twitter.util.logging.Logging -import com.twitter.inject.utils.Handler -import com.twitter.util.Try -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class HomeMixerHttpServerWarmupHandler @Inject() (warmup: HttpWarmup) extends Handler with Logging { - - override def handle(): Unit = { - Try(warmup.send(get("/admin/product-mixer/product-pipelines"), admin = true)()) - .onFailure(e => error(e.getMessage, e)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerServer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerServer.scala deleted file mode 100644 index e635c7a68..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerServer.scala +++ /dev/null @@ -1,128 +0,0 @@ -package com.twitter.home_mixer - -import com.google.inject.Module -import com.twitter.finagle.Filter -import com.twitter.finatra.annotations.DarkTrafficFilterType -import com.twitter.finatra.http.HttpServer -import com.twitter.finatra.http.routing.HttpRouter -import com.twitter.finatra.mtls.http.{Mtls => HttpMtls} -import com.twitter.finatra.mtls.thriftmux.Mtls -import com.twitter.finatra.mtls.thriftmux.modules.MtlsThriftWebFormsModule -import com.twitter.finatra.thrift.ThriftServer -import com.twitter.finatra.thrift.filters._ -import com.twitter.finatra.thrift.routing.ThriftRouter -import com.twitter.home_mixer.controller.HomeThriftController -import com.twitter.home_mixer.federated.HomeMixerColumn -import com.twitter.home_mixer.module._ -import com.twitter.home_mixer.param.GlobalParamConfigModule -import com.twitter.home_mixer.product.HomeMixerProductModule -import com.twitter.home_mixer.{thriftscala => st} -import com.twitter.product_mixer.component_library.module.AccountRecommendationsMixerModule -import com.twitter.product_mixer.component_library.module.DarkTrafficFilterModule -import com.twitter.product_mixer.component_library.module.EarlybirdModule -import com.twitter.product_mixer.component_library.module.ExploreRankerClientModule -import com.twitter.product_mixer.component_library.module.GizmoduckClientModule -import com.twitter.product_mixer.component_library.module.OnboardingTaskServiceModule -import com.twitter.product_mixer.component_library.module.SocialGraphServiceModule -import com.twitter.product_mixer.component_library.module.TimelineRankerClientModule -import com.twitter.product_mixer.component_library.module.TimelineScorerClientModule -import com.twitter.product_mixer.component_library.module.TimelineServiceClientModule -import com.twitter.product_mixer.component_library.module.TweetImpressionStoreModule -import com.twitter.product_mixer.component_library.module.TweetMixerClientModule -import com.twitter.product_mixer.component_library.module.UserSessionStoreModule -import com.twitter.product_mixer.core.controllers.ProductMixerController -import com.twitter.product_mixer.core.module.LoggingThrowableExceptionMapper -import com.twitter.product_mixer.core.module.ProductMixerModule -import com.twitter.product_mixer.core.module.stringcenter.ProductScopeStringCenterModule -import com.twitter.strato.fed.StratoFed -import com.twitter.strato.fed.server.StratoFedServer - -object HomeMixerServerMain extends HomeMixerServer - -class HomeMixerServer - extends StratoFedServer - with ThriftServer - with Mtls - with HttpServer - with HttpMtls { - override val name = "home-mixer-server" - - override val modules: Seq[Module] = Seq( - AccountRecommendationsMixerModule, - AdvertiserBrandSafetySettingsStoreModule, - BlenderClientModule, - ClientSentImpressionsPublisherModule, - ConversationServiceModule, - EarlybirdModule, - ExploreRankerClientModule, - FeedbackHistoryClientModule, - GizmoduckClientModule, - GlobalParamConfigModule, - HomeAdsCandidateSourceModule, - HomeMixerFlagsModule, - HomeMixerProductModule, - HomeMixerResourcesModule, - ImpressionBloomFilterModule, - InjectionHistoryClientModule, - ManhattanClientsModule, - ManhattanFeatureRepositoryModule, - ManhattanTweetImpressionStoreModule, - MemcachedFeatureRepositoryModule, - NaviModelClientModule, - OnboardingTaskServiceModule, - OptimizedStratoClientModule, - PeopleDiscoveryServiceModule, - ProductMixerModule, - RealGraphInNetworkScoresModule, - RealtimeAggregateFeatureRepositoryModule, - ScoredTweetsMemcacheModule, - ScribeEventPublisherModule, - SimClustersRecentEngagementsClientModule, - SocialGraphServiceModule, - StaleTweetsCacheModule, - ThriftFeatureRepositoryModule, - TimelineRankerClientModule, - TimelineScorerClientModule, - TimelineServiceClientModule, - TimelinesPersistenceStoreClientModule, - TopicSocialProofClientModule, - TweetImpressionStoreModule, - TweetMixerClientModule, - TweetypieClientModule, - TweetypieStaticEntitiesCacheClientModule, - UserSessionStoreModule, - new DarkTrafficFilterModule[st.HomeMixer.ReqRepServicePerEndpoint](), - new MtlsThriftWebFormsModule[st.HomeMixer.MethodPerEndpoint](this), - new ProductScopeStringCenterModule() - ) - - override def configureThrift(router: ThriftRouter): Unit = { - router - .filter[LoggingMDCFilter] - .filter[TraceIdMDCFilter] - .filter[ThriftMDCFilter] - .filter[StatsFilter] - .filter[AccessLoggingFilter] - .filter[ExceptionMappingFilter] - .filter[Filter.TypeAgnostic, DarkTrafficFilterType] - .exceptionMapper[LoggingThrowableExceptionMapper] - .exceptionMapper[PipelineFailureExceptionMapper] - .add[HomeThriftController] - } - - override def configureHttp(router: HttpRouter): Unit = - router.add( - ProductMixerController[st.HomeMixer.MethodPerEndpoint]( - this.injector, - st.HomeMixer.ExecutePipeline)) - - override val dest: String = "/s/home-mixer/home-mixer:strato" - - override val columns: Seq[Class[_ <: StratoFed.Column]] = - Seq(classOf[HomeMixerColumn]) - - override protected def warmup(): Unit = { - handle[HomeMixerThriftServerWarmupHandler]() - handle[HomeMixerHttpServerWarmupHandler]() - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerThriftServerWarmupHandler.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerThriftServerWarmupHandler.scala deleted file mode 100644 index 982b77487..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerThriftServerWarmupHandler.scala +++ /dev/null @@ -1,73 +0,0 @@ -package com.twitter.home_mixer - -import com.twitter.finagle.thrift.ClientId -import com.twitter.finatra.thrift.routing.ThriftWarmup -import com.twitter.home_mixer.{thriftscala => st} -import com.twitter.util.logging.Logging -import com.twitter.inject.utils.Handler -import com.twitter.product_mixer.core.{thriftscala => pt} -import com.twitter.scrooge.Request -import com.twitter.scrooge.Response -import com.twitter.util.Return -import com.twitter.util.Throw -import com.twitter.util.Try -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class HomeMixerThriftServerWarmupHandler @Inject() (warmup: ThriftWarmup) - extends Handler - with Logging { - - private val clientId = ClientId("thrift-warmup-client") - - def handle(): Unit = { - val testIds = Seq(1, 2, 3) - try { - clientId.asCurrent { - testIds.foreach { id => - val warmupReq = warmupQuery(id) - info(s"Sending warm-up request to service with query: $warmupReq") - warmup.sendRequest( - method = st.HomeMixer.GetUrtResponse, - req = Request(st.HomeMixer.GetUrtResponse.Args(warmupReq)))(assertWarmupResponse) - } - } - } catch { - case e: Throwable => error(e.getMessage, e) - } - info("Warm-up done.") - } - - private def warmupQuery(userId: Long): st.HomeMixerRequest = { - val clientContext = pt.ClientContext( - userId = Some(userId), - guestId = None, - appId = Some(12345L), - ipAddress = Some("0.0.0.0"), - userAgent = Some("FAKE_USER_AGENT_FOR_WARMUPS"), - countryCode = Some("US"), - languageCode = Some("en"), - isTwoffice = None, - userRoles = None, - deviceId = Some("FAKE_DEVICE_ID_FOR_WARMUPS") - ) - st.HomeMixerRequest( - clientContext = clientContext, - product = st.Product.Following, - productContext = Some(st.ProductContext.Following(st.Following())), - maxResults = Some(3) - ) - } - - private def assertWarmupResponse( - result: Try[Response[st.HomeMixer.GetUrtResponse.SuccessType]] - ): Unit = { - result match { - case Return(_) => // ok - case Throw(exception) => - warn("Error performing warm-up request.") - error(exception.getMessage, exception) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/BUILD.bazel deleted file mode 100644 index 3f738bcb6..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/BUILD.bazel +++ /dev/null @@ -1,26 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfig.scala deleted file mode 100644 index d26843193..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfig.scala +++ /dev/null @@ -1,116 +0,0 @@ -package com.twitter.home_mixer.candidate_pipeline - -import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator -import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter -import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter -import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource -import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSourceRequest -import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata -import com.twitter.product_mixer.component_library.filter.FeatureFilter -import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.gate.BaseGate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.functional_component.transformer.DependentCandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig - -/** - * Candidate Pipeline Config that fetches tweets from the Conversation Service Candidate Source - */ -class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery]( - conversationServiceCandidateSource: ConversationServiceCandidateSource, - tweetypieFeatureHydrator: TweetypieFeatureHydrator, - namesFeatureHydrator: NamesFeatureHydrator, - invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter, - override val gates: Seq[BaseGate[Query]], - override val decorator: Option[CandidateDecorator[Query, TweetCandidate]]) - extends DependentCandidatePipelineConfig[ - Query, - ConversationServiceCandidateSourceRequest, - TweetWithConversationMetadata, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ConversationService") - - private val TweetypieHydratedFilterId = "TweetypieHydrated" - private val QuotedTweetDroppedFilterId = "QuotedTweetDropped" - - override val candidateSource: BaseCandidateSource[ - ConversationServiceCandidateSourceRequest, - TweetWithConversationMetadata - ] = conversationServiceCandidateSource - - override val queryTransformer: DependentCandidatePipelineQueryTransformer[ - Query, - ConversationServiceCandidateSourceRequest - ] = { (_, candidates) => - val tweetsWithConversationMetadata = candidates.map { candidate => - TweetWithConversationMetadata( - tweetId = candidate.candidateIdLong, - userId = candidate.features.getOrElse(AuthorIdFeature, None), - sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None), - sourceUserId = candidate.features.getOrElse(SourceUserIdFeature, None), - inReplyToTweetId = candidate.features.getOrElse(InReplyToTweetIdFeature, None), - conversationId = None, - ancestors = Seq.empty - ) - } - ConversationServiceCandidateSourceRequest(tweetsWithConversationMetadata) - } - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[TweetWithConversationMetadata] - ] = Seq(ConversationServiceResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - TweetWithConversationMetadata, - TweetCandidate - ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } - - override val preFilterFeatureHydrationPhase1: Seq[ - BaseCandidateFeatureHydrator[Query, TweetCandidate, _] - ] = Seq( - tweetypieFeatureHydrator, - InNetworkFeatureHydrator, - ) - - override def filters: Seq[Filter[Query, TweetCandidate]] = Seq( - RetweetDeduplicationFilter, - FeatureFilter.fromFeature(FilterIdentifier(TweetypieHydratedFilterId), IsHydratedFeature), - PredicateFeatureFilter.fromPredicate( - FilterIdentifier(QuotedTweetDroppedFilterId), - shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) } - ), - invalidSubscriptionTweetFilter, - InvalidConversationModuleFilter - ) - - override val postFilterFeatureHydration: Seq[ - BaseCandidateFeatureHydrator[Query, TweetCandidate, _] - ] = Seq(namesFeatureHydrator) - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), - HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfigBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfigBuilder.scala deleted file mode 100644 index bb55f85e3..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfigBuilder.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.home_mixer.candidate_pipeline - -import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator -import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter -import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.gate.BaseGate -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery] @Inject() ( - conversationServiceCandidateSource: ConversationServiceCandidateSource, - tweetypieFeatureHydrator: TweetypieFeatureHydrator, - invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter, - namesFeatureHydrator: NamesFeatureHydrator) { - - def build( - gates: Seq[BaseGate[Query]] = Seq.empty, - decorator: Option[CandidateDecorator[Query, TweetCandidate]] = None - ): ConversationServiceCandidatePipelineConfig[Query] = { - new ConversationServiceCandidatePipelineConfig( - conversationServiceCandidateSource, - tweetypieFeatureHydrator, - namesFeatureHydrator, - invalidSubscriptionTweetFilter, - gates, - decorator - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceResponseFeatureTransformer.scala deleted file mode 100644 index 154c080ad..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceResponseFeatureTransformer.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.twitter.home_mixer.candidate_pipeline - -import com.twitter.home_mixer.model.HomeFeatures._ -import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.timelineservice.suggests.thriftscala.SuggestType - -object ConversationServiceResponseFeatureTransformer - extends CandidateFeatureTransformer[TweetWithConversationMetadata] { - - override val identifier: TransformerIdentifier = - TransformerIdentifier("ConversationServiceResponse") - - override val features: Set[Feature[_, _]] = Set( - AuthorIdFeature, - InReplyToTweetIdFeature, - IsRetweetFeature, - SourceTweetIdFeature, - SourceUserIdFeature, - ConversationModuleFocalTweetIdFeature, - AncestorsFeature, - SuggestTypeFeature - ) - - override def transform(candidate: TweetWithConversationMetadata): FeatureMap = FeatureMapBuilder() - .add(AuthorIdFeature, candidate.userId) - .add(InReplyToTweetIdFeature, candidate.inReplyToTweetId) - .add(IsRetweetFeature, candidate.sourceTweetId.isDefined) - .add(SourceTweetIdFeature, candidate.sourceTweetId) - .add(SourceUserIdFeature, candidate.sourceUserId) - .add(ConversationModuleFocalTweetIdFeature, candidate.conversationId) - .add(AncestorsFeature, candidate.ancestors) - .add(SuggestTypeFeature, Some(SuggestType.RankedOrganicTweet)) - .build() -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/EditedTweetsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/EditedTweetsCandidatePipelineConfig.scala deleted file mode 100644 index d9bb73695..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/EditedTweetsCandidatePipelineConfig.scala +++ /dev/null @@ -1,84 +0,0 @@ -package com.twitter.home_mixer.candidate_pipeline - -import com.twitter.home_mixer.functional_component.candidate_source.StaleTweetsCacheCandidateSource -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder -import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator -import com.twitter.home_mixer.functional_component.query_transformer.EditedTweetsCandidatePipelineQueryTransformer -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.EmptyClientEventInfoBuilder -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.rtf.safety_level.TimelineFocalTweetSafetyLevel -import com.twitter.product_mixer.core.model.marshalling.response.urt.contextual_ref.TweetHydrationContext -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Candidate Pipeline Config that fetches edited tweets from the Stale Tweets Cache - */ -@Singleton -case class EditedTweetsCandidatePipelineConfig @Inject() ( - staleTweetsCacheCandidateSource: StaleTweetsCacheCandidateSource, - namesFeatureHydrator: NamesFeatureHydrator, - homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder) - extends DependentCandidatePipelineConfig[ - PipelineQuery, - Seq[Long], - Long, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("EditedTweets") - - override val candidateSource: BaseCandidateSource[Seq[Long], Long] = - staleTweetsCacheCandidateSource - - override val queryTransformer: CandidatePipelineQueryTransformer[ - PipelineQuery, - Seq[Long] - ] = EditedTweetsCandidatePipelineQueryTransformer - - override val resultTransformer: CandidatePipelineResultsTransformer[ - Long, - TweetCandidate - ] = { candidate => TweetCandidate(id = candidate) } - - override val postFilterFeatureHydration: Seq[ - BaseCandidateFeatureHydrator[PipelineQuery, TweetCandidate, _] - ] = Seq(namesFeatureHydrator) - - override val decorator: Option[CandidateDecorator[PipelineQuery, TweetCandidate]] = { - val tweetItemBuilder = TweetCandidateUrtItemBuilder[PipelineQuery, TweetCandidate]( - clientEventInfoBuilder = EmptyClientEventInfoBuilder, - entryIdToReplaceBuilder = Some((_, candidate, _) => - Some(s"${TweetItem.TweetEntryNamespace}-${candidate.id.toString}")), - contextualTweetRefBuilder = Some( - ContextualTweetRefBuilder( - TweetHydrationContext( - // Apply safety level that includes canonical VF treatments that apply regardless of context. - safetyLevelOverride = Some(TimelineFocalTweetSafetyLevel), - outerTweetContext = None - ) - ) - ), - feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder) - ) - - Some(UrtItemCandidateDecorator(tweetItemBuilder)) - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.5, 50, 60, 60) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/NewTweetsPillCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/NewTweetsPillCandidatePipelineConfig.scala deleted file mode 100644 index e1a92b9ca..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/NewTweetsPillCandidatePipelineConfig.scala +++ /dev/null @@ -1,123 +0,0 @@ -package com.twitter.home_mixer.candidate_pipeline - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.functional_component.gate.RequestContextNotGate -import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature -import com.twitter.home_mixer.model.request.DeviceContext -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.alert.DurationParamBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.alert.ShowAlertCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.alert.StaticShowAlertColorConfigurationBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.alert.StaticShowAlertDisplayLocationBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.alert.StaticShowAlertIconDisplayInfoBuilder -import com.twitter.product_mixer.component_library.gate.FeatureGate -import com.twitter.product_mixer.component_library.model.candidate.ShowAlertCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.functional_component.candidate_source.StaticCandidateSource -import com.twitter.product_mixer.core.functional_component.configapi.StaticParam -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.item.alert.BaseDurationBuilder -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.alert.NewTweets -import com.twitter.product_mixer.core.model.marshalling.response.urt.alert.ShowAlertColorConfiguration -import com.twitter.product_mixer.core.model.marshalling.response.urt.alert.ShowAlertIconDisplayInfo -import com.twitter.product_mixer.core.model.marshalling.response.urt.alert.Top -import com.twitter.product_mixer.core.model.marshalling.response.urt.alert.UpArrow -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.TwitterBlueRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.WhiteRosettaColor -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig -import com.twitter.util.Duration -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Candidate Pipeline Config that creates the New Tweets Pill - */ -@Singleton -class NewTweetsPillCandidatePipelineConfig[Query <: PipelineQuery with HasDeviceContext] @Inject() ( -) extends DependentCandidatePipelineConfig[ - Query, - Unit, - ShowAlertCandidate, - ShowAlertCandidate - ] { - import NewTweetsPillCandidatePipelineConfig._ - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("NewTweetsPill") - - override val gates: Seq[Gate[Query]] = Seq( - RequestContextNotGate(Seq(DeviceContext.RequestContext.PullToRefresh)), - FeatureGate.fromFeature(GetNewerFeature) - ) - - override val candidateSource: CandidateSource[Unit, ShowAlertCandidate] = - StaticCandidateSource( - CandidateSourceIdentifier(identifier.name), - Seq(ShowAlertCandidate(id = identifier.name, userIds = Seq.empty)) - ) - - override val queryTransformer: CandidatePipelineQueryTransformer[Query, Unit] = { _ => Unit } - - override val resultTransformer: CandidatePipelineResultsTransformer[ - ShowAlertCandidate, - ShowAlertCandidate - ] = { candidate => candidate } - - override val decorator: Option[CandidateDecorator[Query, ShowAlertCandidate]] = { - val triggerDelayBuilder = new BaseDurationBuilder[Query] { - override def apply( - query: Query, - candidate: ShowAlertCandidate, - features: FeatureMap - ): Option[Duration] = { - val delay = query.deviceContext.flatMap(_.requestContextValue) match { - case Some(DeviceContext.RequestContext.TweetSelfThread) => 0.millis - case Some(DeviceContext.RequestContext.ManualRefresh) => 0.millis - case _ => TriggerDelay - } - - Some(delay) - } - } - - val homeShowAlertCandidateBuilder = ShowAlertCandidateUrtItemBuilder( - alertType = NewTweets, - colorConfigBuilder = StaticShowAlertColorConfigurationBuilder(DefaultColorConfig), - displayLocationBuilder = StaticShowAlertDisplayLocationBuilder(Top), - triggerDelayBuilder = Some(triggerDelayBuilder), - displayDurationBuilder = Some(DurationParamBuilder(StaticParam(DisplayDuration))), - iconDisplayInfoBuilder = Some(StaticShowAlertIconDisplayInfoBuilder(DefaultIconDisplayInfo)) - ) - - Some(UrtItemCandidateDecorator(homeShowAlertCandidateBuilder)) - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), - HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() - ) -} - -object NewTweetsPillCandidatePipelineConfig { - val DefaultColorConfig: ShowAlertColorConfiguration = ShowAlertColorConfiguration( - background = TwitterBlueRosettaColor, - text = WhiteRosettaColor, - border = Some(WhiteRosettaColor) - ) - - val DefaultIconDisplayInfo: ShowAlertIconDisplayInfo = - ShowAlertIconDisplayInfo(icon = UpArrow, tint = WhiteRosettaColor) - - // Unlimited display time (until user takes action) - val DisplayDuration = -1.millisecond - val TriggerDelay = 4.minutes -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/TimelineServiceResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/TimelineServiceResponseFeatureTransformer.scala deleted file mode 100644 index 3f0a932c9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/TimelineServiceResponseFeatureTransformer.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.home_mixer.candidate_pipeline - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.timelineservice.{thriftscala => t} - -object TimelineServiceResponseFeatureTransformer extends CandidateFeatureTransformer[t.Tweet] { - - override val identifier: TransformerIdentifier = TransformerIdentifier("TimelineServiceResponse") - - override val features: Set[Feature[_, _]] = Set( - AuthorIdFeature, - InReplyToTweetIdFeature, - IsRetweetFeature, - SourceTweetIdFeature, - SourceUserIdFeature, - ) - - override def transform(candidate: t.Tweet): FeatureMap = FeatureMapBuilder() - .add(AuthorIdFeature, candidate.userId) - .add(InReplyToTweetIdFeature, candidate.inReplyToStatusId) - .add(IsRetweetFeature, candidate.sourceStatusId.isDefined) - .add(SourceTweetIdFeature, candidate.sourceStatusId) - .add(SourceUserIdFeature, candidate.sourceUserId) - .build() -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/BUILD.bazel deleted file mode 100644 index dfaff319d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/BUILD.bazel +++ /dev/null @@ -1,17 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift:controller", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt", - "snowflake/src/main/scala/com/twitter/snowflake/id", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/HomeThriftController.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/HomeThriftController.scala deleted file mode 100644 index fc1b7770e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/HomeThriftController.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.home_mixer.controller - -import com.twitter.finatra.thrift.Controller -import com.twitter.home_mixer.marshaller.request.HomeMixerRequestUnmarshaller -import com.twitter.home_mixer.model.request.HomeMixerRequest -import com.twitter.home_mixer.service.ScoredTweetsService -import com.twitter.home_mixer.{thriftscala => t} -import com.twitter.product_mixer.core.controllers.DebugTwitterContext -import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder -import com.twitter.product_mixer.core.service.debug_query.DebugQueryService -import com.twitter.product_mixer.core.service.urt.UrtService -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.Params -import javax.inject.Inject - -class HomeThriftController @Inject() ( - homeRequestUnmarshaller: HomeMixerRequestUnmarshaller, - urtService: UrtService, - scoredTweetsService: ScoredTweetsService, - paramsBuilder: ParamsBuilder) - extends Controller(t.HomeMixer) - with DebugTwitterContext { - - handle(t.HomeMixer.GetUrtResponse) { args: t.HomeMixer.GetUrtResponse.Args => - val request = homeRequestUnmarshaller(args.request) - val params = buildParams(request) - Stitch.run(urtService.getUrtResponse[HomeMixerRequest](request, params)) - } - - handle(t.HomeMixer.GetScoredTweetsResponse) { args: t.HomeMixer.GetScoredTweetsResponse.Args => - val request = homeRequestUnmarshaller(args.request) - val params = buildParams(request) - withDebugTwitterContext(request.clientContext) { - Stitch.run(scoredTweetsService.getScoredTweetsResponse[HomeMixerRequest](request, params)) - } - } - - private def buildParams(request: HomeMixerRequest): Params = { - val userAgeOpt = request.clientContext.userId.map { userId => - SnowflakeId.timeFromIdOpt(userId).map(_.untilNow.inDays).getOrElse(Int.MaxValue) - } - val fsCustomMapInput = userAgeOpt.map("account_age_in_days" -> _).toMap - paramsBuilder.build( - clientContext = request.clientContext, - product = request.product, - featureOverrides = request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty), - fsCustomMapInput = fsCustomMapInput - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/BUILD.bazel deleted file mode 100644 index 30ee81acc..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/BUILD.bazel +++ /dev/null @@ -1,24 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry", - "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", - "src/thrift/com/twitter/gizmoduck:thrift-scala", - "src/thrift/com/twitter/timelines/render:thrift-scala", - "stitch/stitch-repo/src/main/scala", - "strato/config/columns/auth-context:auth-context-strato-client", - "strato/config/columns/gizmoduck:gizmoduck-strato-client", - "strato/config/src/thrift/com/twitter/strato/graphql/timelines:graphql-timelines-scala", - "strato/src/main/scala/com/twitter/strato/callcontext", - "strato/src/main/scala/com/twitter/strato/fed", - "strato/src/main/scala/com/twitter/strato/fed/server", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/HomeMixerColumn.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/HomeMixerColumn.scala deleted file mode 100644 index 0ef27b7a0..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/HomeMixerColumn.scala +++ /dev/null @@ -1,217 +0,0 @@ -package com.twitter.home_mixer.federated - -import com.twitter.gizmoduck.{thriftscala => gd} -import com.twitter.home_mixer.marshaller.request.HomeMixerRequestUnmarshaller -import com.twitter.home_mixer.model.request.HomeMixerRequest -import com.twitter.home_mixer.{thriftscala => hm} -import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder -import com.twitter.product_mixer.core.pipeline.product.ProductPipelineRequest -import com.twitter.product_mixer.core.pipeline.product.ProductPipelineResult -import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry -import com.twitter.product_mixer.core.{thriftscala => pm} -import com.twitter.stitch.Arrow -import com.twitter.stitch.Stitch -import com.twitter.strato.callcontext.CallContext -import com.twitter.strato.catalog.OpMetadata -import com.twitter.strato.config._ -import com.twitter.strato.data._ -import com.twitter.strato.fed.StratoFed -import com.twitter.strato.generated.client.auth_context.AuditIpClientColumn -import com.twitter.strato.generated.client.gizmoduck.CompositeOnUserClientColumn -import com.twitter.strato.graphql.timelines.{thriftscala => gql} -import com.twitter.strato.thrift.ScroogeConv -import com.twitter.timelines.render.{thriftscala => tr} -import com.twitter.util.Try -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class HomeMixerColumn @Inject() ( - homeMixerRequestUnmarshaller: HomeMixerRequestUnmarshaller, - compositeOnUserClientColumn: CompositeOnUserClientColumn, - auditIpClientColumn: AuditIpClientColumn, - paramsBuilder: ParamsBuilder, - productPipelineRegistry: ProductPipelineRegistry) - extends StratoFed.Column(HomeMixerColumn.Path) - with StratoFed.Fetch.Arrow { - - override val contactInfo: ContactInfo = ContactInfo( - contactEmail = "", - ldapGroup = "", - slackRoomId = "" - ) - - override val metadata: OpMetadata = - OpMetadata( - lifecycle = Some(Lifecycle.Production), - description = - Some(Description.PlainText("Federated Strato column for Timelines served via Home Mixer")) - ) - - private val bouncerAccess: Seq[Policy] = Seq(BouncerAccess()) - private val finatraTestServiceIdentifiers: Seq[Policy] = Seq( - ServiceIdentifierPattern( - role = "", - service = "", - env = "", - zone = Seq("")) - ) - - override val policy: Policy = AnyOf(bouncerAccess ++ finatraTestServiceIdentifiers) - - override type Key = gql.TimelineKey - override type View = gql.HomeTimelineView - override type Value = tr.Timeline - - override val keyConv: Conv[Key] = ScroogeConv.fromStruct[gql.TimelineKey] - override val viewConv: Conv[View] = ScroogeConv.fromStruct[gql.HomeTimelineView] - override val valueConv: Conv[Value] = ScroogeConv.fromStruct[tr.Timeline] - - private def createHomeMixerRequestArrow( - compositeOnUserClientColumn: CompositeOnUserClientColumn, - auditIpClientColumn: AuditIpClientColumn - ): Arrow[(Key, View), hm.HomeMixerRequest] = { - - val populateUserRolesAndIp: Arrow[(Key, View), (Option[Set[String]], Option[String])] = { - val gizmoduckView: (gd.LookupContext, Set[gd.QueryFields]) = - (gd.LookupContext(), Set(gd.QueryFields.Roles)) - - val populateUserRoles = Arrow - .flatMap[(Key, View), Option[Set[String]]] { _ => - Stitch.collect { - CallContext.twitterUserId.map { userId => - compositeOnUserClientColumn.fetcher - .callStack(HomeMixerColumn.FetchCallstack) - .fetch(userId, gizmoduckView).map(_.v) - .map { - _.flatMap(_.roles.map(_.roles.toSet)).getOrElse(Set.empty) - } - } - } - } - - val populateIpAddress = Arrow - .flatMap[(Key, View), Option[String]](_ => - auditIpClientColumn.fetcher - .callStack(HomeMixerColumn.FetchCallstack) - .fetch((), ()).map(_.v)) - - Arrow.join( - populateUserRoles, - populateIpAddress - ) - } - - Arrow.zipWithArg(populateUserRolesAndIp).map { - case ((key, view), (roles, ipAddress)) => - val deviceContextOpt = Some( - hm.DeviceContext( - isPolling = CallContext.isPolling, - requestContext = view.requestContext, - latestControlAvailable = view.latestControlAvailable, - autoplayEnabled = view.autoplayEnabled - )) - val seenTweetIds = view.seenTweetIds.filter(_.nonEmpty) - - val (product, productContext) = key match { - case gql.TimelineKey.HomeTimeline(_) | gql.TimelineKey.HomeTimelineV2(_) => - ( - hm.Product.ForYou, - hm.ProductContext.ForYou( - hm.ForYou( - deviceContextOpt, - seenTweetIds, - view.dspClientContext, - view.pushToHomeTweetId - ) - )) - case gql.TimelineKey.HomeLatestTimeline(_) | gql.TimelineKey.HomeLatestTimelineV2(_) => - ( - hm.Product.Following, - hm.ProductContext.Following( - hm.Following(deviceContextOpt, seenTweetIds, view.dspClientContext))) - case gql.TimelineKey.CreatorSubscriptionsTimeline(_) => - ( - hm.Product.Subscribed, - hm.ProductContext.Subscribed(hm.Subscribed(deviceContextOpt, seenTweetIds))) - case _ => throw new UnsupportedOperationException(s"Unknown product: $key") - } - - val clientContext = pm.ClientContext( - userId = CallContext.twitterUserId, - guestId = CallContext.guestId, - guestIdAds = CallContext.guestIdAds, - guestIdMarketing = CallContext.guestIdMarketing, - appId = CallContext.clientApplicationId, - ipAddress = ipAddress, - userAgent = CallContext.userAgent, - countryCode = CallContext.requestCountryCode, - languageCode = CallContext.requestLanguageCode, - isTwoffice = CallContext.isInternalOrTwoffice, - userRoles = roles, - deviceId = CallContext.deviceId, - mobileDeviceId = CallContext.mobileDeviceId, - mobileDeviceAdId = CallContext.adId, - limitAdTracking = CallContext.limitAdTracking - ) - - hm.HomeMixerRequest( - clientContext = clientContext, - product = product, - productContext = Some(productContext), - maxResults = Try(view.count.get.toInt).toOption.orElse(HomeMixerColumn.MaxCount), - cursor = view.cursor.filter(_.nonEmpty) - ) - } - } - - override val fetch: Arrow[(Key, View), Result[Value]] = { - val transformThriftIntoPipelineRequest: Arrow[ - (Key, View), - ProductPipelineRequest[HomeMixerRequest] - ] = { - Arrow - .identity[(Key, View)] - .andThen { - createHomeMixerRequestArrow(compositeOnUserClientColumn, auditIpClientColumn) - } - .map { - case thriftRequest => - val request = homeMixerRequestUnmarshaller(thriftRequest) - val params = paramsBuilder.build( - clientContext = request.clientContext, - product = request.product, - featureOverrides = - request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty), - ) - ProductPipelineRequest(request, params) - } - } - - val underlyingProduct: Arrow[ - ProductPipelineRequest[HomeMixerRequest], - ProductPipelineResult[tr.TimelineResponse] - ] = Arrow - .identity[ProductPipelineRequest[HomeMixerRequest]] - .map { pipelineRequest => - val pipelineArrow = productPipelineRegistry - .getProductPipeline[HomeMixerRequest, tr.TimelineResponse]( - pipelineRequest.request.product) - .arrow - (pipelineArrow, pipelineRequest) - }.applyArrow - - transformThriftIntoPipelineRequest.andThen(underlyingProduct).map { - _.result match { - case Some(result) => found(result.timeline) - case _ => missing - } - } - } -} - -object HomeMixerColumn { - val Path = "home-mixer/homeMixer.Timeline" - private val FetchCallstack = s"$Path:fetch" - private val MaxCount: Option[Int] = Some(100) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/BUILD.bazel deleted file mode 100644 index a40bd4030..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/BUILD.bazel +++ /dev/null @@ -1,19 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finagle/finagle-memcached/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/thrift/com/twitter/search:earlybird-scala", - "stitch/stitch-timelineservice/src/main/scala", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/EarlybirdCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/EarlybirdCandidateSource.scala deleted file mode 100644 index 2ddd05814..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/EarlybirdCandidateSource.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.twitter.home_mixer.functional_component.candidate_source - -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSourceWithExtractedFeatures -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidatesWithSourceFeatures -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.search.earlybird.{thriftscala => t} -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -case object EarlybirdResponseTruncatedFeature - extends FeatureWithDefaultOnFailure[t.EarlybirdRequest, Boolean] { - override val defaultValue: Boolean = false -} - -case object EarlybirdBottomTweetFeature - extends FeatureWithDefaultOnFailure[t.EarlybirdRequest, Option[Long]] { - override val defaultValue: Option[Long] = None -} - -@Singleton -case class EarlybirdCandidateSource @Inject() ( - earlybird: t.EarlybirdService.MethodPerEndpoint) - extends CandidateSourceWithExtractedFeatures[t.EarlybirdRequest, t.ThriftSearchResult] { - - override val identifier = CandidateSourceIdentifier("Earlybird") - - override def apply( - request: t.EarlybirdRequest - ): Stitch[CandidatesWithSourceFeatures[t.ThriftSearchResult]] = { - Stitch.callFuture(earlybird.search(request)).map { response => - val candidates = response.searchResults.map(_.results).getOrElse(Seq.empty) - - val features = FeatureMapBuilder() - .add(EarlybirdResponseTruncatedFeature, candidates.size == request.searchQuery.numResults) - .add(EarlybirdBottomTweetFeature, candidates.lastOption.map(_.id)) - .build() - - CandidatesWithSourceFeatures(candidates, features) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/StaleTweetsCacheCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/StaleTweetsCacheCandidateSource.scala deleted file mode 100644 index 9a346129d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/StaleTweetsCacheCandidateSource.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.home_mixer.functional_component.candidate_source - -import com.google.inject.name.Named -import com.twitter.finagle.memcached.{Client => MemcachedClient} -import com.twitter.home_mixer.param.HomeMixerInjectionNames.StaleTweetsCache -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class StaleTweetsCacheCandidateSource @Inject() ( - @Named(StaleTweetsCache) staleTweetsCache: MemcachedClient) - extends CandidateSource[Seq[Long], Long] { - - override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("StaleTweetsCache") - - private val StaleTweetsCacheKeyPrefix = "v1_" - - override def apply(request: Seq[Long]): Stitch[Seq[Long]] = { - val keys = request.map(StaleTweetsCacheKeyPrefix + _) - - Stitch.callFuture(staleTweetsCache.get(keys).map { tweets => - tweets.map { - case (k, _) => k.replaceFirst(StaleTweetsCacheKeyPrefix, "").toLong - }.toSeq - }) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/BUILD.bazel deleted file mode 100644 index 4df69c9f9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/BUILD.bazel +++ /dev/null @@ -1,28 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "finagle/finagle-core/src/main", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/scala/com/twitter/suggests/controller_data", - "src/thrift/com/twitter/suggests/controller_data:controller_data-scala", - "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala", - "stringcenter/client", - "stringcenter/client/src/main/java", - "timelines/src/main/scala/com/twitter/timelines/injection/scribe", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeConversationServiceCandidateDecorator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeConversationServiceCandidateDecorator.scala deleted file mode 100644 index d145391d4..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeConversationServiceCandidateDecorator.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator - -import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder -import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder -import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.StaticModuleDisplayTypeBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.VerticalConversation -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.injection.scribe.InjectionScribeUtil -import com.twitter.timelineservice.suggests.{thriftscala => st} - -object HomeConversationServiceCandidateDecorator { - - private val ConversationModuleNamespace = EntryNamespace("home-conversation") - - def apply( - homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder - ): Some[UrtMultipleModulesDecorator[PipelineQuery, TweetCandidate, Long]] = { - val suggestType = st.SuggestType.RankedOrganicTweet - val component = InjectionScribeUtil.scribeComponent(suggestType).get - val clientEventInfoBuilder = ClientEventInfoBuilder(component) - val tweetItemBuilder = TweetCandidateUrtItemBuilder( - clientEventInfoBuilder = clientEventInfoBuilder, - timelinesScoreInfoBuilder = Some(HomeTimelinesScoreInfoBuilder), - feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder) - ) - - val moduleBuilder = TimelineModuleBuilder( - entryNamespace = ConversationModuleNamespace, - clientEventInfoBuilder = clientEventInfoBuilder, - displayTypeBuilder = StaticModuleDisplayTypeBuilder(VerticalConversation), - metadataBuilder = Some(HomeConversationModuleMetadataBuilder()) - ) - - Some( - UrtMultipleModulesDecorator( - urtItemCandidateDecorator = UrtItemCandidateDecorator(tweetItemBuilder), - moduleBuilder = moduleBuilder, - groupByKey = (_, _, candidateFeatures) => - candidateFeatures.getOrElse(ConversationModuleFocalTweetIdFeature, None) - )) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeQueryTypePredicates.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeQueryTypePredicates.scala deleted file mode 100644 index a79543269..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeQueryTypePredicates.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator - -import com.twitter.home_mixer.model.HomeFeatures._ -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap - -object HomeQueryTypePredicates { - private[this] val QueryPredicates: Seq[(String, FeatureMap => Boolean)] = Seq( - ("request", _ => true), - ("get_initial", _.getOrElse(GetInitialFeature, false)), - ("get_newer", _.getOrElse(GetNewerFeature, false)), - ("get_older", _.getOrElse(GetOlderFeature, false)), - ("pull_to_refresh", _.getOrElse(PullToRefreshFeature, false)), - ("request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)), - ("request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false)) - ) - - val PredicateMap = QueryPredicates.toMap -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/BUILD.bazel deleted file mode 100644 index cb59d1d73..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/BUILD.bazel +++ /dev/null @@ -1,26 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/twitter/bijection:scrooge", - "finagle/finagle-core/src/main", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "joinkey/src/main/scala/com/twitter/joinkey/context", - "joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/scala/com/twitter/suggests/controller_data", - "src/thrift/com/twitter/suggests/controller_data:controller_data-scala", - "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", - "timelines/src/main/scala/com/twitter/timelines/injection/scribe", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeAdsClientEventDetailsBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeAdsClientEventDetailsBuilder.scala deleted file mode 100644 index eb072f135..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeAdsClientEventDetailsBuilder.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.builder - -import com.twitter.finagle.tracing.Trace -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventDetailsBuilder -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TimelinesDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.suggests.controller_data.home_tweets.v1.{thriftscala => v1ht} -import com.twitter.suggests.controller_data.home_tweets.{thriftscala => ht} -import com.twitter.suggests.controller_data.thriftscala.ControllerData -import com.twitter.suggests.controller_data.v2.thriftscala.{ControllerData => ControllerDataV2} - -case class HomeAdsClientEventDetailsBuilder(injectionType: Option[String]) - extends BaseClientEventDetailsBuilder[PipelineQuery, UniversalNoun[Any]] { - - override def apply( - query: PipelineQuery, - candidate: UniversalNoun[Any], - candidateFeatures: FeatureMap - ): Option[ClientEventDetails] = { - val homeTweetsControllerDataV1 = v1ht.HomeTweetsControllerData( - tweetTypesBitmap = 0L, - traceId = Some(Trace.id.traceId.toLong), - requestJoinId = None) - - val serializedControllerData = HomeClientEventDetailsBuilder.ControllerDataSerializer( - ControllerData.V2( - ControllerDataV2.HomeTweets(ht.HomeTweetsControllerData.V1(homeTweetsControllerDataV1)))) - - val clientEventDetails = ClientEventDetails( - conversationDetails = None, - timelinesDetails = Some( - TimelinesDetails( - injectionType = injectionType, - controllerData = Some(serializedControllerData), - sourceData = None)), - articleDetails = None, - liveEventDetails = None, - commerceDetails = None - ) - - Some(clientEventDetails) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventDetailsBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventDetailsBuilder.scala deleted file mode 100644 index 2e4c60d25..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventDetailsBuilder.scala +++ /dev/null @@ -1,92 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.builder - -import com.twitter.bijection.Base64String -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.bijection.{Injection => Serializer} -import com.twitter.finagle.tracing.Trace -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature -import com.twitter.home_mixer.model.HomeFeatures.PositionFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.joinkey.context.RequestJoinKeyContext -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventDetailsBuilder -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TimelinesDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.suggests.controller_data.Home -import com.twitter.suggests.controller_data.TweetTypeGenerator -import com.twitter.suggests.controller_data.home_tweets.v1.{thriftscala => v1ht} -import com.twitter.suggests.controller_data.home_tweets.{thriftscala => ht} -import com.twitter.suggests.controller_data.thriftscala.ControllerData -import com.twitter.suggests.controller_data.v2.thriftscala.{ControllerData => ControllerDataV2} - -object HomeClientEventDetailsBuilder { - implicit val ByteSerializer: Serializer[ControllerData, Array[Byte]] = - BinaryScalaCodec(ControllerData) - - val ControllerDataSerializer: Serializer[ControllerData, String] = - Serializer.connect[ControllerData, Array[Byte], Base64String, String] - - /** - * define getRequestJoinId as a method(def) rather than a val because each new request - * needs to call the context to update the id. - */ - private def getRequestJoinId(): Option[Long] = - RequestJoinKeyContext.current.flatMap(_.requestJoinId) -} - -case class HomeClientEventDetailsBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( -) extends BaseClientEventDetailsBuilder[Query, Candidate] - with TweetTypeGenerator[FeatureMap] { - - import HomeClientEventDetailsBuilder._ - - override def apply( - query: Query, - candidate: Candidate, - candidateFeatures: FeatureMap - ): Option[ClientEventDetails] = { - - val tweetTypesBitmaps = mkTweetTypesBitmaps( - Home.TweetTypeIdxMap, - HomeTweetTypePredicates.PredicateMap, - candidateFeatures) - - val tweetTypesListBytes = mkItemTypesBitmapsV2( - Home.TweetTypeIdxMap, - HomeTweetTypePredicates.PredicateMap, - candidateFeatures) - - val candidateSourceId = - candidateFeatures.getOrElse(CandidateSourceIdFeature, None).map(_.value.toByte) - - val homeTweetsControllerDataV1 = v1ht.HomeTweetsControllerData( - tweetTypesBitmap = tweetTypesBitmaps.getOrElse(0, 0L), - tweetTypesBitmapContinued1 = tweetTypesBitmaps.get(1), - candidateTweetSourceId = candidateSourceId, - traceId = Some(Trace.id.traceId.toLong), - injectedPosition = candidateFeatures.getOrElse(PositionFeature, None), - tweetTypesListBytes = Some(tweetTypesListBytes), - requestJoinId = getRequestJoinId(), - ) - - val serializedControllerData = ControllerDataSerializer( - ControllerData.V2( - ControllerDataV2.HomeTweets(ht.HomeTweetsControllerData.V1(homeTweetsControllerDataV1)))) - - val clientEventDetails = ClientEventDetails( - conversationDetails = None, - timelinesDetails = Some( - TimelinesDetails( - injectionType = candidateFeatures.getOrElse(SuggestTypeFeature, None).map(_.name), - controllerData = Some(serializedControllerData), - sourceData = None)), - articleDetails = None, - liveEventDetails = None, - commerceDetails = None - ) - - Some(clientEventDetails) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventInfoBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventInfoBuilder.scala deleted file mode 100644 index f79b0d931..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventInfoBuilder.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.builder - -import com.twitter.home_mixer.model.HomeFeatures.EntityTokenFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventDetailsBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventInfo -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.injection.scribe.InjectionScribeUtil - -/** - * Sets the [[ClientEventInfo]] with the `component` field set to the Suggest Type assigned to each candidate - */ -case class HomeClientEventInfoBuilder[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]( - detailsBuilder: Option[BaseClientEventDetailsBuilder[Query, Candidate]] = None) - extends BaseClientEventInfoBuilder[Query, Candidate] { - - override def apply( - query: Query, - candidate: Candidate, - candidateFeatures: FeatureMap, - element: Option[String] - ): Option[ClientEventInfo] = { - val suggestType = candidateFeatures - .getOrElse(SuggestTypeFeature, None) - .getOrElse(throw new UnsupportedOperationException(s"No SuggestType was set")) - - Some( - ClientEventInfo( - component = InjectionScribeUtil.scribeComponent(suggestType), - element = element, - details = detailsBuilder.flatMap(_.apply(query, candidate, candidateFeatures)), - action = None, - /** - * A backend entity encoded by the Client Entities Encoding Library. - * Placeholder string for now - */ - entityToken = candidateFeatures.getOrElse(EntityTokenFeature, None) - ) - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeConversationModuleMetadataBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeConversationModuleMetadataBuilder.scala deleted file mode 100644 index dc6d51327..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeConversationModuleMetadataBuilder.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.builder - -import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature -import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseModuleMetadataBuilder -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleConversationMetadata -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleMetadata -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class HomeConversationModuleMetadataBuilder[ - -Query <: PipelineQuery, - -Candidate <: BaseTweetCandidate -]() extends BaseModuleMetadataBuilder[Query, Candidate] { - - override def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[Candidate]] - ): ModuleMetadata = ModuleMetadata( - adsMetadata = None, - conversationMetadata = Some( - ModuleConversationMetadata( - allTweetIds = Some((candidates.last.candidate.id +: - candidates.last.features.getOrElse(AncestorsFeature, Seq.empty).map(_.tweetId)).reverse), - socialContext = None, - enableDeduplication = Some(true) - )), - gridCarouselMetadata = None - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTimelinesScoreInfoBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTimelinesScoreInfoBuilder.scala deleted file mode 100644 index bc934414c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTimelinesScoreInfoBuilder.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.builder - -import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature -import com.twitter.home_mixer.param.HomeGlobalParams.EnableSendScoresToClient -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.item.tweet.BaseTimelinesScoreInfoBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TimelinesScoreInfo -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object HomeTimelinesScoreInfoBuilder - extends BaseTimelinesScoreInfoBuilder[PipelineQuery, TweetCandidate] { - - private val UndefinedTweetScore = -1.0 - - override def apply( - query: PipelineQuery, - candidate: TweetCandidate, - candidateFeatures: FeatureMap - ): Option[TimelinesScoreInfo] = { - if (query.params(EnableSendScoresToClient)) { - val score = candidateFeatures.getOrElse(ScoreFeature, None).getOrElse(UndefinedTweetScore) - Some(TimelinesScoreInfo(score)) - } else None - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTweetTypePredicates.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTweetTypePredicates.scala deleted file mode 100644 index 7272c360d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTweetTypePredicates.scala +++ /dev/null @@ -1,256 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.builder - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.model.HomeFeatures._ -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType -import com.twitter.timelinemixer.injection.model.candidate.SemanticCoreFeatures -import com.twitter.tweetypie.{thriftscala => tpt} - -object HomeTweetTypePredicates { - - /** - * IMPORTANT: Please avoid logging tweet types that are tied to sensitive - * internal author information / labels (e.g. blink labels, abuse labels, or geo-location). - */ - private[this] val CandidatePredicates: Seq[(String, FeatureMap => Boolean)] = Seq( - ("with_candidate", _ => true), - ("retweet", _.getOrElse(IsRetweetFeature, false)), - ("reply", _.getOrElse(InReplyToTweetIdFeature, None).nonEmpty), - ("image", _.getOrElse(EarlybirdFeature, None).exists(_.hasImage)), - ("video", _.getOrElse(EarlybirdFeature, None).exists(_.hasVideo)), - ("link", _.getOrElse(EarlybirdFeature, None).exists(_.hasVisibleLink)), - ("quote", _.getOrElse(EarlybirdFeature, None).exists(_.hasQuote.contains(true))), - ("like_social_context", _.getOrElse(NonSelfFavoritedByUserIdsFeature, Seq.empty).nonEmpty), - ("protected", _.getOrElse(EarlybirdFeature, None).exists(_.isProtected)), - ( - "has_exclusive_conversation_author_id", - _.getOrElse(ExclusiveConversationAuthorIdFeature, None).nonEmpty), - ("is_eligible_for_connect_boost", _ => false), - ("hashtag", _.getOrElse(EarlybirdFeature, None).exists(_.numHashtags > 0)), - ("has_scheduled_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isScheduled)), - ("has_recorded_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isRecorded)), - ("is_read_from_cache", _.getOrElse(IsReadFromCacheFeature, false)), - ("get_initial", _.getOrElse(GetInitialFeature, false)), - ("get_newer", _.getOrElse(GetNewerFeature, false)), - ("get_middle", _.getOrElse(GetMiddleFeature, false)), - ("get_older", _.getOrElse(GetOlderFeature, false)), - ("pull_to_refresh", _.getOrElse(PullToRefreshFeature, false)), - ("polling", _.getOrElse(PollingFeature, false)), - ("near_empty", _.getOrElse(ServedSizeFeature, None).exists(_ < 3)), - ("is_request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)), - ("mutual_follow", _.getOrElse(EarlybirdFeature, None).exists(_.fromMutualFollow)), - ( - "less_than_10_mins_since_lnpt", - _.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 10.minutes)), - ("served_in_conversation_module", _.getOrElse(ServedInConversationModuleFeature, false)), - ("has_ticketed_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasTickets)), - ("in_utis_top5", _.getOrElse(PositionFeature, None).exists(_ < 5)), - ( - "conversation_module_has_2_displayed_tweets", - _.getOrElse(ConversationModule2DisplayedTweetsFeature, false)), - ("empty_request", _.getOrElse(ServedSizeFeature, None).exists(_ == 0)), - ("served_size_less_than_50", _.getOrElse(ServedSizeFeature, None).exists(_ < 50)), - ( - "served_size_between_50_and_100", - _.getOrElse(ServedSizeFeature, None).exists(size => size >= 50 && size < 100)), - ("authored_by_contextual_user", _.getOrElse(AuthoredByContextualUserFeature, false)), - ( - "is_self_thread_tweet", - _.getOrElse(ConversationFeature, None).exists(_.isSelfThreadTweet.contains(true))), - ("has_ancestors", _.getOrElse(AncestorsFeature, Seq.empty).nonEmpty), - ("full_scoring_succeeded", _.getOrElse(FullScoringSucceededFeature, false)), - ("served_size_less_than_20", _.getOrElse(ServedSizeFeature, None).exists(_ < 20)), - ("served_size_less_than_10", _.getOrElse(ServedSizeFeature, None).exists(_ < 10)), - ("served_size_less_than_5", _.getOrElse(ServedSizeFeature, None).exists(_ < 5)), - ( - "account_age_less_than_30_minutes", - _.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)), - ("conversation_module_has_gap", _.getOrElse(ConversationModuleHasGapFeature, false)), - ( - "directed_at_user_is_in_first_degree", - _.getOrElse(EarlybirdFeature, None).exists(_.directedAtUserIdIsInFirstDegree.contains(true))), - ( - "has_semantic_core_annotation", - _.getOrElse(EarlybirdFeature, None).exists(_.semanticCoreAnnotations.nonEmpty)), - ("is_request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false)), - ( - "account_age_less_than_1_day", - _.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 1.day)), - ( - "account_age_less_than_7_days", - _.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 7.days)), - ( - "part_of_utt", - _.getOrElse(EarlybirdFeature, None) - .exists(_.semanticCoreAnnotations.exists(_.exists(annotation => - annotation.domainId == SemanticCoreFeatures.UnifiedTwitterTaxonomy)))), - ( - "has_home_latest_request_past_week", - _.getOrElse(FollowingLastNonPollingTimeFeature, None).exists(_.untilNow < 7.days)), - ("is_utis_pos0", _.getOrElse(PositionFeature, None).exists(_ == 0)), - ("is_utis_pos1", _.getOrElse(PositionFeature, None).exists(_ == 1)), - ("is_utis_pos2", _.getOrElse(PositionFeature, None).exists(_ == 2)), - ("is_utis_pos3", _.getOrElse(PositionFeature, None).exists(_ == 3)), - ("is_utis_pos4", _.getOrElse(PositionFeature, None).exists(_ == 4)), - ("is_random_tweet", _.getOrElse(IsRandomTweetFeature, false)), - ("has_random_tweet_in_response", _.getOrElse(HasRandomTweetFeature, false)), - ("is_random_tweet_above_in_utis", _.getOrElse(IsRandomTweetAboveFeature, false)), - ( - "has_ancestor_authored_by_viewer", - candidate => - candidate - .getOrElse(AncestorsFeature, Seq.empty).exists(ancestor => - candidate.getOrElse(ViewerIdFeature, 0L) == ancestor.userId)), - ("ancestor", _.getOrElse(IsAncestorCandidateFeature, false)), - ( - "deep_reply", - candidate => - candidate.getOrElse(InReplyToTweetIdFeature, None).nonEmpty && candidate - .getOrElse(AncestorsFeature, Seq.empty).size > 2), - ( - "has_simcluster_embeddings", - _.getOrElse( - SimclustersTweetTopKClustersWithScoresFeature, - Map.empty[String, Double]).nonEmpty), - ( - "tweet_age_less_than_15_seconds", - _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None) - .exists(_.untilNow <= 15.seconds)), - ( - "less_than_1_hour_since_lnpt", - _.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 1.hour)), - ("has_gte_10_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 10))), - ( - "device_language_matches_tweet_language", - candidate => - candidate.getOrElse(TweetLanguageFeature, None) == - candidate.getOrElse(DeviceLanguageFeature, None)), - ( - "root_ancestor", - candidate => - candidate.getOrElse(IsAncestorCandidateFeature, false) && candidate - .getOrElse(InReplyToTweetIdFeature, None).isEmpty), - ("question", _.getOrElse(EarlybirdFeature, None).exists(_.hasQuestion.contains(true))), - ("in_network", _.getOrElse(InNetworkFeature, true)), - ( - "has_political_annotation", - _.getOrElse(EarlybirdFeature, None).exists( - _.semanticCoreAnnotations.exists( - _.exists(annotation => - SemanticCoreFeatures.PoliticalDomains.contains(annotation.domainId) || - (annotation.domainId == SemanticCoreFeatures.UnifiedTwitterTaxonomy && - annotation.entityId == SemanticCoreFeatures.UttPoliticsEntityId))))), - ( - "is_dont_at_me_by_invitation", - _.getOrElse(EarlybirdFeature, None).exists( - _.conversationControl.exists(_.isInstanceOf[tpt.ConversationControl.ByInvitation]))), - ( - "is_dont_at_me_community", - _.getOrElse(EarlybirdFeature, None) - .exists(_.conversationControl.exists(_.isInstanceOf[tpt.ConversationControl.Community]))), - ("has_zero_score", _.getOrElse(ScoreFeature, None).exists(_ == 0.0)), - ( - "is_followed_topic_tweet", - _.getOrElse(TopicContextFunctionalityTypeFeature, None) - .exists(_ == BasicTopicContextFunctionalityType)), - ( - "is_recommended_topic_tweet", - _.getOrElse(TopicContextFunctionalityTypeFeature, None) - .exists(_ == RecommendationTopicContextFunctionalityType)), - ("has_gte_100_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100))), - ("has_gte_1k_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 1000))), - ( - "has_gte_10k_favs", - _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 10000))), - ( - "has_gte_100k_favs", - _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100000))), - ("has_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasSpace)), - ("has_live_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isLive)), - ( - "has_gte_10_retweets", - _.getOrElse(EarlybirdFeature, None).exists(_.retweetCountV2.exists(_ >= 10))), - ( - "has_gte_100_retweets", - _.getOrElse(EarlybirdFeature, None).exists(_.retweetCountV2.exists(_ >= 100))), - ( - "has_gte_1k_retweets", - _.getOrElse(EarlybirdFeature, None).exists(_.retweetCountV2.exists(_ >= 1000))), - ( - "has_us_political_annotation", - _.getOrElse(EarlybirdFeature, None) - .exists(_.semanticCoreAnnotations.exists(_.exists(annotation => - annotation.domainId == SemanticCoreFeatures.UnifiedTwitterTaxonomy && - annotation.entityId == SemanticCoreFeatures.usPoliticalTweetEntityId && - annotation.groupId == SemanticCoreFeatures.UsPoliticalTweetAnnotationGroupIds.BalancedV0)))), - ( - "has_toxicity_score_above_threshold", - _.getOrElse(EarlybirdFeature, None).exists(_.toxicityScore.exists(_ > 0.91))), - ("is_topic_tweet", _.getOrElse(TopicIdSocialContextFeature, None).isDefined), - ( - "text_only", - candidate => - candidate.getOrElse(HasDisplayedTextFeature, false) && - !(candidate.getOrElse(EarlybirdFeature, None).exists(_.hasImage) || - candidate.getOrElse(EarlybirdFeature, None).exists(_.hasVideo) || - candidate.getOrElse(EarlybirdFeature, None).exists(_.hasCard))), - ( - "image_only", - candidate => - candidate.getOrElse(EarlybirdFeature, None).exists(_.hasImage) && - !candidate.getOrElse(HasDisplayedTextFeature, false)), - ("has_1_image", _.getOrElse(NumImagesFeature, None).exists(_ == 1)), - ("has_2_images", _.getOrElse(NumImagesFeature, None).exists(_ == 2)), - ("has_3_images", _.getOrElse(NumImagesFeature, None).exists(_ == 3)), - ("has_4_images", _.getOrElse(NumImagesFeature, None).exists(_ == 4)), - ("has_card", _.getOrElse(EarlybirdFeature, None).exists(_.hasCard)), - ("user_follow_count_gte_50", _.getOrElse(UserFollowingCountFeature, None).exists(_ > 50)), - ( - "has_liked_by_social_context", - candidateFeatures => - candidateFeatures - .getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty) - .exists(candidateFeatures - .getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Seq.empty).toSet.contains)), - ( - "has_followed_by_social_context", - _.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty).nonEmpty), - ( - "has_topic_social_context", - candidateFeatures => - candidateFeatures - .getOrElse(TopicIdSocialContextFeature, None) - .isDefined && - candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None).isDefined), - ("video_lte_10_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ <= 10000)), - ( - "video_bt_10_60_sec", - _.getOrElse(VideoDurationMsFeature, None).exists(duration => - duration > 10000 && duration <= 60000)), - ("video_gt_60_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ > 60000)), - ( - "tweet_age_lte_30_minutes", - _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None) - .exists(_.untilNow <= 30.minutes)), - ( - "tweet_age_lte_1_hour", - _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None) - .exists(_.untilNow <= 1.hour)), - ( - "tweet_age_lte_6_hours", - _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None) - .exists(_.untilNow <= 6.hours)), - ( - "tweet_age_lte_12_hours", - _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None) - .exists(_.untilNow <= 12.hours)), - ( - "tweet_age_gte_24_hours", - _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None) - .exists(_.untilNow >= 24.hours)), - ) - - val PredicateMap = CandidatePredicates.toMap -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/ListClientEventDetailsBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/ListClientEventDetailsBuilder.scala deleted file mode 100644 index cfffee0ca..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/ListClientEventDetailsBuilder.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.builder - -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventDetailsBuilder -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TimelinesDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelineservice.suggests.{thriftscala => st} - -case class ListClientEventDetailsBuilder(suggestType: st.SuggestType) - extends BaseClientEventDetailsBuilder[PipelineQuery, UniversalNoun[Any]] { - - override def apply( - query: PipelineQuery, - candidate: UniversalNoun[Any], - candidateFeatures: FeatureMap - ): Option[ClientEventDetails] = { - val clientEventDetails = ClientEventDetails( - conversationDetails = None, - timelinesDetails = Some( - TimelinesDetails( - injectionType = Some(suggestType.name), - controllerData = None, - sourceData = None)), - articleDetails = None, - liveEventDetails = None, - commerceDetails = None - ) - - Some(clientEventDetails) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AddEntriesWithReplaceAndShowAlertAndShowCoverInstructionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AddEntriesWithReplaceAndShowAlertAndShowCoverInstructionBuilder.scala deleted file mode 100644 index dc3080eb5..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AddEntriesWithReplaceAndShowAlertAndShowCoverInstructionBuilder.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.AlwaysInclude -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.IncludeInstruction -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtInstructionBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.AddEntriesTimelineInstruction -import com.twitter.product_mixer.core.model.marshalling.response.urt.Cover -import com.twitter.product_mixer.core.model.marshalling.response.urt.ShowAlert -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder[Query <: PipelineQuery]( - override val includeInstruction: IncludeInstruction[Query] = AlwaysInclude) - extends UrtInstructionBuilder[Query, AddEntriesTimelineInstruction] { - - override def build( - query: Query, - entries: Seq[TimelineEntry] - ): Seq[AddEntriesTimelineInstruction] = { - if (includeInstruction(query, entries)) { - val entriesToAdd = entries - .filterNot(_.isInstanceOf[ShowAlert]) - .filterNot(_.isInstanceOf[Cover]) - .filter(_.entryIdToReplace.isEmpty) - if (entriesToAdd.nonEmpty) Seq(AddEntriesTimelineInstruction(entriesToAdd)) - else Seq.empty - } else - Seq.empty - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AuthorChildFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AuthorChildFeedbackActionBuilder.scala deleted file mode 100644 index dee273f5f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AuthorChildFeedbackActionBuilder.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import com.twitter.timelines.service.{thriftscala => t} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class AuthorChildFeedbackActionBuilder @Inject() ( - @ProductScoped stringCenter: StringCenter, - externalStrings: HomeMixerExternalStrings) { - - def apply(candidateFeatures: FeatureMap): Option[ChildFeedbackAction] = { - CandidatesUtil.getOriginalAuthorId(candidateFeatures).flatMap { authorId => - FeedbackUtil.buildUserSeeFewerChildFeedbackAction( - userId = authorId, - namesByUserId = candidateFeatures.getOrElse(ScreenNamesFeature, Map.empty[Long, String]), - promptExternalString = externalStrings.showFewerTweetsString, - confirmationExternalString = externalStrings.showFewerTweetsConfirmationString, - engagementType = t.FeedbackEngagementType.Tweet, - stringCenter = stringCenter, - injectionType = candidateFeatures.getOrElse(SuggestTypeFeature, None) - ) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BUILD.bazel deleted file mode 100644 index 3a4f19592..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BUILD.bazel +++ /dev/null @@ -1,19 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", - "src/thrift/com/twitter/timelines/service:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BlockUserChildFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BlockUserChildFeedbackActionBuilder.scala deleted file mode 100644 index a23b57dee..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BlockUserChildFeedbackActionBuilder.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.response.urt.icon -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BottomSheet -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichBehavior -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorBlockUser -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class BlockUserChildFeedbackActionBuilder @Inject() ( - @ProductScoped stringCenter: StringCenter, - externalStrings: HomeMixerExternalStrings) { - - def apply(candidateFeatures: FeatureMap): Option[ChildFeedbackAction] = { - val userIdOpt = - if (candidateFeatures.getOrElse(IsRetweetFeature, false)) - candidateFeatures.getOrElse(SourceUserIdFeature, None) - else candidateFeatures.getOrElse(AuthorIdFeature, None) - - userIdOpt.flatMap { userId => - val screenNamesMap = candidateFeatures.getOrElse(ScreenNamesFeature, Map.empty[Long, String]) - val userScreenNameOpt = screenNamesMap.get(userId) - userScreenNameOpt.map { userScreenName => - val prompt = stringCenter.prepare( - externalStrings.blockUserString, - Map("username" -> userScreenName) - ) - ChildFeedbackAction( - feedbackType = RichBehavior, - prompt = Some(prompt), - confirmation = None, - feedbackUrl = None, - hasUndoAction = Some(true), - confirmationDisplayType = Some(BottomSheet), - clientEventInfo = None, - icon = Some(icon.No), - richBehavior = Some(RichFeedbackBehaviorBlockUser(userId)), - subprompt = None - ) - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/DontLikeFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/DontLikeFeedbackActionBuilder.scala deleted file mode 100644 index 9032b53d9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/DontLikeFeedbackActionBuilder.scala +++ /dev/null @@ -1,87 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.param.HomeGlobalParams.EnableNahFeedbackInfoParam -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.response.urt.icon.Frown -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.DontLike -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackAction -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import com.twitter.timelines.common.{thriftscala => tlc} -import com.twitter.timelineservice.model.FeedbackInfo -import com.twitter.timelineservice.model.FeedbackMetadata -import com.twitter.timelineservice.{thriftscala => tls} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class DontLikeFeedbackActionBuilder @Inject() ( - @ProductScoped stringCenter: StringCenter, - externalStrings: HomeMixerExternalStrings, - authorChildFeedbackActionBuilder: AuthorChildFeedbackActionBuilder, - retweeterChildFeedbackActionBuilder: RetweeterChildFeedbackActionBuilder, - notRelevantChildFeedbackActionBuilder: NotRelevantChildFeedbackActionBuilder, - unfollowUserChildFeedbackActionBuilder: UnfollowUserChildFeedbackActionBuilder, - muteUserChildFeedbackActionBuilder: MuteUserChildFeedbackActionBuilder, - blockUserChildFeedbackActionBuilder: BlockUserChildFeedbackActionBuilder, - reportTweetChildFeedbackActionBuilder: ReportTweetChildFeedbackActionBuilder) { - - def apply( - query: PipelineQuery, - candidate: TweetCandidate, - candidateFeatures: FeatureMap - ): Option[FeedbackAction] = { - CandidatesUtil.getOriginalAuthorId(candidateFeatures).map { authorId => - val feedbackEntities = Seq( - tlc.FeedbackEntity.TweetId(candidate.id), - tlc.FeedbackEntity.UserId(authorId) - ) - val feedbackMetadata = FeedbackMetadata( - engagementType = None, - entityIds = feedbackEntities, - ttl = Some(30.days) - ) - val feedbackUrl = FeedbackInfo.feedbackUrl( - feedbackType = tls.FeedbackType.DontLike, - feedbackMetadata = feedbackMetadata, - injectionType = candidateFeatures.getOrElse(SuggestTypeFeature, None) - ) - val childFeedbackActions = if (query.params(EnableNahFeedbackInfoParam)) { - Seq( - unfollowUserChildFeedbackActionBuilder(candidateFeatures), - muteUserChildFeedbackActionBuilder(candidateFeatures), - blockUserChildFeedbackActionBuilder(candidateFeatures), - reportTweetChildFeedbackActionBuilder(candidate) - ).flatten - } else { - Seq( - authorChildFeedbackActionBuilder(candidateFeatures), - retweeterChildFeedbackActionBuilder(candidateFeatures), - notRelevantChildFeedbackActionBuilder(candidate, candidateFeatures) - ).flatten - } - - FeedbackAction( - feedbackType = DontLike, - prompt = Some(stringCenter.prepare(externalStrings.dontLikeString)), - confirmation = Some(stringCenter.prepare(externalStrings.dontLikeConfirmationString)), - childFeedbackActions = - if (childFeedbackActions.nonEmpty) Some(childFeedbackActions) else None, - feedbackUrl = Some(feedbackUrl), - hasUndoAction = Some(true), - confirmationDisplayType = None, - clientEventInfo = None, - icon = Some(Frown), - richBehavior = None, - subprompt = None, - encodedFeedbackRequest = None - ) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/EngagerSocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/EngagerSocialContextBuilder.scala deleted file mode 100644 index 950271e4a..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/EngagerSocialContextBuilder.scala +++ /dev/null @@ -1,119 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.RealNamesFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stringcenter.client.StringCenter -import com.twitter.stringcenter.client.core.ExternalString - -private[decorator] case class SocialContextIdAndScreenName( - socialContextId: Long, - screenName: String) - -object EngagerSocialContextBuilder { - private val UserIdRequestParamName = "user_id" - private val DirectInjectionContentSourceRequestParamName = "dis" - private val DirectInjectionIdRequestParamName = "diid" - private val DirectInjectionContentSourceSocialProofUsers = "socialproofusers" - private val SocialProofUrl = "" -} - -case class EngagerSocialContextBuilder( - contextType: GeneralContextType, - stringCenter: StringCenter, - oneUserString: ExternalString, - twoUsersString: ExternalString, - moreUsersString: ExternalString, - timelineTitle: ExternalString) { - import EngagerSocialContextBuilder._ - - def apply( - socialContextIds: Seq[Long], - query: PipelineQuery, - candidateFeatures: FeatureMap - ): Option[SocialContext] = { - val realNames = candidateFeatures.getOrElse(RealNamesFeature, Map.empty[Long, String]) - val validSocialContextIdAndScreenNames = socialContextIds.flatMap { socialContextId => - realNames - .get(socialContextId).map(screenName => - SocialContextIdAndScreenName(socialContextId, screenName)) - } - - validSocialContextIdAndScreenNames match { - case Seq(user) => - val socialContextString = - stringCenter.prepare(oneUserString, Map("user" -> user.screenName)) - Some(mkOneUserSocialContext(socialContextString, user.socialContextId)) - case Seq(firstUser, secondUser) => - val socialContextString = - stringCenter - .prepare( - twoUsersString, - Map("user1" -> firstUser.screenName, "user2" -> secondUser.screenName)) - Some( - mkManyUserSocialContext( - socialContextString, - query.getRequiredUserId, - validSocialContextIdAndScreenNames.map(_.socialContextId))) - - case firstUser +: otherUsers => - val otherUsersCount = otherUsers.size - val socialContextString = - stringCenter - .prepare( - moreUsersString, - Map("user" -> firstUser.screenName, "count" -> otherUsersCount)) - Some( - mkManyUserSocialContext( - socialContextString, - query.getRequiredUserId, - validSocialContextIdAndScreenNames.map(_.socialContextId))) - case _ => None - } - } - - private def mkOneUserSocialContext(socialContextString: String, userId: Long): GeneralContext = { - GeneralContext( - contextType = contextType, - text = socialContextString, - url = None, - contextImageUrls = None, - landingUrl = Some( - Url( - urlType = DeepLink, - url = "", - urtEndpointOptions = None - ) - ) - ) - } - - private def mkManyUserSocialContext( - socialContextString: String, - viewerId: Long, - socialContextIds: Seq[Long] - ): GeneralContext = { - GeneralContext( - contextType = contextType, - text = socialContextString, - url = None, - contextImageUrls = None, - landingUrl = Some( - Url( - urlType = UrtEndpoint, - url = SocialProofUrl, - urtEndpointOptions = Some(UrtEndpointOptions( - requestParams = Some(Map( - UserIdRequestParamName -> viewerId.toString, - DirectInjectionContentSourceRequestParamName -> DirectInjectionContentSourceSocialProofUsers, - DirectInjectionIdRequestParamName -> socialContextIds.mkString(",") - )), - title = Some(stringCenter.prepare(timelineTitle)), - cacheId = None, - subtitle = None - )) - )) - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ExtendedReplySocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ExtendedReplySocialContextBuilder.scala deleted file mode 100644 index 74576eae8..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ExtendedReplySocialContextBuilder.scala +++ /dev/null @@ -1,78 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.FocalTweetAuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.FocalTweetRealNamesFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton - -/** - * Use '@A replied' when the root tweet is out-of-network and the reply is in network. - * - * This function should only be called for the root Tweet of convo modules. This is enforced by - * [[HomeTweetSocialContextBuilder]]. - */ -@Singleton -case class ExtendedReplySocialContextBuilder @Inject() ( - externalStrings: HomeMixerExternalStrings, - @ProductScoped stringCenterProvider: Provider[StringCenter]) - extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { - - private val stringCenter = stringCenterProvider.get() - private val extendedReplyString = externalStrings.socialContextExtendedReply - - def apply( - query: PipelineQuery, - candidate: TweetCandidate, - candidateFeatures: FeatureMap - ): Option[SocialContext] = { - - // If these values are missing default to not showing an extended reply banner - val inNetworkRoot = candidateFeatures.getOrElse(InNetworkFeature, true) - - val inNetworkFocalTweet = - candidateFeatures.getOrElse(FocalTweetInNetworkFeature, None).getOrElse(false) - - if (!inNetworkRoot && inNetworkFocalTweet) { - - val focalTweetAuthorIdOpt = candidateFeatures.getOrElse(FocalTweetAuthorIdFeature, None) - val focalTweetRealNames = - candidateFeatures - .getOrElse(FocalTweetRealNamesFeature, None).getOrElse(Map.empty[Long, String]) - val focalTweetAuthorNameOpt = focalTweetAuthorIdOpt.flatMap(focalTweetRealNames.get) - - (focalTweetAuthorIdOpt, focalTweetAuthorNameOpt) match { - case (Some(focalTweetAuthorId), Some(focalTweetAuthorName)) => - Some( - GeneralContext( - contextType = ConversationGeneralContextType, - text = stringCenter - .prepare(extendedReplyString, placeholders = Map("user1" -> focalTweetAuthorName)), - url = None, - contextImageUrls = None, - landingUrl = Some( - Url( - urlType = DeepLink, - url = "", - urtEndpointOptions = None - )) - )) - case _ => - None - } - } else { - None - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackStrings.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackStrings.scala deleted file mode 100644 index 8a9c214aa..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackStrings.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.ExternalStringRegistry -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton - -@Singleton -class FeedbackStrings @Inject() ( - @ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry]) { - private val externalStringRegistry = externalStringRegistryProvider.get() - - val seeLessOftenFeedbackString = - externalStringRegistry.createProdString("Feedback.seeLessOften") - val seeLessOftenConfirmationFeedbackString = - externalStringRegistry.createProdString("Feedback.seeLessOftenConfirmation") -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackUtil.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackUtil.scala deleted file mode 100644 index 08534f26e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackUtil.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.conversions.DurationOps._ -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SeeFewer -import com.twitter.stringcenter.client.StringCenter -import com.twitter.stringcenter.client.core.ExternalString -import com.twitter.timelines.common.{thriftscala => tlc} -import com.twitter.timelines.service.{thriftscala => t} -import com.twitter.timelineservice.model.FeedbackInfo -import com.twitter.timelineservice.model.FeedbackMetadata -import com.twitter.timelineservice.suggests.{thriftscala => st} -import com.twitter.timelineservice.{thriftscala => tlst} - -object FeedbackUtil { - - val FeedbackTtl = 30.days - - def buildUserSeeFewerChildFeedbackAction( - userId: Long, - namesByUserId: Map[Long, String], - promptExternalString: ExternalString, - confirmationExternalString: ExternalString, - engagementType: t.FeedbackEngagementType, - stringCenter: StringCenter, - injectionType: Option[st.SuggestType] - ): Option[ChildFeedbackAction] = { - namesByUserId.get(userId).map { userScreenName => - val prompt = stringCenter.prepare( - promptExternalString, - Map("user" -> userScreenName) - ) - val confirmation = stringCenter.prepare( - confirmationExternalString, - Map("user" -> userScreenName) - ) - val feedbackMetadata = FeedbackMetadata( - engagementType = Some(engagementType), - entityIds = Seq(tlc.FeedbackEntity.UserId(userId)), - ttl = Some(FeedbackTtl)) - val feedbackUrl = FeedbackInfo.feedbackUrl( - feedbackType = tlst.FeedbackType.SeeFewer, - feedbackMetadata = feedbackMetadata, - injectionType = injectionType - ) - - ChildFeedbackAction( - feedbackType = SeeFewer, - prompt = Some(prompt), - confirmation = Some(confirmation), - feedbackUrl = Some(feedbackUrl), - hasUndoAction = Some(true), - confirmationDisplayType = None, - clientEventInfo = None, - icon = None, - richBehavior = None, - subprompt = None - ) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FollowedBySocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FollowedBySocialContextBuilder.scala deleted file mode 100644 index 7554f7b36..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FollowedBySocialContextBuilder.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton - -@Singleton -case class FollowedBySocialContextBuilder @Inject() ( - externalStrings: HomeMixerExternalStrings, - @ProductScoped stringCenterProvider: Provider[StringCenter]) - extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { - - private val stringCenter = stringCenterProvider.get() - - private val engagerSocialContextBuilder = EngagerSocialContextBuilder( - contextType = FollowGeneralContextType, - stringCenter = stringCenter, - oneUserString = externalStrings.socialContextOneUserFollowsString, - twoUsersString = externalStrings.socialContextTwoUsersFollowString, - moreUsersString = externalStrings.socialContextMoreUsersFollowString, - timelineTitle = externalStrings.socialContextFollowedByTimelineTitle - ) - - def apply( - query: PipelineQuery, - candidate: TweetCandidate, - candidateFeatures: FeatureMap - ): Option[SocialContext] = { - // Only apply followed-by social context for OON Tweets - val inNetwork = candidateFeatures.getOrElse(InNetworkFeature, true) - if (!inNetwork) { - val validFollowedByUserIds = - candidateFeatures.getOrElse(SGSValidFollowedByUserIdsFeature, Nil) - engagerSocialContextBuilder( - socialContextIds = validFollowedByUserIds, - query = query, - candidateFeatures = candidateFeatures - ) - } else { - None - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeFeedbackActionInfoBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeFeedbackActionInfoBuilder.scala deleted file mode 100644 index c932f6362..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeFeedbackActionInfoBuilder.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.param.HomeGlobalParams.EnableNahFeedbackInfoParam -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackActionInfo -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.service.{thriftscala => t} -import com.twitter.timelines.util.FeedbackMetadataSerializer -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class HomeFeedbackActionInfoBuilder @Inject() ( - notInterestedTopicFeedbackActionBuilder: NotInterestedTopicFeedbackActionBuilder, - dontLikeFeedbackActionBuilder: DontLikeFeedbackActionBuilder) - extends BaseFeedbackActionInfoBuilder[PipelineQuery, TweetCandidate] { - - override def apply( - query: PipelineQuery, - candidate: TweetCandidate, - candidateFeatures: FeatureMap - ): Option[FeedbackActionInfo] = { - val supportedProduct = query.product match { - case FollowingProduct => query.params(EnableNahFeedbackInfoParam) - case ForYouProduct => true - case _ => false - } - val isAuthoredByViewer = CandidatesUtil.isAuthoredByViewer(query, candidateFeatures) - - if (supportedProduct && !isAuthoredByViewer) { - val feedbackActions = Seq( - notInterestedTopicFeedbackActionBuilder(candidateFeatures), - dontLikeFeedbackActionBuilder(query, candidate, candidateFeatures) - ).flatten - val feedbackMetadata = FeedbackMetadataSerializer.serialize( - t.FeedbackMetadata(injectionType = candidateFeatures.getOrElse(SuggestTypeFeature, None))) - - Some( - FeedbackActionInfo( - feedbackActions = feedbackActions, - feedbackMetadata = Some(feedbackMetadata), - displayContext = None, - clientEventInfo = None - )) - } else None - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeTweetSocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeTweetSocialContextBuilder.scala deleted file mode 100644 index 5138f254a..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeTweetSocialContextBuilder.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleIdFeature -import com.twitter.home_mixer.param.HomeGlobalParams.EnableSocialContextParam -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class HomeTweetSocialContextBuilder @Inject() ( - likedBySocialContextBuilder: LikedBySocialContextBuilder, - listsSocialContextBuilder: ListsSocialContextBuilder, - followedBySocialContextBuilder: FollowedBySocialContextBuilder, - topicSocialContextBuilder: TopicSocialContextBuilder, - extendedReplySocialContextBuilder: ExtendedReplySocialContextBuilder, - receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder, - popularVideoSocialContextBuilder: PopularVideoSocialContextBuilder, - popularInYourAreaSocialContextBuilder: PopularInYourAreaSocialContextBuilder) - extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { - - def apply( - query: PipelineQuery, - candidate: TweetCandidate, - features: FeatureMap - ): Option[SocialContext] = { - if (query.params(EnableSocialContextParam)) { - features.getOrElse(ConversationModuleFocalTweetIdFeature, None) match { - case None => - likedBySocialContextBuilder(query, candidate, features) - .orElse(followedBySocialContextBuilder(query, candidate, features)) - .orElse(topicSocialContextBuilder(query, candidate, features)) - .orElse(popularVideoSocialContextBuilder(query, candidate, features)) - .orElse(listsSocialContextBuilder(query, candidate, features)) - .orElse(popularInYourAreaSocialContextBuilder(query, candidate, features)) - case Some(_) => - val conversationId = features.getOrElse(ConversationModuleIdFeature, None) - // Only hydrate the social context into the root tweet in a conversation module - if (conversationId.contains(candidate.id)) { - extendedReplySocialContextBuilder(query, candidate, features) - .orElse(receivedReplySocialContextBuilder(query, candidate, features)) - } else None - } - } else None - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToFollowFeedbackActionInfoBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToFollowFeedbackActionInfoBuilder.scala deleted file mode 100644 index 49017e8cf..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToFollowFeedbackActionInfoBuilder.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.WhoToFollowFeedbackActionInfoBuilder -import com.twitter.product_mixer.component_library.model.candidate.UserCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackActionInfo -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.service.{thriftscala => tl} -import com.twitter.timelines.util.FeedbackRequestSerializer -import com.twitter.timelineservice.suggests.thriftscala.SuggestType -import com.twitter.timelineservice.thriftscala.FeedbackType - -object HomeWhoToFollowFeedbackActionInfoBuilder { - private val FeedbackMetadata = tl.FeedbackMetadata( - injectionType = Some(SuggestType.WhoToFollow), - engagementType = None, - entityIds = Seq.empty, - ttlMs = None - ) - private val FeedbackRequest = - tl.DefaultFeedbackRequest2(FeedbackType.SeeFewer, FeedbackMetadata) - private val EncodedFeedbackRequest = - FeedbackRequestSerializer.serialize(tl.FeedbackRequest.DefaultFeedbackRequest2(FeedbackRequest)) -} - -@Singleton -case class HomeWhoToFollowFeedbackActionInfoBuilder @Inject() ( - feedbackStrings: FeedbackStrings, - @ProductScoped stringCenterProvider: Provider[StringCenter]) - extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] { - - private val whoToFollowFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder( - seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString, - seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenConfirmationFeedbackString, - stringCenter = stringCenterProvider.get(), - encodedFeedbackRequest = Some(HomeWhoToFollowFeedbackActionInfoBuilder.EncodedFeedbackRequest) - ) - - override def apply( - query: PipelineQuery, - candidate: UserCandidate, - candidateFeatures: FeatureMap - ): Option[FeedbackActionInfo] = - whoToFollowFeedbackActionInfoBuilder.apply(query, candidate, candidateFeatures) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToSubscribeFeedbackActionInfoBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToSubscribeFeedbackActionInfoBuilder.scala deleted file mode 100644 index 270ea66f6..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToSubscribeFeedbackActionInfoBuilder.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.WhoToFollowFeedbackActionInfoBuilder -import com.twitter.product_mixer.component_library.model.candidate.UserCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackActionInfo -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.service.{thriftscala => tl} -import com.twitter.timelines.util.FeedbackRequestSerializer -import com.twitter.timelineservice.suggests.thriftscala.SuggestType -import com.twitter.timelineservice.thriftscala.FeedbackType - -object HomeWhoToSubscribeFeedbackActionInfoBuilder { - private val FeedbackMetadata = tl.FeedbackMetadata( - injectionType = Some(SuggestType.WhoToSubscribe), - engagementType = None, - entityIds = Seq.empty, - ttlMs = None - ) - private val FeedbackRequest = - tl.DefaultFeedbackRequest2(FeedbackType.SeeFewer, FeedbackMetadata) - private val EncodedFeedbackRequest = - FeedbackRequestSerializer.serialize(tl.FeedbackRequest.DefaultFeedbackRequest2(FeedbackRequest)) -} - -@Singleton -case class HomeWhoToSubscribeFeedbackActionInfoBuilder @Inject() ( - feedbackStrings: FeedbackStrings, - @ProductScoped stringCenterProvider: Provider[StringCenter]) - extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] { - - private val whoToSubscribeFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder( - seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString, - seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenConfirmationFeedbackString, - stringCenter = stringCenterProvider.get(), - encodedFeedbackRequest = - Some(HomeWhoToSubscribeFeedbackActionInfoBuilder.EncodedFeedbackRequest) - ) - - override def apply( - query: PipelineQuery, - candidate: UserCandidate, - candidateFeatures: FeatureMap - ): Option[FeedbackActionInfo] = - whoToSubscribeFeedbackActionInfoBuilder.apply(query, candidate, candidateFeatures) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/LikedBySocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/LikedBySocialContextBuilder.scala deleted file mode 100644 index fbc65be13..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/LikedBySocialContextBuilder.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.LikeGeneralContextType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton - -@Singleton -case class LikedBySocialContextBuilder @Inject() ( - externalStrings: HomeMixerExternalStrings, - @ProductScoped stringCenterProvider: Provider[StringCenter]) - extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { - - private val stringCenter = stringCenterProvider.get() - - private val engagerSocialContextBuilder = EngagerSocialContextBuilder( - contextType = LikeGeneralContextType, - stringCenter = stringCenter, - oneUserString = externalStrings.socialContextOneUserLikedString, - twoUsersString = externalStrings.socialContextTwoUsersLikedString, - moreUsersString = externalStrings.socialContextMoreUsersLikedString, - timelineTitle = externalStrings.socialContextLikedByTimelineTitle - ) - - def apply( - query: PipelineQuery, - candidate: TweetCandidate, - candidateFeatures: FeatureMap - ): Option[SocialContext] = { - - // Liked by users are valid only if they pass both the SGS and Perspective filters. - val validLikedByUserIds = - candidateFeatures - .getOrElse(SGSValidLikedByUserIdsFeature, Nil) - .filter( - candidateFeatures.getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Nil).toSet.contains) - - engagerSocialContextBuilder( - socialContextIds = validLikedByUserIds, - query = query, - candidateFeatures = candidateFeatures - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ListsSocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ListsSocialContextBuilder.scala deleted file mode 100644 index e5e746223..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ListsSocialContextBuilder.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.model.HomeFeatures.UserScreenNameFeature -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import com.twitter.timelineservice.suggests.{thriftscala => t} -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton - -/** - * "Your Lists" will be rendered for the context and a url link for your lists. - */ -@Singleton -case class ListsSocialContextBuilder @Inject() ( - externalStrings: HomeMixerExternalStrings, - @ProductScoped stringCenterProvider: Provider[StringCenter]) - extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { - - private val stringCenter = stringCenterProvider.get() - private val listString = externalStrings.ownedSubscribedListsModuleHeaderString - - def apply( - query: PipelineQuery, - candidate: TweetCandidate, - candidateFeatures: FeatureMap - ): Option[SocialContext] = { - candidateFeatures.get(SuggestTypeFeature) match { - case Some(suggestType) if suggestType == t.SuggestType.RankedListTweet => - val userName = query.features.flatMap(_.getOrElse(UserScreenNameFeature, None)) - Some( - GeneralContext( - contextType = ListGeneralContextType, - text = stringCenter.prepare(listString), - url = userName.map(name => ""), - contextImageUrls = None, - landingUrl = None - )) - case _ => None - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/MuteUserChildFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/MuteUserChildFeedbackActionBuilder.scala deleted file mode 100644 index 542ed8a67..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/MuteUserChildFeedbackActionBuilder.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.response.urt.icon -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichBehavior -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleMuteUser -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class MuteUserChildFeedbackActionBuilder @Inject() ( - @ProductScoped stringCenter: StringCenter, - externalStrings: HomeMixerExternalStrings) { - - def apply( - candidateFeatures: FeatureMap - ): Option[ChildFeedbackAction] = { - val userIdOpt = - if (candidateFeatures.getOrElse(IsRetweetFeature, false)) - candidateFeatures.getOrElse(SourceUserIdFeature, None) - else candidateFeatures.getOrElse(AuthorIdFeature, None) - - userIdOpt.flatMap { userId => - val screenNamesMap = candidateFeatures.getOrElse(ScreenNamesFeature, Map.empty[Long, String]) - val userScreenNameOpt = screenNamesMap.get(userId) - userScreenNameOpt.map { userScreenName => - val prompt = stringCenter.prepare( - externalStrings.muteUserString, - Map("username" -> userScreenName) - ) - ChildFeedbackAction( - feedbackType = RichBehavior, - prompt = Some(prompt), - confirmation = None, - feedbackUrl = None, - hasUndoAction = Some(true), - confirmationDisplayType = None, - clientEventInfo = None, - icon = Some(icon.SpeakerOff), - richBehavior = Some(RichFeedbackBehaviorToggleMuteUser(userId)), - subprompt = None - ) - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotInterestedTopicFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotInterestedTopicFeedbackActionBuilder.scala deleted file mode 100644 index 2ec520e11..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotInterestedTopicFeedbackActionBuilder.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature -import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackAction -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecWithEducationTopicContextFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichBehavior -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorMarkNotInterestedTopic -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class NotInterestedTopicFeedbackActionBuilder @Inject() ( - @ProductScoped stringCenter: StringCenter, - externalStrings: HomeMixerExternalStrings) { - - def apply( - candidateFeatures: FeatureMap - ): Option[FeedbackAction] = { - val isOutOfNetwork = !candidateFeatures.getOrElse(InNetworkFeature, true) - val validFollowedByUserIds = - candidateFeatures.getOrElse(SGSValidFollowedByUserIdsFeature, Nil) - val validLikedByUserIds = - candidateFeatures - .getOrElse(SGSValidLikedByUserIdsFeature, Nil) - .filter( - candidateFeatures.getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Nil).toSet.contains) - - if (isOutOfNetwork && validLikedByUserIds.isEmpty && validFollowedByUserIds.isEmpty) { - val topicIdSocialContext = candidateFeatures.getOrElse(TopicIdSocialContextFeature, None) - val topicContextFunctionalityType = - candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None) - - (topicIdSocialContext, topicContextFunctionalityType) match { - case (Some(topicId), Some(topicContextFunctionalityType)) - if topicContextFunctionalityType == RecommendationTopicContextFunctionalityType || - topicContextFunctionalityType == RecWithEducationTopicContextFunctionalityType => - Some( - FeedbackAction( - feedbackType = RichBehavior, - prompt = None, - confirmation = None, - childFeedbackActions = None, - feedbackUrl = None, - hasUndoAction = Some(true), - confirmationDisplayType = None, - clientEventInfo = None, - icon = None, - richBehavior = - Some(RichFeedbackBehaviorMarkNotInterestedTopic(topicId = topicId.toString)), - subprompt = None, - encodedFeedbackRequest = None - ) - ) - case _ => None - } - } else { - None - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotRelevantChildFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotRelevantChildFeedbackActionBuilder.scala deleted file mode 100644 index 39df781ce..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotRelevantChildFeedbackActionBuilder.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.NotRelevant -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import com.twitter.timelines.common.{thriftscala => tlc} -import com.twitter.timelineservice.model.FeedbackInfo -import com.twitter.timelineservice.model.FeedbackMetadata -import com.twitter.timelineservice.{thriftscala => tlst} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class NotRelevantChildFeedbackActionBuilder @Inject() ( - @ProductScoped stringCenter: StringCenter, - externalStrings: HomeMixerExternalStrings) { - - def apply( - candidate: TweetCandidate, - candidateFeatures: FeatureMap - ): Option[ChildFeedbackAction] = { - val prompt = stringCenter.prepare(externalStrings.notRelevantString) - val confirmation = stringCenter.prepare(externalStrings.notRelevantConfirmationString) - val feedbackMetadata = FeedbackMetadata( - engagementType = None, - entityIds = Seq(tlc.FeedbackEntity.TweetId(candidate.id)), - ttl = Some(FeedbackUtil.FeedbackTtl)) - val feedbackUrl = FeedbackInfo.feedbackUrl( - feedbackType = tlst.FeedbackType.NotRelevant, - feedbackMetadata = feedbackMetadata, - injectionType = candidateFeatures.getOrElse(SuggestTypeFeature, None) - ) - - Some( - ChildFeedbackAction( - feedbackType = NotRelevant, - prompt = Some(prompt), - confirmation = Some(confirmation), - feedbackUrl = Some(feedbackUrl), - hasUndoAction = Some(true), - confirmationDisplayType = None, - clientEventInfo = None, - icon = None, - richBehavior = None, - subprompt = None - ) - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularInYourAreaSocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularInYourAreaSocialContextBuilder.scala deleted file mode 100644 index 3782dd115..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularInYourAreaSocialContextBuilder.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import com.twitter.timelineservice.suggests.{thriftscala => st} -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton - -@Singleton -case class PopularInYourAreaSocialContextBuilder @Inject() ( - externalStrings: HomeMixerExternalStrings, - @ProductScoped stringCenterProvider: Provider[StringCenter]) - extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { - - private val stringCenter = stringCenterProvider.get() - private val popularInYourAreaString = externalStrings.socialContextPopularInYourAreaString - - def apply( - query: PipelineQuery, - candidate: TweetCandidate, - candidateFeatures: FeatureMap - ): Option[SocialContext] = { - val suggestTypeOpt = candidateFeatures.getOrElse(SuggestTypeFeature, None) - if (suggestTypeOpt.contains(st.SuggestType.RecommendedTrendTweet)) { - Some( - GeneralContext( - contextType = LocationGeneralContextType, - text = stringCenter.prepare(popularInYourAreaString), - url = None, - contextImageUrls = None, - landingUrl = None - )) - } else None - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularVideoSocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularVideoSocialContextBuilder.scala deleted file mode 100644 index 7eef7f180..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularVideoSocialContextBuilder.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import com.twitter.timelineservice.suggests.{thriftscala => st} -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton - -@Singleton -case class PopularVideoSocialContextBuilder @Inject() ( - externalStrings: HomeMixerExternalStrings, - @ProductScoped stringCenterProvider: Provider[StringCenter]) - extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { - - private val stringCenter = stringCenterProvider.get() - private val popularVideoString = externalStrings.socialContextPopularVideoString - - def apply( - query: PipelineQuery, - candidate: TweetCandidate, - candidateFeatures: FeatureMap - ): Option[SocialContext] = { - val suggestTypeOpt = candidateFeatures.getOrElse(SuggestTypeFeature, None) - if (suggestTypeOpt.contains(st.SuggestType.MediaTweet)) { - Some( - GeneralContext( - contextType = SparkleGeneralContextType, - text = stringCenter.prepare(popularVideoString), - url = None, - contextImageUrls = None, - landingUrl = Some( - Url( - urlType = DeepLink, - url = "" - ) - ) - )) - } else None - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReceivedReplySocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReceivedReplySocialContextBuilder.scala deleted file mode 100644 index f1af07c98..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReceivedReplySocialContextBuilder.scala +++ /dev/null @@ -1,76 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.RealNamesFeature -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton - -/** - * Use '@A received a reply' as social context when the root Tweet is in network and the focal tweet is OON. - * - * This function should only be called for the root Tweet of convo modules. This is enforced by - * [[HomeTweetSocialContextBuilder]]. - */ -@Singleton -case class ReceivedReplySocialContextBuilder @Inject() ( - externalStrings: HomeMixerExternalStrings, - @ProductScoped stringCenterProvider: Provider[StringCenter]) - extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { - - private val stringCenter = stringCenterProvider.get() - private val receivedReplyString = externalStrings.socialContextReceivedReply - - def apply( - query: PipelineQuery, - candidate: TweetCandidate, - candidateFeatures: FeatureMap - ): Option[SocialContext] = { - - // If these values are missing default to not showing a received a reply banner - val inNetwork = candidateFeatures.getOrElse(InNetworkFeature, false) - val inNetworkFocalTweet = - candidateFeatures.getOrElse(FocalTweetInNetworkFeature, None).getOrElse(true) - - if (inNetwork && !inNetworkFocalTweet) { - - val authorIdOpt = candidateFeatures.getOrElse(AuthorIdFeature, None) - val realNames = candidateFeatures.getOrElse(RealNamesFeature, Map.empty[Long, String]) - val authorNameOpt = authorIdOpt.flatMap(realNames.get) - - (authorIdOpt, authorNameOpt) match { - case (Some(authorId), Some(authorName)) => - Some( - GeneralContext( - contextType = ConversationGeneralContextType, - text = stringCenter - .prepare(receivedReplyString, placeholders = Map("user1" -> authorName)), - url = None, - contextImageUrls = None, - landingUrl = Some( - Url( - urlType = DeepLink, - url = "", - urtEndpointOptions = None - ) - ) - ) - ) - case _ => None - } - } else { - None - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReportTweetChildFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReportTweetChildFeedbackActionBuilder.scala deleted file mode 100644 index a4ef0cbb2..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReportTweetChildFeedbackActionBuilder.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.model.marshalling.response.urt.icon -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichBehavior -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorReportTweet -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class ReportTweetChildFeedbackActionBuilder @Inject() ( - @ProductScoped stringCenter: StringCenter, - externalStrings: HomeMixerExternalStrings) { - - def apply( - candidate: TweetCandidate - ): Option[ChildFeedbackAction] = { - Some( - ChildFeedbackAction( - feedbackType = RichBehavior, - prompt = Some(stringCenter.prepare(externalStrings.reportTweetString)), - confirmation = None, - feedbackUrl = None, - hasUndoAction = Some(true), - confirmationDisplayType = None, - clientEventInfo = None, - icon = Some(icon.Flag), - richBehavior = Some(RichFeedbackBehaviorReportTweet(candidate.id)), - subprompt = None - ) - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/RetweeterChildFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/RetweeterChildFeedbackActionBuilder.scala deleted file mode 100644 index 006e93b58..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/RetweeterChildFeedbackActionBuilder.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import com.twitter.timelines.service.{thriftscala => t} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class RetweeterChildFeedbackActionBuilder @Inject() ( - @ProductScoped stringCenter: StringCenter, - externalStrings: HomeMixerExternalStrings) { - - def apply(candidateFeatures: FeatureMap): Option[ChildFeedbackAction] = { - val isRetweet = candidateFeatures.getOrElse(IsRetweetFeature, false) - - if (isRetweet) { - candidateFeatures.getOrElse(AuthorIdFeature, None).flatMap { retweeterId => - FeedbackUtil.buildUserSeeFewerChildFeedbackAction( - userId = retweeterId, - namesByUserId = candidateFeatures.getOrElse(ScreenNamesFeature, Map.empty[Long, String]), - promptExternalString = externalStrings.showFewerRetweetsString, - confirmationExternalString = externalStrings.showFewerRetweetsConfirmationString, - engagementType = t.FeedbackEngagementType.Retweet, - stringCenter = stringCenter, - injectionType = candidateFeatures.getOrElse(SuggestTypeFeature, None) - ) - } - } else None - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/TopicSocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/TopicSocialContextBuilder.scala deleted file mode 100644 index 7d01382c3..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/TopicSocialContextBuilder.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature -import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContext -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class TopicSocialContextBuilder @Inject() () - extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { - - def apply( - query: PipelineQuery, - candidate: TweetCandidate, - candidateFeatures: FeatureMap - ): Option[SocialContext] = { - val inNetwork = candidateFeatures.getOrElse(InNetworkFeature, true) - if (!inNetwork) { - val topicIdSocialContextOpt = candidateFeatures.getOrElse(TopicIdSocialContextFeature, None) - val topicContextFunctionalityTypeOpt = - candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None) - (topicIdSocialContextOpt, topicContextFunctionalityTypeOpt) match { - case (Some(topicId), Some(topicContextFunctionalityType)) => - Some( - TopicContext( - topicId = topicId.toString, - functionalityType = Some(topicContextFunctionalityType) - )) - case _ => None - } - } else { - None - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/UnfollowUserChildFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/UnfollowUserChildFeedbackActionBuilder.scala deleted file mode 100644 index 864ee06d9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/UnfollowUserChildFeedbackActionBuilder.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.response.urt.icon -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichBehavior -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleFollowUser -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class UnfollowUserChildFeedbackActionBuilder @Inject() ( - @ProductScoped stringCenter: StringCenter, - externalStrings: HomeMixerExternalStrings) { - - def apply(candidateFeatures: FeatureMap): Option[ChildFeedbackAction] = { - val isInNetwork = candidateFeatures.getOrElse(InNetworkFeature, false) - val userIdOpt = candidateFeatures.getOrElse(AuthorIdFeature, None) - - if (isInNetwork) { - userIdOpt.flatMap { userId => - val screenNamesMap = - candidateFeatures.getOrElse(ScreenNamesFeature, Map.empty[Long, String]) - val userScreenNameOpt = screenNamesMap.get(userId) - userScreenNameOpt.map { userScreenName => - val prompt = stringCenter.prepare( - externalStrings.unfollowUserString, - Map("username" -> userScreenName) - ) - val confirmation = stringCenter.prepare( - externalStrings.unfollowUserConfirmationString, - Map("username" -> userScreenName) - ) - ChildFeedbackAction( - feedbackType = RichBehavior, - prompt = Some(prompt), - confirmation = Some(confirmation), - feedbackUrl = None, - hasUndoAction = Some(true), - confirmationDisplayType = None, - clientEventInfo = None, - icon = Some(icon.Unfollow), - richBehavior = Some(RichFeedbackBehaviorToggleFollowUser(userId)), - subprompt = None - ) - } - } - } else None - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BUILD.bazel deleted file mode 100644 index 60c104143..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BUILD.bazel +++ /dev/null @@ -1,64 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "configapi/configapi-decider", - "finatra/inject/inject-core/src/main/scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie", - "joinkey/src/main/scala/com/twitter/joinkey/context", - "joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_visibility_reason", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "snowflake/src/main/scala/com/twitter/snowflake/id", - "src/java/com/twitter/search/common/util/lang", - "src/scala/com/twitter/timelines/prediction/adapters/request_context", - "src/thrift/com/twitter/gizmoduck:thrift-scala", - "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", - "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/search/common:constants-java", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "src/thrift/com/twitter/spam/rtf:safety-result-scala", - "src/thrift/com/twitter/timelineranker:thrift-scala", - "src/thrift/com/twitter/timelines/impression:thrift-scala", - "src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala", - "src/thrift/com/twitter/timelines/real_graph:real_graph-scala", - "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", - "src/thrift/com/twitter/tweetypie:service-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - "src/thrift/com/twitter/user_session_store:thrift-java", - "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", - "stitch/stitch-core", - "stitch/stitch-gizmoduck", - "stitch/stitch-socialgraph", - "stitch/stitch-timelineservice", - "stitch/stitch-tweetypie", - "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/feedback", - "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan", - "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/persistence", - "timelines/src/main/scala/com/twitter/timelines/clients/manhattan/store", - "timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter", - "timelines/src/main/scala/com/twitter/timelines/impressionstore/store", - "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", - "user_session_store/src/main/scala/com/twitter/user_session_store", - "util/util-core", - ], - exports = [ - "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", - "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/DismissInfoQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/DismissInfoQueryFeatureHydrator.scala deleted file mode 100644 index 976cd1e30..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/DismissInfoQueryFeatureHydrator.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.DismissInfoFeature -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.timelinemixer.clients.manhattan.InjectionHistoryClient -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelinemixer.clients.manhattan.DismissInfo -import com.twitter.timelineservice.suggests.thriftscala.SuggestType -import javax.inject.Inject -import javax.inject.Singleton - -object DismissInfoQueryFeatureHydrator { - val DismissInfoSuggestTypes = Seq(SuggestType.WhoToFollow) -} - -@Singleton -case class DismissInfoQueryFeatureHydrator @Inject() ( - dismissInfoClient: InjectionHistoryClient) - extends QueryFeatureHydrator[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("DismissInfo") - - override val features: Set[Feature[_, _]] = Set(DismissInfoFeature) - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = - Stitch.callFuture { - dismissInfoClient - .readDismissInfoEntries( - query.getRequiredUserId, - DismissInfoQueryFeatureHydrator.DismissInfoSuggestTypes).map { response => - val dismissInfoMap = response.mapValues(DismissInfo.fromThrift) - FeatureMapBuilder().add(DismissInfoFeature, dismissInfoMap).build() - } - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8, 50, 60, 60) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FeedbackHistoryQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FeedbackHistoryQueryFeatureHydrator.scala deleted file mode 100644 index c0793be4b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FeedbackHistoryQueryFeatureHydrator.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelinemixer.clients.feedback.FeedbackHistoryManhattanClient -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class FeedbackHistoryQueryFeatureHydrator @Inject() ( - feedbackHistoryClient: FeedbackHistoryManhattanClient) - extends QueryFeatureHydrator[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FeedbackHistory") - - override val features: Set[Feature[_, _]] = Set(FeedbackHistoryFeature) - - override def hydrate( - query: PipelineQuery - ): Stitch[FeatureMap] = - Stitch - .callFuture(feedbackHistoryClient.get(query.getRequiredUserId)) - .map { feedbackHistory => - FeatureMapBuilder().add(FeedbackHistoryFeature, feedbackHistory).build() - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GizmoduckUserQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GizmoduckUserQueryFeatureHydrator.scala deleted file mode 100644 index 431711900..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GizmoduckUserQueryFeatureHydrator.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.gizmoduck.{thriftscala => gt} -import com.twitter.home_mixer.model.HomeFeatures.UserFollowingCountFeature -import com.twitter.home_mixer.model.HomeFeatures.UserScreenNameFeature -import com.twitter.home_mixer.model.HomeFeatures.UserTypeFeature -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.stitch.gizmoduck.Gizmoduck -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class GizmoduckUserQueryFeatureHydrator @Inject() (gizmoduck: Gizmoduck) - extends QueryFeatureHydrator[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("GizmoduckUser") - - override val features: Set[Feature[_, _]] = - Set(UserFollowingCountFeature, UserTypeFeature, UserScreenNameFeature) - - private val queryFields: Set[gt.QueryFields] = - Set(gt.QueryFields.Counts, gt.QueryFields.Safety, gt.QueryFields.Profile) - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - val userId = query.getRequiredUserId - gizmoduck - .getUserById( - userId = userId, - queryFields = queryFields, - context = gt.LookupContext(forUserId = Some(userId), includeSoftUsers = true)) - .map { user => - FeatureMapBuilder() - .add(UserFollowingCountFeature, user.counts.map(_.following.toInt)) - .add(UserTypeFeature, Some(user.userType)) - .add(UserScreenNameFeature, user.profile.map(_.screenName)) - .build() - } - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.7) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressionBloomFilterQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressionBloomFilterQueryFeatureHydrator.scala deleted file mode 100644 index 6ece16ce0..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressionBloomFilterQueryFeatureHydrator.scala +++ /dev/null @@ -1,62 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature -import com.twitter.home_mixer.model.request.HasSeenTweetIds -import com.twitter.home_mixer.param.HomeGlobalParams.ImpressionBloomFilterFalsePositiveRateParam -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelines.clients.manhattan.store.ManhattanStoreClient -import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm} -import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilter -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class ImpressionBloomFilterQueryFeatureHydrator[ - Query <: PipelineQuery with HasSeenTweetIds] @Inject() ( - bloomFilterClient: ManhattanStoreClient[ - blm.ImpressionBloomFilterKey, - blm.ImpressionBloomFilterSeq - ]) extends QueryFeatureHydrator[Query] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( - "ImpressionBloomFilter") - - private val ImpressionBloomFilterTTL = 7.day - - override val features: Set[Feature[_, _]] = Set(ImpressionBloomFilterFeature) - - private val SurfaceArea = blm.SurfaceArea.HomeTimeline - - override def hydrate(query: Query): Stitch[FeatureMap] = { - val userId = query.getRequiredUserId - bloomFilterClient - .get(blm.ImpressionBloomFilterKey(userId, SurfaceArea)) - .map(_.getOrElse(blm.ImpressionBloomFilterSeq(Seq.empty))) - .map { bloomFilterSeq => - val updatedBloomFilterSeq = - if (query.seenTweetIds.forall(_.isEmpty)) bloomFilterSeq - else { - ImpressionBloomFilter.addSeenTweetIds( - surfaceArea = SurfaceArea, - tweetIds = query.seenTweetIds.get, - bloomFilterSeq = bloomFilterSeq, - timeToLive = ImpressionBloomFilterTTL, - falsePositiveRate = query.params(ImpressionBloomFilterFalsePositiveRateParam) - ) - } - FeatureMapBuilder().add(ImpressionBloomFilterFeature, updatedBloomFilterSeq).build() - } - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/InNetworkFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/InNetworkFeatureHydrator.scala deleted file mode 100644 index 55f002b1f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/InNetworkFeatureHydrator.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -object InNetworkFeatureHydrator - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("InNetwork") - - override val features: Set[Feature[_, _]] = Set(InNetworkFeature) - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - val viewerId = query.getRequiredUserId - val followedUserIds = query.features.get.get(SGSFollowedUsersFeature).toSet - - val featureMaps = candidates.map { candidate => - // We use authorId and not sourceAuthorId here so that retweets are defined as in network - val isInNetworkOpt = candidate.features.getOrElse(AuthorIdFeature, None).map { authorId => - // Users cannot follow themselves but this is in network by definition - val isSelfTweet = authorId == viewerId - isSelfTweet || followedUserIds.contains(authorId) - } - FeatureMapBuilder().add(InNetworkFeature, isInNetworkOpt.getOrElse(true)).build() - } - Stitch.value(featureMaps) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/LastNonPollingTimeQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/LastNonPollingTimeQueryFeatureHydrator.scala deleted file mode 100644 index 9f3049049..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/LastNonPollingTimeQueryFeatureHydrator.scala +++ /dev/null @@ -1,68 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.FollowingLastNonPollingTimeFeature -import com.twitter.home_mixer.model.HomeFeatures.LastNonPollingTimeFeature -import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.user_session_store.ReadRequest -import com.twitter.user_session_store.ReadWriteUserSessionStore -import com.twitter.user_session_store.UserSessionDataset -import com.twitter.user_session_store.UserSessionDataset.UserSessionDataset -import com.twitter.util.Time - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class LastNonPollingTimeQueryFeatureHydrator @Inject() ( - userSessionStore: ReadWriteUserSessionStore) - extends QueryFeatureHydrator[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("LastNonPollingTime") - - override val features: Set[Feature[_, _]] = Set( - FollowingLastNonPollingTimeFeature, - LastNonPollingTimeFeature, - NonPollingTimesFeature - ) - - private val datasets: Set[UserSessionDataset] = Set(UserSessionDataset.NonPollingTimes) - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - userSessionStore - .read(ReadRequest(query.getRequiredUserId, datasets)) - .map { userSession => - val nonPollingTimestamps = userSession.flatMap(_.nonPollingTimestamps) - - val lastNonPollingTime = nonPollingTimestamps - .flatMap(_.nonPollingTimestampsMs.headOption) - .map(Time.fromMilliseconds) - - val followingLastNonPollingTime = nonPollingTimestamps - .flatMap(_.mostRecentHomeLatestNonPollingTimestampMs) - .map(Time.fromMilliseconds) - - val nonPollingTimes = nonPollingTimestamps - .map(_.nonPollingTimestampsMs) - .getOrElse(Seq.empty) - - FeatureMapBuilder() - .add(FollowingLastNonPollingTimeFeature, followingLastNonPollingTime) - .add(LastNonPollingTimeFeature, lastNonPollingTime) - .add(NonPollingTimesFeature, nonPollingTimes) - .build() - } - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.9) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/NamesFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/NamesFeatureHydrator.scala deleted file mode 100644 index ede075e5c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/NamesFeatureHydrator.scala +++ /dev/null @@ -1,97 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.gizmoduck.{thriftscala => gt} -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.FavoritedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.FollowedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.RealNamesFeature -import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.home_mixer.param.HomeGlobalParams.EnableNahFeedbackInfoParam -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.Conditionally -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.stitch.gizmoduck.Gizmoduck -import com.twitter.util.Return -import javax.inject.Inject -import javax.inject.Singleton - -protected case class ProfileNames(screenName: String, realName: String) - -@Singleton -class NamesFeatureHydrator @Inject() (gizmoduck: Gizmoduck) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] - with Conditionally[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Names") - - override val features: Set[Feature[_, _]] = Set(ScreenNamesFeature, RealNamesFeature) - - override def onlyIf(query: PipelineQuery): Boolean = query.product match { - case FollowingProduct => query.params(EnableNahFeedbackInfoParam) - case _ => true - } - - private val queryFields: Set[gt.QueryFields] = Set(gt.QueryFields.Profile) - - /** - * The UI currently only ever displays the first 2 names in social context lines - * E.g. "User and 3 others like" or "UserA and UserB liked" - */ - private val MaxCountUsers = 2 - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - - val candidateUserIdsMap = candidates.map { candidate => - candidate.candidate.id -> - (candidate.features.getOrElse(FavoritedByUserIdsFeature, Nil).take(MaxCountUsers) ++ - candidate.features.getOrElse(FollowedByUserIdsFeature, Nil).take(MaxCountUsers) ++ - candidate.features.getOrElse(AuthorIdFeature, None) ++ - candidate.features.getOrElse(SourceUserIdFeature, None)).distinct - }.toMap - - val distinctUserIds = candidateUserIdsMap.values.flatten.toSeq.distinct - - Stitch - .collectToTry(distinctUserIds.map(userId => gizmoduck.getUserById(userId, queryFields))) - .map { allUsers => - val idToProfileNamesMap = allUsers.flatMap { - case Return(allUser) => - allUser.profile - .map(profile => allUser.id -> ProfileNames(profile.screenName, profile.name)) - case _ => None - }.toMap - - val validUserIds = idToProfileNamesMap.keySet - - candidates.map { candidate => - val combinedMap = candidateUserIdsMap - .getOrElse(candidate.candidate.id, Nil) - .flatMap { - case userId if validUserIds.contains(userId) => - idToProfileNamesMap.get(userId).map(profileNames => userId -> profileNames) - case _ => None - } - - val perCandidateRealNameMap = combinedMap.map { case (k, v) => k -> v.realName }.toMap - val perCandidateScreenNameMap = combinedMap.map { case (k, v) => k -> v.screenName }.toMap - - FeatureMapBuilder() - .add(ScreenNamesFeature, perCandidateScreenNameMap) - .add(RealNamesFeature, perCandidateRealNameMap) - .build() - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PersistenceStoreQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PersistenceStoreQueryFeatureHydrator.scala deleted file mode 100644 index b7aae811b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PersistenceStoreQueryFeatureHydrator.scala +++ /dev/null @@ -1,118 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.conversions.DurationOps._ -import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature -import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.ServedTweetPreviewIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.WhoToFollowExcludedUserIdsFeature -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelinemixer.clients.persistence.TimelineResponseBatchesClient -import com.twitter.timelinemixer.clients.persistence.TimelineResponseV3 -import com.twitter.timelines.util.client_info.ClientPlatform -import com.twitter.timelineservice.model.TimelineQuery -import com.twitter.timelineservice.model.core.TimelineKind -import com.twitter.timelineservice.model.rich.EntityIdType -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class PersistenceStoreQueryFeatureHydrator @Inject() ( - timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3], - statsReceiver: StatsReceiver) - extends QueryFeatureHydrator[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("PersistenceStore") - - private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) - private val servedTweetIdsSizeStat = scopedStatsReceiver.stat("ServedTweetIdsSize") - - private val WhoToFollowExcludedUserIdsLimit = 1000 - private val ServedTweetIdsDuration = 10.minutes - private val ServedTweetIdsLimit = 100 - private val ServedTweetPreviewIdsDuration = 10.hours - private val ServedTweetPreviewIdsLimit = 10 - - override val features: Set[Feature[_, _]] = - Set( - ServedTweetIdsFeature, - ServedTweetPreviewIdsFeature, - PersistenceEntriesFeature, - WhoToFollowExcludedUserIdsFeature) - - private val supportedClients = Seq( - ClientPlatform.IPhone, - ClientPlatform.IPad, - ClientPlatform.Mac, - ClientPlatform.Android, - ClientPlatform.Web, - ClientPlatform.RWeb, - ClientPlatform.TweetDeckGryphon - ) - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - val timelineKind = query.product match { - case FollowingProduct => TimelineKind.homeLatest - case ForYouProduct => TimelineKind.home - case other => throw new UnsupportedOperationException(s"Unknown product: $other") - } - val timelineQuery = TimelineQuery(id = query.getRequiredUserId, kind = timelineKind) - - Stitch.callFuture { - timelineResponseBatchesClient - .get(query = timelineQuery, clientPlatforms = supportedClients) - .map { timelineResponses => - // Note that the WTF entries are not being scoped by ClientPlatform - val whoToFollowUserIds = timelineResponses - .flatMap { timelineResponse => - timelineResponse.entries - .filter(_.entityIdType == EntityIdType.WhoToFollow) - .flatMap(_.itemIds.toSeq.flatMap(_.flatMap(_.userId))) - }.take(WhoToFollowExcludedUserIdsLimit) - - val clientPlatform = ClientPlatform.fromQueryOptions( - clientAppId = query.clientContext.appId, - userAgent = query.clientContext.userAgent.flatMap(UserAgent.fromString)) - - val servedTweetIds = timelineResponses - .filter(_.clientPlatform == clientPlatform) - .filter(_.servedTime >= Time.now - ServedTweetIdsDuration) - .sortBy(-_.servedTime.inMilliseconds) - .flatMap( - _.entries.flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetIdsLimit)) - - servedTweetIdsSizeStat.add(servedTweetIds.size) - - val servedTweetPreviewIds = timelineResponses - .filter(_.clientPlatform == clientPlatform) - .filter(_.servedTime >= Time.now - ServedTweetPreviewIdsDuration) - .sortBy(-_.servedTime.inMilliseconds) - .flatMap(_.entries - .filter(_.entityIdType == EntityIdType.TweetPreview) - .flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetPreviewIdsLimit)) - - FeatureMapBuilder() - .add(ServedTweetIdsFeature, servedTweetIds) - .add(ServedTweetPreviewIdsFeature, servedTweetPreviewIds) - .add(PersistenceEntriesFeature, timelineResponses) - .add(WhoToFollowExcludedUserIdsFeature, whoToFollowUserIds) - .build() - } - } - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.7, 50, 60, 60) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PerspectiveFilteredSocialContextFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PerspectiveFilteredSocialContextFeatureHydrator.scala deleted file mode 100644 index 602a8b2dc..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PerspectiveFilteredSocialContextFeatureHydrator.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.FavoritedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch -import com.twitter.stitch.timelineservice.TimelineService -import com.twitter.stitch.timelineservice.TimelineService.GetPerspectives -import com.twitter.timelineservice.thriftscala.PerspectiveType -import com.twitter.timelineservice.thriftscala.PerspectiveType.Favorited -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Filter out unlike edges from liked-by tweets - * Useful if the likes come from a cache and because UTEG does not fully remove unlike edges. - */ -@Singleton -class PerspectiveFilteredSocialContextFeatureHydrator @Inject() (timelineService: TimelineService) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("PerspectiveFilteredSocialContext") - - override val features: Set[Feature[_, _]] = Set(PerspectiveFilteredLikedByUserIdsFeature) - - private val MaxCountUsers = 10 - private val favoritePerspectiveSet: Set[PerspectiveType] = Set(Favorited) - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { - val engagingUserIdtoTweetId = candidates.flatMap { candidate => - candidate.features - .getOrElse(FavoritedByUserIdsFeature, Seq.empty).take(MaxCountUsers) - .map(favoritedBy => favoritedBy -> candidate.candidate.id) - } - - val queries = engagingUserIdtoTweetId.map { - case (userId, tweetId) => - GetPerspectives.Query(userId = userId, tweetId = tweetId, types = favoritePerspectiveSet) - } - - Stitch.collect(queries.map(timelineService.getPerspective)).map { perspectiveResults => - val validUserIdTweetIds: Set[(Long, Long)] = - queries - .zip(perspectiveResults) - .collect { case (query, perspective) if perspective.favorited => query } - .map(query => (query.userId, query.tweetId)) - .toSet - - candidates.map { candidate => - val perspectiveFilteredFavoritedByUserIds: Seq[Long] = candidate.features - .getOrElse(FavoritedByUserIdsFeature, Seq.empty).take(MaxCountUsers) - .filter { userId => validUserIdTweetIds.contains((userId, candidate.candidate.id)) } - - FeatureMapBuilder() - .add(PerspectiveFilteredLikedByUserIdsFeature, perspectiveFilteredFavoritedByUserIds) - .build() - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphInNetworkScoresQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphInNetworkScoresQueryFeatureHydrator.scala deleted file mode 100644 index 92ab90f2c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphInNetworkScoresQueryFeatureHydrator.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature -import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealGraphInNetworkScores -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.storehaus.ReadableStore -import com.twitter.wtf.candidate.{thriftscala => wtf} -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -@Singleton -case class RealGraphInNetworkScoresQueryFeatureHydrator @Inject() ( - @Named(RealGraphInNetworkScores) store: ReadableStore[Long, Seq[wtf.Candidate]]) - extends QueryFeatureHydrator[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("RealGraphInNetworkScores") - - override val features: Set[Feature[_, _]] = Set(RealGraphInNetworkScoresFeature) - - private val RealGraphCandidateCount = 1000 - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - Stitch.callFuture(store.get(query.getRequiredUserId)).map { realGraphFollowedUsers => - val realGraphScoresFeatures = realGraphFollowedUsers - .getOrElse(Seq.empty) - .sortBy(-_.score) - .map(candidate => candidate.userId -> scaleScore(candidate.score)) - .take(RealGraphCandidateCount) - .toMap - - FeatureMapBuilder().add(RealGraphInNetworkScoresFeature, realGraphScoresFeatures).build() - } - } - - // Rescale Real Graph v2 scores from [0,1] to the v1 scores distribution [1,2.97] - private def scaleScore(score: Double): Double = - if (score >= 0.0 && score <= 1.0) score * 1.97 + 1.0 else score -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RequestQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RequestQueryFeatureHydrator.scala deleted file mode 100644 index 3968c9532..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RequestQueryFeatureHydrator.scala +++ /dev/null @@ -1,121 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.finagle.tracing.Annotation.BinaryAnnotation -import com.twitter.finagle.tracing.ForwardAnnotation -import com.twitter.home_mixer.model.HomeFeatures._ -import com.twitter.home_mixer.model.request.DeviceContext.RequestContext -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.joinkey.context.RequestJoinKeyContext -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.BottomCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor -import com.twitter.product_mixer.core.pipeline.HasPipelineCursor -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.search.common.util.lang.ThriftLanguageUtil -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.stitch.Stitch -import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter.dowFromTimestamp -import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter.hourFromTimestamp -import java.util.UUID -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class RequestQueryFeatureHydrator[ - Query <: PipelineQuery with HasPipelineCursor[UrtOrderedCursor] with HasDeviceContext] @Inject() ( -) extends QueryFeatureHydrator[Query] { - - override val features: Set[Feature[_, _]] = Set( - AccountAgeFeature, - ClientIdFeature, - DeviceLanguageFeature, - GetInitialFeature, - GetMiddleFeature, - GetNewerFeature, - GetOlderFeature, - GuestIdFeature, - HasDarkRequestFeature, - IsForegroundRequestFeature, - IsLaunchRequestFeature, - PollingFeature, - PullToRefreshFeature, - RequestJoinIdFeature, - ServedRequestIdFeature, - TimestampFeature, - TimestampGMTDowFeature, - TimestampGMTHourFeature, - ViewerIdFeature - ) - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Request") - - private val DarkRequestAnnotation = "clnt/has_dark_request" - - // Convert Language code to ISO 639-3 format - private def getLanguageISOFormatByCode(languageCode: String): String = - ThriftLanguageUtil.getLanguageCodeOf(ThriftLanguageUtil.getThriftLanguageOf(languageCode)) - - private def getRequestJoinId(servedRequestId: Long): Option[Long] = - Some(RequestJoinKeyContext.current.flatMap(_.requestJoinId).getOrElse(servedRequestId)) - - private def hasDarkRequest: Option[Boolean] = ForwardAnnotation.current - .getOrElse(Seq[BinaryAnnotation]()) - .find(_.key == DarkRequestAnnotation) - .map(_.value.asInstanceOf[Boolean]) - - override def hydrate(query: Query): Stitch[FeatureMap] = { - val requestContext = query.deviceContext.flatMap(_.requestContextValue) - val servedRequestId = UUID.randomUUID.getMostSignificantBits - val timestamp = query.queryTime.inMilliseconds - - val featureMap = FeatureMapBuilder() - .add(AccountAgeFeature, query.getOptionalUserId.flatMap(SnowflakeId.timeFromIdOpt)) - .add(ClientIdFeature, query.clientContext.appId) - .add(DeviceLanguageFeature, query.getLanguageCode.map(getLanguageISOFormatByCode)) - .add( - GetInitialFeature, - query.pipelineCursor.forall(cursor => cursor.id.isEmpty && cursor.gapBoundaryId.isEmpty)) - .add( - GetMiddleFeature, - query.pipelineCursor.exists(cursor => - cursor.id.isDefined && cursor.gapBoundaryId.isDefined && - cursor.cursorType.contains(GapCursor))) - .add( - GetNewerFeature, - query.pipelineCursor.exists(cursor => - cursor.id.isDefined && cursor.gapBoundaryId.isEmpty && - cursor.cursorType.contains(TopCursor))) - .add( - GetOlderFeature, - query.pipelineCursor.exists(cursor => - cursor.id.isDefined && cursor.gapBoundaryId.isEmpty && - cursor.cursorType.contains(BottomCursor))) - .add(GuestIdFeature, query.clientContext.guestId) - .add(IsForegroundRequestFeature, requestContext.contains(RequestContext.Foreground)) - .add(IsLaunchRequestFeature, requestContext.contains(RequestContext.Launch)) - .add(PollingFeature, query.deviceContext.exists(_.isPolling.contains(true))) - .add(PullToRefreshFeature, requestContext.contains(RequestContext.PullToRefresh)) - .add(ServedRequestIdFeature, Some(servedRequestId)) - .add(RequestJoinIdFeature, getRequestJoinId(servedRequestId)) - .add(TimestampFeature, timestamp) - .add(TimestampGMTDowFeature, dowFromTimestamp(timestamp)) - .add(TimestampGMTHourFeature, hourFromTimestamp(timestamp)) - .add(HasDarkRequestFeature, hasDarkRequest) - .add( - ViewerIdFeature, - query.getOptionalUserId - .orElse(query.getGuestId).getOrElse( - throw PipelineFailure(BadRequest, "Missing viewer id"))) - .build() - - Stitch.value(featureMap) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSValidSocialContextFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSValidSocialContextFeatureHydrator.scala deleted file mode 100644 index 92c351ecf..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSValidSocialContextFeatureHydrator.scala +++ /dev/null @@ -1,105 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.FavoritedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.FollowedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.socialgraph.{thriftscala => sg} -import com.twitter.stitch.Stitch -import com.twitter.stitch.socialgraph.SocialGraph -import javax.inject.Inject -import javax.inject.Singleton - -/** - * This hydrator takes liked-by and followed-by user ids and checks via SGS that the viewer is - * following the engager, that the viewer is not blocking the engager, that the engager is not - * blocking the viewer, and that the viewer has not muted the engager. - */ -@Singleton -class SGSValidSocialContextFeatureHydrator @Inject() ( - socialGraph: SocialGraph) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("SGSValidSocialContext") - - override val features: Set[Feature[_, _]] = Set( - SGSValidFollowedByUserIdsFeature, - SGSValidLikedByUserIdsFeature - ) - - private val MaxCountUsers = 10 - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { - val allSocialContextUserIds = - candidates.flatMap { candidate => - candidate.features.getOrElse(FavoritedByUserIdsFeature, Nil).take(MaxCountUsers) ++ - candidate.features.getOrElse(FollowedByUserIdsFeature, Nil).take(MaxCountUsers) - }.distinct - - getValidUserIds(query.getRequiredUserId, allSocialContextUserIds).map { validUserIds => - candidates.map { candidate => - val sgsFilteredLikedByUserIds = - candidate.features - .getOrElse(FavoritedByUserIdsFeature, Nil).take(MaxCountUsers) - .filter(validUserIds.contains) - - val sgsFilteredFollowedByUserIds = - candidate.features - .getOrElse(FollowedByUserIdsFeature, Nil).take(MaxCountUsers) - .filter(validUserIds.contains) - - FeatureMapBuilder() - .add(SGSValidFollowedByUserIdsFeature, sgsFilteredFollowedByUserIds) - .add(SGSValidLikedByUserIdsFeature, sgsFilteredLikedByUserIds) - .build() - } - } - } - - private def getValidUserIds( - viewerId: Long, - socialProofUserIds: Seq[Long] - ): Stitch[Seq[Long]] = { - if (socialProofUserIds.nonEmpty) { - val request = sg.IdsRequest( - relationships = Seq( - sg.SrcRelationship( - viewerId, - sg.RelationshipType.Following, - targets = Some(socialProofUserIds), - hasRelationship = true), - sg.SrcRelationship( - viewerId, - sg.RelationshipType.Blocking, - targets = Some(socialProofUserIds), - hasRelationship = false), - sg.SrcRelationship( - viewerId, - sg.RelationshipType.BlockedBy, - targets = Some(socialProofUserIds), - hasRelationship = false), - sg.SrcRelationship( - viewerId, - sg.RelationshipType.Muting, - targets = Some(socialProofUserIds), - hasRelationship = false) - ), - pageRequest = Some(sg.PageRequest(selectAll = Some(true))) - ) - socialGraph.ids(request).map(_.ids) - } else Stitch.Nil - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetImpressionsQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetImpressionsQueryFeatureHydrator.scala deleted file mode 100644 index 0fc6fd1f5..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetImpressionsQueryFeatureHydrator.scala +++ /dev/null @@ -1,87 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.model.HomeFeatures.TweetImpressionsFeature -import com.twitter.home_mixer.model.request.HasSeenTweetIds -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelines.impression.{thriftscala => t} -import com.twitter.timelines.impressionstore.store.ManhattanTweetImpressionStoreClient -import com.twitter.util.Duration -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class TweetImpressionsQueryFeatureHydrator[ - Query <: PipelineQuery with HasSeenTweetIds] @Inject() ( - manhattanTweetImpressionStoreClient: ManhattanTweetImpressionStoreClient) - extends QueryFeatureHydrator[Query] { - - private val TweetImpressionTTL = 2.days - private val TweetImpressionCap = 5000 - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetImpressions") - - override val features: Set[Feature[_, _]] = Set(TweetImpressionsFeature) - - override def hydrate(query: Query): Stitch[FeatureMap] = { - manhattanTweetImpressionStoreClient.get(query.getRequiredUserId).map { entriesOpt => - val entries = entriesOpt.map(_.entries).toSeq.flatten - val updatedImpressions = - if (query.seenTweetIds.forall(_.isEmpty)) entries - else updateTweetImpressions(entries, query.seenTweetIds.get) - - FeatureMapBuilder().add(TweetImpressionsFeature, updatedImpressions).build() - } - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8) - ) - - /** - * 1) Check timestamps and remove expired tweets based on [[TweetImpressionTTL]] - * 2) Filter duplicates between current tweets and those in the impression store (remove older ones) - * 3) Prepend new (Timestamp, Seq[TweetIds]) to the tweets from the impression store - * 4) Truncate older tweets if sum of all tweets across timestamps >= [[TweetImpressionCap]], - */ - private[feature_hydrator] def updateTweetImpressions( - tweetImpressionsFromStore: Seq[t.TweetImpressionsEntry], - seenIdsFromClient: Seq[Long], - currentTime: Long = Time.now.inMilliseconds, - tweetImpressionTTL: Duration = TweetImpressionTTL, - tweetImpressionCap: Int = TweetImpressionCap, - ): Seq[t.TweetImpressionsEntry] = { - val seenIdsFromClientSet = seenIdsFromClient.toSet - val dedupedTweetImpressionsFromStore: Seq[t.TweetImpressionsEntry] = tweetImpressionsFromStore - .collect { - case t.TweetImpressionsEntry(ts, tweetIds) - if Time.fromMilliseconds(ts).untilNow < tweetImpressionTTL => - t.TweetImpressionsEntry(ts, tweetIds.filterNot(seenIdsFromClientSet.contains)) - }.filter { _.tweetIds.nonEmpty } - - val mergedTweetImpressionsEntries = - t.TweetImpressionsEntry(currentTime, seenIdsFromClient) +: dedupedTweetImpressionsFromStore - val initialTweetImpressionsWithCap = (Seq.empty[t.TweetImpressionsEntry], tweetImpressionCap) - - val (truncatedTweetImpressionsEntries: Seq[t.TweetImpressionsEntry], _) = - mergedTweetImpressionsEntries - .foldLeft(initialTweetImpressionsWithCap) { - case ( - (tweetImpressions: Seq[t.TweetImpressionsEntry], remainingCap), - t.TweetImpressionsEntry(ts, tweetIds)) if remainingCap > 0 => - ( - t.TweetImpressionsEntry(ts, tweetIds.take(remainingCap)) +: tweetImpressions, - remainingCap - tweetIds.size) - case (tweetImpressionsWithCap, _) => tweetImpressionsWithCap - } - truncatedTweetImpressionsEntries.reverse - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetypieFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetypieFeatureHydrator.scala deleted file mode 100644 index f2effa1b2..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetypieFeatureHydrator.scala +++ /dev/null @@ -1,179 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature -import com.twitter.home_mixer.model.HomeFeatures.IsNsfwFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature -import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.model.request.ListTweetsProduct -import com.twitter.home_mixer.model.request.ScoredTweetsProduct -import com.twitter.home_mixer.model.request.SubscribedProduct -import com.twitter.home_mixer.util.tweetypie.RequestFields -import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_is_nsfw.IsNsfw -import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason.VisibilityReason -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.spam.rtf.{thriftscala => rtf} -import com.twitter.stitch.Stitch -import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient} -import com.twitter.tweetypie.{thriftscala => tp} -import com.twitter.util.logging.Logging -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class TweetypieFeatureHydrator @Inject() ( - tweetypieStitchClient: TweetypieStitchClient, - statsReceiver: StatsReceiver) - extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] - with Logging { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Tweetypie") - - override val features: Set[Feature[_, _]] = Set( - AuthorIdFeature, - ExclusiveConversationAuthorIdFeature, - InReplyToTweetIdFeature, - IsHydratedFeature, - IsNsfw, - IsNsfwFeature, - IsRetweetFeature, - QuotedTweetDroppedFeature, - QuotedTweetIdFeature, - QuotedUserIdFeature, - SourceTweetIdFeature, - SourceUserIdFeature, - TweetTextFeature, - TweetLanguageFeature, - VisibilityReason - ) - - private val DefaultFeatureMap = FeatureMapBuilder() - .add(IsHydratedFeature, false) - .add(IsNsfw, None) - .add(IsNsfwFeature, false) - .add(QuotedTweetDroppedFeature, false) - .add(TweetTextFeature, None) - .add(VisibilityReason, None) - .build() - - override def apply( - query: PipelineQuery, - candidate: TweetCandidate, - existingFeatures: FeatureMap - ): Stitch[FeatureMap] = { - val safetyLevel = query.product match { - case FollowingProduct => rtf.SafetyLevel.TimelineHomeLatest - case ForYouProduct => - val inNetwork = existingFeatures.getOrElse(InNetworkFeature, true) - if (inNetwork) rtf.SafetyLevel.TimelineHome else rtf.SafetyLevel.TimelineHomeRecommendations - case ScoredTweetsProduct => rtf.SafetyLevel.TimelineHome - case ListTweetsProduct => rtf.SafetyLevel.TimelineLists - case SubscribedProduct => rtf.SafetyLevel.TimelineHomeSubscribed - case unknown => throw new UnsupportedOperationException(s"Unknown product: $unknown") - } - - val tweetFieldsOptions = tp.GetTweetFieldsOptions( - tweetIncludes = RequestFields.TweetTPHydrationFields, - includeRetweetedTweet = true, - includeQuotedTweet = true, - visibilityPolicy = tp.TweetVisibilityPolicy.UserVisible, - safetyLevel = Some(safetyLevel), - forUserId = query.getOptionalUserId - ) - - val exclusiveAuthorIdOpt = - existingFeatures.getOrElse(ExclusiveConversationAuthorIdFeature, None) - - tweetypieStitchClient.getTweetFields(tweetId = candidate.id, options = tweetFieldsOptions).map { - case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), quoteOpt, _) => - val coreData = found.tweet.coreData - val isNsfwAdmin = coreData.exists(_.nsfwAdmin) - val isNsfwUser = coreData.exists(_.nsfwUser) - - val quotedTweetDropped = quoteOpt.exists { - case _: tp.TweetFieldsResultState.Filtered => true - case _: tp.TweetFieldsResultState.NotFound => true - case _ => false - } - val quotedTweetIsNsfw = quoteOpt.exists { - case quoteTweet: tp.TweetFieldsResultState.Found => - quoteTweet.found.tweet.coreData.exists(data => data.nsfwAdmin || data.nsfwUser) - case _ => false - } - - val sourceTweetIsNsfw = - found.retweetedTweet.exists(_.coreData.exists(data => data.nsfwAdmin || data.nsfwUser)) - - val tweetText = coreData.map(_.text) - val tweetLanguage = found.tweet.language.map(_.language) - - val tweetAuthorId = coreData.map(_.userId) - val inReplyToTweetId = coreData.flatMap(_.reply.flatMap(_.inReplyToStatusId)) - val retweetedTweetId = found.retweetedTweet.map(_.id) - val quotedTweetId = quoteOpt.flatMap { - case quoteTweet: tp.TweetFieldsResultState.Found => - Some(quoteTweet.found.tweet.id) - case _ => None - } - - val retweetedTweetUserId = found.retweetedTweet.flatMap(_.coreData).map(_.userId) - val quotedTweetUserId = quoteOpt.flatMap { - case quoteTweet: tp.TweetFieldsResultState.Found => - quoteTweet.found.tweet.coreData.map(_.userId) - case _ => None - } - - val isNsfw = isNsfwAdmin || isNsfwUser || sourceTweetIsNsfw || quotedTweetIsNsfw - - FeatureMapBuilder() - .add(AuthorIdFeature, tweetAuthorId) - .add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt) - .add(InReplyToTweetIdFeature, inReplyToTweetId) - .add(IsHydratedFeature, true) - .add(IsNsfw, Some(isNsfw)) - .add(IsNsfwFeature, isNsfw) - .add(IsRetweetFeature, retweetedTweetId.isDefined) - .add(QuotedTweetDroppedFeature, quotedTweetDropped) - .add(QuotedTweetIdFeature, quotedTweetId) - .add(QuotedUserIdFeature, quotedTweetUserId) - .add(SourceTweetIdFeature, retweetedTweetId) - .add(SourceUserIdFeature, retweetedTweetUserId) - .add(TweetLanguageFeature, tweetLanguage) - .add(TweetTextFeature, tweetText) - .add(VisibilityReason, found.suppressReason) - .build() - - // If no tweet result found, return default and pre-existing features - case _ => - DefaultFeatureMap ++ FeatureMapBuilder() - .add(AuthorIdFeature, existingFeatures.getOrElse(AuthorIdFeature, None)) - .add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt) - .add(InReplyToTweetIdFeature, existingFeatures.getOrElse(InReplyToTweetIdFeature, None)) - .add(IsRetweetFeature, existingFeatures.getOrElse(IsRetweetFeature, false)) - .add(QuotedTweetIdFeature, existingFeatures.getOrElse(QuotedTweetIdFeature, None)) - .add(QuotedUserIdFeature, existingFeatures.getOrElse(QuotedUserIdFeature, None)) - .add(SourceTweetIdFeature, existingFeatures.getOrElse(SourceTweetIdFeature, None)) - .add(SourceUserIdFeature, existingFeatures.getOrElse(SourceUserIdFeature, None)) - .add(TweetLanguageFeature, existingFeatures.getOrElse(TweetLanguageFeature, None)) - .build() - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/BUILD.bazel deleted file mode 100644 index 59284224b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/BUILD.bazel +++ /dev/null @@ -1,29 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter", - "src/thrift/com/twitter/spam/rtf:safety-result-scala", - "src/thrift/com/twitter/timelines/impression:thrift-scala", - "src/thrift/com/twitter/tweetypie:service-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - "stitch/stitch-core", - "stitch/stitch-socialgraph", - "stitch/stitch-tweetypie", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", - "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", - "util/util-slf4j-api/src/main/scala", - ], - exports = [ - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/DropMaxCandidatesFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/DropMaxCandidatesFilter.scala deleted file mode 100644 index 94943fc4a..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/DropMaxCandidatesFilter.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.home_mixer.functional_component.filter - -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.FSBoundedParam - -case class DropMaxCandidatesFilter[Candidate <: UniversalNoun[Any]]( - maxCandidatesParam: FSBoundedParam[Int]) - extends Filter[PipelineQuery, Candidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("DropMaxCandidates") - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[Candidate]] - ): Stitch[FilterResult[Candidate]] = { - val maxCandidates = query.params(maxCandidatesParam) - val (kept, removed) = candidates.map(_.candidate).splitAt(maxCandidates) - - Stitch.value(FilterResult(kept, removed)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/FeedbackFatigueFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/FeedbackFatigueFilter.scala deleted file mode 100644 index 582583e7f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/FeedbackFatigueFilter.scala +++ /dev/null @@ -1,89 +0,0 @@ -package com.twitter.home_mixer.functional_component.filter - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelines.common.thriftscala.FeedbackEntity -import com.twitter.timelineservice.model.FeedbackEntry -import com.twitter.timelineservice.{thriftscala => tls} - -object FeedbackFatigueFilter - extends Filter[PipelineQuery, TweetCandidate] - with Filter.Conditionally[PipelineQuery, TweetCandidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("FeedbackFatigue") - - override def onlyIf( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Boolean = - query.features.exists(_.getOrElse(FeedbackHistoryFeature, Seq.empty).nonEmpty) - - private val DurationForFiltering = 14.days - - override def apply( - query: pipeline.PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - val feedbackEntriesByEngagementType = - query.features - .getOrElse(FeatureMap.empty).getOrElse(FeedbackHistoryFeature, Seq.empty) - .filter { entry => - val timeSinceFeedback = query.queryTime.minus(entry.timestamp) - timeSinceFeedback < DurationForFiltering && - entry.feedbackType == tls.FeedbackType.SeeFewer - }.groupBy(_.engagementType) - - val authorsToFilter = - getUserIds( - feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Tweet, Seq.empty)) - val likersToFilter = - getUserIds( - feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Like, Seq.empty)) - val followersToFilter = - getUserIds( - feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Follow, Seq.empty)) - val retweetersToFilter = - getUserIds( - feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Retweet, Seq.empty)) - - val (removed, kept) = candidates.partition { candidate => - val originalAuthorId = CandidatesUtil.getOriginalAuthorId(candidate.features) - val authorId = candidate.features.getOrElse(AuthorIdFeature, None) - - val likers = candidate.features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty) - val eligibleLikers = likers.filterNot(likersToFilter.contains) - - val followers = candidate.features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty) - val eligibleFollowers = followers.filterNot(followersToFilter.contains) - - originalAuthorId.exists(authorsToFilter.contains) || - (likers.nonEmpty && eligibleLikers.isEmpty) || - (followers.nonEmpty && eligibleFollowers.isEmpty && likers.isEmpty) || - (candidate.features.getOrElse(IsRetweetFeature, false) && - authorId.exists(retweetersToFilter.contains)) - } - - Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) - } - - private def getUserIds( - feedbackEntries: Seq[FeedbackEntry], - ): Set[Long] = - feedbackEntries.collect { - case FeedbackEntry(_, _, FeedbackEntity.UserId(userId), _, _) => userId - }.toSet -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidConversationModuleFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidConversationModuleFilter.scala deleted file mode 100644 index 0d3f1ca1f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidConversationModuleFilter.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.home_mixer.functional_component.filter - -import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -/** - * Exclude conversation modules where Tweets have been dropped by other filters - * - * Largest conversation modules have 3 Tweets, so if all 3 are present, module is valid. - * For 2 Tweet modules, check if the head is the root (not a reply) and the last item - * is actually replying to the root directly with no missing intermediate tweets - */ -object InvalidConversationModuleFilter extends Filter[PipelineQuery, TweetCandidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("InvalidConversationModule") - - val ValidThreeTweetModuleSize = 3 - val ValidTwoTweetModuleSize = 2 - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - val allowedTweetIds = candidates - .groupBy(_.features.getOrElse(ConversationModuleFocalTweetIdFeature, None)) - .map { case (id, candidates) => (id, candidates.sortBy(_.candidate.id)) } - .filter { - case (Some(_), conversation) if conversation.size == ValidThreeTweetModuleSize => true - case (Some(focalId), conversation) if conversation.size == ValidTwoTweetModuleSize => - conversation.head.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty && - conversation.last.candidate.id == focalId && - conversation.last.features - .getOrElse(InReplyToTweetIdFeature, None) - .contains(conversation.head.candidate.id) - case (None, _) => true - case _ => false - }.values.flatten.toSeq.map(_.candidate.id).toSet - - val (kept, removed) = - candidates.map(_.candidate).partition(candidate => allowedTweetIds.contains(candidate.id)) - Stitch.value(FilterResult(kept = kept, removed = removed)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidSubscriptionTweetFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidSubscriptionTweetFilter.scala deleted file mode 100644 index 285eec49f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidSubscriptionTweetFilter.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.twitter.home_mixer.functional_component.filter - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.tracing.Trace -import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.socialgraph.{thriftscala => sg} -import com.twitter.stitch.Stitch -import com.twitter.stitch.socialgraph.SocialGraph -import com.twitter.util.logging.Logging - -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Exclude invalid subscription tweets - cases where the viewer is not subscribed to the author - * - * If SGS hydration fails, `SGSInvalidSubscriptionTweetFeature` will be set to None for - * subscription tweets, so we explicitly filter those tweets out. - */ -@Singleton -case class InvalidSubscriptionTweetFilter @Inject() ( - socialGraphClient: SocialGraph, - statsReceiver: StatsReceiver) - extends Filter[PipelineQuery, TweetCandidate] - with Logging { - - override val identifier: FilterIdentifier = FilterIdentifier("InvalidSubscriptionTweet") - - private val scopedStatsReceiver = statsReceiver.scope(identifier.toString) - private val validCounter = scopedStatsReceiver.counter("validExclusiveTweet") - private val invalidCounter = scopedStatsReceiver.counter("invalidExclusiveTweet") - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = Stitch - .traverse(candidates) { candidate => - val exclusiveAuthorId = - candidate.features.getOrElse(ExclusiveConversationAuthorIdFeature, None) - - if (exclusiveAuthorId.isDefined) { - val request = sg.ExistsRequest( - source = query.getRequiredUserId, - target = exclusiveAuthorId.get, - relationships = - Seq(sg.Relationship(sg.RelationshipType.TierOneSuperFollowing, hasRelationship = true)), - ) - socialGraphClient.exists(request).map(_.exists).map { valid => - if (!valid) invalidCounter.incr() else validCounter.incr() - valid - } - } else Stitch.value(true) - }.map { validResults => - val (kept, removed) = candidates - .map(_.candidate) - .zip(validResults) - .partition { case (candidate, valid) => valid } - - val keptCandidates = kept.map { case (candidate, _) => candidate } - val removedCandidates = removed.map { case (candidate, _) => candidate } - - FilterResult(kept = keptCandidates, removed = removedCandidates) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PredicateGatedFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PredicateGatedFilter.scala deleted file mode 100644 index b263a87fc..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PredicateGatedFilter.scala +++ /dev/null @@ -1,47 +0,0 @@ -package com.twitter.home_mixer.functional_component.filter - -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.Conditionally -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -trait FilterPredicate[-Query <: PipelineQuery] { - def apply(query: Query): Boolean -} - -/** - * A [[Filter]] with [[Conditionally]] based on a [[FilterPredicate]] - * - * @param predicate the predicate to turn this filter on and off - * @param filter the underlying filter to run when `predicate` is true - * @tparam Query The domain model for the query or request - * @tparam Candidate The type of the candidates - */ -case class PredicateGatedFilter[-Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]( - predicate: FilterPredicate[Query], - filter: Filter[Query, Candidate]) - extends Filter[Query, Candidate] - with Filter.Conditionally[Query, Candidate] { - - override val identifier: FilterIdentifier = FilterIdentifier( - PredicateGatedFilter.IdentifierPrefix + filter.identifier.name) - - override val alerts: Seq[Alert] = filter.alerts - - override def onlyIf(query: Query, candidates: Seq[CandidateWithFeatures[Candidate]]): Boolean = - Conditionally.and(Filter.Input(query, candidates), filter, predicate(query)) - - override def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[Candidate]] - ): Stitch[FilterResult[Candidate]] = filter.apply(query, candidates) -} - -object PredicateGatedFilter { - val IdentifierPrefix = "PredicateGated" -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslySeenTweetsFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslySeenTweetsFilter.scala deleted file mode 100644 index 047233a41..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslySeenTweetsFilter.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.home_mixer.functional_component.filter - -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.home_mixer.util.TweetImpressionsHelper -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -/** - * Filter out users' previously seen tweets from 2 sources: - * 1. Heron Topology Impression Store in Memcache; - * 2. Manhattan Impression Store; - */ -object PreviouslySeenTweetsFilter extends Filter[PipelineQuery, TweetCandidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("PreviouslySeenTweets") - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - - val seenTweetIds = - query.features.map(TweetImpressionsHelper.tweetImpressions).getOrElse(Set.empty) - - val (removed, kept) = candidates.partition { candidate => - val tweetIdAndSourceId = CandidatesUtil.getTweetIdAndSourceId(candidate) - tweetIdAndSourceId.exists(seenTweetIds.contains) - } - - Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedAncestorsFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedAncestorsFilter.scala deleted file mode 100644 index 42b0ad51a..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedAncestorsFilter.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.twitter.home_mixer.functional_component.filter - -import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent -import com.twitter.home_mixer.model.HomeFeatures.IsAncestorCandidateFeature -import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelinemixer.injection.store.persistence.TimelinePersistenceUtils -import com.twitter.timelines.util.client_info.ClientPlatform - -object PreviouslyServedAncestorsFilter - extends Filter[PipelineQuery, TweetCandidate] - with TimelinePersistenceUtils { - - override val identifier: FilterIdentifier = FilterIdentifier("PreviouslyServedAncestors") - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - val clientPlatform = ClientPlatform.fromQueryOptions( - clientAppId = query.clientContext.appId, - userAgent = query.clientContext.userAgent.flatMap(UserAgent.fromString)) - val entries = - query.features.map(_.getOrElse(PersistenceEntriesFeature, Seq.empty)).toSeq.flatten - val tweetIds = applicableResponses(clientPlatform, entries) - .flatMap(_.entries.flatMap(_.tweetIds(includeSourceTweets = true))).toSet - val ancestorIds = - candidates - .filter(_.features.getOrElse(IsAncestorCandidateFeature, false)).map(_.candidate.id).toSet - - val (removed, kept) = - candidates - .map(_.candidate).partition(candidate => - tweetIds.contains(candidate.id) && ancestorIds.contains(candidate.id)) - - Stitch.value(FilterResult(kept = kept, removed = removed)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetPreviewsFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetPreviewsFilter.scala deleted file mode 100644 index 0ff820479..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetPreviewsFilter.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.home_mixer.functional_component.filter - -import com.twitter.home_mixer.model.HomeFeatures.ServedTweetPreviewIdsFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -object PreviouslyServedTweetPreviewsFilter extends Filter[PipelineQuery, TweetCandidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("PreviouslyServedTweetPreviews") - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - - val servedTweetPreviewIds = - query.features.map(_.getOrElse(ServedTweetPreviewIdsFeature, Seq.empty)).toSeq.flatten.toSet - - val (removed, kept) = candidates.partition { candidate => - servedTweetPreviewIds.contains(candidate.candidate.id) - } - - Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetsFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetsFilter.scala deleted file mode 100644 index 56b6a1c0b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetsFilter.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.home_mixer.functional_component.filter - -import com.twitter.home_mixer.model.HomeFeatures.GetOlderFeature -import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -object PreviouslyServedTweetsFilter - extends Filter[PipelineQuery, TweetCandidate] - with Filter.Conditionally[PipelineQuery, TweetCandidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("PreviouslyServedTweets") - - override def onlyIf( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Boolean = { - query.features.exists(_.getOrElse(GetOlderFeature, false)) - } - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - - val servedTweetIds = - query.features.map(_.getOrElse(ServedTweetIdsFeature, Seq.empty)).toSeq.flatten.toSet - - val (removed, kept) = candidates.partition { candidate => - val tweetIdAndSourceId = CandidatesUtil.getTweetIdAndSourceId(candidate) - tweetIdAndSourceId.exists(servedTweetIds.contains) - } - - Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RejectTweetFromViewerFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RejectTweetFromViewerFilter.scala deleted file mode 100644 index c25d9ec2b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RejectTweetFromViewerFilter.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.home_mixer.functional_component.filter - -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -object RejectTweetFromViewerFilter extends Filter[PipelineQuery, TweetCandidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("RejectTweetFromViewer") - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - val (removed, kept) = candidates.partition(candidate => - CandidatesUtil.isAuthoredByViewer(query, candidate.features)) - Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ReplyFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ReplyFilter.scala deleted file mode 100644 index a1b99df66..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ReplyFilter.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.home_mixer.functional_component.filter - -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -object ReplyFilter extends Filter[PipelineQuery, TweetCandidate] { - override val identifier: FilterIdentifier = FilterIdentifier("Reply") - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - - val (kept, removed) = candidates - .partition { candidate => - candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty - } - - val filterResult = FilterResult( - kept = kept.map(_.candidate), - removed = removed.map(_.candidate) - ) - - Stitch.value(filterResult) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetDeduplicationFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetDeduplicationFilter.scala deleted file mode 100644 index 1e1f7f03a..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetDeduplicationFilter.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.home_mixer.functional_component.filter - -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import scala.collection.mutable - -object RetweetDeduplicationFilter extends Filter[PipelineQuery, TweetCandidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("RetweetDeduplication") - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - // If there are 2 retweets of the same native tweet, we will choose the first one - // The tweets are returned in descending score order, so we will choose the higher scored tweet - val dedupedTweetIdsSet = - candidates.partition(_.features.getOrElse(IsRetweetFeature, false)) match { - case (retweets, nativeTweets) => - val nativeTweetIds = nativeTweets.map(_.candidate.id) - val seenTweetIds = mutable.Set[Long]() ++ nativeTweetIds - val dedupedRetweets = retweets.filter { retweet => - val tweetIdAndSourceId = CandidatesUtil.getTweetIdAndSourceId(retweet) - val retweetIsUnique = tweetIdAndSourceId.forall(!seenTweetIds.contains(_)) - if (retweetIsUnique) { - seenTweetIds ++= tweetIdAndSourceId - } - retweetIsUnique - } - (nativeTweets ++ dedupedRetweets).map(_.candidate.id).toSet - } - - val (kept, removed) = - candidates - .map(_.candidate).partition(candidate => dedupedTweetIdsSet.contains(candidate.id)) - Stitch.value(FilterResult(kept = kept, removed = removed)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetFilter.scala deleted file mode 100644 index 0ffc4b00c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetFilter.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.home_mixer.functional_component.filter - -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -object RetweetFilter extends Filter[PipelineQuery, TweetCandidate] { - override val identifier: FilterIdentifier = FilterIdentifier("Retweet") - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - - val (kept, removed) = candidates - .partition { candidate => - !candidate.features.getOrElse(IsRetweetFeature, false) - } - - val filterResult = FilterResult( - kept = kept.map(_.candidate), - removed = removed.map(_.candidate) - ) - - Stitch.value(filterResult) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/BUILD.bazel deleted file mode 100644 index 6be06dee9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/BUILD.bazel +++ /dev/null @@ -1,20 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", - "src/thrift/com/twitter/gizmoduck:thrift-scala", - "stitch/stitch-socialgraph", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", - "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/DismissFatigueGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/DismissFatigueGate.scala deleted file mode 100644 index b35a6cda5..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/DismissFatigueGate.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.twitter.home_mixer.functional_component.gate - -import com.twitter.conversions.DurationOps._ -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelinemixer.clients.manhattan.DismissInfo -import com.twitter.timelineservice.suggests.thriftscala.SuggestType -import com.twitter.util.Duration - -object DismissFatigueGate { - // how long a dismiss action from user needs to be respected - val DefaultBaseDismissDuration = 7.days - val MaximumDismissalCountMultiplier = 4 -} - -case class DismissFatigueGate( - suggestType: SuggestType, - dismissInfoFeature: Feature[PipelineQuery, Map[SuggestType, Option[DismissInfo]]], - baseDismissDuration: Duration = DismissFatigueGate.DefaultBaseDismissDuration, -) extends Gate[PipelineQuery] { - - override val identifier: GateIdentifier = GateIdentifier("DismissFatigue") - - override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { - val dismissInfoMap = query.features.map( - _.getOrElse(dismissInfoFeature, Map.empty[SuggestType, Option[DismissInfo]])) - - val isVisible = dismissInfoMap - .flatMap(_.get(suggestType)) - .flatMap(_.map { info => - val currentDismissalDuration = query.queryTime.since(info.lastDismissed) - val targetDismissalDuration = dismissDurationForCount(info.count, baseDismissDuration) - - currentDismissalDuration > targetDismissalDuration - }).getOrElse(true) - Stitch.value(isVisible) - } - - private def dismissDurationForCount( - dismissCount: Int, - dismissDuration: Duration - ): Duration = - // limit to maximum dismissal duration - dismissDuration * Math.min(dismissCount, DismissFatigueGate.MaximumDismissalCountMultiplier) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/ExcludeSoftUserGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/ExcludeSoftUserGate.scala deleted file mode 100644 index a86159036..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/ExcludeSoftUserGate.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.home_mixer.functional_component.gate - -import com.twitter.gizmoduck.{thriftscala => t} -import com.twitter.home_mixer.model.HomeFeatures.UserTypeFeature -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -/** - * A Soft User is a user who is in the gradual onboarding state. This gate can be - * used to turn off certain functionality like ads for these users. - */ -object ExcludeSoftUserGate extends Gate[PipelineQuery] { - - override val identifier: GateIdentifier = GateIdentifier("ExcludeSoftUser") - - override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { - val softUser = query.features - .exists(_.getOrElse(UserTypeFeature, None).exists(_ == t.UserType.Soft)) - Stitch.value(!softUser) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RequestContextGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RequestContextGate.scala deleted file mode 100644 index 0ffe6793e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RequestContextGate.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.home_mixer.functional_component.gate - -import com.twitter.home_mixer.model.request.DeviceContext.RequestContext -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -/** - * Gate that fetches the request context from the device context and - * continues if the request context matches *any* of the specified ones. - */ -case class RequestContextGate(requestContexts: Seq[RequestContext.Value]) - extends Gate[PipelineQuery with HasDeviceContext] { - - override val identifier: GateIdentifier = GateIdentifier("RequestContext") - - override def shouldContinue(query: PipelineQuery with HasDeviceContext): Stitch[Boolean] = - Stitch.value( - requestContexts.exists(query.deviceContext.flatMap(_.requestContextValue).contains)) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RequestContextNotGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RequestContextNotGate.scala deleted file mode 100644 index c52f05f75..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RequestContextNotGate.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.home_mixer.functional_component.gate - -import com.twitter.home_mixer.model.request.DeviceContext.RequestContext -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -/** - * Gate that fetches the request context from the device context and - * continues if the request context does not match any of the specified ones. - * - * If no input request context is specified, the gate continues - */ -case class RequestContextNotGate(requestContexts: Seq[RequestContext.Value]) - extends Gate[PipelineQuery with HasDeviceContext] { - - override val identifier: GateIdentifier = GateIdentifier("RequestContextNot") - - override def shouldContinue(query: PipelineQuery with HasDeviceContext): Stitch[Boolean] = - Stitch.value( - !requestContexts.exists(query.deviceContext.flatMap(_.requestContextValue).contains)) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/SupportedLanguagesGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/SupportedLanguagesGate.scala deleted file mode 100644 index 53b681236..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/SupportedLanguagesGate.scala +++ /dev/null @@ -1,68 +0,0 @@ -package com.twitter.home_mixer.functional_component.gate - -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -object SupportedLanguagesGate extends Gate[PipelineQuery] { - - override val identifier: GateIdentifier = GateIdentifier("SupportedLanguages") - - // Production languages which have high translation coverage for strings used in Home Timeline. - private val supportedLanguages: Set[String] = Set( - "ar", // Arabic - "ar-x-fm", // Arabic (Female) - "bg", // Bulgarian - "bn", // Bengali - "ca", // Catalan - "cs", // Czech - "da", // Danish - "de", // German - "el", // Greek - "en", // English - "en-gb", // British English - "en-ss", // English Screen shot - "en-xx", // English Pseudo - "es", // Spanish - "eu", // Basque - "fa", // Farsi (Persian) - "fi", // Finnish - "fil", // Filipino - "fr", // French - "ga", // Irish - "gl", // Galician - "gu", // Gujarati - "he", // Hebrew - "hi", // Hindi - "hr", // Croatian - "hu", // Hungarian - "id", // Indonesian - "it", // Italian - "ja", // Japanese - "kn", // Kannada - "ko", // Korean - "mr", // Marathi - "msa", // Malay - "nl", // Dutch - "no", // Norwegian - "pl", // Polish - "pt", // Portuguese - "ro", // Romanian - "ru", // Russian - "sk", // Slovak - "sr", // Serbian - "sv", // Swedish - "ta", // Tamil - "th", // Thai - "tr", // Turkish - "uk", // Ukrainian - "ur", // Urdu - "vi", // Vietnamese - "zh-cn", // Simplified Chinese - "zh-tw" // Traditional Chinese - ) - - override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = - Stitch.value(query.getLanguageCode.forall(supportedLanguages.contains)) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/TimelinesPersistenceStoreLastInjectionGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/TimelinesPersistenceStoreLastInjectionGate.scala deleted file mode 100644 index 31fff2306..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/TimelinesPersistenceStoreLastInjectionGate.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.home_mixer.functional_component.gate - -import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelinemixer.clients.persistence.TimelineResponseV3 -import com.twitter.timelinemixer.injection.store.persistence.TimelinePersistenceUtils -import com.twitter.timelines.configapi.Param -import com.twitter.timelines.util.client_info.ClientPlatform -import com.twitter.timelineservice.model.rich.EntityIdType -import com.twitter.util.Duration -import com.twitter.util.Time - -/** - * Gate used to reduce the frequency of injections. Note that the actual interval between injections may be - * less than the specified minInjectionIntervalParam if data is unavailable or missing. For example, being deleted by - * the persistence store via a TTL or similar mechanism. - * - * @param minInjectionIntervalParam the desired minimum interval between injections - * @param persistenceEntriesFeature the feature for retrieving persisted timeline responses - */ -case class TimelinesPersistenceStoreLastInjectionGate( - minInjectionIntervalParam: Param[Duration], - persistenceEntriesFeature: Feature[PipelineQuery, Seq[TimelineResponseV3]], - entityIdType: EntityIdType.Value) - extends Gate[PipelineQuery] - with TimelinePersistenceUtils { - - override val identifier: GateIdentifier = GateIdentifier("TimelinesPersistenceStoreLastInjection") - - override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = - Stitch( - query.queryTime.since(getLastInjectionTime(query)) > query.params(minInjectionIntervalParam)) - - private def getLastInjectionTime(query: PipelineQuery) = query.features - .flatMap { featureMap => - val timelineResponses = featureMap.getOrElse(persistenceEntriesFeature, Seq.empty) - val clientPlatform = ClientPlatform.fromQueryOptions( - clientAppId = query.clientContext.appId, - userAgent = query.clientContext.userAgent.flatMap(UserAgent.fromString) - ) - val sortedResponses = responseByClient(clientPlatform, timelineResponses) - val latestResponseWithEntityIdTypeEntry = - sortedResponses.find(_.entries.exists(_.entityIdType == entityIdType)) - - latestResponseWithEntityIdTypeEntry.map(_.servedTime) - }.getOrElse(Time.Bottom) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer/BUILD.bazel deleted file mode 100644 index 67763235b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "common-internal/analytics/twitter-client-user-agent-parser/src/main/scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", - "timelineservice/common:model", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer/EditedTweetsCandidatePipelineQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer/EditedTweetsCandidatePipelineQueryTransformer.scala deleted file mode 100644 index 8753f2f28..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer/EditedTweetsCandidatePipelineQueryTransformer.scala +++ /dev/null @@ -1,85 +0,0 @@ -package com.twitter.home_mixer.functional_component.query_transformer - -import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelinemixer.clients.persistence.EntryWithItemIds -import com.twitter.timelines.persistence.thriftscala.RequestType -import com.twitter.timelines.util.client_info.ClientPlatform -import com.twitter.timelineservice.model.rich.EntityIdType -import com.twitter.util.Time - -object EditedTweetsCandidatePipelineQueryTransformer - extends CandidatePipelineQueryTransformer[PipelineQuery, Seq[Long]] { - - override val identifier: TransformerIdentifier = TransformerIdentifier("EditedTweets") - - // The time window for which a tweet remains editable after creation. - private val EditTimeWindow = 60.minutes - - override def transform(query: PipelineQuery): Seq[Long] = { - val applicableCandidates = getApplicableCandidates(query) - - if (applicableCandidates.nonEmpty) { - // Include the response corresponding with the Previous Timeline Load (PTL). - // Any tweets in it could have become stale since being served. - val previousTimelineLoadTime = applicableCandidates.head.servedTime - - // The time window for editing a tweet is 60 minutes, - // so we ignore responses older than (PTL Time - 60 mins). - val inWindowCandidates: Seq[PersistenceStoreEntry] = applicableCandidates - .takeWhile(_.servedTime.until(previousTimelineLoadTime) < EditTimeWindow) - - // Exclude the tweet IDs for which ReplaceEntry instructions have already been sent. - val (tweetsAlreadyReplaced, tweetsToCheck) = inWindowCandidates - .partition(_.entryWithItemIds.itemIds.exists(_.head.entryIdToReplace.nonEmpty)) - - val tweetIdFromEntry: PartialFunction[PersistenceStoreEntry, Long] = { - case entry if entry.tweetId.nonEmpty => entry.tweetId.get - } - - val tweetIdsAlreadyReplaced: Set[Long] = tweetsAlreadyReplaced.collect(tweetIdFromEntry).toSet - val tweetIdsToCheck: Seq[Long] = tweetsToCheck.collect(tweetIdFromEntry) - - tweetIdsToCheck.filterNot(tweetIdsAlreadyReplaced.contains).distinct - } else Seq.empty - } - - // The candidates here come from the Timelines Persistence Store, via a query feature - private def getApplicableCandidates(query: PipelineQuery): Seq[PersistenceStoreEntry] = { - val userAgent = UserAgent.fromString(query.clientContext.userAgent.getOrElse("")) - val clientPlatform = ClientPlatform.fromQueryOptions(query.clientContext.appId, userAgent) - - val sortedResponses = query.features - .getOrElse(FeatureMap.empty) - .getOrElse(PersistenceEntriesFeature, Seq.empty) - .filter(_.clientPlatform == clientPlatform) - .sortBy(-_.servedTime.inMilliseconds) - - val recentResponses = sortedResponses.indexWhere(_.requestType == RequestType.Initial) match { - case -1 => sortedResponses - case lastGetInitialIndex => sortedResponses.take(lastGetInitialIndex + 1) - } - - recentResponses.flatMap { r => - r.entries.collect { - case entry if entry.entityIdType == EntityIdType.Tweet => - PersistenceStoreEntry(entry, r.servedTime, r.clientPlatform, r.requestType) - } - }.distinct - } -} - -case class PersistenceStoreEntry( - entryWithItemIds: EntryWithItemIds, - servedTime: Time, - clientPlatform: ClientPlatform, - requestType: RequestType) { - - // Timelines Persistence Store currently includes 1 tweet ID per entryWithItemIds for tweets - val tweetId: Option[Long] = entryWithItemIds.itemIds.flatMap(_.head.tweetId) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/BUILD.bazel deleted file mode 100644 index 99605204c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/BUILD.bazel +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "src/thrift/com/twitter/timelines/common:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", - "timelineservice/common:model", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/FeedbackFatigueScorer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/FeedbackFatigueScorer.scala deleted file mode 100644 index ceb71139e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/FeedbackFatigueScorer.scala +++ /dev/null @@ -1,144 +0,0 @@ -package com.twitter.home_mixer.functional_component.scorer - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.Conditionally -import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelines.common.{thriftscala => tl} -import com.twitter.timelineservice.model.FeedbackEntry -import com.twitter.timelineservice.{thriftscala => tls} -import com.twitter.util.Time -import scala.collection.mutable - -object FeedbackFatigueScorer - extends Scorer[PipelineQuery, TweetCandidate] - with Conditionally[PipelineQuery] { - - override val identifier: ScorerIdentifier = ScorerIdentifier("FeedbackFatigue") - - override def features: Set[Feature[_, _]] = Set(ScoreFeature) - - override def onlyIf(query: PipelineQuery): Boolean = - query.features.exists(_.getOrElse(FeedbackHistoryFeature, Seq.empty).nonEmpty) - - val DurationForFiltering = 14.days - val DurationForDiscounting = 140.days - private val ScoreMultiplierLowerBound = 0.2 - private val ScoreMultiplierUpperBound = 1.0 - private val ScoreMultiplierIncrementsCount = 4 - private val ScoreMultiplierIncrement = - (ScoreMultiplierUpperBound - ScoreMultiplierLowerBound) / ScoreMultiplierIncrementsCount - private val ScoreMultiplierIncrementDurationInDays = - DurationForDiscounting.inDays / ScoreMultiplierIncrementsCount.toDouble - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - val feedbackEntriesByEngagementType = - query.features - .getOrElse(FeatureMap.empty).getOrElse(FeedbackHistoryFeature, Seq.empty) - .filter { entry => - val timeSinceFeedback = query.queryTime.minus(entry.timestamp) - timeSinceFeedback < DurationForFiltering + DurationForDiscounting && - entry.feedbackType == tls.FeedbackType.SeeFewer - }.groupBy(_.engagementType) - - val authorsToDiscount = - getUserDiscounts( - query.queryTime, - feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Tweet, Seq.empty)) - val likersToDiscount = - getUserDiscounts( - query.queryTime, - feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Like, Seq.empty)) - val followersToDiscount = - getUserDiscounts( - query.queryTime, - feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Follow, Seq.empty)) - val retweetersToDiscount = - getUserDiscounts( - query.queryTime, - feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Retweet, Seq.empty)) - - val featureMaps = candidates.map { candidate => - val multiplier = getScoreMultiplier( - candidate, - authorsToDiscount, - likersToDiscount, - followersToDiscount, - retweetersToDiscount - ) - val score = candidate.features.getOrElse(ScoreFeature, None) - FeatureMapBuilder().add(ScoreFeature, score.map(_ * multiplier)).build() - } - - Stitch.value(featureMaps) - } - - def getScoreMultiplier( - candidate: CandidateWithFeatures[TweetCandidate], - authorsToDiscount: Map[Long, Double], - likersToDiscount: Map[Long, Double], - followersToDiscount: Map[Long, Double], - retweetersToDiscount: Map[Long, Double], - ): Double = { - val originalAuthorId = - CandidatesUtil.getOriginalAuthorId(candidate.features).getOrElse(0L) - val originalAuthorMultiplier = authorsToDiscount.getOrElse(originalAuthorId, 1.0) - - val likers = candidate.features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty) - val likerMultipliers = likers.flatMap(likersToDiscount.get) - val likerMultiplier = - if (likerMultipliers.nonEmpty && likers.size == likerMultipliers.size) - likerMultipliers.max - else 1.0 - - val followers = candidate.features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty) - val followerMultipliers = followers.flatMap(followersToDiscount.get) - val followerMultiplier = - if (followerMultipliers.nonEmpty && followers.size == followerMultipliers.size && - likers.isEmpty) - followerMultipliers.max - else 1.0 - - val authorId = candidate.features.getOrElse(AuthorIdFeature, None).getOrElse(0L) - val retweeterMultiplier = - if (candidate.features.getOrElse(IsRetweetFeature, false)) - retweetersToDiscount.getOrElse(authorId, 1.0) - else 1.0 - - originalAuthorMultiplier * likerMultiplier * followerMultiplier * retweeterMultiplier - } - - def getUserDiscounts( - queryTime: Time, - feedbackEntries: Seq[FeedbackEntry], - ): Map[Long, Double] = { - val userDiscounts = mutable.Map[Long, Double]() - feedbackEntries - .collect { - case FeedbackEntry(_, _, tl.FeedbackEntity.UserId(userId), timestamp, _) => - val timeSinceFeedback = queryTime.minus(timestamp) - val timeSinceDiscounting = timeSinceFeedback - DurationForFiltering - val multiplier = ((timeSinceDiscounting.inDays / ScoreMultiplierIncrementDurationInDays) - * ScoreMultiplierIncrement + ScoreMultiplierLowerBound) - userDiscounts.update(userId, multiplier) - } - userDiscounts.toMap - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/OONTweetScalingScorer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/OONTweetScalingScorer.scala deleted file mode 100644 index 4b1cec4a5..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/OONTweetScalingScorer.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.twitter.home_mixer.functional_component.scorer - -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -/** - * Scales scores of each out-of-network tweet by the specified scale factor - */ -object OONTweetScalingScorer extends Scorer[PipelineQuery, TweetCandidate] { - - override val identifier: ScorerIdentifier = ScorerIdentifier("OONTweetScaling") - - override val features: Set[Feature[_, _]] = Set(ScoreFeature) - - private val ScaleFactor = 0.75 - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - Stitch.value { - candidates.map { candidate => - val score = candidate.features.getOrElse(ScoreFeature, None) - val updatedScore = if (selector(candidate)) score.map(_ * ScaleFactor) else score - FeatureMapBuilder().add(ScoreFeature, updatedScore).build() - } - } - } - - /** - * We should only be applying this multiplier to Out-Of-Network tweets. - * In-Network Retweets of Out-Of-Network tweets should not have this multiplier applied - */ - private def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = { - !candidate.features.getOrElse(InNetworkFeature, false) && - !candidate.features.getOrElse(IsRetweetFeature, false) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/BUILD.bazel deleted file mode 100644 index 3689de6ea..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/BUILD.bazel +++ /dev/null @@ -1,25 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/item", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/scala/com/twitter/suggests/controller_data", - "stringcenter/client", - "stringcenter/client/src/main/java", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/DebunchCandidates.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/DebunchCandidates.scala deleted file mode 100644 index 648174273..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/DebunchCandidates.scala +++ /dev/null @@ -1,83 +0,0 @@ -package com.twitter.home_mixer.functional_component.selector - -import com.twitter.home_mixer.functional_component.selector.DebunchCandidates.TrailingTweetsMinSize -import com.twitter.home_mixer.functional_component.selector.DebunchCandidates.TrailingTweetsPortionToKeep -import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.common.CandidateScope.PartitionedCandidates -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -trait MustDebunch { - def apply(candidate: CandidateWithDetails): Boolean -} - -object DebunchCandidates { - val TrailingTweetsMinSize = 5 - val TrailingTweetsPortionToKeep = 0.1 -} - -/** - * This selector rearranges the candidates to only allow bunches of size [[maxBunchSize]], where a - * bunch is a consecutive sequence of candidates that meet [[mustDebunch]]. - */ -case class DebunchCandidates( - override val pipelineScope: CandidateScope, - mustDebunch: MustDebunch, - maxBunchSize: Int) - extends Selector[PipelineQuery] { - - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val PartitionedCandidates(selectedCandidates, otherCandidates) = - pipelineScope.partition(remainingCandidates) - val mutableCandidates = collection.mutable.ListBuffer(selectedCandidates: _*) - - var candidatePointer = 0 - var nonDebunchPointer = 0 - var bunchSize = 0 - var finalNonDebunch = -1 - - while (candidatePointer < mutableCandidates.size) { - if (mustDebunch(mutableCandidates(candidatePointer))) bunchSize += 1 - else { - bunchSize = 0 - finalNonDebunch = candidatePointer - } - - if (bunchSize > maxBunchSize) { - nonDebunchPointer = Math.max(candidatePointer, nonDebunchPointer) - while (nonDebunchPointer < mutableCandidates.size && - mustDebunch(mutableCandidates(nonDebunchPointer))) { - nonDebunchPointer += 1 - } - if (nonDebunchPointer == mutableCandidates.size) - candidatePointer = mutableCandidates.size - else { - val nextNonDebunch = mutableCandidates(nonDebunchPointer) - mutableCandidates.remove(nonDebunchPointer) - mutableCandidates.insert(candidatePointer, nextNonDebunch) - bunchSize = 0 - finalNonDebunch = candidatePointer - } - } - - candidatePointer += 1 - } - - val debunchedCandidates = if (query.features.exists(_.getOrElse(GetNewerFeature, false))) { - val trailingTweetsSize = mutableCandidates.size - finalNonDebunch - 1 - val keepCandidates = finalNonDebunch + 1 + - Math.max(TrailingTweetsMinSize, TrailingTweetsPortionToKeep * trailingTweetsSize).toInt - mutableCandidates.toList.take(keepCandidates) - } else mutableCandidates.toList - - val updatedCandidates = otherCandidates ++ debunchedCandidates - SelectorResult(remainingCandidates = updatedCandidates, result = result) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateConversationModuleId.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateConversationModuleId.scala deleted file mode 100644 index bbde0ed0d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateConversationModuleId.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.home_mixer.functional_component.selector - -import com.twitter.product_mixer.component_library.model.presentation.urt.UrtModulePresentation -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.common.CandidateScope.PartitionedCandidates -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * This selector updates the id of the conversation modules to be the head of the module's id. - */ -case class UpdateConversationModuleId( - override val pipelineScope: CandidateScope) - extends Selector[PipelineQuery] { - - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val PartitionedCandidates(selectedCandidates, otherCandidates) = - pipelineScope.partition(remainingCandidates) - - val updatedCandidates = selectedCandidates.map { - case module @ ModuleCandidateWithDetails(candidates, presentationOpt, _) => - val updatedPresentation = presentationOpt.map { - case urtModule @ UrtModulePresentation(timelineModule) => - urtModule.copy(timelineModule = - timelineModule.copy(id = candidates.head.candidateIdLong)) - } - module.copy(presentation = updatedPresentation) - case candidate => candidate - } - - SelectorResult(remainingCandidates = updatedCandidates ++ otherCandidates, result = result) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateHomeClientEventDetails.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateHomeClientEventDetails.scala deleted file mode 100644 index c7944b506..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateHomeClientEventDetails.scala +++ /dev/null @@ -1,137 +0,0 @@ -package com.twitter.home_mixer.functional_component.selector - -import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventDetailsBuilder -import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature -import com.twitter.home_mixer.model.HomeFeatures.ConversationModule2DisplayedTweetsFeature -import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleHasGapFeature -import com.twitter.home_mixer.model.HomeFeatures.HasRandomTweetFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRandomTweetAboveFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRandomTweetFeature -import com.twitter.home_mixer.model.HomeFeatures.PositionFeature -import com.twitter.home_mixer.model.HomeFeatures.ServedInConversationModuleFeature -import com.twitter.home_mixer.model.HomeFeatures.ServedSizeFeature -import com.twitter.product_mixer.component_library.model.presentation.urt.UrtItemPresentation -import com.twitter.product_mixer.component_library.model.presentation.urt.UrtModulePresentation -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * Builds serialized tweet type metrics controller data and updates Client Event Details - * and Candidate Presentations with this info. - * - * Currently only updates presentation of Item Candidates. This needs to be updated - * when modules are added. - * - * This is implemented as a Selector instead of a Decorator in the Candidate Pipeline - * because we need to add controller data that looks at the final timeline as a whole - * (e.g. served size, final candidate positions). - * - * @param candidatePipelines - only candidates from the specified pipeline will be updated - */ -case class UpdateHomeClientEventDetails(candidatePipelines: Set[CandidatePipelineIdentifier]) - extends Selector[PipelineQuery] { - - override val pipelineScope: CandidateScope = SpecificPipelines(candidatePipelines) - - private val detailsBuilder = HomeClientEventDetailsBuilder() - - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val selectedCandidates = result.filter(pipelineScope.contains) - - val randomTweetsByPosition = result - .map(_.features.getOrElse(IsRandomTweetFeature, false)) - .zipWithIndex.map(_.swap).toMap - - val resultFeatures = FeatureMapBuilder() - .add(ServedSizeFeature, Some(selectedCandidates.size)) - .add(HasRandomTweetFeature, randomTweetsByPosition.valuesIterator.contains(true)) - .build() - - val updatedResult = result.zipWithIndex.map { - case (item @ ItemCandidateWithDetails(candidate, _, _), position) - if pipelineScope.contains(item) => - val resultCandidateFeatures = FeatureMapBuilder() - .add(PositionFeature, Some(position)) - .add(IsRandomTweetAboveFeature, randomTweetsByPosition.getOrElse(position - 1, false)) - .build() - - updateItemPresentation(query, item, resultFeatures, resultCandidateFeatures) - - case (module @ ModuleCandidateWithDetails(candidates, presentation, features), position) - if pipelineScope.contains(module) => - val resultCandidateFeatures = FeatureMapBuilder() - .add(PositionFeature, Some(position)) - .add(IsRandomTweetAboveFeature, randomTweetsByPosition.getOrElse(position - 1, false)) - .add(ServedInConversationModuleFeature, true) - .add(ConversationModule2DisplayedTweetsFeature, module.candidates.size == 2) - .add( - ConversationModuleHasGapFeature, - module.candidates.last.features.getOrElse(AncestorsFeature, Seq.empty).size > 2) - .build() - - val updatedItemCandidates = - candidates.map(updateItemPresentation(query, _, resultFeatures, resultCandidateFeatures)) - - val updatedCandidateFeatures = features ++ resultFeatures ++ resultCandidateFeatures - - val updatedPresentation = presentation.map { - case urtModule @ UrtModulePresentation(timelineModule) => - val clientEventDetails = - detailsBuilder( - query, - candidates.last.candidate, - query.features.get ++ updatedCandidateFeatures) - val updatedClientEventInfo = - timelineModule.clientEventInfo.map(_.copy(details = clientEventDetails)) - val updatedTimelineModule = - timelineModule.copy(clientEventInfo = updatedClientEventInfo) - urtModule.copy(timelineModule = updatedTimelineModule) - } - - module.copy( - candidates = updatedItemCandidates, - presentation = updatedPresentation, - features = updatedCandidateFeatures - ) - - case (any, position) => any - } - - SelectorResult(remainingCandidates = remainingCandidates, result = updatedResult) - } - - private def updateItemPresentation( - query: PipelineQuery, - item: ItemCandidateWithDetails, - resultCandidateFeatures: FeatureMap, - resultFeatures: FeatureMap, - ): ItemCandidateWithDetails = { - val updatedItemCandidateFeatures = item.features ++ resultFeatures ++ resultCandidateFeatures - - val updatedPresentation = item.presentation.map { - case urtItem @ UrtItemPresentation(timelineItem: TweetItem, _) => - val clientEventDetails = - detailsBuilder(query, item.candidate, query.features.get ++ updatedItemCandidateFeatures) - val updatedClientEventInfo = - timelineItem.clientEventInfo.map(_.copy(details = clientEventDetails)) - val updatedTimelineItem = timelineItem.copy(clientEventInfo = updatedClientEventInfo) - urtItem.copy(timelineItem = updatedTimelineItem) - case any => any - } - item.copy(presentation = updatedPresentation, features = updatedItemCandidateFeatures) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateNewTweetsPillDecoration.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateNewTweetsPillDecoration.scala deleted file mode 100644 index a805ae1a7..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateNewTweetsPillDecoration.scala +++ /dev/null @@ -1,80 +0,0 @@ -package com.twitter.home_mixer.functional_component.selector - -import com.twitter.home_mixer.functional_component.selector.UpdateNewTweetsPillDecoration.NumAvatars -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.home_mixer.param.HomeGlobalParams.EnableNewTweetsPillAvatarsParam -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.product_mixer.component_library.model.candidate.ShowAlertCandidate -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.component_library.model.presentation.urt.UrtItemPresentation -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.ShowAlert -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichText -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stringcenter.client.StringCenter -import com.twitter.stringcenter.client.core.ExternalString - -object UpdateNewTweetsPillDecoration { - val NumAvatars = 3 -} - -case class UpdateNewTweetsPillDecoration[Query <: PipelineQuery with HasDeviceContext]( - override val pipelineScope: CandidateScope, - stringCenter: StringCenter, - seeNewTweetsString: ExternalString, - tweetedString: ExternalString) - extends Selector[Query] { - - override def apply( - query: Query, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val (alerts, otherCandidates) = - remainingCandidates.partition(candidate => - candidate.isCandidateType[ShowAlertCandidate]() && pipelineScope.contains(candidate)) - val updatedCandidates = alerts - .collectFirst { - case newTweetsPill: ItemCandidateWithDetails => - val userIds = CandidatesUtil - .getItemCandidatesWithOnlyModuleLast(result) - .filter(candidate => - candidate.isCandidateType[TweetCandidate]() && pipelineScope.contains(candidate)) - .filterNot(_.features.getOrElse(IsRetweetFeature, false)) - .flatMap(_.features.getOrElse(AuthorIdFeature, None)) - .filterNot(_ == query.getRequiredUserId) - .distinct - - val updatedPresentation = newTweetsPill.presentation.map { - case presentation: UrtItemPresentation => - presentation.timelineItem match { - case alert: ShowAlert => - val text = if (useAvatars(query, userIds)) tweetedString else seeNewTweetsString - val richText = RichText( - text = stringCenter.prepare(text), - entities = List.empty, - rtl = None, - alignment = None) - - val updatedAlert = - alert.copy(userIds = Some(userIds.take(NumAvatars)), richText = Some(richText)) - presentation.copy(timelineItem = updatedAlert) - } - } - otherCandidates :+ newTweetsPill.copy(presentation = updatedPresentation) - }.getOrElse(remainingCandidates) - - SelectorResult(remainingCandidates = updatedCandidates, result = result) - } - - private def useAvatars(query: Query, userIds: Seq[Long]): Boolean = { - val enableAvatars = query.params(EnableNewTweetsPillAvatarsParam) - enableAvatars && userIds.size >= NumAvatars - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/BUILD.bazel deleted file mode 100644 index f352ce63a..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/BUILD.bazel +++ /dev/null @@ -1,47 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "eventbus/client/src/main/scala/com/twitter/eventbus/client", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_subscribe_module", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "src/scala/com/twitter/timelines/prediction/common/adapters", - "src/scala/com/twitter/timelines/prediction/features/common", - "src/thrift/com/twitter/timelines/impression:thrift-scala", - "src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala", - "src/thrift/com/twitter/timelines/impression_store:thrift-scala", - "src/thrift/com/twitter/timelines/served_candidates_logging:served_candidates_logging-scala", - "src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java", - "src/thrift/com/twitter/timelines/timeline_logging:thrift-scala", - "src/thrift/com/twitter/user_session_store:thrift-scala", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", - "timelines/ml:kafka", - "timelines/ml/cont_train/common/client/src/main/scala/com/twitter/timelines/ml/cont_train/common/client/kafka", - "timelines/ml/cont_train/common/domain/src/main/scala/com/twitter/timelines/ml/cont_train/common/domain/non_scalding", - "timelines/src/main/scala/com/twitter/timelines/clientconfig", - "timelines/src/main/scala/com/twitter/timelines/clients/manhattan/store", - "timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter", - "timelines/src/main/scala/com/twitter/timelines/impressionstore/store", - "timelines/src/main/scala/com/twitter/timelines/injection/scribe", - "timelineservice/common:model", - "user_session_store/src/main/scala/com/twitter/user_session_store", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ClientEventsBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ClientEventsBuilder.scala deleted file mode 100644 index a5cf739d3..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ClientEventsBuilder.scala +++ /dev/null @@ -1,185 +0,0 @@ -package com.twitter.home_mixer.functional_component.side_effect - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.functional_component.decorator.HomeQueryTypePredicates -import com.twitter.home_mixer.functional_component.decorator.builder.HomeTweetTypePredicates -import com.twitter.home_mixer.model.HomeFeatures.AccountAgeFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.model.HomeFeatures.VideoDurationMsFeature -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.model.request.ListTweetsProduct -import com.twitter.home_mixer.model.request.SubscribedProduct -import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.ClientEvent -import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.EventNamespace -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.injection.scribe.InjectionScribeUtil - -private[side_effect] sealed trait ClientEventsBuilder { - private val FollowingSection = Some("latest") - private val ForYouSection = Some("home") - private val ListTweetsSection = Some("list") - private val SubscribedSection = Some("subscribed") - - protected def section(query: PipelineQuery): Option[String] = { - query.product match { - case FollowingProduct => FollowingSection - case ForYouProduct => ForYouSection - case ListTweetsProduct => ListTweetsSection - case SubscribedProduct => SubscribedSection - case other => throw new UnsupportedOperationException(s"Unknown product: $other") - } - } - - protected def count( - candidates: Seq[CandidateWithDetails], - predicate: FeatureMap => Boolean = _ => true, - queryFeatures: FeatureMap = FeatureMap.empty - ): Option[Long] = Some(candidates.view.count(item => predicate(item.features ++ queryFeatures))) - - protected def sum( - candidates: Seq[CandidateWithDetails], - predicate: FeatureMap => Option[Int], - queryFeatures: FeatureMap = FeatureMap.empty - ): Option[Long] = - Some(candidates.view.flatMap(item => predicate(item.features ++ queryFeatures)).sum) -} - -private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder { - - private val ServedTweetsAction = Some("served_tweets") - private val ServedUsersAction = Some("served_users") - private val InjectedComponent = Some("injected") - private val PromotedComponent = Some("promoted") - private val WhoToFollowComponent = Some("who_to_follow") - private val WhoToSubscribeComponent = Some("who_to_subscribe") - private val WithVideoDurationComponent = Some("with_video_duration") - private val VideoDurationSumElement = Some("video_duration_sum") - private val NumVideosElement = Some("num_videos") - - def build( - query: PipelineQuery, - injectedTweets: Seq[ItemCandidateWithDetails], - promotedTweets: Seq[ItemCandidateWithDetails], - whoToFollowUsers: Seq[ItemCandidateWithDetails], - whoToSubscribeUsers: Seq[ItemCandidateWithDetails] - ): Seq[ClientEvent] = { - val baseEventNamespace = EventNamespace( - section = section(query), - action = ServedTweetsAction - ) - val overallServedEvents = Seq( - ClientEvent(baseEventNamespace, eventValue = count(injectedTweets ++ promotedTweets)), - ClientEvent( - baseEventNamespace.copy(component = InjectedComponent), - eventValue = count(injectedTweets)), - ClientEvent( - baseEventNamespace.copy(component = PromotedComponent), - eventValue = count(promotedTweets)), - ClientEvent( - baseEventNamespace.copy(component = WhoToFollowComponent, action = ServedUsersAction), - eventValue = count(whoToFollowUsers)), - ClientEvent( - baseEventNamespace.copy(component = WhoToSubscribeComponent, action = ServedUsersAction), - eventValue = count(whoToSubscribeUsers)), - ) - - val tweetTypeServedEvents = HomeTweetTypePredicates.PredicateMap.map { - case (tweetType, predicate) => - ClientEvent( - baseEventNamespace.copy(component = InjectedComponent, element = Some(tweetType)), - eventValue = count(injectedTweets, predicate, query.features.getOrElse(FeatureMap.empty)) - ) - }.toSeq - - val suggestTypeServedEvents = injectedTweets - .flatMap(_.features.getOrElse(SuggestTypeFeature, None)) - .map { - InjectionScribeUtil.scribeComponent - } - .groupBy(identity).map { - case (suggestType, group) => - ClientEvent( - baseEventNamespace.copy(component = suggestType), - eventValue = Some(group.size.toLong)) - }.toSeq - - // Video duration events - val numVideosEvent = ClientEvent( - baseEventNamespace.copy(component = WithVideoDurationComponent, element = NumVideosElement), - eventValue = count(injectedTweets, _.getOrElse(VideoDurationMsFeature, None).nonEmpty) - ) - val videoDurationSumEvent = ClientEvent( - baseEventNamespace - .copy(component = WithVideoDurationComponent, element = VideoDurationSumElement), - eventValue = sum(injectedTweets, _.getOrElse(VideoDurationMsFeature, None)) - ) - val videoEvents = Seq(numVideosEvent, videoDurationSumEvent) - - overallServedEvents ++ tweetTypeServedEvents ++ suggestTypeServedEvents ++ videoEvents - } -} - -private[side_effect] object EmptyTimelineEventsBuilder extends ClientEventsBuilder { - private val EmptyAction = Some("empty") - private val AccountAgeLessThan30MinutesComponent = Some("account_age_less_than_30_minutes") - private val ServedNonPromotedTweetElement = Some("served_non_promoted_tweet") - - def build( - query: PipelineQuery, - injectedTweets: Seq[ItemCandidateWithDetails] - ): Seq[ClientEvent] = { - val baseEventNamespace = EventNamespace( - section = section(query), - action = EmptyAction - ) - - // Empty timeline events - val accountAgeLessThan30Minutes = query.features - .flatMap(_.getOrElse(AccountAgeFeature, None)) - .exists(_.untilNow < 30.minutes) - val isEmptyTimeline = count(injectedTweets).contains(0L) - val predicates = Seq( - None -> isEmptyTimeline, - AccountAgeLessThan30MinutesComponent -> (isEmptyTimeline && accountAgeLessThan30Minutes) - ) - for { - (component, predicate) <- predicates - if predicate - } yield ClientEvent( - baseEventNamespace.copy(component = component, element = ServedNonPromotedTweetElement)) - } -} - -private[side_effect] object QueryEventsBuilder extends ClientEventsBuilder { - - private val ServedSizePredicateMap: Map[String, Int => Boolean] = Map( - ("size_is_empty", _ <= 0), - ("size_at_most_5", _ <= 5), - ("size_at_most_10", _ <= 10), - ("size_at_most_35", _ <= 35) - ) - - def build( - query: PipelineQuery, - injectedTweets: Seq[ItemCandidateWithDetails] - ): Seq[ClientEvent] = { - val baseEventNamespace = EventNamespace( - section = section(query) - ) - val queryFeatureMap = query.features.getOrElse(FeatureMap.empty) - val servedSizeQueryEvents = - for { - (queryPredicateName, queryPredicate) <- HomeQueryTypePredicates.PredicateMap - if queryPredicate(queryFeatureMap) - (servedSizePredicateName, servedSizePredicate) <- ServedSizePredicateMap - if servedSizePredicate(injectedTweets.size) - } yield ClientEvent( - baseEventNamespace - .copy(component = Some(servedSizePredicateName), action = Some(queryPredicateName))) - servedSizeQueryEvents.toSeq - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeClientEventSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeClientEventSideEffect.scala deleted file mode 100644 index a8feef707..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeClientEventSideEffect.scala +++ /dev/null @@ -1,75 +0,0 @@ -package com.twitter.home_mixer.functional_component.side_effect - -import com.twitter.clientapp.thriftscala.LogEvent -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.logpipeline.client.common.EventPublisher -import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * Side effect that logs served tweet metrics to Scribe as client events. - */ -case class HomeScribeClientEventSideEffect( - enableScribeClientEvents: Boolean, - override val logPipelinePublisher: EventPublisher[LogEvent], - injectedTweetsCandidatePipelineIdentifiers: Seq[CandidatePipelineIdentifier], - adsCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None, - whoToFollowCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None, - whoToSubscribeCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None) - extends ScribeClientEventSideEffect[PipelineQuery, Timeline] - with PipelineResultSideEffect.Conditionally[ - PipelineQuery, - Timeline - ] { - - override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeClientEvent") - - override val page = "timelinemixer" - - override def onlyIf( - query: PipelineQuery, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: Timeline - ): Boolean = enableScribeClientEvents - - override def buildClientEvents( - query: PipelineQuery, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: Timeline - ): Seq[ScribeClientEventSideEffect.ClientEvent] = { - - val itemCandidates = CandidatesUtil.getItemCandidates(selectedCandidates) - val sources = itemCandidates.groupBy(_.source) - val injectedTweets = - injectedTweetsCandidatePipelineIdentifiers.flatMap(sources.getOrElse(_, Seq.empty)) - val promotedTweets = adsCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten - - // WhoToFollow and WhoToSubscribe modules are not required for all home-mixer products, e.g. list tweets timeline. - val whoToFollowUsers = whoToFollowCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten - val whoToSubscribeUsers = - whoToSubscribeCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten - - val servedEvents = ServedEventsBuilder - .build(query, injectedTweets, promotedTweets, whoToFollowUsers, whoToSubscribeUsers) - - val emptyTimelineEvents = EmptyTimelineEventsBuilder.build(query, injectedTweets) - - val queryEvents = QueryEventsBuilder.build(query, injectedTweets) - - (servedEvents ++ emptyTimelineEvents ++ queryEvents).filter(_.eventValue.forall(_ > 0)) - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.9) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeServedCandidatesSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeServedCandidatesSideEffect.scala deleted file mode 100644 index 3f19dfc99..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeServedCandidatesSideEffect.scala +++ /dev/null @@ -1,245 +0,0 @@ -package com.twitter.home_mixer.functional_component.side_effect - -import com.twitter.finagle.tracing.Trace -import com.twitter.home_mixer.marshaller.timeline_logging.PromotedTweetDetailsMarshaller -import com.twitter.home_mixer.marshaller.timeline_logging.TweetDetailsMarshaller -import com.twitter.home_mixer.marshaller.timeline_logging.WhoToFollowDetailsMarshaller -import com.twitter.home_mixer.model.HomeFeatures.GetInitialFeature -import com.twitter.home_mixer.model.HomeFeatures.GetMiddleFeature -import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature -import com.twitter.home_mixer.model.HomeFeatures.GetOlderFeature -import com.twitter.home_mixer.model.HomeFeatures.HasDarkRequestFeature -import com.twitter.home_mixer.model.HomeFeatures.RequestJoinIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature -import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature -import com.twitter.home_mixer.model.request.DeviceContext.RequestContext -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.home_mixer.model.request.HasSeenTweetIds -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.model.request.SubscribedProduct -import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeServedCandidatesFlag -import com.twitter.home_mixer.param.HomeGlobalParams.EnableScribeServedCandidatesParam -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.inject.annotations.Flag -import com.twitter.logpipeline.client.common.EventPublisher -import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate -import com.twitter.product_mixer.component_library.model.candidate.BaseUserCandidate -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidateDecorator -import com.twitter.product_mixer.component_library.side_effect.ScribeLogEventSideEffect -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.AddEntriesTimelineInstruction -import com.twitter.product_mixer.core.model.marshalling.response.urt.ModuleItem -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.timeline_logging.{thriftscala => thrift} -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Side effect that logs home timeline served candidates to Scribe. - */ -@Singleton -class HomeScribeServedCandidatesSideEffect @Inject() ( - @Flag(ScribeServedCandidatesFlag) enableScribeServedCandidates: Boolean, - scribeEventPublisher: EventPublisher[thrift.ServedEntry]) - extends ScribeLogEventSideEffect[ - thrift.ServedEntry, - PipelineQuery with HasSeenTweetIds with HasDeviceContext, - Timeline - ] - with PipelineResultSideEffect.Conditionally[ - PipelineQuery with HasSeenTweetIds with HasDeviceContext, - Timeline - ] { - - override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeServedCandidates") - - override def onlyIf( - query: PipelineQuery with HasSeenTweetIds with HasDeviceContext, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: Timeline - ): Boolean = enableScribeServedCandidates && query.params(EnableScribeServedCandidatesParam) - - override def buildLogEvents( - query: PipelineQuery with HasSeenTweetIds with HasDeviceContext, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: Timeline - ): Seq[thrift.ServedEntry] = { - val timelineType = query.product match { - case FollowingProduct => thrift.TimelineType.HomeLatest - case ForYouProduct => thrift.TimelineType.Home - case SubscribedProduct => thrift.TimelineType.HomeSubscribed - case other => throw new UnsupportedOperationException(s"Unknown product: $other") - } - val requestProvenance = query.deviceContext.map { deviceContext => - deviceContext.requestContextValue match { - case RequestContext.Foreground => thrift.RequestProvenance.Foreground - case RequestContext.Launch => thrift.RequestProvenance.Launch - case RequestContext.PullToRefresh => thrift.RequestProvenance.Ptr - case _ => thrift.RequestProvenance.Other - } - } - val queryType = query.features.map { featureMap => - if (featureMap.getOrElse(GetOlderFeature, false)) thrift.QueryType.GetOlder - else if (featureMap.getOrElse(GetNewerFeature, false)) thrift.QueryType.GetNewer - else if (featureMap.getOrElse(GetMiddleFeature, false)) thrift.QueryType.GetMiddle - else if (featureMap.getOrElse(GetInitialFeature, false)) thrift.QueryType.GetInitial - else thrift.QueryType.Other - } - val requestInfo = thrift.RequestInfo( - requestTimeMs = query.queryTime.inMilliseconds, - traceId = Trace.id.traceId.toLong, - userId = query.getOptionalUserId, - clientAppId = query.clientContext.appId, - hasDarkRequest = query.features.flatMap(_.getOrElse(HasDarkRequestFeature, None)), - parentId = Some(Trace.id.parentId.toLong), - spanId = Some(Trace.id.spanId.toLong), - timelineType = Some(timelineType), - ipAddress = query.clientContext.ipAddress, - userAgent = query.clientContext.userAgent, - queryType = queryType, - requestProvenance = requestProvenance, - languageCode = query.clientContext.languageCode, - countryCode = query.clientContext.countryCode, - requestEndTimeMs = Some(Time.now.inMilliseconds), - servedRequestId = query.features.flatMap(_.getOrElse(ServedRequestIdFeature, None)), - requestJoinId = query.features.flatMap(_.getOrElse(RequestJoinIdFeature, None)) - ) - - val tweetIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] = - selectedCandidates.flatMap { - case item: ItemCandidateWithDetails if item.candidate.isInstanceOf[BaseTweetCandidate] => - Seq((item.candidateIdLong, item)) - case module: ModuleCandidateWithDetails - if module.candidates.headOption.exists(_.candidate.isInstanceOf[BaseTweetCandidate]) => - module.candidates.map(item => (item.candidateIdLong, item)) - case _ => Seq.empty - }.toMap - - val userIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] = - selectedCandidates.flatMap { - case module: ModuleCandidateWithDetails - if module.candidates.forall(_.candidate.isInstanceOf[BaseUserCandidate]) => - module.candidates.map { item => - (item.candidateIdLong, item) - } - case _ => Seq.empty - }.toMap - - response.instructions.zipWithIndex - .collect { - case (AddEntriesTimelineInstruction(entries), index) => - entries.collect { - case entry: TweetItem if entry.promotedMetadata.isDefined => - val promotedTweetDetails = PromotedTweetDetailsMarshaller(entry, index) - Seq( - thrift.EntryInfo( - id = entry.id, - position = index.shortValue(), - entryId = entry.entryIdentifier, - entryType = thrift.EntryType.PromotedTweet, - sortIndex = entry.sortIndex, - verticalSize = Some(1), - displayType = Some(entry.displayType.toString), - details = Some(thrift.ItemDetails.PromotedTweetDetails(promotedTweetDetails)) - ) - ) - case entry: TweetItem => - val candidate = tweetIdToItemCandidateMap(entry.id) - val tweetDetails = TweetDetailsMarshaller(entry, candidate) - Seq( - thrift.EntryInfo( - id = candidate.candidateIdLong, - position = index.shortValue(), - entryId = entry.entryIdentifier, - entryType = thrift.EntryType.Tweet, - sortIndex = entry.sortIndex, - verticalSize = Some(1), - score = candidate.features.getOrElse(ScoreFeature, None), - displayType = Some(entry.displayType.toString), - details = Some(thrift.ItemDetails.TweetDetails(tweetDetails)) - ) - ) - case module: TimelineModule - if module.entryNamespace.toString == WhoToFollowCandidateDecorator.EntryNamespaceString => - module.items.collect { - case ModuleItem(entry: UserItem, _, _) => - val candidate = userIdToItemCandidateMap(entry.id) - val whoToFollowDetails = WhoToFollowDetailsMarshaller(entry, candidate) - thrift.EntryInfo( - id = entry.id, - position = index.shortValue(), - entryId = module.entryIdentifier, - entryType = thrift.EntryType.WhoToFollowModule, - sortIndex = module.sortIndex, - score = candidate.features.getOrElse(ScoreFeature, None), - displayType = Some(entry.displayType.toString), - details = Some(thrift.ItemDetails.WhoToFollowDetails(whoToFollowDetails)) - ) - } - case module: TimelineModule - if module.entryNamespace.toString == WhoToSubscribeCandidateDecorator.EntryNamespaceString => - module.items.collect { - case ModuleItem(entry: UserItem, _, _) => - val candidate = userIdToItemCandidateMap(entry.id) - val whoToSubscribeDetails = WhoToFollowDetailsMarshaller(entry, candidate) - thrift.EntryInfo( - id = entry.id, - position = index.shortValue(), - entryId = module.entryIdentifier, - entryType = thrift.EntryType.WhoToSubscribeModule, - sortIndex = module.sortIndex, - score = candidate.features.getOrElse(ScoreFeature, None), - displayType = Some(entry.displayType.toString), - details = Some(thrift.ItemDetails.WhoToFollowDetails(whoToSubscribeDetails)) - ) - } - case module: TimelineModule - if module.sortIndex.isDefined && module.items.headOption.exists( - _.item.isInstanceOf[TweetItem]) => - module.items.collect { - case ModuleItem(entry: TweetItem, _, _) => - val candidate = tweetIdToItemCandidateMap(entry.id) - thrift.EntryInfo( - id = entry.id, - position = index.shortValue(), - entryId = module.entryIdentifier, - entryType = thrift.EntryType.ConversationModule, - sortIndex = module.sortIndex, - score = candidate.features.getOrElse(ScoreFeature, None), - displayType = Some(entry.displayType.toString) - ) - } - case _ => Seq.empty - }.flatten - // Other instructions - case _ => Seq.empty[thrift.EntryInfo] - }.flatten.map { entryInfo => - thrift.ServedEntry( - entry = Some(entryInfo), - request = requestInfo - ) - } - } - - override val logPipelinePublisher: EventPublisher[thrift.ServedEntry] = - scribeEventPublisher - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert() - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsEventBusSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsEventBusSideEffect.scala deleted file mode 100644 index c0437767e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsEventBusSideEffect.scala +++ /dev/null @@ -1,95 +0,0 @@ -package com.twitter.home_mixer.functional_component.side_effect - -import com.twitter.eventbus.client.EventBusPublisher -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.model.request.SubscribedProduct -import com.twitter.home_mixer.model.request.HasSeenTweetIds -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelines.impressionstore.thriftscala.Impression -import com.twitter.timelines.impressionstore.thriftscala.ImpressionList -import com.twitter.timelines.impressionstore.thriftscala.PublishedImpressionList -import com.twitter.timelines.impressionstore.thriftscala.SurfaceArea -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Singleton - -object PublishClientSentImpressionsEventBusSideEffect { - val HomeSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeTimeline)) - val HomeLatestSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeLatestTimeline)) - val HomeSubscribedSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeSubscribed)) -} - -/** - * Side effect that publishes seen tweet IDs sent from clients. The seen tweet IDs are sent to a - * heron topology which writes to a memcache dataset. - */ -@Singleton -class PublishClientSentImpressionsEventBusSideEffect @Inject() ( - eventBusPublisher: EventBusPublisher[PublishedImpressionList]) - extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, HasMarshalling] - with PipelineResultSideEffect.Conditionally[ - PipelineQuery with HasSeenTweetIds, - HasMarshalling - ] { - import PublishClientSentImpressionsEventBusSideEffect._ - - override val identifier: SideEffectIdentifier = - SideEffectIdentifier("PublishClientSentImpressionsEventBus") - - override def onlyIf( - query: PipelineQuery with HasSeenTweetIds, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: HasMarshalling - ): Boolean = query.seenTweetIds.exists(_.nonEmpty) - - def buildEvents( - query: PipelineQuery with HasSeenTweetIds, - currentTime: Long - ): Option[Seq[Impression]] = { - val surfaceArea = query.product match { - case ForYouProduct => HomeSurfaceArea - case FollowingProduct => HomeLatestSurfaceArea - case SubscribedProduct => HomeSubscribedSurfaceArea - case _ => None - } - query.seenTweetIds.map { seenTweetIds => - seenTweetIds.map { tweetId => - Impression( - tweetId = tweetId, - impressionTime = Some(currentTime), - surfaceAreas = surfaceArea - ) - } - } - } - - final override def apply( - inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, HasMarshalling] - ): Stitch[Unit] = { - val currentTime = Time.now.inMilliseconds - val impressions = buildEvents(inputs.query, currentTime) - - Stitch.callFuture( - eventBusPublisher.publish( - PublishedImpressionList( - inputs.query.getRequiredUserId, - ImpressionList(impressions), - currentTime - ) - ) - ) - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.4) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsManhattanSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsManhattanSideEffect.scala deleted file mode 100644 index bf365f1a6..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsManhattanSideEffect.scala +++ /dev/null @@ -1,65 +0,0 @@ -package com.twitter.home_mixer.functional_component.side_effect - -import com.twitter.home_mixer.model.HomeFeatures.TweetImpressionsFeature -import com.twitter.home_mixer.model.request.HasSeenTweetIds -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelines.impression.{thriftscala => t} -import com.twitter.timelines.impressionstore.store.ManhattanTweetImpressionStoreClient -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Side effect that updates the timelines tweet impression - * store (Manhattan) with seen tweet IDs sent from clients - */ -@Singleton -class PublishClientSentImpressionsManhattanSideEffect @Inject() ( - manhattanTweetImpressionStoreClient: ManhattanTweetImpressionStoreClient) - extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, HasMarshalling] - with PipelineResultSideEffect.Conditionally[ - PipelineQuery with HasSeenTweetIds, - HasMarshalling - ] { - - override val identifier: SideEffectIdentifier = - SideEffectIdentifier("PublishClientSentImpressionsManhattan") - - override def onlyIf( - query: PipelineQuery with HasSeenTweetIds, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: HasMarshalling - ): Boolean = query.seenTweetIds.exists(_.nonEmpty) - - def buildEvents(query: PipelineQuery): Option[(Long, t.TweetImpressionsEntries)] = { - query.features.flatMap { featureMap => - val impressions = featureMap.getOrElse(TweetImpressionsFeature, Seq.empty) - if (impressions.nonEmpty) - Some((query.getRequiredUserId, t.TweetImpressionsEntries(impressions))) - else None - } - } - - final override def apply( - inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, HasMarshalling] - ): Stitch[Unit] = { - val events = buildEvents(inputs.query) - - Stitch - .traverse(events) { - case (key, value) => manhattanTweetImpressionStoreClient.write(key, value) - } - .unit - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.4) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishImpressionBloomFilterSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishImpressionBloomFilterSideEffect.scala deleted file mode 100644 index f965bafac..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishImpressionBloomFilterSideEffect.scala +++ /dev/null @@ -1,65 +0,0 @@ -package com.twitter.home_mixer.functional_component.side_effect - -import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature -import com.twitter.home_mixer.model.request.HasSeenTweetIds -import com.twitter.home_mixer.param.HomeGlobalParams.EnableImpressionBloomFilter -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelines.clients.manhattan.store.ManhattanStoreClient -import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class PublishImpressionBloomFilterSideEffect @Inject() ( - bloomFilterClient: ManhattanStoreClient[ - blm.ImpressionBloomFilterKey, - blm.ImpressionBloomFilterSeq - ]) extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, HasMarshalling] - with PipelineResultSideEffect.Conditionally[ - PipelineQuery with HasSeenTweetIds, - HasMarshalling - ] { - - override val identifier: SideEffectIdentifier = - SideEffectIdentifier("PublishImpressionBloomFilter") - - private val SurfaceArea = blm.SurfaceArea.HomeTimeline - - override def onlyIf( - query: PipelineQuery with HasSeenTweetIds, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: HasMarshalling - ): Boolean = - query.params.getBoolean(EnableImpressionBloomFilter) && query.seenTweetIds.exists(_.nonEmpty) - - def buildEvents(query: PipelineQuery): Option[blm.ImpressionBloomFilterSeq] = { - query.features.flatMap { featureMap => - val impressionBloomFilterSeq = featureMap.get(ImpressionBloomFilterFeature) - if (impressionBloomFilterSeq.entries.nonEmpty) Some(impressionBloomFilterSeq) - else None - } - } - - override def apply( - inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, HasMarshalling] - ): Stitch[Unit] = { - buildEvents(inputs.query) - .map { updatedBloomFilterSeq => - bloomFilterClient.write( - blm.ImpressionBloomFilterKey(inputs.query.getRequiredUserId, SurfaceArea), - updatedBloomFilterSeq) - }.getOrElse(Stitch.Unit) - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/TruncateTimelinesPersistenceStoreSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/TruncateTimelinesPersistenceStoreSideEffect.scala deleted file mode 100644 index 63855d3b8..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/TruncateTimelinesPersistenceStoreSideEffect.scala +++ /dev/null @@ -1,68 +0,0 @@ -package com.twitter.home_mixer.functional_component.side_effect - -import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.param.HomeGlobalParams.TimelinesPersistenceStoreMaxEntriesPerClient -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelinemixer.clients.persistence.TimelineResponseBatchesClient -import com.twitter.timelinemixer.clients.persistence.TimelineResponseV3 -import com.twitter.timelineservice.model.TimelineQuery -import com.twitter.timelineservice.model.core.TimelineKind -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Side effect that truncates entries in the Timelines Persistence store - * based on the number of entries per client. - */ -@Singleton -class TruncateTimelinesPersistenceStoreSideEffect @Inject() ( - timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3]) - extends PipelineResultSideEffect[PipelineQuery, Timeline] { - - override val identifier: SideEffectIdentifier = - SideEffectIdentifier("TruncateTimelinesPersistenceStore") - - def getResponsesToDelete(query: PipelineQuery): Seq[TimelineResponseV3] = { - val responses = - query.features.map(_.getOrElse(PersistenceEntriesFeature, Seq.empty)).toSeq.flatten - val responsesByClient = responses.groupBy(_.clientPlatform).values.toSeq - val maxEntriesPerClient = query.params(TimelinesPersistenceStoreMaxEntriesPerClient) - - responsesByClient.flatMap { - _.sortBy(_.servedTime.inMilliseconds) - .foldRight((Seq.empty[TimelineResponseV3], maxEntriesPerClient)) { - case (response, (responsesToDelete, remainingCap)) => - if (remainingCap > 0) (responsesToDelete, remainingCap - response.entries.size) - else (response +: responsesToDelete, remainingCap) - } match { case (responsesToDelete, _) => responsesToDelete } - } - } - - final override def apply( - inputs: PipelineResultSideEffect.Inputs[PipelineQuery, Timeline] - ): Stitch[Unit] = { - val timelineKind = inputs.query.product match { - case FollowingProduct => TimelineKind.homeLatest - case ForYouProduct => TimelineKind.home - case other => throw new UnsupportedOperationException(s"Unknown product: $other") - } - val timelineQuery = TimelineQuery(id = inputs.query.getRequiredUserId, kind = timelineKind) - - val responsesToDelete = getResponsesToDelete(inputs.query) - - if (responsesToDelete.nonEmpty) - Stitch.callFuture(timelineResponseBatchesClient.delete(timelineQuery, responsesToDelete)) - else Stitch.Unit - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateLastNonPollingTimeSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateLastNonPollingTimeSideEffect.scala deleted file mode 100644 index 24199a79c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateLastNonPollingTimeSideEffect.scala +++ /dev/null @@ -1,78 +0,0 @@ -package com.twitter.home_mixer.functional_component.side_effect - -import com.twitter.home_mixer.model.HomeFeatures.FollowingLastNonPollingTimeFeature -import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature -import com.twitter.home_mixer.model.HomeFeatures.PollingFeature -import com.twitter.home_mixer.model.request.DeviceContext -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.side_effect.UserSessionStoreUpdateSideEffect -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelineservice.model.util.FinagleRequestContext -import com.twitter.user_session_store.ReadWriteUserSessionStore -import com.twitter.user_session_store.WriteRequest -import com.twitter.user_session_store.thriftscala.NonPollingTimestamps -import com.twitter.user_session_store.thriftscala.UserSessionField -import com.twitter.util.Time - -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Side effect that updates the User Session Store (Manhattan) with the timestamps of non polling requests. - */ -@Singleton -class UpdateLastNonPollingTimeSideEffect[ - Query <: PipelineQuery with HasDeviceContext, - ResponseType <: HasMarshalling] @Inject() ( - override val userSessionStore: ReadWriteUserSessionStore) - extends UserSessionStoreUpdateSideEffect[ - WriteRequest, - Query, - ResponseType - ] { - private val MaxNonPollingTimes = 10 - - override val identifier: SideEffectIdentifier = SideEffectIdentifier("UpdateLastNonPollingTime") - - /** - * When the request is non polling and is not a background fetch request, update - * the list of non polling timestamps with the timestamp of the current request - */ - override def buildWriteRequest(query: Query): Option[WriteRequest] = { - val isBackgroundFetch = query.deviceContext - .exists(_.requestContextValue.contains(DeviceContext.RequestContext.BackgroundFetch)) - - if (!query.features.exists(_.getOrElse(PollingFeature, false)) && !isBackgroundFetch) { - val fields = Seq(UserSessionField.NonPollingTimestamps(makeLastNonPollingTimestamps(query))) - Some(WriteRequest(query.getRequiredUserId, fields)) - } else None - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.96) - ) - - private def makeLastNonPollingTimestamps(query: Query): NonPollingTimestamps = { - val priorNonPollingTimestamps = - query.features.map(_.getOrElse(NonPollingTimesFeature, Seq.empty)).toSeq.flatten - - val lastNonPollingTimeMs = - FinagleRequestContext.default.requestStartTime.get.getOrElse(Time.now).inMillis - - val followingLastNonPollingTime = query.features - .flatMap(features => features.getOrElse(FollowingLastNonPollingTimeFeature, None)) - .map(_.inMillis) - - NonPollingTimestamps( - nonPollingTimestampsMs = - (lastNonPollingTimeMs +: priorNonPollingTimestamps).take(MaxNonPollingTimes), - mostRecentHomeLatestNonPollingTimestampMs = - if (query.product == FollowingProduct) Some(lastNonPollingTimeMs) - else followingLastNonPollingTime - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateTimelinesPersistenceStoreSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateTimelinesPersistenceStoreSideEffect.scala deleted file mode 100644 index 5a1bb0f6b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateTimelinesPersistenceStoreSideEffect.scala +++ /dev/null @@ -1,263 +0,0 @@ -package com.twitter.home_mixer.functional_component.side_effect - -import com.twitter.home_mixer.model.HomeFeatures._ -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.model.HomeFeatures.IsTweetPreviewFeature -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidateDecorator -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.AddEntriesTimelineInstruction -import com.twitter.product_mixer.core.model.marshalling.response.urt.ReplaceEntryTimelineInstruction -import com.twitter.product_mixer.core.model.marshalling.response.urt.ShowCoverInstruction -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelinemixer.clients.persistence.EntryWithItemIds -import com.twitter.timelinemixer.clients.persistence.ItemIds -import com.twitter.timelinemixer.clients.persistence.TimelineResponseBatchesClient -import com.twitter.timelinemixer.clients.persistence.TimelineResponseV3 -import com.twitter.timelines.persistence.thriftscala.TweetScoreV1 -import com.twitter.timelines.persistence.{thriftscala => persistence} -import com.twitter.timelineservice.model.TimelineQuery -import com.twitter.timelineservice.model.TimelineQueryOptions -import com.twitter.timelineservice.model.TweetScore -import com.twitter.timelineservice.model.core.TimelineKind -import com.twitter.timelineservice.model.rich.EntityIdType -import com.twitter.util.Time -import com.twitter.{timelineservice => tls} -import javax.inject.Inject -import javax.inject.Singleton - -object UpdateTimelinesPersistenceStoreSideEffect { - val EmptyItemIds = ItemIds( - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None) -} - -/** - * Side effect that updates the Timelines Persistence Store (Manhattan) with the entries being returned. - */ -@Singleton -class UpdateTimelinesPersistenceStoreSideEffect @Inject() ( - timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3]) - extends PipelineResultSideEffect[PipelineQuery, Timeline] { - - override val identifier: SideEffectIdentifier = - SideEffectIdentifier("UpdateTimelinesPersistenceStore") - - final override def apply( - inputs: PipelineResultSideEffect.Inputs[PipelineQuery, Timeline] - ): Stitch[Unit] = { - if (inputs.response.instructions.nonEmpty) { - val timelineKind = inputs.query.product match { - case FollowingProduct => TimelineKind.homeLatest - case ForYouProduct => TimelineKind.home - case other => throw new UnsupportedOperationException(s"Unknown product: $other") - } - val timelineQuery = TimelineQuery( - id = inputs.query.getRequiredUserId, - kind = timelineKind, - options = TimelineQueryOptions( - contextualUserId = inputs.query.getOptionalUserId, - deviceContext = tls.DeviceContext.empty.copy( - userAgent = inputs.query.clientContext.userAgent, - clientAppId = inputs.query.clientContext.appId) - ) - ) - - val tweetIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] = - inputs.selectedCandidates.flatMap { - case item: ItemCandidateWithDetails if item.candidate.id.isInstanceOf[Long] => - Seq((item.candidateIdLong, item)) - case module: ModuleCandidateWithDetails - if module.candidates.headOption.exists(_.candidate.id.isInstanceOf[Long]) => - module.candidates.map(item => (item.candidateIdLong, item)) - case _ => Seq.empty - }.toMap - - val entries = inputs.response.instructions.collect { - case AddEntriesTimelineInstruction(entries) => - entries.collect { - // includes tweets, tweet previews, and promoted tweets - case entry: TweetItem if entry.sortIndex.isDefined => { - Seq( - buildTweetEntryWithItemIds( - tweetIdToItemCandidateMap(entry.id), - entry.sortIndex.get - )) - } - // tweet conversation modules are flattened to individual tweets in the persistence store - case module: TimelineModule - if module.sortIndex.isDefined && module.items.headOption.exists( - _.item.isInstanceOf[TweetItem]) => - module.items.map { item => - buildTweetEntryWithItemIds( - tweetIdToItemCandidateMap(item.item.id.asInstanceOf[Long]), - module.sortIndex.get) - } - case module: TimelineModule - if module.sortIndex.isDefined && module.entryNamespace.toString == WhoToFollowCandidateDecorator.EntryNamespaceString => - val userIds = module.items - .map(item => - UpdateTimelinesPersistenceStoreSideEffect.EmptyItemIds.copy(userId = - Some(item.item.id.asInstanceOf[Long]))) - Seq( - EntryWithItemIds( - entityIdType = EntityIdType.WhoToFollow, - sortIndex = module.sortIndex.get, - size = module.items.size.toShort, - itemIds = Some(userIds) - )) - case module: TimelineModule - if module.sortIndex.isDefined && module.entryNamespace.toString == WhoToSubscribeCandidateDecorator.EntryNamespaceString => - val userIds = module.items - .map(item => - UpdateTimelinesPersistenceStoreSideEffect.EmptyItemIds.copy(userId = - Some(item.item.id.asInstanceOf[Long]))) - Seq( - EntryWithItemIds( - entityIdType = EntityIdType.WhoToSubscribe, - sortIndex = module.sortIndex.get, - size = module.items.size.toShort, - itemIds = Some(userIds) - )) - }.flatten - case ShowCoverInstruction(cover) => - Seq( - EntryWithItemIds( - entityIdType = EntityIdType.Prompt, - sortIndex = cover.sortIndex.get, - size = 1, - itemIds = None - ) - ) - case ReplaceEntryTimelineInstruction(entry) => - val namespaceLength = TweetItem.TweetEntryNamespace.toString.length - Seq( - EntryWithItemIds( - entityIdType = EntityIdType.Tweet, - sortIndex = entry.sortIndex.get, - size = 1, - itemIds = Some( - Seq( - ItemIds( - tweetId = - entry.entryIdToReplace.map(e => e.substring(namespaceLength + 1).toLong), - sourceTweetId = None, - quoteTweetId = None, - sourceAuthorId = None, - quoteAuthorId = None, - inReplyToTweetId = None, - inReplyToAuthorId = None, - semanticCoreId = None, - articleId = None, - hasRelevancePrompt = None, - promptData = None, - tweetScore = None, - entryIdToReplace = entry.entryIdToReplace, - tweetReactiveData = None, - userId = None - ) - )) - ) - ) - - }.flatten - - val response = TimelineResponseV3( - clientPlatform = timelineQuery.clientPlatform, - servedTime = Time.now, - requestType = requestTypeFromQuery(inputs.query), - entries = entries) - - Stitch.callFuture(timelineResponseBatchesClient.insertResponse(timelineQuery, response)) - } else Stitch.Unit - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8) - ) - - private def buildTweetEntryWithItemIds( - candidate: ItemCandidateWithDetails, - sortIndex: Long - ): EntryWithItemIds = { - val features = candidate.features - val sourceAuthorId = - if (features.getOrElse(IsRetweetFeature, false)) features.getOrElse(SourceUserIdFeature, None) - else features.getOrElse(AuthorIdFeature, None) - val quoteAuthorId = - if (features.getOrElse(QuotedTweetIdFeature, None).nonEmpty) - features.getOrElse(SourceUserIdFeature, None) - else None - val tweetScore = features.getOrElse(ScoreFeature, None).map { score => - TweetScore.fromThrift(persistence.TweetScore.TweetScoreV1(TweetScoreV1(score))) - } - - val itemIds = ItemIds( - tweetId = Some(candidate.candidateIdLong), - sourceTweetId = features.getOrElse(SourceTweetIdFeature, None), - quoteTweetId = features.getOrElse(QuotedTweetIdFeature, None), - sourceAuthorId = sourceAuthorId, - quoteAuthorId = quoteAuthorId, - inReplyToTweetId = features.getOrElse(InReplyToTweetIdFeature, None), - inReplyToAuthorId = features.getOrElse(DirectedAtUserIdFeature, None), - semanticCoreId = features.getOrElse(SemanticCoreIdFeature, None), - articleId = None, - hasRelevancePrompt = None, - promptData = None, - tweetScore = tweetScore, - entryIdToReplace = None, - tweetReactiveData = None, - userId = None - ) - - val isPreview = features.getOrElse(IsTweetPreviewFeature, default = false) - val entityType = if (isPreview) EntityIdType.TweetPreview else EntityIdType.Tweet - - EntryWithItemIds( - entityIdType = entityType, - sortIndex = sortIndex, - size = 1.toShort, - itemIds = Some(Seq(itemIds)) - ) - } - - private def requestTypeFromQuery(query: PipelineQuery): persistence.RequestType = { - val features = query.features.getOrElse(FeatureMap.empty) - - val featureToRequestType = Seq( - (PollingFeature, persistence.RequestType.Polling), - (GetInitialFeature, persistence.RequestType.Initial), - (GetNewerFeature, persistence.RequestType.Newer), - (GetMiddleFeature, persistence.RequestType.Middle), - (GetOlderFeature, persistence.RequestType.Older) - ) - - featureToRequestType - .collectFirst { - case (feature, requestType) if features.getOrElse(feature, false) => requestType - }.getOrElse(persistence.RequestType.Other) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/BUILD.bazel deleted file mode 100644 index fd35daeeb..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/BUILD.bazel +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - ], - exports = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/DeviceContextUnmarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/DeviceContextUnmarshaller.scala deleted file mode 100644 index b2c6e6347..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/DeviceContextUnmarshaller.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.home_mixer.marshaller.request - -import com.twitter.home_mixer.model.request.DeviceContext -import com.twitter.home_mixer.{thriftscala => t} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class DeviceContextUnmarshaller @Inject() () { - - def apply(deviceContext: t.DeviceContext): DeviceContext = { - DeviceContext( - isPolling = deviceContext.isPolling, - requestContext = deviceContext.requestContext, - latestControlAvailable = deviceContext.latestControlAvailable, - autoplayEnabled = deviceContext.autoplayEnabled - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerDebugParamsUnmarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerDebugParamsUnmarshaller.scala deleted file mode 100644 index 81a9abf2b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerDebugParamsUnmarshaller.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.home_mixer.marshaller.request - -import com.twitter.home_mixer.model.request.HomeMixerDebugOptions -import com.twitter.home_mixer.{thriftscala => t} -import com.twitter.product_mixer.core.functional_component.marshaller.request.FeatureValueUnmarshaller -import com.twitter.product_mixer.core.model.marshalling.request.DebugParams -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class HomeMixerDebugParamsUnmarshaller @Inject() ( - featureValueUnmarshaller: FeatureValueUnmarshaller) { - - def apply(debugParams: t.DebugParams): DebugParams = { - DebugParams( - featureOverrides = debugParams.featureOverrides.map { map => - map.mapValues(featureValueUnmarshaller(_)).toMap - }, - debugOptions = debugParams.debugOptions.map { options => - HomeMixerDebugOptions( - requestTimeOverride = options.requestTimeOverrideMillis.map(Time.fromMilliseconds) - ) - } - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductContextUnmarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductContextUnmarshaller.scala deleted file mode 100644 index ec3a183b9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductContextUnmarshaller.scala +++ /dev/null @@ -1,62 +0,0 @@ -package com.twitter.home_mixer.marshaller.request - -import com.twitter.home_mixer.model.request.FollowingProductContext -import com.twitter.home_mixer.model.request.ForYouProductContext -import com.twitter.home_mixer.model.request.ListRecommendedUsersProductContext -import com.twitter.home_mixer.model.request.ListTweetsProductContext -import com.twitter.home_mixer.model.request.ScoredTweetsProductContext -import com.twitter.home_mixer.model.request.SubscribedProductContext -import com.twitter.home_mixer.{thriftscala => t} -import com.twitter.product_mixer.core.model.marshalling.request.ProductContext -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class HomeMixerProductContextUnmarshaller @Inject() ( - deviceContextUnmarshaller: DeviceContextUnmarshaller) { - - def apply(productContext: t.ProductContext): ProductContext = productContext match { - case t.ProductContext.Following(p) => - FollowingProductContext( - deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)), - seenTweetIds = p.seenTweetIds, - dspClientContext = p.dspClientContext - ) - case t.ProductContext.ForYou(p) => - ForYouProductContext( - deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)), - seenTweetIds = p.seenTweetIds, - dspClientContext = p.dspClientContext, - pushToHomeTweetId = p.pushToHomeTweetId - ) - case t.ProductContext.ListManagement(p) => - throw new UnsupportedOperationException(s"This product is no longer used") - case t.ProductContext.ScoredTweets(p) => - ScoredTweetsProductContext( - deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)), - seenTweetIds = p.seenTweetIds, - servedTweetIds = p.servedTweetIds, - backfillTweetIds = p.backfillTweetIds - ) - case t.ProductContext.ListTweets(p) => - ListTweetsProductContext( - listId = p.listId, - deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)), - dspClientContext = p.dspClientContext - ) - case t.ProductContext.ListRecommendedUsers(p) => - ListRecommendedUsersProductContext( - listId = p.listId, - selectedUserIds = p.selectedUserIds, - excludedUserIds = p.excludedUserIds, - listName = p.listName - ) - case t.ProductContext.Subscribed(p) => - SubscribedProductContext( - deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)), - seenTweetIds = p.seenTweetIds, - ) - case t.ProductContext.UnknownUnionField(field) => - throw new UnsupportedOperationException(s"Unknown display context: ${field.field.name}") - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductUnmarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductUnmarshaller.scala deleted file mode 100644 index f5d0d002b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductUnmarshaller.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.home_mixer.marshaller.request - -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.model.request.ListRecommendedUsersProduct -import com.twitter.home_mixer.model.request.ListTweetsProduct -import com.twitter.home_mixer.model.request.ScoredTweetsProduct -import com.twitter.home_mixer.model.request.SubscribedProduct -import com.twitter.home_mixer.{thriftscala => t} -import com.twitter.product_mixer.core.model.marshalling.request.Product -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class HomeMixerProductUnmarshaller @Inject() () { - - def apply(product: t.Product): Product = product match { - case t.Product.Following => FollowingProduct - case t.Product.ForYou => ForYouProduct - case t.Product.ListManagement => - throw new UnsupportedOperationException(s"This product is no longer used") - case t.Product.ScoredTweets => ScoredTweetsProduct - case t.Product.ListTweets => ListTweetsProduct - case t.Product.ListRecommendedUsers => ListRecommendedUsersProduct - case t.Product.Subscribed => SubscribedProduct - case t.Product.EnumUnknownProduct(value) => - throw new UnsupportedOperationException(s"Unknown product: $value") - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerRequestUnmarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerRequestUnmarshaller.scala deleted file mode 100644 index b8894c8b0..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerRequestUnmarshaller.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.home_mixer.marshaller.request - -import com.twitter.home_mixer.model.request.HomeMixerRequest -import com.twitter.home_mixer.{thriftscala => t} -import com.twitter.product_mixer.core.functional_component.marshaller.request.ClientContextUnmarshaller -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class HomeMixerRequestUnmarshaller @Inject() ( - clientContextUnmarshaller: ClientContextUnmarshaller, - homeProductUnmarshaller: HomeMixerProductUnmarshaller, - homeProductContextUnmarshaller: HomeMixerProductContextUnmarshaller, - homeDebugParamsUnmarshaller: HomeMixerDebugParamsUnmarshaller) { - - def apply(homeRequest: t.HomeMixerRequest): HomeMixerRequest = { - HomeMixerRequest( - clientContext = clientContextUnmarshaller(homeRequest.clientContext), - product = homeProductUnmarshaller(homeRequest.product), - productContext = homeRequest.productContext.map(homeProductContextUnmarshaller(_)), - // Avoid de-serializing cursors in the request unmarshaller. The unmarshaller should never - // fail, which is often a possibility when trying to de-serialize a cursor. Cursors can also - // be product-specific and more appropriately handled in individual product pipelines. - serializedRequestCursor = homeRequest.cursor, - maxResults = homeRequest.maxResults, - debugParams = homeRequest.debugParams.map(homeDebugParamsUnmarshaller(_)), - homeRequestParam = false - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/BUILD.bazel deleted file mode 100644 index efcad840b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/item", - "src/thrift/com/twitter/timelines/timeline_logging:thrift-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/PromotedTweetDetailsMarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/PromotedTweetDetailsMarshaller.scala deleted file mode 100644 index d913f8d64..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/PromotedTweetDetailsMarshaller.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.home_mixer.marshaller.timeline_logging - -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog} - -object PromotedTweetDetailsMarshaller { - - def apply(entry: TweetItem, position: Int): thriftlog.PromotedTweetDetails = { - thriftlog.PromotedTweetDetails( - advertiserId = Some(entry.promotedMetadata.map(_.advertiserId).getOrElse(0L)), - insertPosition = Some(position), - impressionId = entry.promotedMetadata.flatMap(_.impressionString) - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/TweetDetailsMarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/TweetDetailsMarshaller.scala deleted file mode 100644 index 8e1c475d5..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/TweetDetailsMarshaller.scala +++ /dev/null @@ -1,47 +0,0 @@ -package com.twitter.home_mixer.marshaller.timeline_logging - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.product_mixer.component_library.model.presentation.urt.UrtItemPresentation -import com.twitter.product_mixer.component_library.model.presentation.urt.UrtModulePresentation -import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.metadata.GeneralContextTypeMarshaller -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ConversationGeneralContextType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.GeneralContext -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContext -import com.twitter.timelines.service.{thriftscala => tst} -import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog} - -object TweetDetailsMarshaller { - - private val generalContextTypeMarshaller = new GeneralContextTypeMarshaller() - - def apply(entry: TweetItem, candidate: CandidateWithDetails): thriftlog.TweetDetails = { - val socialContext = candidate.presentation.flatMap { - case _ @UrtItemPresentation(timelineItem: TweetItem, _) => timelineItem.socialContext - case _ @UrtModulePresentation(timelineModule) => - timelineModule.items.head.item match { - case timelineItem: TweetItem => timelineItem.socialContext - case _ => Some(ConversationGeneralContextType) - } - } - - val socialContextType = socialContext match { - case Some(GeneralContext(contextType, _, _, _, _)) => - Some(generalContextTypeMarshaller(contextType).value.toShort) - case Some(TopicContext(_, _)) => Some(tst.ContextType.Topic.value.toShort) - case _ => None - } - - thriftlog.TweetDetails( - sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None), - socialContextType = socialContextType, - suggestType = candidate.features.getOrElse(SuggestTypeFeature, None).map(_.name), - authorId = candidate.features.getOrElse(AuthorIdFeature, None), - sourceAuthorId = candidate.features.getOrElse(SourceUserIdFeature, None) - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/WhoToFollowDetailsMarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/WhoToFollowDetailsMarshaller.scala deleted file mode 100644 index 4c0b4dd1b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/WhoToFollowDetailsMarshaller.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.home_mixer.marshaller.timeline_logging - -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem -import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog} - -object WhoToFollowDetailsMarshaller { - - def apply(entry: UserItem, candidate: ItemCandidateWithDetails): thriftlog.WhoToFollowDetails = - thriftlog.WhoToFollowDetails( - enableReactiveBlending = entry.enableReactiveBlending, - impressionId = entry.promotedMetadata.flatMap(_.impressionString), - advertiserId = entry.promotedMetadata.map(_.advertiserId) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/BUILD.bazel deleted file mode 100644 index ebf305ead..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/BUILD.bazel +++ /dev/null @@ -1,11 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "src/thrift/com/twitter/timelineservice:thrift-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/ChronologicalCursorMarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/ChronologicalCursorMarshaller.scala deleted file mode 100644 index 40e25a8ab..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/ChronologicalCursorMarshaller.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.home_mixer.marshaller.timelines - -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.BottomCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor -import com.twitter.timelines.service.{thriftscala => t} - -object ChronologicalCursorMarshaller { - - def apply(cursor: UrtOrderedCursor): Option[t.ChronologicalCursor] = { - cursor.cursorType match { - case Some(TopCursor) => Some(t.ChronologicalCursor(bottom = cursor.id)) - case Some(BottomCursor) => Some(t.ChronologicalCursor(top = cursor.id)) - case Some(GapCursor) => - Some(t.ChronologicalCursor(top = cursor.id, bottom = cursor.gapBoundaryId)) - case _ => None - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/ChronologicalCursorUnmarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/ChronologicalCursorUnmarshaller.scala deleted file mode 100644 index 739c490ea..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/ChronologicalCursorUnmarshaller.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.home_mixer.marshaller.timelines - -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.BottomCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor -import com.twitter.timelines.service.{thriftscala => t} - -object ChronologicalCursorUnmarshaller { - - def apply(requestCursor: t.RequestCursor): Option[UrtOrderedCursor] = { - requestCursor match { - case t.RequestCursor.ChronologicalCursor(cursor) => - (cursor.top, cursor.bottom) match { - case (Some(top), None) => - Some(UrtOrderedCursor(top, cursor.top, Some(BottomCursor))) - case (None, Some(bottom)) => - Some(UrtOrderedCursor(bottom, cursor.bottom, Some(TopCursor))) - case (Some(top), Some(bottom)) => - Some(UrtOrderedCursor(top, cursor.top, Some(GapCursor), cursor.bottom)) - case _ => None - } - case _ => None - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/DeviceContextMarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/DeviceContextMarshaller.scala deleted file mode 100644 index 097d8dea4..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/DeviceContextMarshaller.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.home_mixer.marshaller.timelines - -import com.twitter.home_mixer.model.request.DeviceContext -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext -import com.twitter.timelineservice.{thriftscala => t} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class DeviceContextMarshaller @Inject() () { - - def apply(deviceContext: DeviceContext, clientContext: ClientContext): t.DeviceContext = { - t.DeviceContext( - countryCode = clientContext.countryCode, - languageCode = clientContext.languageCode, - clientAppId = clientContext.appId, - ipAddress = clientContext.ipAddress, - guestId = clientContext.guestId, - userAgent = clientContext.userAgent, - deviceId = clientContext.deviceId, - isPolling = deviceContext.isPolling, - requestContext = deviceContext.requestContext, - referrer = None, - tfeAuthHeader = None, - mobileDeviceId = clientContext.mobileDeviceId, - isSessionStart = None, - latestControlAvailable = deviceContext.latestControlAvailable, - guestIdMarketing = clientContext.guestIdMarketing, - isInternalOrTwoffice = clientContext.isTwoffice, - guestIdAds = clientContext.guestIdAds, - isUrtRequest = Some(true) - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/RecommendedUsersCursorUnmarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/RecommendedUsersCursorUnmarshaller.scala deleted file mode 100644 index 693684558..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/RecommendedUsersCursorUnmarshaller.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.home_mixer.marshaller.timelines - -import com.twitter.product_mixer.component_library.model.cursor.UrtUnorderedExcludeIdsCursor -import com.twitter.timelines.service.{thriftscala => t} -import com.twitter.util.Time - -object RecommendedUsersCursorUnmarshaller { - - def apply(requestCursor: t.RequestCursor): Option[UrtUnorderedExcludeIdsCursor] = { - requestCursor match { - case t.RequestCursor.RecommendedUsersCursor(cursor) => - Some( - UrtUnorderedExcludeIdsCursor( - initialSortIndex = cursor.minSortIndex.getOrElse(Time.now.inMilliseconds), - excludedIds = cursor.previouslyRecommendedUserIds - )) - case _ => None - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/TimelineServiceCursorMarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/TimelineServiceCursorMarshaller.scala deleted file mode 100644 index cbc956ad4..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/TimelineServiceCursorMarshaller.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.home_mixer.marshaller.timelines - -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.BottomCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor -import com.twitter.timelineservice.{thriftscala => t} - -object TimelineServiceCursorMarshaller { - - def apply(cursor: UrtOrderedCursor): Option[t.Cursor2] = { - val id = cursor.id.map(_.toString) - val gapBoundaryId = cursor.gapBoundaryId.map(_.toString) - cursor.cursorType match { - case Some(TopCursor) => Some(t.Cursor2(bottom = id)) - case Some(BottomCursor) => Some(t.Cursor2(top = id)) - case Some(GapCursor) => Some(t.Cursor2(top = id, bottom = gapBoundaryId)) - case _ => None - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/TopicContextFunctionalityTypeUnmarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/TopicContextFunctionalityTypeUnmarshaller.scala deleted file mode 100644 index f542cd535..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/TopicContextFunctionalityTypeUnmarshaller.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.home_mixer.marshaller.timelines - -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecWithEducationTopicContextFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContextFunctionalityType -import com.twitter.timelines.render.{thriftscala => urt} - -object TopicContextFunctionalityTypeUnmarshaller { - - def apply( - topicContextFunctionalityType: urt.TopicContextFunctionalityType - ): TopicContextFunctionalityType = topicContextFunctionalityType match { - case urt.TopicContextFunctionalityType.Basic => BasicTopicContextFunctionalityType - case urt.TopicContextFunctionalityType.Recommendation => - RecommendationTopicContextFunctionalityType - case urt.TopicContextFunctionalityType.RecWithEducation => - RecWithEducationTopicContextFunctionalityType - case urt.TopicContextFunctionalityType.EnumUnknownTopicContextFunctionalityType(field) => - throw new UnsupportedOperationException(s"Unknown topic context functionality type: $field") - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/BUILD.bazel deleted file mode 100644 index 65ece62a3..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/BUILD.bazel +++ /dev/null @@ -1,42 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/java/com/twitter/ml/api:api-base", - "src/java/com/twitter/ml/api/constant", - "src/scala/com/twitter/ml/api:api-base", - "src/scala/com/twitter/timelines/prediction/features/common", - "src/scala/com/twitter/timelines/prediction/features/recap", - "src/scala/com/twitter/timelines/prediction/features/request_context", - "src/thrift/com/twitter/escherbird:tweet-annotation-scala", - "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/timelines/conversation_features:conversation_features-scala", - "src/thrift/com/twitter/timelines/impression:thrift-scala", - "src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala", - "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan", - "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/persistence", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", - "topic-social-proof/server/src/main/thrift:thrift-scala", - "tweetconvosvc/common/src/main/thrift/com/twitter/tweetconvosvc/tweet_ancestor:thrift-scala", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/thrift/com/twitter/timelines/impression:thrift-scala", - "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", - "tweetconvosvc/common/src/main/thrift/com/twitter/tweetconvosvc/tweet_ancestor:thrift-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ClearCacheIncludeInstruction.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ClearCacheIncludeInstruction.scala deleted file mode 100644 index 85154e55b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ClearCacheIncludeInstruction.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.twitter.home_mixer.model - -import com.twitter.home_mixer.model.request.DeviceContext.RequestContext -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.IncludeInstruction -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam - -/** - * Include a clear cache timeline instruction when we satisfy these criteria: - * - Request Provenance is "pull to refresh" - * - Atleast N non-ad tweet entries in the response - * - * This is to ensure that we have sufficient new content to justify jumping users to the - * top of the new timelines response and don't add unnecessary load to backend systems - */ -case class ClearCacheIncludeInstruction( - enableParam: FSParam[Boolean], - minEntriesParam: FSBoundedParam[Int]) - extends IncludeInstruction[PipelineQuery with HasDeviceContext] { - - override def apply( - query: PipelineQuery with HasDeviceContext, - entries: Seq[TimelineEntry] - ): Boolean = { - val enabled = query.params(enableParam) - - val ptr = - query.deviceContext.flatMap(_.requestContextValue).contains(RequestContext.PullToRefresh) - - val minTweets = query.params(minEntriesParam) <= entries.collect { - case item: TweetItem if item.promotedMetadata.isEmpty => 1 - case module: TimelineModule if module.items.head.item.isInstanceOf[TweetItem] => - module.items.size - }.sum - - enabled && ptr && minTweets - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ContentFeatures.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ContentFeatures.scala deleted file mode 100644 index f141d67d1..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ContentFeatures.scala +++ /dev/null @@ -1,144 +0,0 @@ -package com.twitter.home_mixer.model - -import com.twitter.escherbird.{thriftscala => esb} -import com.twitter.search.common.features.{thriftscala => sc} -import com.twitter.tweetypie.{thriftscala => tp} - -object ContentFeatures { - val Empty: ContentFeatures = ContentFeatures( - 0.toShort, - false, - 0.toShort, - 0.toShort, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None - ) - - def fromThrift(ebFeatures: sc.ThriftTweetFeatures): ContentFeatures = - ContentFeatures( - length = ebFeatures.tweetLength.getOrElse(0).toShort, - hasQuestion = ebFeatures.hasQuestion.getOrElse(false), - numCaps = ebFeatures.numCaps.getOrElse(0).toShort, - numWhiteSpaces = ebFeatures.numWhitespaces.getOrElse(0).toShort, - numNewlines = ebFeatures.numNewlines, - videoDurationMs = ebFeatures.videoDurationMs, - bitRate = ebFeatures.bitRate, - aspectRatioNum = ebFeatures.aspectRatioNum, - aspectRatioDen = ebFeatures.aspectRatioDen, - widths = ebFeatures.widths.map(_.map(_.toShort)), - heights = ebFeatures.heights.map(_.map(_.toShort)), - resizeMethods = ebFeatures.resizeMethods.map(_.map(_.toShort)), - numMediaTags = ebFeatures.numMediaTags.map(_.toShort), - mediaTagScreenNames = ebFeatures.mediaTagScreenNames, - emojiTokens = ebFeatures.emojiTokens.map(_.toSet), - emoticonTokens = ebFeatures.emoticonTokens.map(_.toSet), - faceAreas = ebFeatures.faceAreas, - dominantColorRed = ebFeatures.dominantColorRed, - dominantColorBlue = ebFeatures.dominantColorBlue, - dominantColorGreen = ebFeatures.dominantColorGreen, - numColors = ebFeatures.numColors.map(_.toShort), - stickerIds = ebFeatures.stickerIds, - mediaOriginProviders = ebFeatures.mediaOriginProviders, - isManaged = ebFeatures.isManaged, - is360 = ebFeatures.is360, - viewCount = ebFeatures.viewCount, - isMonetizable = ebFeatures.isMonetizable, - isEmbeddable = ebFeatures.isEmbeddable, - hasSelectedPreviewImage = ebFeatures.hasSelectedPreviewImage, - hasTitle = ebFeatures.hasTitle, - hasDescription = ebFeatures.hasDescription, - hasVisitSiteCallToAction = ebFeatures.hasVisitSiteCallToAction, - hasAppInstallCallToAction = ebFeatures.hasAppInstallCallToAction, - hasWatchNowCallToAction = ebFeatures.hasWatchNowCallToAction, - dominantColorPercentage = ebFeatures.dominantColorPercentage, - posUnigrams = ebFeatures.posUnigrams.map(_.toSet), - posBigrams = ebFeatures.posBigrams.map(_.toSet), - semanticCoreAnnotations = ebFeatures.semanticCoreAnnotations, - tokens = ebFeatures.textTokens.map(_.toSeq), - conversationControl = ebFeatures.conversationControl, - // media and selfThreadMetadata not carried by ThriftTweetFeatures - media = None, - selfThreadMetadata = None - ) -} - -case class ContentFeatures( - length: Short, - hasQuestion: Boolean, - numCaps: Short, - numWhiteSpaces: Short, - numNewlines: Option[Short], - videoDurationMs: Option[Int], - bitRate: Option[Int], - aspectRatioNum: Option[Short], - aspectRatioDen: Option[Short], - widths: Option[Seq[Short]], - heights: Option[Seq[Short]], - resizeMethods: Option[Seq[Short]], - numMediaTags: Option[Short], - mediaTagScreenNames: Option[Seq[String]], - emojiTokens: Option[Set[String]], - emoticonTokens: Option[Set[String]], - faceAreas: Option[Seq[Int]], - dominantColorRed: Option[Short], - dominantColorBlue: Option[Short], - dominantColorGreen: Option[Short], - numColors: Option[Short], - stickerIds: Option[Seq[Long]], - mediaOriginProviders: Option[Seq[String]], - isManaged: Option[Boolean], - is360: Option[Boolean], - viewCount: Option[Long], - isMonetizable: Option[Boolean], - isEmbeddable: Option[Boolean], - hasSelectedPreviewImage: Option[Boolean], - hasTitle: Option[Boolean], - hasDescription: Option[Boolean], - hasVisitSiteCallToAction: Option[Boolean], - hasAppInstallCallToAction: Option[Boolean], - hasWatchNowCallToAction: Option[Boolean], - media: Option[Seq[tp.MediaEntity]], - dominantColorPercentage: Option[Double], - posUnigrams: Option[Set[String]], - posBigrams: Option[Set[String]], - semanticCoreAnnotations: Option[Seq[esb.TweetEntityAnnotation]], - selfThreadMetadata: Option[tp.SelfThreadMetadata], - tokens: Option[Seq[String]], - conversationControl: Option[tp.ConversationControl], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/GapIncludeInstruction.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/GapIncludeInstruction.scala deleted file mode 100644 index c11631fc8..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/GapIncludeInstruction.scala +++ /dev/null @@ -1,63 +0,0 @@ -package com.twitter.home_mixer.model - -import com.twitter.home_mixer.functional_component.candidate_source.EarlybirdBottomTweetFeature -import com.twitter.home_mixer.functional_component.candidate_source.EarlybirdResponseTruncatedFeature -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.IncludeInstruction -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor -import com.twitter.product_mixer.core.pipeline.HasPipelineCursor -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * Determine whether to include a Gap Cursor in the response based on whether a timeline - * is truncated because it has more entries than the max response size. - * There are two ways this can happen: - * 1) There are unused entries in Earlybird. This is determined by a flag returned from Earlybird. - * We respect the Earlybird flag only if there are some entries after deduping and filtering - * to ensure that we do not get stuck repeatedly serving gaps which lead to no tweets. - * 2) Ads injection can take the response size over the max count. Goldfinch truncates tweet - * entries in this case. We can check if the bottom tweet from Earlybird is in the response to - * determine if all Earlybird tweets have been used. - * - * While scrolling down to get older tweets (BottomCursor), responses will generally be - * truncated, but we don't want to render a gap cursor there, so we need to ensure we only - * apply the truncation check to newer (TopCursor) or middle (GapCursor) requests. - * - * We return either a Gap Cursor or a Bottom Cursor, but not both, so the include instruction - * for Bottom should be the inverse of Gap. - */ -object GapIncludeInstruction - extends IncludeInstruction[PipelineQuery with HasPipelineCursor[UrtOrderedCursor]] { - - override def apply( - query: PipelineQuery with HasPipelineCursor[UrtOrderedCursor], - entries: Seq[TimelineEntry] - ): Boolean = { - val wasTruncated = query.features.exists(_.getOrElse(EarlybirdResponseTruncatedFeature, false)) - - // Get oldest tweet or tweets within oldest conversation module - val tweetEntries = entries.view.reverse - .collectFirst { - case item: TweetItem if item.promotedMetadata.isEmpty => Seq(item.id.toString) - case module: TimelineModule if module.items.head.item.isInstanceOf[TweetItem] => - module.items.map(_.item.id.toString) - }.toSeq.flatten - - val bottomCursor = - query.features.flatMap(_.getOrElse(EarlybirdBottomTweetFeature, None)).map(_.toString) - - // Ads truncation happened if we have at least max count entries and bottom tweet is missing - val adsTruncation = query.requestedMaxResults.exists(_ <= entries.size) && - !bottomCursor.exists(tweetEntries.contains) - - query.pipelineCursor.exists(_.cursorType match { - case Some(TopCursor) | Some(GapCursor) => - (wasTruncated && tweetEntries.nonEmpty) || adsTruncation - case _ => false - }) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeAdsQuery.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeAdsQuery.scala deleted file mode 100644 index bfbe722fc..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeAdsQuery.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.home_mixer.model - -import com.twitter.adserver.thriftscala.RequestTriggerType -import com.twitter.home_mixer.model.HomeFeatures.GetInitialFeature -import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature -import com.twitter.home_mixer.model.HomeFeatures.GetOlderFeature -import com.twitter.home_mixer.model.HomeFeatures.PollingFeature -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.component_library.model.query.ads.AdsQuery -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.pipeline.HasPipelineCursor -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * These are for feeds needed for ads only. - */ -trait HomeAdsQuery - extends AdsQuery - with PipelineQuery - with HasDeviceContext - with HasPipelineCursor[UrtOrderedCursor] { - - private val featureToRequestTriggerType = Seq( - (GetInitialFeature, RequestTriggerType.Initial), - (GetNewerFeature, RequestTriggerType.Scroll), - (GetOlderFeature, RequestTriggerType.Scroll), - (PollingFeature, RequestTriggerType.AutoRefresh) - ) - - override val autoplayEnabled: Option[Boolean] = deviceContext.flatMap(_.autoplayEnabled) - - override def requestTriggerType: Option[RequestTriggerType] = { - val features = this.features.getOrElse(FeatureMap.empty) - - featureToRequestTriggerType.collectFirst { - case (feature, requestType) if features.get(feature) => Some(requestType) - }.flatten - } - - override val disableNsfwAvoidance: Option[Boolean] = Some(true) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeFeatures.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeFeatures.scala deleted file mode 100644 index fe085b1f9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeFeatures.scala +++ /dev/null @@ -1,325 +0,0 @@ -package com.twitter.home_mixer.model - -import com.twitter.core_workflows.user_model.{thriftscala => um} -import com.twitter.dal.personal_data.{thriftjava => pd} -import com.twitter.gizmoduck.{thriftscala => gt} -import com.twitter.home_mixer.{thriftscala => hmt} -import com.twitter.ml.api.constant.SharedFeatures -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.BoolDataRecordCompatible -import com.twitter.product_mixer.core.feature.datarecord.DataRecordFeature -import com.twitter.product_mixer.core.feature.datarecord.DataRecordOptionalFeature -import com.twitter.product_mixer.core.feature.datarecord.DoubleDataRecordCompatible -import com.twitter.product_mixer.core.feature.datarecord.LongDiscreteDataRecordCompatible -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContextFunctionalityType -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.search.common.features.{thriftscala => sc} -import com.twitter.search.earlybird.{thriftscala => eb} -import com.twitter.timelinemixer.clients.manhattan.DismissInfo -import com.twitter.timelinemixer.clients.persistence.TimelineResponseV3 -import com.twitter.timelinemixer.injection.model.candidate.AudioSpaceMetaData -import com.twitter.timelines.conversation_features.v1.thriftscala.ConversationFeatures -import com.twitter.timelines.impression.{thriftscala => imp} -import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm} -import com.twitter.timelines.model.UserId -import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures -import com.twitter.timelines.prediction.features.engagement_features.EngagementDataRecordFeatures -import com.twitter.timelines.prediction.features.recap.RecapFeatures -import com.twitter.timelines.prediction.features.request_context.RequestContextFeatures -import com.twitter.timelines.service.{thriftscala => tst} -import com.twitter.timelineservice.model.FeedbackEntry -import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts} -import com.twitter.timelineservice.suggests.{thriftscala => st} -import com.twitter.tsp.{thriftscala => tsp} -import com.twitter.tweetconvosvc.tweet_ancestor.{thriftscala => ta} -import com.twitter.util.Time - -object HomeFeatures { - // Candidate Features - object AncestorsFeature extends Feature[TweetCandidate, Seq[ta.TweetAncestor]] - object AudioSpaceMetaDataFeature extends Feature[TweetCandidate, Option[AudioSpaceMetaData]] - object TwitterListIdFeature extends Feature[TweetCandidate, Option[Long]] - - /** - * For Retweets, this should refer to the retweeting user. Use [[SourceUserIdFeature]] if you want to know - * who created the Tweet that was retweeted. - */ - object AuthorIdFeature - extends DataRecordOptionalFeature[TweetCandidate, Long] - with LongDiscreteDataRecordCompatible { - override val featureName: String = SharedFeatures.AUTHOR_ID.getFeatureName - override val personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.UserId) - } - - object AuthorIsBlueVerifiedFeature extends Feature[TweetCandidate, Boolean] - object AuthorIsGoldVerifiedFeature extends Feature[TweetCandidate, Boolean] - object AuthorIsGrayVerifiedFeature extends Feature[TweetCandidate, Boolean] - object AuthorIsLegacyVerifiedFeature extends Feature[TweetCandidate, Boolean] - object AuthorIsCreatorFeature extends Feature[TweetCandidate, Boolean] - object AuthorIsProtectedFeature extends Feature[TweetCandidate, Boolean] - - object AuthoredByContextualUserFeature extends Feature[TweetCandidate, Boolean] - object CachedCandidatePipelineIdentifierFeature extends Feature[TweetCandidate, Option[String]] - object CandidateSourceIdFeature - extends Feature[TweetCandidate, Option[cts.CandidateTweetSourceId]] - object ConversationFeature extends Feature[TweetCandidate, Option[ConversationFeatures]] - - /** - * This field should be set to the focal Tweet's tweetId for all tweets which are expected to - * be rendered in the same convo module. For non-convo module Tweets, this will be - * set to None. Note this is different from how TweetyPie defines ConversationId which is defined - * on all Tweets and points to the root tweet. This feature is used for grouping convo modules together. - */ - object ConversationModuleFocalTweetIdFeature extends Feature[TweetCandidate, Option[Long]] - - /** - * This field should always be set to the root Tweet in a conversation for all Tweets. For replies, this will - * point back to the root Tweet. For non-replies, this will be the candidate's Tweet id. This is consistent with - * the TweetyPie definition of ConversationModuleId. - */ - object ConversationModuleIdFeature extends Feature[TweetCandidate, Option[Long]] - object DirectedAtUserIdFeature extends Feature[TweetCandidate, Option[Long]] - object EarlybirdFeature extends Feature[TweetCandidate, Option[sc.ThriftTweetFeatures]] - object EarlybirdScoreFeature extends Feature[TweetCandidate, Option[Double]] - object EarlybirdSearchResultFeature extends Feature[TweetCandidate, Option[eb.ThriftSearchResult]] - object EntityTokenFeature extends Feature[TweetCandidate, Option[String]] - object ExclusiveConversationAuthorIdFeature extends Feature[TweetCandidate, Option[Long]] - object FavoritedByCountFeature - extends DataRecordFeature[TweetCandidate, Double] - with DoubleDataRecordCompatible { - override val featureName: String = - EngagementDataRecordFeatures.InNetworkFavoritesCount.getFeatureName - override val personalDataTypes: Set[pd.PersonalDataType] = - Set(pd.PersonalDataType.CountOfPrivateLikes, pd.PersonalDataType.CountOfPublicLikes) - } - object FavoritedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] - object FeedbackHistoryFeature extends Feature[TweetCandidate, Seq[FeedbackEntry]] - object RetweetedByCountFeature - extends DataRecordFeature[TweetCandidate, Double] - with DoubleDataRecordCompatible { - override val featureName: String = - EngagementDataRecordFeatures.InNetworkRetweetsCount.getFeatureName - override val personalDataTypes: Set[pd.PersonalDataType] = - Set(pd.PersonalDataType.CountOfPrivateRetweets, pd.PersonalDataType.CountOfPublicRetweets) - } - object RetweetedByEngagerIdsFeature extends Feature[TweetCandidate, Seq[Long]] - object RepliedByCountFeature - extends DataRecordFeature[TweetCandidate, Double] - with DoubleDataRecordCompatible { - override val featureName: String = - EngagementDataRecordFeatures.InNetworkRepliesCount.getFeatureName - override val personalDataTypes: Set[pd.PersonalDataType] = - Set(pd.PersonalDataType.CountOfPrivateReplies, pd.PersonalDataType.CountOfPublicReplies) - } - object RepliedByEngagerIdsFeature extends Feature[TweetCandidate, Seq[Long]] - object FollowedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] - - object TopicIdSocialContextFeature extends Feature[TweetCandidate, Option[Long]] - object TopicContextFunctionalityTypeFeature - extends Feature[TweetCandidate, Option[TopicContextFunctionalityType]] - object FromInNetworkSourceFeature extends Feature[TweetCandidate, Boolean] - - object FullScoringSucceededFeature extends Feature[TweetCandidate, Boolean] - object HasDisplayedTextFeature extends Feature[TweetCandidate, Boolean] - object InReplyToTweetIdFeature extends Feature[TweetCandidate, Option[Long]] - object InReplyToUserIdFeature extends Feature[TweetCandidate, Option[Long]] - object IsAncestorCandidateFeature extends Feature[TweetCandidate, Boolean] - object IsExtendedReplyFeature - extends DataRecordFeature[TweetCandidate, Boolean] - with BoolDataRecordCompatible { - override val featureName: String = RecapFeatures.IS_EXTENDED_REPLY.getFeatureName - override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty - } - object IsRandomTweetFeature - extends DataRecordFeature[TweetCandidate, Boolean] - with BoolDataRecordCompatible { - override val featureName: String = TimelinesSharedFeatures.IS_RANDOM_TWEET.getFeatureName - override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty - } - object IsReadFromCacheFeature extends Feature[TweetCandidate, Boolean] - object IsRetweetFeature extends Feature[TweetCandidate, Boolean] - object IsRetweetedReplyFeature extends Feature[TweetCandidate, Boolean] - object IsSupportAccountReplyFeature extends Feature[TweetCandidate, Boolean] - object LastScoredTimestampMsFeature extends Feature[TweetCandidate, Option[Long]] - object NonSelfFavoritedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] - object NumImagesFeature extends Feature[TweetCandidate, Option[Int]] - object OriginalTweetCreationTimeFromSnowflakeFeature extends Feature[TweetCandidate, Option[Time]] - object PositionFeature extends Feature[TweetCandidate, Option[Int]] - // Internal id generated per prediction service request - object PredictionRequestIdFeature extends Feature[TweetCandidate, Option[Long]] - object QuotedTweetIdFeature extends Feature[TweetCandidate, Option[Long]] - object QuotedUserIdFeature extends Feature[TweetCandidate, Option[Long]] - object ScoreFeature extends Feature[TweetCandidate, Option[Double]] - object SemanticCoreIdFeature extends Feature[TweetCandidate, Option[Long]] - // Key for kafka logging - object ServedIdFeature extends Feature[TweetCandidate, Option[Long]] - object SimclustersTweetTopKClustersWithScoresFeature - extends Feature[TweetCandidate, Map[String, Double]] - object SocialContextFeature extends Feature[TweetCandidate, Option[tst.SocialContext]] - object SourceTweetIdFeature - extends DataRecordOptionalFeature[TweetCandidate, Long] - with LongDiscreteDataRecordCompatible { - override val featureName: String = TimelinesSharedFeatures.SOURCE_TWEET_ID.getFeatureName - override val personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.TweetId) - } - object SourceUserIdFeature extends Feature[TweetCandidate, Option[Long]] - object StreamToKafkaFeature extends Feature[TweetCandidate, Boolean] - object SuggestTypeFeature extends Feature[TweetCandidate, Option[st.SuggestType]] - object TSPMetricTagFeature extends Feature[TweetCandidate, Set[tsp.MetricTag]] - object TweetLanguageFeature extends Feature[TweetCandidate, Option[String]] - object TweetUrlsFeature extends Feature[TweetCandidate, Seq[String]] - object VideoDurationMsFeature extends Feature[TweetCandidate, Option[Int]] - object ViewerIdFeature - extends DataRecordFeature[TweetCandidate, Long] - with LongDiscreteDataRecordCompatible { - override def featureName: String = SharedFeatures.USER_ID.getFeatureName - override def personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.UserId) - } - object WeightedModelScoreFeature extends Feature[TweetCandidate, Option[Double]] - object MentionUserIdFeature extends Feature[TweetCandidate, Seq[Long]] - object MentionScreenNameFeature extends Feature[TweetCandidate, Seq[String]] - object HasImageFeature extends Feature[TweetCandidate, Boolean] - object HasVideoFeature extends Feature[TweetCandidate, Boolean] - - // Tweetypie VF Features - object IsHydratedFeature extends Feature[TweetCandidate, Boolean] - object IsNsfwFeature extends Feature[TweetCandidate, Boolean] - object QuotedTweetDroppedFeature extends Feature[TweetCandidate, Boolean] - // Raw Tweet Text from Tweetypie - object TweetTextFeature extends Feature[TweetCandidate, Option[String]] - - object AuthorEnabledPreviewsFeature extends Feature[TweetCandidate, Boolean] - object IsTweetPreviewFeature extends Feature[TweetCandidate, Boolean] - - // SGS Features - /** - * By convention, this is set to true for retweets of non-followed authors - * E.g. where somebody the viewer follows retweets a Tweet from somebody the viewer doesn't follow - */ - object InNetworkFeature extends FeatureWithDefaultOnFailure[TweetCandidate, Boolean] { - override val defaultValue: Boolean = true - } - - // Query Features - object AccountAgeFeature extends Feature[PipelineQuery, Option[Time]] - object ClientIdFeature - extends DataRecordOptionalFeature[PipelineQuery, Long] - with LongDiscreteDataRecordCompatible { - override def featureName: String = SharedFeatures.CLIENT_ID.getFeatureName - override def personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.ClientType) - } - object CachedScoredTweetsFeature extends Feature[PipelineQuery, Seq[hmt.ScoredTweet]] - object DeviceLanguageFeature extends Feature[PipelineQuery, Option[String]] - object DismissInfoFeature - extends FeatureWithDefaultOnFailure[PipelineQuery, Map[st.SuggestType, Option[DismissInfo]]] { - override def defaultValue: Map[st.SuggestType, Option[DismissInfo]] = Map.empty - } - object FollowingLastNonPollingTimeFeature extends Feature[PipelineQuery, Option[Time]] - object GetInitialFeature - extends DataRecordFeature[PipelineQuery, Boolean] - with BoolDataRecordCompatible { - override def featureName: String = RequestContextFeatures.IS_GET_INITIAL.getFeatureName - override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty - } - object GetMiddleFeature - extends DataRecordFeature[PipelineQuery, Boolean] - with BoolDataRecordCompatible { - override def featureName: String = RequestContextFeatures.IS_GET_MIDDLE.getFeatureName - override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty - } - object GetNewerFeature - extends DataRecordFeature[PipelineQuery, Boolean] - with BoolDataRecordCompatible { - override def featureName: String = RequestContextFeatures.IS_GET_NEWER.getFeatureName - override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty - } - object GetOlderFeature - extends DataRecordFeature[PipelineQuery, Boolean] - with BoolDataRecordCompatible { - override def featureName: String = RequestContextFeatures.IS_GET_OLDER.getFeatureName - override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty - } - object GuestIdFeature - extends DataRecordOptionalFeature[PipelineQuery, Long] - with LongDiscreteDataRecordCompatible { - override def featureName: String = SharedFeatures.GUEST_ID.getFeatureName - override def personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.GuestId) - } - object HasDarkRequestFeature extends Feature[PipelineQuery, Option[Boolean]] - object ImpressionBloomFilterFeature - extends FeatureWithDefaultOnFailure[PipelineQuery, blm.ImpressionBloomFilterSeq] { - override def defaultValue: blm.ImpressionBloomFilterSeq = - blm.ImpressionBloomFilterSeq(Seq.empty) - } - object IsForegroundRequestFeature extends Feature[PipelineQuery, Boolean] - object IsLaunchRequestFeature extends Feature[PipelineQuery, Boolean] - object LastNonPollingTimeFeature extends Feature[PipelineQuery, Option[Time]] - object NonPollingTimesFeature extends Feature[PipelineQuery, Seq[Long]] - object PersistenceEntriesFeature extends Feature[PipelineQuery, Seq[TimelineResponseV3]] - object PollingFeature extends Feature[PipelineQuery, Boolean] - object PullToRefreshFeature extends Feature[PipelineQuery, Boolean] - // Scores from Real Graph representing the relationship between the viewer and another user - object RealGraphInNetworkScoresFeature extends Feature[PipelineQuery, Map[UserId, Double]] - object RequestJoinIdFeature extends Feature[TweetCandidate, Option[Long]] - // Internal id generated per request, mainly to deduplicate re-served cached tweets in logging - object ServedRequestIdFeature extends Feature[PipelineQuery, Option[Long]] - object ServedTweetIdsFeature extends Feature[PipelineQuery, Seq[Long]] - object ServedTweetPreviewIdsFeature extends Feature[PipelineQuery, Seq[Long]] - object TimelineServiceTweetsFeature extends Feature[PipelineQuery, Seq[Long]] - object TimestampFeature - extends DataRecordFeature[PipelineQuery, Long] - with LongDiscreteDataRecordCompatible { - override def featureName: String = SharedFeatures.TIMESTAMP.getFeatureName - override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty - } - object TimestampGMTDowFeature - extends DataRecordFeature[PipelineQuery, Long] - with LongDiscreteDataRecordCompatible { - override def featureName: String = RequestContextFeatures.TIMESTAMP_GMT_DOW.getFeatureName - override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty - } - object TimestampGMTHourFeature - extends DataRecordFeature[PipelineQuery, Long] - with LongDiscreteDataRecordCompatible { - override def featureName: String = RequestContextFeatures.TIMESTAMP_GMT_HOUR.getFeatureName - override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty - } - object TweetImpressionsFeature extends Feature[PipelineQuery, Seq[imp.TweetImpressionsEntry]] - object UserFollowedTopicsCountFeature extends Feature[PipelineQuery, Option[Int]] - object UserFollowingCountFeature extends Feature[PipelineQuery, Option[Int]] - object UserScreenNameFeature extends Feature[PipelineQuery, Option[String]] - object UserStateFeature extends Feature[PipelineQuery, Option[um.UserState]] - object UserTypeFeature extends Feature[PipelineQuery, Option[gt.UserType]] - object WhoToFollowExcludedUserIdsFeature - extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] { - override def defaultValue = Seq.empty - } - - // Result Features - object ServedSizeFeature extends Feature[PipelineQuery, Option[Int]] - object HasRandomTweetFeature extends Feature[PipelineQuery, Boolean] - object IsRandomTweetAboveFeature extends Feature[TweetCandidate, Boolean] - object ServedInConversationModuleFeature extends Feature[TweetCandidate, Boolean] - object ConversationModule2DisplayedTweetsFeature extends Feature[TweetCandidate, Boolean] - object ConversationModuleHasGapFeature extends Feature[TweetCandidate, Boolean] - object SGSValidLikedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] - object SGSValidFollowedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] - object PerspectiveFilteredLikedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] - object ScreenNamesFeature extends Feature[TweetCandidate, Map[Long, String]] - object RealNamesFeature extends Feature[TweetCandidate, Map[Long, String]] - - /** - * Features around the focal Tweet for Tweets which should be rendered in convo modules. - * These are needed in order to render social context above the root tweet in a convo modules. - * For example if we have a convo module A-B-C (A Tweets, B replies to A, C replies to B), the descendant features are - * for the Tweet C. These features are None except for the root Tweet for Tweets which should render into - * convo modules. - */ - object FocalTweetAuthorIdFeature extends Feature[TweetCandidate, Option[Long]] - object FocalTweetInNetworkFeature extends Feature[TweetCandidate, Option[Boolean]] - object FocalTweetRealNamesFeature extends Feature[TweetCandidate, Option[Map[Long, String]]] - object FocalTweetScreenNamesFeature extends Feature[TweetCandidate, Option[Map[Long, String]]] - object MediaUnderstandingAnnotationIdsFeature extends Feature[TweetCandidate, Seq[Long]] -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/BUILD.bazel deleted file mode 100644 index 3883e454e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/BUILD.bazel +++ /dev/null @@ -1,15 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "dspbidder/thrift/src/main/thrift/com/twitter/dspbidder/commons:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "timelineservice/common:model", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/DeviceContext.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/DeviceContext.scala deleted file mode 100644 index ef96865ca..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/DeviceContext.scala +++ /dev/null @@ -1,74 +0,0 @@ -package com.twitter.home_mixer.model.request - -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext -import com.twitter.{timelineservice => tls} - -case class DeviceContext( - isPolling: Option[Boolean], - requestContext: Option[String], - latestControlAvailable: Option[Boolean], - autoplayEnabled: Option[Boolean]) { - - lazy val requestContextValue: Option[DeviceContext.RequestContext.Value] = - requestContext.flatMap { value => - val normalizedValue = value.trim.toLowerCase() - DeviceContext.RequestContext.values.find(_.toString == normalizedValue) - } - - def toTimelineServiceDeviceContext(clientContext: ClientContext): tls.DeviceContext = - tls.DeviceContext( - countryCode = clientContext.countryCode, - languageCode = clientContext.languageCode, - clientAppId = clientContext.appId, - ipAddress = clientContext.ipAddress, - guestId = clientContext.guestId, - sessionId = None, - timezone = None, - userAgent = clientContext.userAgent, - deviceId = clientContext.deviceId, - isPolling = isPolling, - requestProvenance = requestContext, - referrer = None, - tfeAuthHeader = None, - mobileDeviceId = clientContext.mobileDeviceId, - isSessionStart = None, - displaySize = None, - isURTRequest = Some(true), - latestControlAvailable = latestControlAvailable, - guestIdMarketing = clientContext.guestIdMarketing, - isInternalOrTwoffice = clientContext.isTwoffice, - browserNotificationPermission = None, - guestIdAds = clientContext.guestIdAds, - ) -} - -object DeviceContext { - val Empty: DeviceContext = DeviceContext( - isPolling = None, - requestContext = None, - latestControlAvailable = None, - autoplayEnabled = None - ) - - /** - * Constants which reflect valid client request provenances (why a request was initiated, encoded - * by the "request_context" HTTP parameter). - */ - object RequestContext extends Enumeration { - val Auto = Value("auto") - val Foreground = Value("foreground") - val Gap = Value("gap") - val Launch = Value("launch") - val ManualRefresh = Value("manual_refresh") - val Navigate = Value("navigate") - val Polling = Value("polling") - val PullToRefresh = Value("ptr") - val Signup = Value("signup") - val TweetSelfThread = Value("tweet_self_thread") - val BackgroundFetch = Value("background_fetch") - } -} - -trait HasDeviceContext { - def deviceContext: Option[DeviceContext] -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HasListId.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HasListId.scala deleted file mode 100644 index ece09a0d4..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HasListId.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.home_mixer.model.request - -/** - * [[HasListId]] enables shared components to access the list id shared by all list timeline products. - */ -trait HasListId { - def listId: Long -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HasSeenTweetIds.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HasSeenTweetIds.scala deleted file mode 100644 index 0ec657384..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HasSeenTweetIds.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.home_mixer.model.request - -/** - * [[HasSeenTweetIds]] enables shared components to access the list of impressed tweet IDs - * sent by clients across different Home Mixer query types (e.g. FollowingQuery, ForYouQuery) - */ -trait HasSeenTweetIds { - def seenTweetIds: Option[Seq[Long]] -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerDebugOptions.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerDebugOptions.scala deleted file mode 100644 index 818380946..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerDebugOptions.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.home_mixer.model.request - -import com.twitter.product_mixer.core.model.marshalling.request.DebugOptions -import com.twitter.util.Time - -case class HomeMixerDebugOptions( - override val requestTimeOverride: Option[Time]) - extends DebugOptions diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProduct.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProduct.scala deleted file mode 100644 index 7c27c50d5..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProduct.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.home_mixer.model.request - -import com.twitter.product_mixer.core.model.common.identifier.ProductIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.Product - -/** - * Identifier names on products can be used to create Feature Switch rules by product, - * which useful if bucketing occurs in a component shared by multiple products. - * @see [[Product.identifier]] - */ - -case object FollowingProduct extends Product { - override val identifier: ProductIdentifier = ProductIdentifier("Following") - override val stringCenterProject: Option[String] = Some("timelinemixer") -} - -case object ForYouProduct extends Product { - override val identifier: ProductIdentifier = ProductIdentifier("ForYou") - override val stringCenterProject: Option[String] = Some("timelinemixer") -} - -case object ScoredTweetsProduct extends Product { - override val identifier: ProductIdentifier = ProductIdentifier("ScoredTweets") - override val stringCenterProject: Option[String] = Some("timelinemixer") -} - -case object ListTweetsProduct extends Product { - override val identifier: ProductIdentifier = ProductIdentifier("ListTweets") - override val stringCenterProject: Option[String] = Some("timelinemixer") -} - -case object ListRecommendedUsersProduct extends Product { - override val identifier: ProductIdentifier = ProductIdentifier("ListRecommendedUsers") - override val stringCenterProject: Option[String] = Some("timelinemixer") -} - -case object SubscribedProduct extends Product { - override val identifier: ProductIdentifier = ProductIdentifier("Subscribed") - override val stringCenterProject: Option[String] = Some("timelinemixer") -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProductContext.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProductContext.scala deleted file mode 100644 index dddd733f3..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProductContext.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.home_mixer.model.request - -import com.twitter.dspbidder.commons.thriftscala.DspClientContext -import com.twitter.product_mixer.core.model.marshalling.request.ProductContext - -case class FollowingProductContext( - deviceContext: Option[DeviceContext], - seenTweetIds: Option[Seq[Long]], - dspClientContext: Option[DspClientContext]) - extends ProductContext - -case class ForYouProductContext( - deviceContext: Option[DeviceContext], - seenTweetIds: Option[Seq[Long]], - dspClientContext: Option[DspClientContext], - pushToHomeTweetId: Option[Long]) - extends ProductContext - -case class ScoredTweetsProductContext( - deviceContext: Option[DeviceContext], - seenTweetIds: Option[Seq[Long]], - servedTweetIds: Option[Seq[Long]], - backfillTweetIds: Option[Seq[Long]]) - extends ProductContext - -case class ListTweetsProductContext( - listId: Long, - deviceContext: Option[DeviceContext], - dspClientContext: Option[DspClientContext]) - extends ProductContext - -case class ListRecommendedUsersProductContext( - listId: Long, - selectedUserIds: Option[Seq[Long]], - excludedUserIds: Option[Seq[Long]], - listName: Option[String]) - extends ProductContext - -case class SubscribedProductContext( - deviceContext: Option[DeviceContext], - seenTweetIds: Option[Seq[Long]]) - extends ProductContext diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerRequest.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerRequest.scala deleted file mode 100644 index 34595c34c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerRequest.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.home_mixer.model.request - -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext -import com.twitter.product_mixer.core.model.marshalling.request.DebugParams -import com.twitter.product_mixer.core.model.marshalling.request.Product -import com.twitter.product_mixer.core.model.marshalling.request.ProductContext -import com.twitter.product_mixer.core.model.marshalling.request.Request - -case class HomeMixerRequest( - override val clientContext: ClientContext, - override val product: Product, - // Product-specific parameters should be placed in the Product Context - override val productContext: Option[ProductContext], - override val serializedRequestCursor: Option[String], - override val maxResults: Option[Int], - override val debugParams: Option[DebugParams], - // Parameters that apply to all products can be promoted to the request-level - homeRequestParam: Boolean) - extends Request diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/AdvertiserBrandSafetySettingsStoreModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/AdvertiserBrandSafetySettingsStoreModule.scala deleted file mode 100644 index d095d2054..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/AdvertiserBrandSafetySettingsStoreModule.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides - -import com.twitter.adserver.{thriftscala => ads} -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.storage.client.manhattan.kv.Guarantee -import com.twitter.storehaus.ReadableStore -import com.twitter.storehaus_internal.manhattan.ManhattanCluster -import com.twitter.storehaus_internal.manhattan.ManhattanClusters -import com.twitter.timelines.clients.ads.AdvertiserBrandSafetySettingsStore -import com.twitter.timelines.clients.manhattan.mhv3.ManhattanClientBuilder -import com.twitter.timelines.clients.manhattan.mhv3.ManhattanClientConfigWithDataset -import com.twitter.util.Duration - -import javax.inject.Singleton - -object AdvertiserBrandSafetySettingsStoreModule extends TwitterModule { - - @Provides - @Singleton - def providesAdvertiserBrandSafetySettingsStore( - injectedServiceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): ReadableStore[Long, ads.AdvertiserBrandSafetySettings] = { - val advertiserBrandSafetySettingsManhattanClientConfig = new ManhattanClientConfigWithDataset { - override val cluster: ManhattanCluster = ManhattanClusters.apollo - override val appId: String = "brand_safety_apollo" - override val dataset = "advertiser_brand_safety_settings" - override val statsScope: String = "AdvertiserBrandSafetySettingsManhattanClient" - override val defaultGuarantee = Guarantee.Weak - override val defaultMaxTimeout: Duration = 100.milliseconds - override val maxRetryCount: Int = 1 - override val isReadOnly: Boolean = true - override val serviceIdentifier: ServiceIdentifier = injectedServiceIdentifier - } - - val advertiserBrandSafetySettingsManhattanEndpoint = ManhattanClientBuilder - .buildManhattanEndpoint(advertiserBrandSafetySettingsManhattanClientConfig, statsReceiver) - - val advertiserBrandSafetySettingsStore: ReadableStore[Long, ads.AdvertiserBrandSafetySettings] = - AdvertiserBrandSafetySettingsStore - .cached( - advertiserBrandSafetySettingsManhattanEndpoint, - advertiserBrandSafetySettingsManhattanClientConfig.dataset, - ttl = 60.minutes, - maxKeys = 100000, - windowSize = 10L - )(statsReceiver) - - advertiserBrandSafetySettingsStore - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BUILD.bazel deleted file mode 100644 index b7fbca0d3..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BUILD.bazel +++ /dev/null @@ -1,90 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/twitter/bijection:scrooge", - "3rdparty/jvm/com/twitter/bijection:thrift", - "3rdparty/jvm/com/twitter/src/java/com/twitter/logpipeline/client:logpipeline-event-publisher-thin", - "3rdparty/jvm/com/twitter/storehaus:core", - "3rdparty/jvm/io/netty:netty4-tcnative-boringssl-static", - "eventbus/client/src/main/scala/com/twitter/eventbus/client", - "finagle-internal/finagle-grpc/src/main/scala", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/client", - "finagle/finagle-core/src/main", - "finagle/finagle-memcached/src/main/scala", - "finagle/finagle-mux/src/main/scala", - "finagle/finagle-thriftmux/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/store", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "interests-service/thrift/src/main/thrift:thrift-scala", - "people-discovery/api/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/manhattan_client", - "product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/memcached_client", - "product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/thrift_client", - "servo/client/src/main/scala/com/twitter/servo/client", - "servo/manhattan", - "servo/util", - "socialgraph/server/src/main/scala/com/twitter/socialgraph/util", - "src/scala/com/twitter/ml/featurestore/lib", - "src/scala/com/twitter/scalding_internal/multiformat/format", - "src/scala/com/twitter/storehaus_internal", - "src/scala/com/twitter/summingbird_internal/bijection:bijection-implicits", - "src/scala/com/twitter/timelines/util", - "src/thrift/com/twitter/ads/adserver:adserver_rpc-scala", - "src/thrift/com/twitter/clientapp/gen:clientapp-scala", - "src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala", - "src/thrift/com/twitter/manhattan:v1-scala", - "src/thrift/com/twitter/manhattan:v2-scala", - "src/thrift/com/twitter/onboarding/relevance/features:features-java", - "src/thrift/com/twitter/search:blender-scala", - "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/service/metastore/gen:thrift-scala", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "src/thrift/com/twitter/timelines/author_features:thrift-java", - "src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala", - "src/thrift/com/twitter/timelines/impression_store:thrift-scala", - "src/thrift/com/twitter/timelines/real_graph:real_graph-scala", - "src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java", - "src/thrift/com/twitter/timelines/timeline_logging:thrift-scala", - "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", - "src/thrift/com/twitter/topic_recos:topic_recos-thrift-java", - "src/thrift/com/twitter/user_session_store:thrift-java", - "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", - "stitch/stitch-socialgraph", - "stitch/stitch-tweetypie", - "strato/src/main/scala/com/twitter/strato/client", - "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/feedback", - "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", - "timelines:decider", - "timelines/src/main/scala/com/twitter/timelines/clients/ads", - "timelines/src/main/scala/com/twitter/timelines/clients/manhattan", - "timelines/src/main/scala/com/twitter/timelines/clients/manhattan/store", - "timelines/src/main/scala/com/twitter/timelines/clients/predictionservice", - "timelines/src/main/scala/com/twitter/timelines/clients/strato", - "timelines/src/main/scala/com/twitter/timelines/clients/strato/topics", - "timelines/src/main/scala/com/twitter/timelines/clients/strato/twistly", - "timelines/src/main/scala/com/twitter/timelines/config", - "timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter", - "timelines/src/main/scala/com/twitter/timelines/impressionstore/store", - "timelines/src/main/scala/com/twitter/timelines/util/stats", - "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", - "tweetconvosvc/client/src/main/scala/com/twitter/tweetconvosvc/client/builder", - "twitter-config/yaml", - ], - exports = [ - "timelines/src/main/scala/com/twitter/timelines/clients/predictionservice", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BlenderClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BlenderClientModule.scala deleted file mode 100644 index bc4045181..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BlenderClientModule.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.inject.TwitterModule -import com.twitter.product_mixer.shared_library.thrift_client.FinagleThriftClientBuilder -import com.twitter.product_mixer.shared_library.thrift_client.NonIdempotent -import com.twitter.search.blender.thriftscala.BlenderService -import javax.inject.Singleton - -object BlenderClientModule extends TwitterModule { - - @Singleton - @Provides - def providesBlenderClient( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): BlenderService.MethodPerEndpoint = { - val clientId = serviceIdentifier.environment.toLowerCase match { - case "prod" => ClientId("") - case _ => ClientId("") - } - - FinagleThriftClientBuilder.buildFinagleMethodPerEndpoint[ - BlenderService.ServicePerEndpoint, - BlenderService.MethodPerEndpoint - ]( - serviceIdentifier = serviceIdentifier, - clientId = clientId, - dest = "/s/blender-universal/blender", - label = "blender", - statsReceiver = statsReceiver, - idempotency = NonIdempotent, - timeoutPerRequest = 1000.milliseconds, - timeoutTotal = 1000.milliseconds, - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ClientSentImpressionsPublisherModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ClientSentImpressionsPublisherModule.scala deleted file mode 100644 index f7c5e9bc7..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ClientSentImpressionsPublisherModule.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.eventbus.client.EventBusPublisher -import com.twitter.eventbus.client.EventBusPublisherBuilder -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.timelines.config.ConfigUtils -import com.twitter.timelines.config.Env -import com.twitter.timelines.impressionstore.thriftscala.PublishedImpressionList -import javax.inject.Singleton - -object ClientSentImpressionsPublisherModule extends TwitterModule with ConfigUtils { - private val serviceName = "home-mixer" - - @Singleton - @Provides - def providesClientSentImpressionsPublisher( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): EventBusPublisher[PublishedImpressionList] = { - val env = serviceIdentifier.environment.toLowerCase match { - case "prod" => Env.prod - case "staging" => Env.staging - case "local" => Env.local - case _ => Env.devel - } - - val streamName = env match { - case Env.prod => "timelinemixer_client_sent_impressions_prod" - case _ => "timelinemixer_client_sent_impressions_devel" - } - - EventBusPublisherBuilder() - .clientId(clientIdWithScopeOpt(serviceName, env)) - .serviceIdentifier(serviceIdentifier) - .streamName(streamName) - .statsReceiver(statsReceiver.scope("eventbus")) - .thriftStruct(PublishedImpressionList) - .tcpConnectTimeout(20.milliseconds) - .connectTimeout(100.milliseconds) - .requestTimeout(1.second) - .publishTimeout(1.second) - .build() - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ConversationServiceModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ConversationServiceModule.scala deleted file mode 100644 index 9a7d8771c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ConversationServiceModule.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.home_mixer.module - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.ThriftMux -import com.twitter.finagle.thriftmux.MethodBuilder -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.inject.Injector -import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule -import com.twitter.tweetconvosvc.thriftscala.ConversationService -import com.twitter.util.Duration -import org.apache.thrift.protocol.TCompactProtocol - -object ConversationServiceModule - extends ThriftMethodBuilderClientModule[ - ConversationService.ServicePerEndpoint, - ConversationService.MethodPerEndpoint - ] - with MtlsClient { - - override val label: String = "tweetconvosvc" - override val dest: String = "/s/tweetconvosvc/tweetconvosvc" - - override protected def configureMethodBuilder( - injector: Injector, - methodBuilder: MethodBuilder - ): MethodBuilder = methodBuilder.withTimeoutPerRequest(100.milliseconds) - - override def configureThriftMuxClient( - injector: Injector, - client: ThriftMux.Client - ): ThriftMux.Client = - super - .configureThriftMuxClient(injector, client) - .withProtocolFactory(new TCompactProtocol.Factory()) - - override protected def sessionAcquisitionTimeout: Duration = 500.milliseconds -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/FeedbackHistoryClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/FeedbackHistoryClientModule.scala deleted file mode 100644 index c065f7b35..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/FeedbackHistoryClientModule.scala +++ /dev/null @@ -1,47 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.timelinemixer.clients.feedback.FeedbackHistoryManhattanClient -import com.twitter.timelinemixer.clients.feedback.FeedbackHistoryManhattanClientConfig -import com.twitter.timelines.clients.manhattan.mhv3.ManhattanClientBuilder -import com.twitter.util.Duration -import javax.inject.Singleton - -object FeedbackHistoryClientModule extends TwitterModule { - private val ProdDataset = "feedback_history" - private val StagingDataset = "feedback_history_nonprod" - private final val Timeout = "mh_feedback_history.timeout" - - flag[Duration](Timeout, 150.millis, "Timeout per request") - - @Provides - @Singleton - def providesFeedbackHistoryClient( - @Flag(Timeout) timeout: Duration, - serviceId: ServiceIdentifier, - statsReceiver: StatsReceiver - ) = { - val manhattanDataset = serviceId.environment.toLowerCase match { - case "prod" => ProdDataset - case _ => StagingDataset - } - - val config = new FeedbackHistoryManhattanClientConfig { - val dataset = manhattanDataset - val isReadOnly = true - val serviceIdentifier = serviceId - override val defaultMaxTimeout = timeout - } - - new FeedbackHistoryManhattanClient( - ManhattanClientBuilder.buildManhattanEndpoint(config, statsReceiver), - manhattanDataset, - statsReceiver - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeAdsCandidateSourceModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeAdsCandidateSourceModule.scala deleted file mode 100644 index 73e1500a0..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeAdsCandidateSourceModule.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.home_mixer.module - -import com.twitter.adserver.thriftscala.NewAdServer -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.thriftmux.MethodBuilder -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.inject.Injector -import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule -import com.twitter.util.Duration - -object HomeAdsCandidateSourceModule - extends ThriftMethodBuilderClientModule[ - NewAdServer.ServicePerEndpoint, - NewAdServer.MethodPerEndpoint - ] - with MtlsClient { - - override val label = "adserver" - override val dest = "/s/ads/adserver" - - override protected def configureMethodBuilder( - injector: Injector, - methodBuilder: MethodBuilder - ): MethodBuilder = { - methodBuilder - .withTimeoutPerRequest(1200.milliseconds) - .withTimeoutTotal(1200.milliseconds) - .withMaxRetries(2) - } - - override protected def sessionAcquisitionTimeout: Duration = 150.milliseconds -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerFlagsModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerFlagsModule.scala deleted file mode 100644 index e31d2d9bc..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerFlagsModule.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.twitter.home_mixer.module - -import com.twitter.conversions.DurationOps.RichDuration -import com.twitter.home_mixer.param.HomeMixerFlagName -import com.twitter.inject.TwitterModule -import com.twitter.util.Duration - -object HomeMixerFlagsModule extends TwitterModule { - - import HomeMixerFlagName._ - - flag[Boolean]( - name = ScribeClientEventsFlag, - default = false, - help = "Toggles logging client events to Scribe" - ) - - flag[Boolean]( - name = ScribeServedCandidatesFlag, - default = false, - help = "Toggles logging served candidates to Scribe" - ) - - flag[Boolean]( - name = ScribeScoredCandidatesFlag, - default = false, - help = "Toggles logging scored candidates to Scribe" - ) - - flag[Boolean]( - name = ScribeServedCommonFeaturesAndCandidateFeaturesFlag, - default = false, - help = "Toggles logging served common features and candidates features to Scribe" - ) - - flag[String]( - name = DataRecordMetadataStoreConfigsYmlFlag, - default = "", - help = "The YML file that contains the necessary info for creating metadata store MySQL client." - ) - - flag[String]( - name = DarkTrafficFilterDeciderKey, - default = "dark_traffic_filter", - help = "Dark traffic filter decider key" - ) - - flag[Duration]( - TargetFetchLatency, - 300.millis, - "Target fetch latency from candidate sources for Quality Factor" - ) - - flag[Duration]( - TargetScoringLatency, - 700.millis, - "Target scoring latency for Quality Factor" - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerResourcesModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerResourcesModule.scala deleted file mode 100644 index b68e5b105..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerResourcesModule.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.home_mixer.module - -import com.twitter.inject.TwitterModule - -object HomeMixerResourcesModule extends TwitterModule {} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ImpressionBloomFilterModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ImpressionBloomFilterModule.scala deleted file mode 100644 index f37531483..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ImpressionBloomFilterModule.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.storage.client.manhattan.kv.Guarantee -import com.twitter.storehaus_internal.manhattan.ManhattanClusters -import com.twitter.timelines.clients.manhattan.store._ -import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm} -import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilterManhattanKeyValueDescriptor -import com.twitter.util.Duration -import javax.inject.Singleton - -object ImpressionBloomFilterModule extends TwitterModule { - - private val ProdAppId = "impression_bloom_filter_store" - private val ProdDataset = "impression_bloom_filter" - private val StagingAppId = "impression_bloom_filter_store_staging" - private val StagingDataset = "impression_bloom_filter_staging" - private val ClientStatsScope = "tweetBloomFilterImpressionManhattanClient" - private val DefaultTTL = 7.days - private final val Timeout = "mh_impression_store_bloom_filter.timeout" - - flag[Duration](Timeout, 150.millis, "Timeout per request") - - @Provides - @Singleton - def providesImpressionBloomFilter( - @Flag(Timeout) timeout: Duration, - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): ManhattanStoreClient[blm.ImpressionBloomFilterKey, blm.ImpressionBloomFilterSeq] = { - val (appId, dataset) = serviceIdentifier.environment.toLowerCase match { - case "prod" => (ProdAppId, ProdDataset) - case _ => (StagingAppId, StagingDataset) - } - - implicit val manhattanKeyValueDescriptor: ImpressionBloomFilterManhattanKeyValueDescriptor = - ImpressionBloomFilterManhattanKeyValueDescriptor( - dataset = dataset, - ttl = DefaultTTL - ) - - ManhattanStoreClientBuilder.buildManhattanClient( - serviceIdentifier = serviceIdentifier, - cluster = ManhattanClusters.nash, - appId = appId, - defaultMaxTimeout = timeout, - maxRetryCount = 2, - defaultGuarantee = Some(Guarantee.SoftDcReadMyWrites), - isReadOnly = false, - statsScope = ClientStatsScope, - statsReceiver = statsReceiver - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/InjectionHistoryClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/InjectionHistoryClientModule.scala deleted file mode 100644 index fe274ff1d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/InjectionHistoryClientModule.scala +++ /dev/null @@ -1,88 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.ThriftMux -import com.twitter.finagle.builder.ClientBuilder -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.mtls.client.MtlsStackClient._ -import com.twitter.finagle.service.RetryPolicy -import com.twitter.finagle.ssl.OpportunisticTls -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.manhattan.v2.thriftscala.{ManhattanCoordinator => ManhattanV2} -import com.twitter.timelinemixer.clients.manhattan.InjectionHistoryClient -import com.twitter.timelinemixer.clients.manhattan.ManhattanDatasetConfig -import com.twitter.timelines.clients.manhattan.Dataset -import com.twitter.timelines.clients.manhattan.ManhattanClient -import com.twitter.timelines.util.stats.RequestScope -import javax.inject.Singleton -import org.apache.thrift.protocol.TBinaryProtocol -import com.twitter.timelines.config.TimelinesUnderlyingClientConfiguration.ConnectTimeout -import com.twitter.timelines.config.TimelinesUnderlyingClientConfiguration.TCPConnectTimeout - -object InjectionHistoryClientModule extends TwitterModule { - private val ProdDataset = "suggestion_history" - private val StagingDataset = "suggestion_history_nonprod" - private val AppId = "twitter_suggests" - private val ServiceName = "manhattan.omega" - private val OmegaManhattanDest = "/s/manhattan/omega.native-thrift" - private val InjectionRequestScope = RequestScope("injectionHistoryClient") - private val RequestTimeout = 75.millis - private val Timeout = 150.millis - - val retryPolicy = RetryPolicy.tries( - 2, - RetryPolicy.TimeoutAndWriteExceptionsOnly - .orElse(RetryPolicy.ChannelClosedExceptionsOnly)) - - @Provides - @Singleton - def providesInjectionHistoryClient( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ) = { - val dataset = serviceIdentifier.environment.toLowerCase match { - case "prod" => ProdDataset - case _ => StagingDataset - } - - val thriftMuxClient = ClientBuilder() - .name(ServiceName) - .daemon(daemonize = true) - .failFast(enabled = true) - .retryPolicy(retryPolicy) - .tcpConnectTimeout(TCPConnectTimeout) - .connectTimeout(ConnectTimeout) - .dest(OmegaManhattanDest) - .requestTimeout(RequestTimeout) - .timeout(Timeout) - .stack(ThriftMux.client - .withMutualTls(serviceIdentifier) - .withOpportunisticTls(OpportunisticTls.Required)) - .build() - - val manhattanOmegaClient = new ManhattanV2.FinagledClient( - service = thriftMuxClient, - protocolFactory = new TBinaryProtocol.Factory(), - serviceName = ServiceName, - ) - - val readOnlyMhClient = new ManhattanClient( - appId = AppId, - manhattan = manhattanOmegaClient, - requestScope = InjectionRequestScope, - serviceName = ServiceName, - statsReceiver = statsReceiver - ).readOnly - - val mhDatasetConfig = new ManhattanDatasetConfig { - override val SuggestionHistoryDataset = Dataset(dataset) - } - - new InjectionHistoryClient( - readOnlyMhClient, - mhDatasetConfig - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanClientsModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanClientsModule.scala deleted file mode 100644 index fc0e282af..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanClientsModule.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealGraphManhattanEndpoint -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.storage.client.manhattan.kv._ -import com.twitter.timelines.config.ConfigUtils -import com.twitter.util.Duration -import javax.inject.Named -import javax.inject.Singleton - -object ManhattanClientsModule extends TwitterModule with ConfigUtils { - - private val ApolloDest = "/s/manhattan/apollo.native-thrift" - private final val Timeout = "mh_real_graph.timeout" - - flag[Duration](Timeout, 150.millis, "Timeout total") - - @Provides - @Singleton - @Named(RealGraphManhattanEndpoint) - def providesRealGraphManhattanEndpoint( - @Flag(Timeout) timeout: Duration, - serviceIdentifier: ServiceIdentifier - ): ManhattanKVEndpoint = { - lazy val client = ManhattanKVClient( - appId = "real_graph", - dest = ApolloDest, - mtlsParams = ManhattanKVClientMtlsParams(serviceIdentifier = serviceIdentifier), - label = "real-graph-data" - ) - - ManhattanKVEndpointBuilder(client) - .maxRetryCount(2) - .defaultMaxTimeout(timeout) - .build() - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanFeatureRepositoryModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanFeatureRepositoryModule.scala deleted file mode 100644 index 5668ba0ee..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanFeatureRepositoryModule.scala +++ /dev/null @@ -1,468 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.bijection.Injection -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.bijection.scrooge.CompactScalaCodec -import com.twitter.bijection.thrift.ThriftCodec -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.home_mixer.param.HomeMixerInjectionNames._ -import com.twitter.home_mixer.util.InjectionTransformerImplicits._ -import com.twitter.home_mixer.util.LanguageUtil -import com.twitter.home_mixer.util.TensorFlowUtil -import com.twitter.inject.TwitterModule -import com.twitter.manhattan.v1.{thriftscala => mh} -import com.twitter.ml.api.{thriftscala => ml} -import com.twitter.ml.featurestore.lib.UserId -import com.twitter.ml.featurestore.{thriftscala => fs} -import com.twitter.onboarding.relevance.features.{thriftjava => rf} -import com.twitter.product_mixer.shared_library.manhattan_client.ManhattanClientBuilder -import com.twitter.scalding_internal.multiformat.format.keyval.KeyValInjection.ScalaBinaryThrift -import com.twitter.search.common.constants.{thriftscala => scc} -import com.twitter.service.metastore.gen.{thriftscala => smg} -import com.twitter.servo.cache._ -import com.twitter.servo.manhattan.ManhattanKeyValueRepository -import com.twitter.servo.repository.CachingKeyValueRepository -import com.twitter.servo.repository.ChunkingStrategy -import com.twitter.servo.repository.KeyValueRepository -import com.twitter.servo.repository.Repository -import com.twitter.servo.repository.keysAsQuery -import com.twitter.servo.util.Transformer -import com.twitter.storage.client.manhattan.bijections.Bijections -import com.twitter.storehaus_internal.manhattan.ManhattanClusters -import com.twitter.timelines.author_features.v1.{thriftjava => af} -import com.twitter.timelines.suggests.common.dense_data_record.{thriftscala => ddr} -import com.twitter.user_session_store.{thriftscala => uss_scala} -import com.twitter.user_session_store.{thriftjava => uss} -import com.twitter.util.Duration -import com.twitter.util.Try -import java.nio.ByteBuffer -import javax.inject.Named -import javax.inject.Singleton -import org.apache.thrift.protocol.TCompactProtocol -import org.apache.thrift.transport.TMemoryInputTransport -import org.apache.thrift.transport.TTransport - -object ManhattanFeatureRepositoryModule extends TwitterModule { - - private val DEFAULT_RPC_CHUNK_SIZE = 50 - - private val ThriftEntityIdInjection = ScalaBinaryThrift(fs.EntityId) - - private val FeatureStoreUserIdKeyTransformer = new Transformer[Long, ByteBuffer] { - override def to(userId: Long): Try[ByteBuffer] = { - Try(ByteBuffer.wrap(ThriftEntityIdInjection.apply(UserId(userId).toThrift))) - } - override def from(b: ByteBuffer): Try[Long] = ??? - } - - private val FloatTensorTransformer = new Transformer[ByteBuffer, ml.FloatTensor] { - override def to(input: ByteBuffer): Try[ml.FloatTensor] = { - val floatTensor = TensorFlowUtil.embeddingByteBufferToFloatTensor(input) - Try(floatTensor) - } - - override def from(b: ml.FloatTensor): Try[ByteBuffer] = ??? - } - - private val LanguageTransformer = new Transformer[ByteBuffer, Seq[scc.ThriftLanguage]] { - override def to(input: ByteBuffer): Try[Seq[scc.ThriftLanguage]] = { - Try.fromScala( - Bijections - .BinaryScalaInjection(smg.UserLanguages) - .andThen(Bijections.byteBuffer2Buf.inverse) - .invert(input).map(LanguageUtil.computeLanguages(_))) - } - - override def from(b: Seq[scc.ThriftLanguage]): Try[ByteBuffer] = ??? - } - - private val LongKeyTransformer = Injection - .connect[Long, Array[Byte]] - .toByteBufferTransformer() - - // manhattan clients - - @Provides - @Singleton - @Named(ManhattanApolloClient) - def providesManhattanApolloClient( - serviceIdentifier: ServiceIdentifier - ): mh.ManhattanCoordinator.MethodPerEndpoint = { - ManhattanClientBuilder - .buildManhattanV1FinagleClient( - ManhattanClusters.apollo, - serviceIdentifier - ) - } - - @Provides - @Singleton - @Named(ManhattanAthenaClient) - def providesManhattanAthenaClient( - serviceIdentifier: ServiceIdentifier - ): mh.ManhattanCoordinator.MethodPerEndpoint = { - ManhattanClientBuilder - .buildManhattanV1FinagleClient( - ManhattanClusters.athena, - serviceIdentifier - ) - } - - @Provides - @Singleton - @Named(ManhattanOmegaClient) - def providesManhattanOmegaClient( - serviceIdentifier: ServiceIdentifier - ): mh.ManhattanCoordinator.MethodPerEndpoint = { - ManhattanClientBuilder - .buildManhattanV1FinagleClient( - ManhattanClusters.omega, - serviceIdentifier - ) - } - - @Provides - @Singleton - @Named(ManhattanStarbuckClient) - def providesManhattanStarbuckClient( - serviceIdentifier: ServiceIdentifier - ): mh.ManhattanCoordinator.MethodPerEndpoint = { - ManhattanClientBuilder - .buildManhattanV1FinagleClient( - ManhattanClusters.starbuck, - serviceIdentifier - ) - } - - // non-cached manhattan repositories - - @Provides - @Singleton - @Named(MetricCenterUserCountingFeatureRepository) - def providesMetricCenterUserCountingFeatureRepository( - @Named(ManhattanStarbuckClient) client: mh.ManhattanCoordinator.MethodPerEndpoint - ): KeyValueRepository[Seq[Long], Long, rf.MCUserCountingFeatures] = { - - val valueTransformer = ThriftCodec - .toBinary[rf.MCUserCountingFeatures] - .toByteBufferTransformer() - .flip - - batchedManhattanKeyValueRepository[Long, rf.MCUserCountingFeatures]( - client = client, - keyTransformer = LongKeyTransformer, - valueTransformer = valueTransformer, - appId = "wtf_ml", - dataset = "mc_user_counting_features_v0_starbuck", - timeoutInMillis = 100 - ) - } - - /** - * A repository of the offline aggregate feature metadata necessary to decode - * DenseCompactDataRecords. - * - * This repository is expected to virtually always pick up the metadata form the local cache with - * nearly 0 latency. - */ - @Provides - @Singleton - @Named(TimelineAggregateMetadataRepository) - def providesTimelineAggregateMetadataRepository( - @Named(ManhattanAthenaClient) client: mh.ManhattanCoordinator.MethodPerEndpoint - ): Repository[Int, Option[ddr.DenseFeatureMetadata]] = { - - val keyTransformer = Injection - .connect[Int, Array[Byte]] - .toByteBufferTransformer() - - val valueTransformer = new Transformer[ByteBuffer, ddr.DenseFeatureMetadata] { - private val compactProtocolFactory = new TCompactProtocol.Factory - - def to(buffer: ByteBuffer): Try[ddr.DenseFeatureMetadata] = Try { - val transport = transportFromByteBuffer(buffer) - ddr.DenseFeatureMetadata.decode(compactProtocolFactory.getProtocol(transport)) - } - - // Encoding intentionally not implemented as it is never used - def from(metadata: ddr.DenseFeatureMetadata): Try[ByteBuffer] = ??? - } - - val inProcessCache: Cache[Int, Cached[ddr.DenseFeatureMetadata]] = InProcessLruCacheFactory( - ttl = Duration.fromMinutes(20), - lruSize = 30 - ).apply(serializer = Transformer(_ => ???, _ => ???)) // Serialization is not necessary here. - - val keyValueRepository = new ManhattanKeyValueRepository( - client = client, - keyTransformer = keyTransformer, - valueTransformer = valueTransformer, - appId = "timelines_dense_aggregates_encoding_metadata", // Expected QPS is negligible. - dataset = "user_session_dense_feature_metadata", - timeoutInMillis = 100 - ) - - KeyValueRepository - .singular( - new CachingKeyValueRepository[Seq[Int], Int, ddr.DenseFeatureMetadata]( - keyValueRepository, - new NonLockingCache(inProcessCache), - keysAsQuery[Int] - ) - ) - } - - @Provides - @Singleton - @Named(RealGraphFeatureRepository) - def providesRealGraphFeatureRepository( - @Named(ManhattanAthenaClient) client: mh.ManhattanCoordinator.MethodPerEndpoint - ): Repository[Long, Option[uss_scala.UserSession]] = { - val valueTransformer = CompactScalaCodec(uss_scala.UserSession).toByteBufferTransformer().flip - - KeyValueRepository.singular( - new ManhattanKeyValueRepository( - client = client, - keyTransformer = LongKeyTransformer, - valueTransformer = valueTransformer, - appId = "real_graph", - dataset = "split_real_graph_features", - timeoutInMillis = 100, - ) - ) - } - - // cached manhattan repositories - - @Provides - @Singleton - @Named(AuthorFeatureRepository) - def providesAuthorFeatureRepository( - @Named(ManhattanAthenaClient) client: mh.ManhattanCoordinator.MethodPerEndpoint, - @Named(HomeAuthorFeaturesCacheClient) cacheClient: Memcache - ): KeyValueRepository[Seq[Long], Long, af.AuthorFeatures] = { - - val valueInjection = ThriftCodec - .toCompact[af.AuthorFeatures] - - val keyValueRepository = batchedManhattanKeyValueRepository( - client = client, - keyTransformer = LongKeyTransformer, - valueTransformer = valueInjection.toByteBufferTransformer().flip, - appId = "timelines_author_feature_store_athena", - dataset = "timelines_author_features", - timeoutInMillis = 100 - ) - - val remoteCacheRepo = buildMemCachedRepository( - keyValueRepository = keyValueRepository, - cacheClient = cacheClient, - cachePrefix = "AuthorFeatureHydrator", - ttl = 12.hours, - valueInjection = valueInjection) - - buildInProcessCachedRepository( - keyValueRepository = remoteCacheRepo, - ttl = 15.minutes, - size = 8000, - valueInjection = valueInjection - ) - } - - @Provides - @Singleton - @Named(TwhinAuthorFollowFeatureRepository) - def providesTwhinAuthorFollowFeatureRepository( - @Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint, - @Named(TwhinAuthorFollowFeatureCacheClient) cacheClient: Memcache - ): KeyValueRepository[Seq[Long], Long, ml.FloatTensor] = { - val keyValueRepository = - batchedManhattanKeyValueRepository( - client = client, - keyTransformer = FeatureStoreUserIdKeyTransformer, - valueTransformer = FloatTensorTransformer, - appId = "ml_features_apollo", - dataset = "twhin_author_follow_embedding_fsv1__v1_thrift__embedding", - timeoutInMillis = 100 - ) - - val valueInjection: Injection[ml.FloatTensor, Array[Byte]] = - BinaryScalaCodec(ml.FloatTensor) - - buildMemCachedRepository( - keyValueRepository = keyValueRepository, - cacheClient = cacheClient, - cachePrefix = "twhinAuthorFollows", - ttl = 24.hours, - valueInjection = valueInjection - ) - } - - @Provides - @Singleton - @Named(UserLanguagesRepository) - def providesUserLanguagesFeatureRepository( - @Named(ManhattanStarbuckClient) client: mh.ManhattanCoordinator.MethodPerEndpoint - ): KeyValueRepository[Seq[Long], Long, Seq[scc.ThriftLanguage]] = { - batchedManhattanKeyValueRepository( - client = client, - keyTransformer = LongKeyTransformer, - valueTransformer = LanguageTransformer, - appId = "user_metadata", - dataset = "languages", - timeoutInMillis = 70 - ) - } - - @Provides - @Singleton - @Named(TwhinUserFollowFeatureRepository) - def providesTwhinUserFollowFeatureRepository( - @Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint - ): KeyValueRepository[Seq[Long], Long, ml.FloatTensor] = { - batchedManhattanKeyValueRepository( - client = client, - keyTransformer = FeatureStoreUserIdKeyTransformer, - valueTransformer = FloatTensorTransformer, - appId = "ml_features_apollo", - dataset = "twhin_user_follow_embedding_fsv1__v1_thrift__embedding", - timeoutInMillis = 100 - ) - } - - @Provides - @Singleton - @Named(TimelineAggregatePartARepository) - def providesTimelineAggregatePartARepository( - @Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint, - ): Repository[Long, Option[uss.UserSession]] = - timelineAggregateRepository( - mhClient = client, - mhDataset = "timelines_aggregates_v2_features_by_user_part_a_apollo", - mhAppId = "timelines_aggregates_v2_features_by_user_part_a_apollo" - ) - - @Provides - @Singleton - @Named(TimelineAggregatePartBRepository) - def providesTimelineAggregatePartBRepository( - @Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint, - ): Repository[Long, Option[uss.UserSession]] = - timelineAggregateRepository( - mhClient = client, - mhDataset = "timelines_aggregates_v2_features_by_user_part_b_apollo", - mhAppId = "timelines_aggregates_v2_features_by_user_part_b_apollo" - ) - - @Provides - @Singleton - @Named(TwhinUserEngagementFeatureRepository) - def providesTwhinUserEngagementFeatureRepository( - @Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint - ): KeyValueRepository[Seq[Long], Long, ml.FloatTensor] = { - - batchedManhattanKeyValueRepository( - client = client, - keyTransformer = FeatureStoreUserIdKeyTransformer, - valueTransformer = FloatTensorTransformer, - appId = "ml_features_apollo", - dataset = "twhin_user_engagement_embedding_fsv1__v1_thrift__embedding", - timeoutInMillis = 100 - ) - } - - private def buildMemCachedRepository[K, V]( - keyValueRepository: KeyValueRepository[Seq[K], K, V], - cacheClient: Memcache, - cachePrefix: String, - ttl: Duration, - valueInjection: Injection[V, Array[Byte]] - ): CachingKeyValueRepository[Seq[K], K, V] = { - val cachedSerializer = CachedSerializer.binary( - valueInjection.toByteArrayTransformer() - ) - - val cache = MemcacheCacheFactory( - cacheClient, - ttl, - PrefixKeyTransformerFactory(cachePrefix) - )[K, Cached[V]](cachedSerializer) - - new CachingKeyValueRepository( - keyValueRepository, - new NonLockingCache(cache), - keysAsQuery[K] - ) - } - - private def buildInProcessCachedRepository[K, V]( - keyValueRepository: KeyValueRepository[Seq[K], K, V], - ttl: Duration, - size: Int, - valueInjection: Injection[V, Array[Byte]] - ): CachingKeyValueRepository[Seq[K], K, V] = { - val cachedSerializer = CachedSerializer.binary( - valueInjection.toByteArrayTransformer() - ) - - val cache = InProcessLruCacheFactory( - ttl = ttl, - lruSize = size - )[K, Cached[V]](cachedSerializer) - - new CachingKeyValueRepository( - keyValueRepository, - new NonLockingCache(cache), - keysAsQuery[K] - ) - } - - private def batchedManhattanKeyValueRepository[K, V]( - client: mh.ManhattanCoordinator.MethodPerEndpoint, - keyTransformer: Transformer[K, ByteBuffer], - valueTransformer: Transformer[ByteBuffer, V], - appId: String, - dataset: String, - timeoutInMillis: Int, - chunkSize: Int = DEFAULT_RPC_CHUNK_SIZE - ): KeyValueRepository[Seq[K], K, V] = - KeyValueRepository.chunked( - new ManhattanKeyValueRepository( - client = client, - keyTransformer = keyTransformer, - valueTransformer = valueTransformer, - appId = appId, - dataset = dataset, - timeoutInMillis = timeoutInMillis - ), - chunker = ChunkingStrategy.equalSize(chunkSize) - ) - - private def transportFromByteBuffer(buffer: ByteBuffer): TTransport = - new TMemoryInputTransport( - buffer.array(), - buffer.arrayOffset() + buffer.position(), - buffer.remaining()) - - private def timelineAggregateRepository( - mhClient: mh.ManhattanCoordinator.MethodPerEndpoint, - mhDataset: String, - mhAppId: String - ): Repository[Long, Option[uss.UserSession]] = { - val valueInjection = ThriftCodec - .toCompact[uss.UserSession] - - KeyValueRepository.singular( - new ManhattanKeyValueRepository( - client = mhClient, - keyTransformer = LongKeyTransformer, - valueTransformer = valueInjection.toByteBufferTransformer().flip, - appId = mhAppId, - dataset = mhDataset, - timeoutInMillis = 100 - ) - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanTweetImpressionStoreModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanTweetImpressionStoreModule.scala deleted file mode 100644 index c6782665a..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanTweetImpressionStoreModule.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.storage.client.manhattan.kv.Guarantee -import com.twitter.storehaus_internal.manhattan.ManhattanClusters -import com.twitter.timelines.clients.manhattan.mhv3.ManhattanClientBuilder -import com.twitter.timelines.impressionstore.store.ManhattanTweetImpressionStoreClientConfig -import com.twitter.timelines.impressionstore.store.ManhattanTweetImpressionStoreClient -import com.twitter.util.Duration -import javax.inject.Singleton - -object ManhattanTweetImpressionStoreModule extends TwitterModule { - - private val ProdAppId = "timelines_tweet_impression_store_v2" - private val ProdDataset = "timelines_tweet_impressions_v2" - private val StagingAppId = "timelines_tweet_impression_store_staging" - private val StagingDataset = "timelines_tweet_impressions_staging" - private val StatsScope = "manhattanTweetImpressionStoreClient" - private val DefaultTTL = 2.days - private final val Timeout = "mh_impression_store.timeout" - - flag[Duration](Timeout, 150.millis, "Timeout per request") - - @Provides - @Singleton - def providesManhattanTweetImpressionStoreClient( - @Flag(Timeout) timeout: Duration, - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): ManhattanTweetImpressionStoreClient = { - - val (appId, dataset) = serviceIdentifier.environment.toLowerCase match { - case "prod" => (ProdAppId, ProdDataset) - case _ => (StagingAppId, StagingDataset) - } - - val config = ManhattanTweetImpressionStoreClientConfig( - cluster = ManhattanClusters.nash, - appId = appId, - dataset = dataset, - statsScope = StatsScope, - defaultGuarantee = Guarantee.SoftDcReadMyWrites, - defaultMaxTimeout = timeout, - maxRetryCount = 2, - isReadOnly = false, - serviceIdentifier = serviceIdentifier, - ttl = DefaultTTL - ) - - val manhattanEndpoint = ManhattanClientBuilder.buildManhattanEndpoint(config, statsReceiver) - ManhattanTweetImpressionStoreClient(config, manhattanEndpoint, statsReceiver) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MemcachedFeatureRepositoryModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MemcachedFeatureRepositoryModule.scala deleted file mode 100644 index 8afafbfb7..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MemcachedFeatureRepositoryModule.scala +++ /dev/null @@ -1,117 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.Memcached -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.HomeAuthorFeaturesCacheClient -import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealTimeInteractionGraphUserVertexClient -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelinesRealTimeAggregateClient -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinAuthorFollowFeatureCacheClient -import com.twitter.inject.TwitterModule -import com.twitter.product_mixer.shared_library.memcached_client.MemcachedClientBuilder -import com.twitter.servo.cache.FinagleMemcacheFactory -import com.twitter.servo.cache.Memcache -import javax.inject.Named -import javax.inject.Singleton - -object MemcachedFeatureRepositoryModule extends TwitterModule { - - // This must match the respective parameter on the write path. Note that servo sets a different - // hasher by default. See [[com.twitter.hashing.KeyHasher]] for the list of other available - // hashers. - private val memcacheKeyHasher = "ketama" - - @Provides - @Singleton - @Named(TimelinesRealTimeAggregateClient) - def providesTimelinesRealTimeAggregateClient( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): Memcache = { - val rawClient = MemcachedClientBuilder.buildRawMemcachedClient( - numTries = 3, - numConnections = 1, - requestTimeout = 100.milliseconds, - globalTimeout = 300.milliseconds, - connectTimeout = 200.milliseconds, - acquisitionTimeout = 200.milliseconds, - serviceIdentifier = serviceIdentifier, - statsReceiver = statsReceiver - ) - - buildMemcacheClient(rawClient, "/s/cache/timelines_real_time_aggregates:twemcaches") - } - - @Provides - @Singleton - @Named(HomeAuthorFeaturesCacheClient) - def providesHomeAuthorFeaturesCacheClient( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): Memcache = { - val cacheClient = MemcachedClientBuilder.buildRawMemcachedClient( - numTries = 2, - numConnections = 1, - requestTimeout = 150.milliseconds, - globalTimeout = 300.milliseconds, - connectTimeout = 200.milliseconds, - acquisitionTimeout = 200.milliseconds, - serviceIdentifier = serviceIdentifier, - statsReceiver = statsReceiver - ) - - buildMemcacheClient(cacheClient, "/s/cache/timelines_author_features:twemcaches") - } - - @Provides - @Singleton - @Named(TwhinAuthorFollowFeatureCacheClient) - def providesTwhinAuthorFollowFeatureCacheClient( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): Memcache = { - val cacheClient = MemcachedClientBuilder.buildRawMemcachedClient( - numTries = 2, - numConnections = 1, - requestTimeout = 150.milliseconds, - globalTimeout = 300.milliseconds, - connectTimeout = 200.milliseconds, - acquisitionTimeout = 200.milliseconds, - serviceIdentifier = serviceIdentifier, - statsReceiver = statsReceiver - ) - - buildMemcacheClient(cacheClient, "/s/cache/home_twhin_author_features:twemcaches") - } - - @Provides - @Singleton - @Named(RealTimeInteractionGraphUserVertexClient) - def providesRealTimeInteractionGraphUserVertexClient( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): Memcache = { - val cacheClient = MemcachedClientBuilder.buildRawMemcachedClient( - numTries = 2, - numConnections = 1, - requestTimeout = 150.milliseconds, - globalTimeout = 300.milliseconds, - connectTimeout = 200.milliseconds, - acquisitionTimeout = 200.milliseconds, - serviceIdentifier = serviceIdentifier, - statsReceiver = statsReceiver - ) - - buildMemcacheClient(cacheClient, "/s/cache/realtime_interactive_graph_prod_v2:twemcaches") - } - - private def buildMemcacheClient(cacheClient: Memcached.Client, dest: String): Memcache = - FinagleMemcacheFactory( - client = cacheClient, - dest = dest, - hashName = memcacheKeyHasher - )() - -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/NaviModelClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/NaviModelClientModule.scala deleted file mode 100644 index 60d580a73..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/NaviModelClientModule.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.Http -import com.twitter.finagle.grpc.FinagleChannelBuilder -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.mtls.client.MtlsStackClient.MtlsStackClientSyntax -import com.twitter.inject.TwitterModule -import com.twitter.timelines.clients.predictionservice.PredictionGRPCService -import com.twitter.util.Duration -import io.grpc.ManagedChannel -import javax.inject.Singleton - -object NaviModelClientModule extends TwitterModule { - - @Singleton - @Provides - def providesPredictionGRPCService( - serviceIdentifier: ServiceIdentifier, - ): PredictionGRPCService = { - // Wily path to the ML Model service (e.g. /s/ml-serving/navi-explore-ranker). - val modelPath = "/s/ml-serving/navi_home_recap_onnx" - - val MaxPredictionTimeoutMs: Duration = 500.millis - val ConnectTimeoutMs: Duration = 200.millis - val AcquisitionTimeoutMs: Duration = 500.millis - val MaxRetryAttempts: Int = 2 - - val client = Http.client - .withLabel(modelPath) - .withMutualTls(serviceIdentifier) - .withRequestTimeout(MaxPredictionTimeoutMs) - .withTransport.connectTimeout(ConnectTimeoutMs) - .withSession.acquisitionTimeout(AcquisitionTimeoutMs) - .withHttpStats - - val channel: ManagedChannel = FinagleChannelBuilder - .forTarget(modelPath) - .overrideAuthority("rustserving") - .maxRetryAttempts(MaxRetryAttempts) - .enableRetryForStatus(io.grpc.Status.RESOURCE_EXHAUSTED) - .enableRetryForStatus(io.grpc.Status.UNKNOWN) - .enableUnsafeFullyBufferingMode() - .httpClient(client) - .build() - - new PredictionGRPCService(channel) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/OptimizedStratoClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/OptimizedStratoClientModule.scala deleted file mode 100644 index b9e315acc..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/OptimizedStratoClientModule.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.service.Retries -import com.twitter.finagle.service.RetryPolicy -import com.twitter.finagle.ssl.OpportunisticTls -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithModerateTimeout -import com.twitter.inject.TwitterModule -import com.twitter.strato.client.Client -import com.twitter.strato.client.Strato -import com.twitter.util.Try -import javax.inject.Named -import javax.inject.Singleton - -object OptimizedStratoClientModule extends TwitterModule { - - private val ModerateStratoServerClientRequestTimeout = 500.millis - - private val DefaultRetryPartialFunction: PartialFunction[Try[Nothing], Boolean] = - RetryPolicy.TimeoutAndWriteExceptionsOnly - .orElse(RetryPolicy.ChannelClosedExceptionsOnly) - - protected def mkRetryPolicy(tries: Int): RetryPolicy[Try[Nothing]] = - RetryPolicy.tries(tries, DefaultRetryPartialFunction) - - @Singleton - @Provides - @Named(BatchedStratoClientWithModerateTimeout) - def providesStratoClient( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): Client = { - Strato.client - .withMutualTls(serviceIdentifier, opportunisticLevel = OpportunisticTls.Required) - .withSession.acquisitionTimeout(500.milliseconds) - .withRequestTimeout(ModerateStratoServerClientRequestTimeout) - .withPerRequestTimeout(ModerateStratoServerClientRequestTimeout) - .withRpcBatchSize(5) - .configured(Retries.Policy(mkRetryPolicy(1))) - .withStatsReceiver(statsReceiver.scope("strato_client")) - .build() - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PeopleDiscoveryServiceModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PeopleDiscoveryServiceModule.scala deleted file mode 100644 index 47353afa7..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PeopleDiscoveryServiceModule.scala +++ /dev/null @@ -1,35 +0,0 @@ -package com.twitter.home_mixer.module - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.thriftmux.MethodBuilder -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.inject.Injector -import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule -import com.twitter.peoplediscovery.api.thriftscala.ThriftPeopleDiscoveryService -import com.twitter.util.Duration - -/** - * Copy of com.twitter.product_mixer.component_library.module.PeopleDiscoveryServiceModule - */ -object PeopleDiscoveryServiceModule - extends ThriftMethodBuilderClientModule[ - ThriftPeopleDiscoveryService.ServicePerEndpoint, - ThriftPeopleDiscoveryService.MethodPerEndpoint - ] - with MtlsClient { - - override val label: String = "people-discovery-api" - - override val dest: String = "/s/people-discovery-api/people-discovery-api:thrift" - - override protected def configureMethodBuilder( - injector: Injector, - methodBuilder: MethodBuilder - ): MethodBuilder = { - methodBuilder - .withTimeoutPerRequest(350.millis) - .withTimeoutTotal(350.millis) - } - - override protected def sessionAcquisitionTimeout: Duration = 500.milliseconds -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PipelineFailureExceptionMapper.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PipelineFailureExceptionMapper.scala deleted file mode 100644 index ad15988cb..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PipelineFailureExceptionMapper.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.home_mixer.module - -import com.twitter.finatra.thrift.exceptions.ExceptionMapper -import com.twitter.home_mixer.{thriftscala => t} -import com.twitter.util.logging.Logging -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.pipeline_failure.ProductDisabled -import com.twitter.scrooge.ThriftException -import com.twitter.util.Future -import javax.inject.Singleton - -@Singleton -class PipelineFailureExceptionMapper - extends ExceptionMapper[PipelineFailure, ThriftException] - with Logging { - - def handleException(throwable: PipelineFailure): Future[ThriftException] = { - throwable match { - // SliceService (unlike UrtService) throws an exception when the requested product is disabled - case PipelineFailure(ProductDisabled, reason, _, _) => - Future.exception( - t.ValidationExceptionList(errors = - Seq(t.ValidationException(t.ValidationErrorCode.ProductDisabled, reason)))) - case _ => - error("Unhandled PipelineFailure", throwable) - Future.exception(throwable) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealGraphInNetworkScoresModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealGraphInNetworkScoresModule.scala deleted file mode 100644 index 7dc6a072d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealGraphInNetworkScoresModule.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.google.inject.name.Named -import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealGraphInNetworkScores -import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealGraphManhattanEndpoint -import com.twitter.home_mixer.store.RealGraphInNetworkScoresStore -import com.twitter.inject.TwitterModule -import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpoint -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.util.CommonTypes.ViewerId -import com.twitter.wtf.candidate.thriftscala.Candidate - -import javax.inject.Singleton - -object RealGraphInNetworkScoresModule extends TwitterModule { - - @Provides - @Singleton - @Named(RealGraphInNetworkScores) - def providesRealGraphInNetworkScoresFeaturesStore( - @Named(RealGraphManhattanEndpoint) realGraphInNetworkScoresManhattanKVEndpoint: ManhattanKVEndpoint - ): ReadableStore[ViewerId, Seq[Candidate]] = { - new RealGraphInNetworkScoresStore(realGraphInNetworkScoresManhattanKVEndpoint) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealtimeAggregateFeatureRepositoryModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealtimeAggregateFeatureRepositoryModule.scala deleted file mode 100644 index c3c545819..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealtimeAggregateFeatureRepositoryModule.scala +++ /dev/null @@ -1,231 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.google.inject.name.Named -import com.twitter.bijection.Injection -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.bijection.thrift.ThriftCodec -import com.twitter.home_mixer.param.HomeMixerInjectionNames.EngagementsReceivedByAuthorCache -import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealTimeInteractionGraphUserVertexCache -import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealTimeInteractionGraphUserVertexClient -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelinesRealTimeAggregateClient -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TopicCountryEngagementCache -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TopicEngagementCache -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetCountryEngagementCache -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetEngagementCache -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwitterListEngagementCache -import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserAuthorEngagementCache -import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserEngagementCache -import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserTopicEngagementForNewUserCache -import com.twitter.home_mixer.util.InjectionTransformerImplicits._ -import com.twitter.inject.TwitterModule -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.Feature -import com.twitter.ml.{api => ml} -import com.twitter.servo.cache.KeyValueTransformingReadCache -import com.twitter.servo.cache.Memcache -import com.twitter.servo.cache.ReadCache -import com.twitter.servo.util.Transformer -import com.twitter.storehaus_internal.memcache.MemcacheHelper -import com.twitter.summingbird.batch.Batcher -import com.twitter.summingbird_internal.bijection.BatchPairImplicits -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregationKey -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregationKeyInjection -import com.twitter.wtf.real_time_interaction_graph.{thriftscala => ig} - -import javax.inject.Singleton - -object RealtimeAggregateFeatureRepositoryModule - extends TwitterModule - with RealtimeAggregateHelpers { - - private val authorIdFeature = new Feature.Discrete("entities.source_author_id").getFeatureId - private val countryCodeFeature = new Feature.Text("geo.user_location.country_code").getFeatureId - private val listIdFeature = new Feature.Discrete("list.id").getFeatureId - private val userIdFeature = new Feature.Discrete("meta.user_id").getFeatureId - private val topicIdFeature = new Feature.Discrete("entities.topic_id").getFeatureId - private val tweetIdFeature = new Feature.Discrete("entities.source_tweet_id").getFeatureId - - @Provides - @Singleton - @Named(UserTopicEngagementForNewUserCache) - def providesUserTopicEngagementForNewUserCache( - @Named(TimelinesRealTimeAggregateClient) client: Memcache - ): ReadCache[(Long, Long), ml.DataRecord] = { - new KeyValueTransformingReadCache( - client, - dataRecordValueTransformer, - keyTransformD2(userIdFeature, topicIdFeature) - ) - } - - @Provides - @Singleton - @Named(TwitterListEngagementCache) - def providesTwitterListEngagementCache( - @Named(TimelinesRealTimeAggregateClient) client: Memcache - ): ReadCache[Long, ml.DataRecord] = { - new KeyValueTransformingReadCache( - client, - dataRecordValueTransformer, - keyTransformD1(listIdFeature) - ) - } - - @Provides - @Singleton - @Named(TopicEngagementCache) - def providesTopicEngagementCache( - @Named(TimelinesRealTimeAggregateClient) client: Memcache - ): ReadCache[Long, ml.DataRecord] = { - new KeyValueTransformingReadCache( - client, - dataRecordValueTransformer, - keyTransformD1(topicIdFeature) - ) - } - - @Provides - @Singleton - @Named(UserAuthorEngagementCache) - def providesUserAuthorEngagementCache( - @Named(TimelinesRealTimeAggregateClient) client: Memcache - ): ReadCache[(Long, Long), ml.DataRecord] = { - new KeyValueTransformingReadCache( - client, - dataRecordValueTransformer, - keyTransformD2(userIdFeature, authorIdFeature) - ) - } - - @Provides - @Singleton - @Named(UserEngagementCache) - def providesUserEngagementCache( - @Named(TimelinesRealTimeAggregateClient) client: Memcache - ): ReadCache[Long, ml.DataRecord] = { - new KeyValueTransformingReadCache( - client, - dataRecordValueTransformer, - keyTransformD1(userIdFeature) - ) - } - - @Provides - @Singleton - @Named(TweetCountryEngagementCache) - def providesTweetCountryEngagementCache( - @Named(TimelinesRealTimeAggregateClient) client: Memcache - ): ReadCache[(Long, String), ml.DataRecord] = { - - new KeyValueTransformingReadCache( - client, - dataRecordValueTransformer, - keyTransformD1T1(tweetIdFeature, countryCodeFeature) - ) - } - - @Provides - @Singleton - @Named(TweetEngagementCache) - def providesTweetEngagementCache( - @Named(TimelinesRealTimeAggregateClient) client: Memcache - ): ReadCache[Long, ml.DataRecord] = { - new KeyValueTransformingReadCache( - client, - dataRecordValueTransformer, - keyTransformD1(tweetIdFeature) - ) - } - - @Provides - @Singleton - @Named(EngagementsReceivedByAuthorCache) - def providesEngagementsReceivedByAuthorCache( - @Named(TimelinesRealTimeAggregateClient) client: Memcache - ): ReadCache[Long, ml.DataRecord] = { - new KeyValueTransformingReadCache( - client, - dataRecordValueTransformer, - keyTransformD1(authorIdFeature) - ) - } - - @Provides - @Singleton - @Named(TopicCountryEngagementCache) - def providesTopicCountryEngagementCache( - @Named(TimelinesRealTimeAggregateClient) client: Memcache - ): ReadCache[(Long, String), ml.DataRecord] = { - new KeyValueTransformingReadCache( - client, - dataRecordValueTransformer, - keyTransformD1T1(topicIdFeature, countryCodeFeature) - ) - } - - @Provides - @Singleton - @Named(RealTimeInteractionGraphUserVertexCache) - def providesRealTimeInteractionGraphUserVertexCache( - @Named(RealTimeInteractionGraphUserVertexClient) client: Memcache - ): ReadCache[Long, ig.UserVertex] = { - - val valueTransformer = BinaryScalaCodec(ig.UserVertex).toByteArrayTransformer() - - val underlyingKey: Long => String = { - val cacheKeyPrefix = "user_vertex" - val defaultBatchID = Batcher.unit.currentBatch - val batchPairInjection = BatchPairImplicits.keyInjection(Injection.connect[Long, Array[Byte]]) - MemcacheHelper - .keyEncoder(cacheKeyPrefix)(batchPairInjection) - .compose((k: Long) => (k, defaultBatchID)) - } - - new KeyValueTransformingReadCache( - client, - valueTransformer, - underlyingKey - ) - } -} - -trait RealtimeAggregateHelpers { - - private def customKeyBuilder[K](prefix: String, f: K => Array[Byte]): K => String = { - // intentionally not implementing injection inverse because it is never used - def g(arr: Array[Byte]) = ??? - - MemcacheHelper.keyEncoder(prefix)(Injection.build(f)(g)) - } - - private val keyEncoder: AggregationKey => String = { - val cacheKeyPrefix = "" - val defaultBatchID = Batcher.unit.currentBatch - - val batchPairInjection = BatchPairImplicits.keyInjection(AggregationKeyInjection) - customKeyBuilder(cacheKeyPrefix, batchPairInjection) - .compose((k: AggregationKey) => (k, defaultBatchID)) - } - - protected def keyTransformD1(f1: Long)(key: Long): String = { - val aggregationKey = AggregationKey(Map(f1 -> key), Map.empty) - keyEncoder(aggregationKey) - } - - protected def keyTransformD2(f1: Long, f2: Long)(keys: (Long, Long)): String = { - val (k1, k2) = keys - val aggregationKey = AggregationKey(Map(f1 -> k1, f2 -> k2), Map.empty) - keyEncoder(aggregationKey) - } - - protected def keyTransformD1T1(f1: Long, f2: Long)(keys: (Long, String)): String = { - val (k1, k2) = keys - val aggregationKey = AggregationKey(Map(f1 -> k1), Map(f2 -> k2)) - keyEncoder(aggregationKey) - } - - protected val dataRecordValueTransformer: Transformer[DataRecord, Array[Byte]] = ThriftCodec - .toCompact[ml.DataRecord] - .toByteArrayTransformer() -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScoredTweetsMemcacheModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScoredTweetsMemcacheModule.scala deleted file mode 100644 index 4bed31c5c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScoredTweetsMemcacheModule.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.{thriftscala => t} -import com.twitter.inject.TwitterModule -import com.twitter.product_mixer.shared_library.memcached_client.MemcachedClientBuilder -import com.twitter.servo.cache.FinagleMemcache -import com.twitter.servo.cache.KeyTransformer -import com.twitter.servo.cache.KeyValueTransformingTtlCache -import com.twitter.servo.cache.Serializer -import com.twitter.servo.cache.ThriftSerializer -import com.twitter.servo.cache.TtlCache -import com.twitter.timelines.model.UserId -import org.apache.thrift.protocol.TCompactProtocol - -import javax.inject.Singleton - -object ScoredTweetsMemcacheModule extends TwitterModule { - - private val ScopeName = "ScoredTweetsCache" - private val ProdDestName = "/srv#/prod/local/cache/home_scored_tweets:twemcaches" - private val StagingDestName = "/srv#/test/local/cache/twemcache_home_scored_tweets:twemcaches" - private val scoredTweetsSerializer: Serializer[t.ScoredTweetsResponse] = - new ThriftSerializer[t.ScoredTweetsResponse]( - t.ScoredTweetsResponse, - new TCompactProtocol.Factory()) - private val userIdKeyTransformer: KeyTransformer[UserId] = (userId: UserId) => userId.toString - - @Singleton - @Provides - def providesScoredTweetsCache( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): TtlCache[UserId, t.ScoredTweetsResponse] = { - val destName = serviceIdentifier.environment.toLowerCase match { - case "prod" => ProdDestName - case _ => StagingDestName - } - val client = MemcachedClientBuilder.buildMemcachedClient( - destName = destName, - numTries = 2, - numConnections = 1, - requestTimeout = 200.milliseconds, - globalTimeout = 400.milliseconds, - connectTimeout = 100.milliseconds, - acquisitionTimeout = 100.milliseconds, - serviceIdentifier = serviceIdentifier, - statsReceiver = statsReceiver.scope(ScopeName) - ) - val underlyingCache = new FinagleMemcache(client) - - new KeyValueTransformingTtlCache( - underlyingCache = underlyingCache, - transformer = scoredTweetsSerializer, - underlyingKey = userIdKeyTransformer - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScribeEventPublisherModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScribeEventPublisherModule.scala deleted file mode 100644 index 99bf61630..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScribeEventPublisherModule.scala +++ /dev/null @@ -1,77 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.clientapp.{thriftscala => ca} -import com.twitter.home_mixer.param.HomeMixerInjectionNames.CandidateFeaturesScribeEventPublisher -import com.twitter.home_mixer.param.HomeMixerInjectionNames.CommonFeaturesScribeEventPublisher -import com.twitter.home_mixer.param.HomeMixerInjectionNames.MinimumFeaturesScribeEventPublisher -import com.twitter.inject.TwitterModule -import com.twitter.logpipeline.client.EventPublisherManager -import com.twitter.logpipeline.client.common.EventPublisher -import com.twitter.logpipeline.client.serializers.EventLogMsgTBinarySerializer -import com.twitter.logpipeline.client.serializers.EventLogMsgThriftStructSerializer -import com.twitter.timelines.suggests.common.poly_data_record.{thriftjava => pldr} -import com.twitter.timelines.timeline_logging.{thriftscala => tl} -import javax.inject.Named -import javax.inject.Singleton - -object ScribeEventPublisherModule extends TwitterModule { - - val ClientEventLogCategory = "client_event" - val ServedCandidatesLogCategory = "home_timeline_served_candidates_flattened" - val ScoredCandidatesLogCategory = "home_timeline_scored_candidates" - val ServedCommonFeaturesLogCategory = "tq_served_common_features_offline" - val ServedCandidateFeaturesLogCategory = "tq_served_candidate_features_offline" - val ServedMinimumFeaturesLogCategory = "tq_served_minimum_features_offline" - - @Provides - @Singleton - def providesClientEventsScribeEventPublisher: EventPublisher[ca.LogEvent] = { - val serializer = EventLogMsgThriftStructSerializer.getNewSerializer[ca.LogEvent]() - EventPublisherManager.buildScribeLogPipelinePublisher(ClientEventLogCategory, serializer) - } - - @Provides - @Singleton - @Named(CommonFeaturesScribeEventPublisher) - def providesCommonFeaturesScribeEventPublisher: EventPublisher[pldr.PolyDataRecord] = { - val serializer = EventLogMsgTBinarySerializer.getNewSerializer - EventPublisherManager.buildScribeLogPipelinePublisher( - ServedCommonFeaturesLogCategory, - serializer) - } - - @Provides - @Singleton - @Named(CandidateFeaturesScribeEventPublisher) - def providesCandidateFeaturesScribeEventPublisher: EventPublisher[pldr.PolyDataRecord] = { - val serializer = EventLogMsgTBinarySerializer.getNewSerializer - EventPublisherManager.buildScribeLogPipelinePublisher( - ServedCandidateFeaturesLogCategory, - serializer) - } - - @Provides - @Singleton - @Named(MinimumFeaturesScribeEventPublisher) - def providesMinimumFeaturesScribeEventPublisher: EventPublisher[pldr.PolyDataRecord] = { - val serializer = EventLogMsgTBinarySerializer.getNewSerializer - EventPublisherManager.buildScribeLogPipelinePublisher( - ServedMinimumFeaturesLogCategory, - serializer) - } - - @Provides - @Singleton - def providesServedCandidatesScribeEventPublisher: EventPublisher[tl.ServedEntry] = { - val serializer = EventLogMsgThriftStructSerializer.getNewSerializer[tl.ServedEntry]() - EventPublisherManager.buildScribeLogPipelinePublisher(ServedCandidatesLogCategory, serializer) - } - - @Provides - @Singleton - def provideScoredCandidatesScribeEventPublisher: EventPublisher[tl.ScoredCandidate] = { - val serializer = EventLogMsgThriftStructSerializer.getNewSerializer[tl.ScoredCandidate]() - EventPublisherManager.buildScribeLogPipelinePublisher(ScoredCandidatesLogCategory, serializer) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/SimClustersRecentEngagementsClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/SimClustersRecentEngagementsClientModule.scala deleted file mode 100644 index 7e819d51e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/SimClustersRecentEngagementsClientModule.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithModerateTimeout -import com.twitter.inject.TwitterModule -import com.twitter.strato.client.Client -import com.twitter.timelines.clients.strato.twistly.SimClustersRecentEngagementSimilarityClient -import com.twitter.timelines.clients.strato.twistly.SimClustersRecentEngagementSimilarityClientImpl -import javax.inject.Named -import javax.inject.Singleton - -object SimClustersRecentEngagementsClientModule extends TwitterModule { - @Singleton - @Provides - def providesSimilarityClient( - @Named(BatchedStratoClientWithModerateTimeout) - stratoClient: Client, - statsReceiver: StatsReceiver - ): SimClustersRecentEngagementSimilarityClient = { - new SimClustersRecentEngagementSimilarityClientImpl(stratoClient, statsReceiver) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/StaleTweetsCacheModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/StaleTweetsCacheModule.scala deleted file mode 100644 index 301dc51c2..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/StaleTweetsCacheModule.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.google.inject.name.Named -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.memcached.{Client => MemcachedClient} -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.hashing.KeyHasher -import com.twitter.home_mixer.param.HomeMixerInjectionNames.StaleTweetsCache -import com.twitter.inject.TwitterModule -import com.twitter.product_mixer.shared_library.memcached_client.MemcachedClientBuilder -import javax.inject.Singleton - -object StaleTweetsCacheModule extends TwitterModule { - - @Singleton - @Provides - @Named(StaleTweetsCache) - def providesCache( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): MemcachedClient = { - MemcachedClientBuilder.buildMemcachedClient( - destName = "/srv#/prod/local/cache/staletweetscache:twemcaches", - numTries = 3, - numConnections = 1, - requestTimeout = 200.milliseconds, - globalTimeout = 500.milliseconds, - connectTimeout = 200.milliseconds, - acquisitionTimeout = 200.milliseconds, - serviceIdentifier = serviceIdentifier, - statsReceiver = statsReceiver, - failureAccrualPolicy = None, - keyHasher = Some(KeyHasher.FNV1_32) - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ThriftFeatureRepositoryModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ThriftFeatureRepositoryModule.scala deleted file mode 100644 index dcbf62451..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ThriftFeatureRepositoryModule.scala +++ /dev/null @@ -1,375 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.conversions.PercentOps._ -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.graph_feature_service.{thriftscala => gfs} -import com.twitter.home_mixer.param.HomeMixerInjectionNames.EarlybirdRepository -import com.twitter.home_mixer.param.HomeMixerInjectionNames.GraphTwoHopRepository -import com.twitter.home_mixer.param.HomeMixerInjectionNames.InterestsThriftServiceClient -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetypieContentRepository -import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserFollowedTopicIdsRepository -import com.twitter.home_mixer.param.HomeMixerInjectionNames.UtegSocialProofRepository -import com.twitter.home_mixer.util.earlybird.EarlybirdRequestUtil -import com.twitter.home_mixer.util.tweetypie.RequestFields -import com.twitter.inject.TwitterModule -import com.twitter.interests.{thriftscala => int} -import com.twitter.product_mixer.shared_library.memcached_client.MemcachedClientBuilder -import com.twitter.product_mixer.shared_library.thrift_client.FinagleThriftClientBuilder -import com.twitter.product_mixer.shared_library.thrift_client.Idempotent -import com.twitter.recos.recos_common.{thriftscala => rc} -import com.twitter.recos.user_tweet_entity_graph.{thriftscala => uteg} -import com.twitter.search.earlybird.{thriftscala => eb} -import com.twitter.servo.cache.Cached -import com.twitter.servo.cache.CachedSerializer -import com.twitter.servo.cache.FinagleMemcacheFactory -import com.twitter.servo.cache.MemcacheCacheFactory -import com.twitter.servo.cache.NonLockingCache -import com.twitter.servo.cache.ThriftSerializer -import com.twitter.servo.keyvalue.KeyValueResultBuilder -import com.twitter.servo.repository.CachingKeyValueRepository -import com.twitter.servo.repository.ChunkingStrategy -import com.twitter.servo.repository.KeyValueRepository -import com.twitter.servo.repository.KeyValueResult -import com.twitter.servo.repository.keysAsQuery -import com.twitter.spam.rtf.{thriftscala => sp} -import com.twitter.tweetypie.{thriftscala => tp} -import com.twitter.util.Future -import com.twitter.util.Return -import javax.inject.Named -import javax.inject.Singleton -import org.apache.thrift.protocol.TCompactProtocol - -object ThriftFeatureRepositoryModule extends TwitterModule { - - private val DefaultRPCChunkSize = 50 - private val GFSInteractionIdsLimit = 10 - - type EarlybirdQuery = (Seq[Long], Long) - type UtegQuery = (Seq[Long], (Long, Map[Long, Double])) - - @Provides - @Singleton - @Named(InterestsThriftServiceClient) - def providesInterestsThriftServiceClient( - clientId: ClientId, - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): int.InterestsThriftService.MethodPerEndpoint = { - FinagleThriftClientBuilder - .buildFinagleMethodPerEndpoint[ - int.InterestsThriftService.ServicePerEndpoint, - int.InterestsThriftService.MethodPerEndpoint]( - serviceIdentifier = serviceIdentifier, - clientId = clientId, - dest = "/s/interests-thrift-service/interests-thrift-service", - label = "interests", - statsReceiver = statsReceiver, - idempotency = Idempotent(1.percent), - timeoutPerRequest = 350.milliseconds, - timeoutTotal = 350.milliseconds - ) - } - - @Provides - @Singleton - @Named(UserFollowedTopicIdsRepository) - def providesUserFollowedTopicIdsRepository( - @Named(InterestsThriftServiceClient) client: int.InterestsThriftService.MethodPerEndpoint - ): KeyValueRepository[Seq[Long], Long, Seq[Long]] = { - - val lookupContext = Some( - int.ExplicitInterestLookupContext(Some(Seq(int.InterestRelationType.Followed))) - ) - - def lookup(userId: Long): Future[Seq[Long]] = { - client.getUserExplicitInterests(userId, lookupContext).map { interests => - interests.flatMap { - _.interestId match { - case int.InterestId.SemanticCore(semanticCoreInterest) => Some(semanticCoreInterest.id) - case _ => None - } - } - } - } - - val keyValueRepository = toRepository(lookup) - - keyValueRepository - } - - @Provides - @Singleton - @Named(UtegSocialProofRepository) - def providesUtegSocialProofRepository( - clientId: ClientId, - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): KeyValueRepository[UtegQuery, Long, uteg.TweetRecommendation] = { - val client = FinagleThriftClientBuilder.buildFinagleMethodPerEndpoint[ - uteg.UserTweetEntityGraph.ServicePerEndpoint, - uteg.UserTweetEntityGraph.MethodPerEndpoint]( - serviceIdentifier = serviceIdentifier, - clientId = clientId, - dest = "/s/cassowary/user_tweet_entity_graph", - label = "uteg-social-proof-repo", - statsReceiver = statsReceiver, - idempotency = Idempotent(1.percent), - timeoutPerRequest = 150.milliseconds, - timeoutTotal = 250.milliseconds - ) - - val utegSocialProofTypes = Seq( - rc.SocialProofType.Favorite, - rc.SocialProofType.Retweet, - rc.SocialProofType.Reply - ) - - def lookup( - tweetIds: Seq[Long], - view: (Long, Map[Long, Double]) - ): Future[Seq[Option[uteg.TweetRecommendation]]] = { - val (userId, seedsWithWeights) = view - val socialProofRequest = uteg.SocialProofRequest( - requesterId = Some(userId), - seedsWithWeights = seedsWithWeights, - inputTweets = tweetIds, - socialProofTypes = Some(utegSocialProofTypes) - ) - client.findTweetSocialProofs(socialProofRequest).map { result => - val resultMap = result.socialProofResults.map(t => t.tweetId -> t).toMap - tweetIds.map(resultMap.get) - } - } - - toRepositoryBatchWithView(lookup, chunkSize = 200) - } - - @Provides - @Singleton - @Named(TweetypieContentRepository) - def providesTweetypieContentRepository( - clientId: ClientId, - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): KeyValueRepository[Seq[Long], Long, tp.Tweet] = { - val client = FinagleThriftClientBuilder - .buildFinagleMethodPerEndpoint[ - tp.TweetService.ServicePerEndpoint, - tp.TweetService.MethodPerEndpoint]( - serviceIdentifier = serviceIdentifier, - clientId = clientId, - dest = "/s/tweetypie/tweetypie", - label = "tweetypie-content-repo", - statsReceiver = statsReceiver, - idempotency = Idempotent(1.percent), - timeoutPerRequest = 300.milliseconds, - timeoutTotal = 500.milliseconds - ) - - def lookup(tweetIds: Seq[Long]): Future[Seq[Option[tp.Tweet]]] = { - val getTweetFieldsOptions = tp.GetTweetFieldsOptions( - tweetIncludes = RequestFields.ContentFields, - includeRetweetedTweet = false, - includeQuotedTweet = false, - forUserId = None, - safetyLevel = Some(sp.SafetyLevel.FilterNone), - visibilityPolicy = tp.TweetVisibilityPolicy.NoFiltering - ) - - val request = tp.GetTweetFieldsRequest(tweetIds = tweetIds, options = getTweetFieldsOptions) - - client.getTweetFields(request).map { results => - results.map { - case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), _, _) => - Some(found.tweet) - case _ => None - } - } - } - - val keyValueRepository = toRepositoryBatch(lookup, chunkSize = 20) - - val cacheClient = MemcachedClientBuilder.buildRawMemcachedClient( - numTries = 1, - numConnections = 1, - requestTimeout = 200.milliseconds, - globalTimeout = 200.milliseconds, - connectTimeout = 200.milliseconds, - acquisitionTimeout = 200.milliseconds, - serviceIdentifier = serviceIdentifier, - statsReceiver = statsReceiver - ) - - val finagleMemcacheFactory = - FinagleMemcacheFactory(cacheClient, "/s/cache/home_content_features:twemcaches") - val cacheValueTransformer = - new ThriftSerializer[tp.Tweet](tp.Tweet, new TCompactProtocol.Factory()) - val cachedSerializer = CachedSerializer.binary(cacheValueTransformer) - - val cache = MemcacheCacheFactory( - memcache = finagleMemcacheFactory(), - ttl = 48.hours - )[Long, Cached[tp.Tweet]](cachedSerializer) - - val lockingCache = new NonLockingCache(cache) - val cachedKeyValueRepository = new CachingKeyValueRepository( - keyValueRepository, - lockingCache, - keysAsQuery[Long] - ) - cachedKeyValueRepository - } - - @Provides - @Singleton - @Named(GraphTwoHopRepository) - def providesGraphTwoHopRepository( - clientId: ClientId, - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): KeyValueRepository[(Seq[Long], Long), Long, Seq[gfs.IntersectionValue]] = { - val client = FinagleThriftClientBuilder - .buildFinagleMethodPerEndpoint[gfs.Server.ServicePerEndpoint, gfs.Server.MethodPerEndpoint]( - serviceIdentifier = serviceIdentifier, - clientId = clientId, - dest = "/s/cassowary/graph_feature_service-server", - label = "gfs-repo", - statsReceiver = statsReceiver, - idempotency = Idempotent(1.percent), - timeoutPerRequest = 350.milliseconds, - timeoutTotal = 500.milliseconds - ) - - def lookup( - userIds: Seq[Long], - viewerId: Long - ): Future[Seq[Option[Seq[gfs.IntersectionValue]]]] = { - val gfsIntersectionRequest = gfs.GfsPresetIntersectionRequest( - userId = viewerId, - candidateUserIds = userIds, - presetFeatureTypes = gfs.PresetFeatureTypes.HtlTwoHop, - intersectionIdLimit = Some(GFSInteractionIdsLimit) - ) - - client - .getPresetIntersection(gfsIntersectionRequest) - .map { graphFeatureServiceResponse => - val resultMap = graphFeatureServiceResponse.results - .map(result => result.candidateUserId -> result.intersectionValues).toMap - userIds.map(resultMap.get(_)) - } - } - - toRepositoryBatchWithView(lookup, chunkSize = 200) - } - - @Provides - @Singleton - @Named(EarlybirdRepository) - def providesEarlybirdSearchRepository( - client: eb.EarlybirdService.MethodPerEndpoint, - clientId: ClientId - ): KeyValueRepository[EarlybirdQuery, Long, eb.ThriftSearchResult] = { - - def lookup( - tweetIds: Seq[Long], - viewerId: Long - ): Future[Seq[Option[eb.ThriftSearchResult]]] = { - val request = EarlybirdRequestUtil.getTweetsFeaturesRequest( - userId = Some(viewerId), - tweetIds = Some(tweetIds), - clientId = Some(clientId.name), - authorScoreMap = None, - tensorflowModel = Some("timelines_rectweet_replica") - ) - - client - .search(request).map { response => - val resultMap = response.searchResults - .map(_.results.map { result => result.id -> result }.toMap).getOrElse(Map.empty) - tweetIds.map(resultMap.get) - } - } - toRepositoryBatchWithView(lookup) - } - - protected def toRepository[K, V]( - hydrate: K => Future[V] - ): KeyValueRepository[Seq[K], K, V] = { - def asRepository(keys: Seq[K]): Future[KeyValueResult[K, V]] = { - Future.collect(keys.map(hydrate(_).liftToTry)).map { results => - keys - .zip(results) - .foldLeft(new KeyValueResultBuilder[K, V]()) { - case (bldr, (k, result)) => - result match { - case Return(v) => bldr.addFound(k, v) - case _ => bldr.addNotFound(k) - } - }.result - } - } - - asRepository - } - - protected def toRepositoryBatch[K, V]( - hydrate: Seq[K] => Future[Seq[Option[V]]], - chunkSize: Int = DefaultRPCChunkSize - ): KeyValueRepository[Seq[K], K, V] = { - def repository(keys: Seq[K]): Future[KeyValueResult[K, V]] = - batchRepositoryProcess(keys, hydrate(keys)) - - KeyValueRepository.chunked(repository, ChunkingStrategy.equalSize(chunkSize)) - } - - protected def toRepositoryBatchWithView[K, T, V]( - hydrate: (Seq[K], T) => Future[Seq[Option[V]]], - chunkSize: Int = DefaultRPCChunkSize - ): KeyValueRepository[(Seq[K], T), K, V] = { - def repository(input: (Seq[K], T)): Future[KeyValueResult[K, V]] = { - val (keys, view) = input - batchRepositoryProcess(keys, hydrate(keys, view)) - } - - KeyValueRepository.chunked(repository, CustomChunkingStrategy.equalSizeWithView(chunkSize)) - } - - private def batchRepositoryProcess[K, V]( - keys: Seq[K], - f: Future[Seq[Option[V]]] - ): Future[KeyValueResult[K, V]] = { - f.liftToTry - .map { - case Return(values) => - keys - .zip(values) - .foldLeft(new KeyValueResultBuilder[K, V]()) { - case (bldr, (k, value)) => - value match { - case Some(v) => bldr.addFound(k, v) - case _ => bldr.addNotFound(k) - } - }.result - case _ => - keys - .foldLeft(new KeyValueResultBuilder[K, V]()) { - case (bldr, k) => bldr.addNotFound(k) - }.result - } - } - - // Use only for cases not already covered by Servo's [[ChunkingStrategy]] - object CustomChunkingStrategy { - def equalSizeWithView[K, T](maxSize: Int): ((Seq[K], T)) => Seq[(Seq[K], T)] = { - case (keys, view) => - ChunkingStrategy - .equalSize[K](maxSize)(keys) - .map { chunk: Seq[K] => (chunk, view) } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TimelinesPersistenceStoreClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TimelinesPersistenceStoreClientModule.scala deleted file mode 100644 index 4af39fd32..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TimelinesPersistenceStoreClientModule.scala +++ /dev/null @@ -1,49 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.timelinemixer.clients.persistence.TimelinePersistenceManhattanClientBuilder -import com.twitter.timelinemixer.clients.persistence.TimelinePersistenceManhattanClientConfig -import com.twitter.timelinemixer.clients.persistence.TimelineResponseBatchesClient -import com.twitter.timelinemixer.clients.persistence.TimelineResponseV3 -import com.twitter.util.Duration -import javax.inject.Singleton - -object TimelinesPersistenceStoreClientModule extends TwitterModule { - private val StagingDataset = "timeline_response_batches_v5_nonprod" - private val ProdDataset = "timeline_response_batches_v5" - private final val Timeout = "mh_persistence_store.timeout" - - flag[Duration](Timeout, 300.millis, "Timeout per request") - - @Provides - @Singleton - def providesTimelinesPersistenceStoreClient( - @Flag(Timeout) timeout: Duration, - injectedServiceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): TimelineResponseBatchesClient[TimelineResponseV3] = { - val timelineResponseBatchesDataset = - injectedServiceIdentifier.environment.toLowerCase match { - case "prod" => ProdDataset - case _ => StagingDataset - } - - val timelineResponseBatchesConfig = new TimelinePersistenceManhattanClientConfig { - val dataset = timelineResponseBatchesDataset - val isReadOnly = false - val serviceIdentifier = injectedServiceIdentifier - override val defaultMaxTimeout = timeout - override val maxRetryCount = 2 - } - - TimelinePersistenceManhattanClientBuilder.buildTimelineResponseV3BatchesClient( - timelineResponseBatchesConfig, - statsReceiver - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TopicSocialProofClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TopicSocialProofClientModule.scala deleted file mode 100644 index 9333e0f84..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TopicSocialProofClientModule.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithModerateTimeout -import com.twitter.inject.TwitterModule -import com.twitter.strato.client.Client -import com.twitter.timelines.clients.strato.topics.TopicSocialProofClient -import com.twitter.timelines.clients.strato.topics.TopicSocialProofClientImpl -import javax.inject.Named -import javax.inject.Singleton - -object TopicSocialProofClientModule extends TwitterModule { - - @Singleton - @Provides - def providesSimilarityClient( - @Named(BatchedStratoClientWithModerateTimeout) - stratoClient: Client, - statsReceiver: StatsReceiver - ): TopicSocialProofClient = new TopicSocialProofClientImpl(stratoClient, statsReceiver) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetyPieClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetyPieClientModule.scala deleted file mode 100644 index 1eb49206c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetyPieClientModule.scala +++ /dev/null @@ -1,63 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.thrift.ClientId -import com.twitter.finagle.thriftmux.MethodBuilder -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.inject.Injector -import com.twitter.inject.annotations.Flags -import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule -import com.twitter.stitch.tweetypie.TweetyPie -import com.twitter.tweetypie.thriftscala.TweetService -import com.twitter.util.Duration -import javax.inject.Singleton - -/** - * Idempotent Tweetypie Thrift and Stitch client. - */ -object TweetypieClientModule - extends ThriftMethodBuilderClientModule[ - TweetService.ServicePerEndpoint, - TweetService.MethodPerEndpoint - ] - with MtlsClient { - - private val TimeoutRequest = "tweetypie.timeout_request" - private val TimeoutTotal = "tweetypie.timeout_total" - - flag[Duration](TimeoutRequest, 1000.millis, "Timeout per request") - flag[Duration](TimeoutTotal, 1000.millis, "Total timeout") - - override val label: String = "tweetypie" - override val dest: String = "/s/tweetypie/tweetypie" - - @Singleton - @Provides - def providesTweetypieStitchClient(tweetService: TweetService.MethodPerEndpoint): TweetyPie = - new TweetyPie(tweetService) - - /** - * TweetyPie client id must be in the form of {service.env} or it will not be treated as an - * unauthorized client - */ - override protected def clientId(injector: Injector): ClientId = { - val serviceIdentifier = injector.instance[ServiceIdentifier] - ClientId(s"${serviceIdentifier.service}.${serviceIdentifier.environment}") - } - - override protected def configureMethodBuilder( - injector: Injector, - methodBuilder: MethodBuilder - ): MethodBuilder = { - val timeoutRequest = injector.instance[Duration](Flags.named(TimeoutRequest)) - val timeoutTotal = injector.instance[Duration](Flags.named(TimeoutTotal)) - - methodBuilder - .withTimeoutPerRequest(timeoutRequest) - .withTimeoutTotal(timeoutTotal) - } - - override protected def sessionAcquisitionTimeout: Duration = 500.millis -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetypieStaticEntitiesCacheClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetypieStaticEntitiesCacheClientModule.scala deleted file mode 100644 index 13283e4b3..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetypieStaticEntitiesCacheClientModule.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.name.Named -import com.google.inject.Provides -import com.twitter.conversions.DurationOps.RichDuration -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetypieStaticEntitiesCache -import com.twitter.inject.TwitterModule -import com.twitter.product_mixer.shared_library.memcached_client.MemcachedClientBuilder -import com.twitter.servo.cache.FinagleMemcache -import com.twitter.servo.cache.KeyTransformer -import com.twitter.servo.cache.KeyValueTransformingTtlCache -import com.twitter.servo.cache.ObservableTtlCache -import com.twitter.servo.cache.Serializer -import com.twitter.servo.cache.ThriftSerializer -import com.twitter.servo.cache.TtlCache -import com.twitter.tweetypie.{thriftscala => tp} -import javax.inject.Singleton -import org.apache.thrift.protocol.TCompactProtocol - -object TweetypieStaticEntitiesCacheClientModule extends TwitterModule { - - private val ScopeName = "TweetypieStaticEntitiesMemcache" - private val ProdDest = "/srv#/prod/local/cache/timelinescorer_tweet_core_data:twemcaches" - - private val tweetsSerializer: Serializer[tp.Tweet] = { - new ThriftSerializer[tp.Tweet](tp.Tweet, new TCompactProtocol.Factory()) - } - private val keyTransformer: KeyTransformer[Long] = { tweetId => tweetId.toString } - - @Provides - @Singleton - @Named(TweetypieStaticEntitiesCache) - def providesTweetypieStaticEntitiesCache( - statsReceiver: StatsReceiver, - serviceIdentifier: ServiceIdentifier - ): TtlCache[Long, tp.Tweet] = { - val memCacheClient = MemcachedClientBuilder.buildMemcachedClient( - destName = ProdDest, - numTries = 1, - numConnections = 1, - requestTimeout = 50.milliseconds, - globalTimeout = 100.milliseconds, - connectTimeout = 100.milliseconds, - acquisitionTimeout = 100.milliseconds, - serviceIdentifier = serviceIdentifier, - statsReceiver = statsReceiver - ) - mkCache(new FinagleMemcache(memCacheClient), statsReceiver) - } - - private def mkCache( - finagleMemcache: FinagleMemcache, - statsReceiver: StatsReceiver - ): TtlCache[Long, tp.Tweet] = { - val baseCache: KeyValueTransformingTtlCache[Long, String, tp.Tweet, Array[Byte]] = - new KeyValueTransformingTtlCache( - underlyingCache = finagleMemcache, - transformer = tweetsSerializer, - underlyingKey = keyTransformer - ) - ObservableTtlCache( - underlyingCache = baseCache, - statsReceiver = statsReceiver.scope(ScopeName), - windowSize = 1000, - name = ScopeName - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/BUILD.bazel deleted file mode 100644 index 25e9a2e31..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/GlobalParamConfigModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/GlobalParamConfigModule.scala deleted file mode 100644 index 304e7bdc8..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/GlobalParamConfigModule.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.home_mixer.param - -import com.twitter.inject.TwitterModule -import com.twitter.product_mixer.core.functional_component.configapi.registry.GlobalParamConfig - -object GlobalParamConfigModule extends TwitterModule { - override def configure(): Unit = { - bind[GlobalParamConfig].to[HomeGlobalParamConfig] - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParamConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParamConfig.scala deleted file mode 100644 index 6fbd28fce..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParamConfig.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.home_mixer.param - -import com.twitter.home_mixer.param.HomeGlobalParams._ -import com.twitter.product_mixer.core.functional_component.configapi.registry.GlobalParamConfig -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Register Params that do not relate to a specific product. See GlobalParamConfig -> ParamConfig - * for hooks to register Params based on type. - */ -@Singleton -class HomeGlobalParamConfig @Inject() () extends GlobalParamConfig { - - override val booleanFSOverrides = Seq( - AdsDisableInjectionBasedOnUserRoleParam, - EnableAdvertiserBrandSafetySettingsFeatureHydratorParam, - EnableImpressionBloomFilter, - EnableNahFeedbackInfoParam, - EnableNewTweetsPillAvatarsParam, - EnableScribeServedCandidatesParam, - EnableSendScoresToClient, - EnableSocialContextParam, - ) - - override val boundedIntFSOverrides = Seq( - MaxNumberReplaceInstructionsParam, - TimelinesPersistenceStoreMaxEntriesPerClient, - ) - - override val boundedDoubleFSOverrides = Seq( - ImpressionBloomFilterFalsePositiveRateParam - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParams.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParams.scala deleted file mode 100644 index f19817bc9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParams.scala +++ /dev/null @@ -1,86 +0,0 @@ -package com.twitter.home_mixer.param - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam - -/** - * Instantiate Params that do not relate to a specific product. - * - * @see [[com.twitter.product_mixer.core.product.ProductParamConfig.supportedClientFSName]] - */ -object HomeGlobalParams { - - /** - * This param is used to disable ads injection for timelines served by home-mixer. - * It is currently used to maintain user-role based no-ads lists for automation accounts, - * and should NOT be used for other purposes. - */ - object AdsDisableInjectionBasedOnUserRoleParam - extends FSParam("home_mixer_ads_disable_injection_based_on_user_role", false) - - object EnableSendScoresToClient - extends FSParam[Boolean]( - name = "home_mixer_enable_send_scores_to_client", - default = false - ) - - object EnableNahFeedbackInfoParam - extends FSParam[Boolean]( - name = "home_mixer_enable_nah_feedback_info", - default = false - ) - - object MaxNumberReplaceInstructionsParam - extends FSBoundedParam[Int]( - name = "home_mixer_max_number_replace_instructions", - default = 100, - min = 0, - max = 200 - ) - - object TimelinesPersistenceStoreMaxEntriesPerClient - extends FSBoundedParam[Int]( - name = "home_mixer_timelines_persistence_store_max_entries_per_client", - default = 1800, - min = 500, - max = 5000 - ) - - object EnableNewTweetsPillAvatarsParam - extends FSParam[Boolean]( - name = "home_mixer_enable_new_tweets_pill_avatars", - default = true - ) - - object EnableSocialContextParam - extends FSParam[Boolean]( - name = "home_mixer_enable_social_context", - default = true - ) - - object EnableAdvertiserBrandSafetySettingsFeatureHydratorParam - extends FSParam[Boolean]( - name = "home_mixer_enable_advertiser_brand_safety_settings_feature_hydrator", - default = true - ) - - object EnableImpressionBloomFilter - extends FSParam[Boolean]( - name = "home_mixer_enable_impression_bloom_filter", - default = false - ) - - object ImpressionBloomFilterFalsePositiveRateParam - extends FSBoundedParam[Double]( - name = "home_mixer_impression_bloom_filter_false_positive_rate", - default = 0.005, - min = 0.001, - max = 0.01 - ) - - object EnableScribeServedCandidatesParam - extends FSParam[Boolean]( - name = "home_mixer_served_tweets_enable_scribing", - default = true - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerFlagName.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerFlagName.scala deleted file mode 100644 index dc5f5513f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerFlagName.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.home_mixer.param - -object HomeMixerFlagName { - final val ScribeClientEventsFlag = "scribe.client_events" - final val ScribeServedCandidatesFlag = "scribe.served_candidates" - final val ScribeScoredCandidatesFlag = "scribe.scored_candidates" - final val ScribeServedCommonFeaturesAndCandidateFeaturesFlag = - "scribe.served_common_features_and_candidate_features" - final val DataRecordMetadataStoreConfigsYmlFlag = "data.record.metadata.store.configs.yml" - final val DarkTrafficFilterDeciderKey = "thrift.dark.traffic.filter.decider_key" - final val TargetFetchLatency = "target.fetch.latency" - final val TargetScoringLatency = "target.scoring.latency" -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerInjectionNames.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerInjectionNames.scala deleted file mode 100644 index 5ea87f5ab..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerInjectionNames.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.home_mixer.param - -object HomeMixerInjectionNames { - final val AuthorFeatureRepository = "AuthorFeatureRepository" - final val CandidateFeaturesScribeEventPublisher = "CandidateFeaturesScribeEventPublisher" - final val CommonFeaturesScribeEventPublisher = "CommonFeaturesScribeEventPublisher" - final val EarlybirdRepository = "EarlybirdRepository" - final val EngagementsReceivedByAuthorCache = "EngagementsReceivedByAuthorCache" - final val GraphTwoHopRepository = "GraphTwoHopRepository" - final val HomeAuthorFeaturesCacheClient = "HomeAuthorFeaturesCacheClient" - final val InterestsThriftServiceClient = "InterestsThriftServiceClient" - final val BatchedStratoClientWithModerateTimeout = "BatchedStratoClientWithModerateTimeout" - final val ManhattanApolloClient = "ManhattanApolloClient" - final val ManhattanAthenaClient = "ManhattanAthenaClient" - final val ManhattanOmegaClient = "ManhattanOmegaClient" - final val ManhattanStarbuckClient = "ManhattanStarbuckClient" - final val MetricCenterUserCountingFeatureRepository = "MetricCenterUserCountingFeatureRepository" - final val MinimumFeaturesScribeEventPublisher = "MinimumFeaturesScribeEventPublisher" - final val RealGraphInNetworkScores = "RealGraphInNetworkScores" - final val RealGraphManhattanEndpoint = "RealGraphFeaturesManhattanEndpoint" - final val RealGraphFeatureRepository = "RealGraphFeatureRepository" - final val RealTimeInteractionGraphUserVertexCache = "RealTimeInteractionGraphUserVertexCache" - final val RealTimeInteractionGraphUserVertexClient = "RealTimeInteractionGraphUserVertexClient" - final val StaleTweetsCache = "StaleTweetsCache" - final val TimelineAggregateMetadataRepository = "TimelineAggregateMetadataRepository" - final val TimelineAggregatePartARepository = "TimelineAggregatePartARepository" - final val TimelineAggregatePartBRepository = "TimelineAggregatePartBRepository" - final val TimelinesRealTimeAggregateClient = "TimelinesRealTimeAggregateClient" - final val TopicCountryEngagementCache = "TopicCountryEngagementCache" - final val TopicEngagementCache = "TopicEngagementCache" - final val TweetCountryEngagementCache = "TweetCountryEngagementCache" - final val TweetEngagementCache = "TweetEngagementCache" - final val TweetypieContentRepository = "TweetypieContentRepository" - final val TweetypieStaticEntitiesCache = "TweetypieStaticEntitiesCache" - final val TwhinAuthorFollowFeatureCacheClient = "TwhinAuthorFollowFeatureCacheClient" - final val TwhinAuthorFollowFeatureRepository = "TwhinAuthorFollowFeatureRepository" - final val TwhinUserEngagementFeatureRepository = "TwhinUserEngagementFeatureRepository" - final val TwhinUserFollowFeatureRepository = "TwhinUserFollowFeatureRepository" - final val TwitterListEngagementCache = "TwitterListEngagementCache" - final val UserAuthorEngagementCache = "UserAuthorEngagementCache" - final val UserEngagementCache = "UserEngagementCache" - final val UserFollowedTopicIdsRepository = "UserFollowedTopicIdsRepository" - final val UserLanguagesRepository = "UserLanguagesRepository" - final val UserTopicEngagementForNewUserCache = "UserTopicEngagementForNewUserCache" - final val UtegSocialProofRepository = "UtegSocialProofRepository" -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/BUILD.bazel deleted file mode 100644 index 5974c186f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/BUILD.bazel +++ /dev/null @@ -1,10 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "servo/decider", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/DeciderKey.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/DeciderKey.scala deleted file mode 100644 index 91f7646d9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/DeciderKey.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.home_mixer.param.decider - -import com.twitter.servo.decider.DeciderKeyEnum - -/** - * These values must correspond to the deciders configured in the - * home-mixer/server/src/main/resources/config/decider.yml file - * - * @see [[com.twitter.product_mixer.core.product.ProductParamConfig.enabledDeciderKey]] - */ -object DeciderKey extends DeciderKeyEnum { - // Products - val EnableForYouProduct = Value("enable_for_you_product") - - val EnableFollowingProduct = Value("enable_following_product") - - val EnableScoredTweetsProduct = Value("enable_scored_tweets_product") - - val EnableListTweetsProduct = Value("enable_list_tweets_product") - - val EnableListRecommendedUsersProduct = Value("enable_list_recommended_users_product") - - val EnableSubscribedProduct = Value("enable_subscribed_product") - - // Candidate Pipelines - val EnableForYouScoredTweetsCandidatePipeline = - Value("enable_for_you_scored_tweets_candidate_pipeline") - - val EnableScoredTweetsTweetMixerCandidatePipeline = - Value("enable_scored_tweets_tweet_mixer_candidate_pipeline") - - val EnableScoredTweetsInNetworkCandidatePipeline = - Value("enable_scored_tweets_in_network_candidate_pipeline") - - val EnableScoredTweetsUtegCandidatePipeline = - Value("enable_scored_tweets_uteg_candidate_pipeline") - - val EnableScoredTweetsFrsCandidatePipeline = - Value("enable_scored_tweets_frs_candidate_pipeline") - - val EnableScoredTweetsListsCandidatePipeline = - Value("enable_scored_tweets_lists_candidate_pipeline") - - val EnableScoredTweetsPopularVideosCandidatePipeline = - Value("enable_scored_tweets_popular_videos_candidate_pipeline") - - val EnableScoredTweetsBackfillCandidatePipeline = - Value("enable_scored_tweets_backfill_candidate_pipeline") - - val EnableSimClustersSimilarityFeatureHydration = - Value("enable_simclusters_similarity_feature_hydration") -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/BUILD.bazel deleted file mode 100644 index e4fa669d2..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/BUILD.bazel +++ /dev/null @@ -1,28 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/model", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/HomeMixerProductModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/HomeMixerProductModule.scala deleted file mode 100644 index cff4da4fb..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/HomeMixerProductModule.scala +++ /dev/null @@ -1,11 +0,0 @@ -package com.twitter.home_mixer.product - -import com.twitter.inject.TwitterModule -import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistryConfig - -object HomeMixerProductModule extends TwitterModule { - - override def configure(): Unit = { - bind[ProductPipelineRegistryConfig].to[HomeProductPipelineRegistryConfig] - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/HomeProductPipelineRegistryConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/HomeProductPipelineRegistryConfig.scala deleted file mode 100644 index 5db4bd7f6..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/HomeProductPipelineRegistryConfig.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.home_mixer.product - -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.model.request.ListRecommendedUsersProduct -import com.twitter.home_mixer.model.request.ListTweetsProduct -import com.twitter.home_mixer.model.request.ScoredTweetsProduct -import com.twitter.home_mixer.model.request.SubscribedProduct -import com.twitter.home_mixer.product.following.FollowingProductPipelineConfig -import com.twitter.home_mixer.product.for_you.ForYouProductPipelineConfig -import com.twitter.home_mixer.product.list_recommended_users.ListRecommendedUsersProductPipelineConfig -import com.twitter.home_mixer.product.scored_tweets.ScoredTweetsProductPipelineConfig -import com.twitter.home_mixer.product.list_tweets.ListTweetsProductPipelineConfig -import com.twitter.home_mixer.product.subscribed.SubscribedProductPipelineConfig -import com.twitter.inject.Injector -import com.twitter.product_mixer.core.product.guice.ProductScope -import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistryConfig -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class HomeProductPipelineRegistryConfig @Inject() ( - injector: Injector, - productScope: ProductScope) - extends ProductPipelineRegistryConfig { - - private val followingProductPipelineConfig = productScope.let(FollowingProduct) { - injector.instance[FollowingProductPipelineConfig] - } - - private val forYouProductPipelineConfig = productScope.let(ForYouProduct) { - injector.instance[ForYouProductPipelineConfig] - } - - private val scoredTweetsProductPipelineConfig = productScope.let(ScoredTweetsProduct) { - injector.instance[ScoredTweetsProductPipelineConfig] - } - - private val listTweetsProductPipelineConfig = productScope.let(ListTweetsProduct) { - injector.instance[ListTweetsProductPipelineConfig] - } - - private val listRecommendedUsersProductPipelineConfig = - productScope.let(ListRecommendedUsersProduct) { - injector.instance[ListRecommendedUsersProductPipelineConfig] - } - - private val subscribedProductPipelineConfig = productScope.let(SubscribedProduct) { - injector.instance[SubscribedProductPipelineConfig] - } - - override val productPipelineConfigs = Seq( - followingProductPipelineConfig, - forYouProductPipelineConfig, - scoredTweetsProductPipelineConfig, - listTweetsProductPipelineConfig, - listRecommendedUsersProductPipelineConfig, - subscribedProductPipelineConfig, - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/BUILD.bazel deleted file mode 100644 index 4771fe655..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/BUILD.bazel +++ /dev/null @@ -1,95 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "ads-injection/lib/src/main/scala/com/twitter/goldfinch/api", - "finagle/finagle-memcached/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/account_recommendations_mixer", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/hermit", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/async", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/param_gated", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", - "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/item", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", - "src/java/com/twitter/search/common/schema/base", - "src/java/com/twitter/search/common/schema/earlybird", - "src/java/com/twitter/search/common/util/lang", - "src/java/com/twitter/search/queryparser/query:core-query-nodes", - "src/java/com/twitter/search/queryparser/query/search:search-query-nodes", - "src/scala/com/twitter/suggests/controller_data", - "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/search/common:constants-java", - "src/thrift/com/twitter/suggests/controller_data:controller_data-scala", - "src/thrift/com/twitter/timelinemixer:thrift-scala", - "src/thrift/com/twitter/timelines/render:thrift-scala", - "src/thrift/com/twitter/timelinescorer:thrift-scala", - "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", - "src/thrift/com/twitter/timelinescorer/server/internal:thrift-scala", - "src/thrift/com/twitter/tweetypie:service-scala", - "stitch/stitch-gizmoduck", - "stitch/stitch-tweetypie", - "stringcenter/client", - "stringcenter/client/src/main/java", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", - "timelines/src/main/scala/com/twitter/timelines/clients/relevance_search", - "timelines/src/main/scala/com/twitter/timelines/injection/scribe", - ], - exports = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "src/thrift/com/twitter/timelines/render:thrift-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingAdsCandidatePipelineBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingAdsCandidatePipelineBuilder.scala deleted file mode 100644 index d4898eb31..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingAdsCandidatePipelineBuilder.scala +++ /dev/null @@ -1,108 +0,0 @@ -package com.twitter.home_mixer.product.following - -import com.twitter.adserver.{thriftscala => ads} -import com.twitter.home_mixer.functional_component.decorator.builder.HomeAdsClientEventDetailsBuilder -import com.twitter.home_mixer.functional_component.gate.ExcludeSoftUserGate -import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature -import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature -import com.twitter.home_mixer.param.HomeGlobalParams -import com.twitter.home_mixer.param.HomeGlobalParams.EnableAdvertiserBrandSafetySettingsFeatureHydratorParam -import com.twitter.home_mixer.product.following.model.FollowingQuery -import com.twitter.home_mixer.product.following.param.FollowingParam.EnableAdsCandidatePipelineParam -import com.twitter.home_mixer.product.following.param.FollowingParam.EnableFastAds -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.candidate_source.ads.AdsProdThriftCandidateSource -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.ad.AdsCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder -import com.twitter.product_mixer.component_library.feature_hydrator.candidate.ads.AdvertiserBrandSafetySettingsFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedCandidateFeatureHydrator -import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate -import com.twitter.product_mixer.component_library.model.candidate.ads.AdsCandidate -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsDependentCandidatePipelineConfig -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsDependentCandidatePipelineConfigBuilder -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.CountCandidatesFromPipelines -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.PipelineScopedOrganicItems -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.ValidAdImpressionIdFilter -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.gate.ParamNotGate -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.rtf.safety_level.TimelineHomePromotedHydrationSafetyLevel -import com.twitter.product_mixer.core.model.marshalling.response.urt.contextual_ref.TweetHydrationContext -import com.twitter.timelines.injection.scribe.InjectionScribeUtil -import com.twitter.timelineservice.suggests.{thriftscala => st} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class FollowingAdsCandidatePipelineBuilder @Inject() ( - adsCandidatePipelineConfigBuilder: AdsDependentCandidatePipelineConfigBuilder, - adsCandidateSource: AdsProdThriftCandidateSource, - advertiserBrandSafetySettingsFeatureHydrator: AdvertiserBrandSafetySettingsFeatureHydrator[ - FollowingQuery, - AdsCandidate - ]) { - - private val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("FollowingAds") - - private val suggestType = st.SuggestType.Promoted - - private val clientEventInfoBuilder = ClientEventInfoBuilder( - component = InjectionScribeUtil.scribeComponent(suggestType).get, - detailsBuilder = Some(HomeAdsClientEventDetailsBuilder(Some(suggestType.name))) - ) - - private val contextualTweetRefBuilder = ContextualTweetRefBuilder( - TweetHydrationContext( - safetyLevelOverride = Some(TimelineHomePromotedHydrationSafetyLevel), - outerTweetContext = None - )) - - private val decorator = UrtItemCandidateDecorator( - AdsCandidateUrtItemBuilder( - tweetClientEventInfoBuilder = Some(clientEventInfoBuilder), - contextualTweetRefBuilder = Some(contextualTweetRefBuilder) - )) - - private val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), - HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() - ) - - def build( - organicCandidatePipelines: CandidateScope - ): AdsDependentCandidatePipelineConfig[FollowingQuery] = - adsCandidatePipelineConfigBuilder.build[FollowingQuery]( - adsCandidateSource = adsCandidateSource, - identifier = identifier, - adsDisplayLocationBuilder = query => - if (query.params.getBoolean(EnableFastAds)) ads.DisplayLocation.TimelineHomeReverseChron - else ads.DisplayLocation.TimelineHome, - getOrganicItems = PipelineScopedOrganicItems( - pipelines = organicCandidatePipelines, - textFeature = TweetTextFeature, - languageFeature = TweetLanguageFeature - ), - countNumOrganicItems = CountCandidatesFromPipelines(organicCandidatePipelines), - supportedClientParam = Some(EnableAdsCandidatePipelineParam), - gates = Seq( - ParamNotGate( - name = "AdsDisableInjectionBasedOnUserRole", - param = HomeGlobalParams.AdsDisableInjectionBasedOnUserRoleParam - ), - ExcludeSoftUserGate, - NonEmptyCandidatesGate(organicCandidatePipelines) - ), - filters = Seq(ValidAdImpressionIdFilter), - postFilterFeatureHydration = Seq( - ParamGatedCandidateFeatureHydrator( - EnableAdvertiserBrandSafetySettingsFeatureHydratorParam, - advertiserBrandSafetySettingsFeatureHydrator - ) - ), - decorator = Some(decorator), - alerts = alerts, - urtRequest = Some(true), - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdCandidatePipelineConfig.scala deleted file mode 100644 index addb298c2..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdCandidatePipelineConfig.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.home_mixer.product.following - -import com.twitter.home_mixer.candidate_pipeline.FollowingEarlybirdResponseFeatureTransformer -import com.twitter.home_mixer.functional_component.candidate_source.EarlybirdCandidateSource -import com.twitter.home_mixer.product.following.model.FollowingQuery -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature -import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.search.earlybird.{thriftscala => t} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class FollowingEarlybirdCandidatePipelineConfig @Inject() ( - earlybirdCandidateSource: EarlybirdCandidateSource, - followingEarlybirdQueryTransformer: FollowingEarlybirdQueryTransformer) - extends CandidatePipelineConfig[ - FollowingQuery, - t.EarlybirdRequest, - t.ThriftSearchResult, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("FollowingEarlybird") - - override val candidateSource: BaseCandidateSource[t.EarlybirdRequest, t.ThriftSearchResult] = - earlybirdCandidateSource - - override val gates: Seq[Gate[FollowingQuery]] = Seq( - NonEmptySeqFeatureGate(SGSFollowedUsersFeature) - ) - - override val queryTransformer: CandidatePipelineQueryTransformer[ - FollowingQuery, - t.EarlybirdRequest - ] = followingEarlybirdQueryTransformer - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[t.ThriftSearchResult] - ] = Seq(FollowingEarlybirdResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - t.ThriftSearchResult, - TweetCandidate - ] = { sourceResult => TweetCandidate(id = sourceResult.id) } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdQueryTransformer.scala deleted file mode 100644 index c388cfa92..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdQueryTransformer.scala +++ /dev/null @@ -1,84 +0,0 @@ -package com.twitter.home_mixer.product.following - -import com.twitter.finagle.thrift.ClientId -import com.twitter.finagle.tracing.Trace -import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature -import com.twitter.home_mixer.product.following.model.FollowingQuery -import com.twitter.home_mixer.product.following.param.FollowingParam.ServerMaxResultsParam -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.BottomCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor -import com.twitter.product_mixer.core.pipeline.pipeline_failure.MalformedCursor -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.search.common.schema.earlybird.EarlybirdFieldConstants.EarlybirdFieldConstant -import com.twitter.search.earlybird.{thriftscala => t} -import com.twitter.search.queryparser.query.Conjunction -import com.twitter.search.queryparser.query.search.SearchOperator -import javax.inject.Inject -import javax.inject.Singleton -import scala.jdk.CollectionConverters.asJavaIterableConverter - -@Singleton -case class FollowingEarlybirdQueryTransformer @Inject() (clientId: ClientId) - extends CandidatePipelineQueryTransformer[FollowingQuery, t.EarlybirdRequest] { - - override def transform(query: FollowingQuery): t.EarlybirdRequest = { - val followedUserIds = - query.features.map(_.get(SGSFollowedUsersFeature)).getOrElse(Seq.empty).toSet - val realGraphInNetworkFollowedUserIds = - query.features.map(_.get(RealGraphInNetworkScoresFeature)).getOrElse(Map.empty).keySet - val userId = query.getRequiredUserId - val combinedUserIds = userId +: followedUserIds.toSeq - - val baseFollowedUsersSearchOperator = new SearchOperator.Builder() - .setType(SearchOperator.Type.FEATURE_VALUE_IN_ACCEPT_LIST_OR_UNSET) - .addOperand(EarlybirdFieldConstant.DIRECTED_AT_USER_ID_CSF.getFieldName) - - val followedUsersQuery = - baseFollowedUsersSearchOperator.addOperands(combinedUserIds.map(_.toString).asJava).build() - - val searchQuery = query.pipelineCursor - .map { cursor => - val sinceIdQuery = - (id: Long) => new SearchOperator(SearchOperator.Type.SINCE_ID, id.toString) - val maxIdQuery = // max ID is inclusive, so subtract 1 - (id: Long) => new SearchOperator(SearchOperator.Type.MAX_ID, (id - 1).toString) - - (cursor.cursorType, cursor.id, cursor.gapBoundaryId) match { - case (Some(TopCursor), Some(sinceId), _) => - new Conjunction(sinceIdQuery(sinceId), followedUsersQuery) - case (Some(BottomCursor), Some(maxId), _) => - new Conjunction(maxIdQuery(maxId), followedUsersQuery) - case (Some(GapCursor), Some(maxId), Some(sinceId)) => - new Conjunction(sinceIdQuery(sinceId), maxIdQuery(maxId), followedUsersQuery) - case (Some(GapCursor), _, _) => - throw PipelineFailure(MalformedCursor, "Invalid cursor " + cursor.toString) - case _ => followedUsersQuery - } - }.getOrElse(followedUsersQuery) - - val metadataOptions = t.ThriftSearchResultMetadataOptions( - getInReplyToStatusId = true, - getReferencedTweetAuthorId = true, - getFromUserId = true - ) - - t.EarlybirdRequest( - searchQuery = t.ThriftSearchQuery( - serializedQuery = Some(searchQuery.serialize), - fromUserIDFilter64 = Some(combinedUserIds), - numResults = query.requestedMaxResults.getOrElse(query.params(ServerMaxResultsParam)), - rankingMode = t.ThriftSearchRankingMode.Recency, - resultMetadataOptions = Some(metadataOptions), - searcherId = query.getOptionalUserId, - ), - getOlderResults = Some(true), // needed for archive access to older tweets - clientRequestID = Some(s"${Trace.id.traceId}"), - followedUserIds = Some(combinedUserIds), - numResultsToReturnAtRoot = Some(query.params(ServerMaxResultsParam)), - clientId = Some(clientId.name), - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdResponseFeatureTransformer.scala deleted file mode 100644 index 0169e6dd7..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdResponseFeatureTransformer.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.home_mixer.candidate_pipeline - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.search.earlybird.{thriftscala => t} - -object FollowingEarlybirdResponseFeatureTransformer - extends CandidateFeatureTransformer[t.ThriftSearchResult] { - - override val identifier: TransformerIdentifier = - TransformerIdentifier("FollowingEarlybirdResponse") - - override val features: Set[Feature[_, _]] = Set( - AuthorIdFeature, - InReplyToTweetIdFeature, - IsRetweetFeature, - SourceTweetIdFeature, - SourceUserIdFeature, - ) - - override def transform(candidate: t.ThriftSearchResult): FeatureMap = FeatureMapBuilder() - .add(AuthorIdFeature, candidate.tweetypieTweet.flatMap(_.coreData.map(_.userId))) - .add( - InReplyToTweetIdFeature, - candidate.tweetypieTweet.flatMap(_.coreData.flatMap(_.reply.flatMap(_.inReplyToStatusId)))) - .add(IsRetweetFeature, candidate.metadata.exists(_.isRetweet.contains(true))) - .add(SourceTweetIdFeature, candidate.sourceTweetypieTweet.map(_.id)) - .add(SourceUserIdFeature, candidate.sourceTweetypieTweet.flatMap(_.coreData.map(_.userId))) - .build() -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingMixerPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingMixerPipelineConfig.scala deleted file mode 100644 index efda03595..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingMixerPipelineConfig.scala +++ /dev/null @@ -1,299 +0,0 @@ -package com.twitter.home_mixer.product.following - -import com.twitter.clientapp.{thriftscala => ca} -import com.twitter.goldfinch.api.AdsInjectionSurfaceAreas -import com.twitter.home_mixer.candidate_pipeline.ConversationServiceCandidatePipelineConfigBuilder -import com.twitter.home_mixer.candidate_pipeline.EditedTweetsCandidatePipelineConfig -import com.twitter.home_mixer.candidate_pipeline.NewTweetsPillCandidatePipelineConfig -import com.twitter.home_mixer.functional_component.decorator.HomeConversationServiceCandidateDecorator -import com.twitter.home_mixer.functional_component.decorator.urt.builder.AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder -import com.twitter.home_mixer.functional_component.feature_hydrator._ -import com.twitter.home_mixer.functional_component.selector.UpdateHomeClientEventDetails -import com.twitter.home_mixer.functional_component.selector.UpdateNewTweetsPillDecoration -import com.twitter.home_mixer.functional_component.side_effect._ -import com.twitter.home_mixer.model.GapIncludeInstruction -import com.twitter.home_mixer.param.HomeGlobalParams.EnableImpressionBloomFilter -import com.twitter.home_mixer.param.HomeGlobalParams.MaxNumberReplaceInstructionsParam -import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeClientEventsFlag -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator -import com.twitter.home_mixer.product.following.model.FollowingQuery -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.home_mixer.product.following.param.FollowingParam.EnableFlipInjectionModuleCandidatePipelineParam -import com.twitter.home_mixer.product.following.param.FollowingParam.FlipInlineInjectionModulePosition -import com.twitter.home_mixer.product.following.param.FollowingParam.ServerMaxResultsParam -import com.twitter.home_mixer.product.following.param.FollowingParam.WhoToFollowPositionParam -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.inject.annotations.Flag -import com.twitter.logpipeline.client.common.EventPublisher -import com.twitter.product_mixer.component_library.feature_hydrator.query.async.AsyncQueryFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweetsQueryFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.AsyncParamGatedQueryFeatureHydrator -import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.FlipPromptDependentCandidatePipelineConfigBuilder -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmCandidatePipelineConfig -import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedBottomCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedGapCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedTopCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceAllEntries -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceEntryInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowAlertInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowCoverInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.StaticTimelineScribeConfigBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtMetadataBuilder -import com.twitter.product_mixer.component_library.selector.DropMaxCandidates -import com.twitter.product_mixer.component_library.selector.DropMaxModuleItemCandidates -import com.twitter.product_mixer.component_library.selector.DropModuleTooFewModuleItemResults -import com.twitter.product_mixer.component_library.selector.InsertAppendResults -import com.twitter.product_mixer.component_library.selector.InsertFixedPositionResults -import com.twitter.product_mixer.component_library.selector.SelectConditionally -import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates -import com.twitter.product_mixer.component_library.selector.ads.AdsInjector -import com.twitter.product_mixer.component_library.selector.ads.InsertAdResults -import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.configapi.StaticParam -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller -import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.UrtTransportMarshaller -import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.product_mixer.core.pipeline.FailOpenPolicy -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import com.twitter.timelines.render.{thriftscala => urt} -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton - -@Singleton -class FollowingMixerPipelineConfig @Inject() ( - followingEarlybirdCandidatePipelineConfig: FollowingEarlybirdCandidatePipelineConfig, - conversationServiceCandidatePipelineConfigBuilder: ConversationServiceCandidatePipelineConfigBuilder[ - FollowingQuery - ], - homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder, - followingAdsCandidatePipelineBuilder: FollowingAdsCandidatePipelineBuilder, - followingWhoToFollowCandidatePipelineConfigBuilder: FollowingWhoToFollowCandidatePipelineConfigBuilder, - flipPromptDependentCandidatePipelineConfigBuilder: FlipPromptDependentCandidatePipelineConfigBuilder, - editedTweetsCandidatePipelineConfig: EditedTweetsCandidatePipelineConfig, - newTweetsPillCandidatePipelineConfig: NewTweetsPillCandidatePipelineConfig[FollowingQuery], - dismissInfoQueryFeatureHydrator: DismissInfoQueryFeatureHydrator, - gizmoduckUserQueryFeatureHydrator: GizmoduckUserQueryFeatureHydrator, - persistenceStoreQueryFeatureHydrator: PersistenceStoreQueryFeatureHydrator, - realGraphInNetworkSourceQueryHydrator: RealGraphInNetworkScoresQueryFeatureHydrator, - requestQueryFeatureHydrator: RequestQueryFeatureHydrator[FollowingQuery], - sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, - impressionBloomFilterQueryFeatureHydrator: ImpressionBloomFilterQueryFeatureHydrator[ - FollowingQuery - ], - manhattanTweetImpressionsQueryFeatureHydrator: TweetImpressionsQueryFeatureHydrator[ - FollowingQuery - ], - memcacheTweetImpressionsQueryFeatureHydrator: ImpressedTweetsQueryFeatureHydrator, - lastNonPollingTimeQueryFeatureHydrator: LastNonPollingTimeQueryFeatureHydrator, - adsInjector: AdsInjector, - updateLastNonPollingTimeSideEffect: UpdateLastNonPollingTimeSideEffect[FollowingQuery, Timeline], - publishClientSentImpressionsEventBusSideEffect: PublishClientSentImpressionsEventBusSideEffect, - publishClientSentImpressionsManhattanSideEffect: PublishClientSentImpressionsManhattanSideEffect, - publishImpressionBloomFilterSideEffect: PublishImpressionBloomFilterSideEffect, - updateTimelinesPersistenceStoreSideEffect: UpdateTimelinesPersistenceStoreSideEffect, - truncateTimelinesPersistenceStoreSideEffect: TruncateTimelinesPersistenceStoreSideEffect, - homeTimelineServedCandidatesSideEffect: HomeScribeServedCandidatesSideEffect, - clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent], - externalStrings: HomeMixerExternalStrings, - @ProductScoped stringCenterProvider: Provider[StringCenter], - urtTransportMarshaller: UrtTransportMarshaller, - @Flag(ScribeClientEventsFlag) enableScribeClientEvents: Boolean) - extends MixerPipelineConfig[FollowingQuery, Timeline, urt.TimelineResponse] { - - override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("Following") - - private val dependentCandidatesStep = MixerPipelineConfig.dependentCandidatePipelinesStep - private val resultSelectorsStep = MixerPipelineConfig.resultSelectorsStep - - override val fetchQueryFeatures: Seq[QueryFeatureHydrator[FollowingQuery]] = Seq( - requestQueryFeatureHydrator, - sgsFollowedUsersQueryFeatureHydrator, - realGraphInNetworkSourceQueryHydrator, - AsyncQueryFeatureHydrator(dependentCandidatesStep, dismissInfoQueryFeatureHydrator), - AsyncQueryFeatureHydrator(dependentCandidatesStep, gizmoduckUserQueryFeatureHydrator), - AsyncQueryFeatureHydrator(dependentCandidatesStep, persistenceStoreQueryFeatureHydrator), - AsyncQueryFeatureHydrator(dependentCandidatesStep, lastNonPollingTimeQueryFeatureHydrator), - AsyncParamGatedQueryFeatureHydrator( - EnableImpressionBloomFilter, - resultSelectorsStep, - impressionBloomFilterQueryFeatureHydrator), - AsyncQueryFeatureHydrator(resultSelectorsStep, manhattanTweetImpressionsQueryFeatureHydrator), - AsyncQueryFeatureHydrator(resultSelectorsStep, memcacheTweetImpressionsQueryFeatureHydrator) - ) - - private val earlybirdCandidatePipelineScope = - SpecificPipeline(followingEarlybirdCandidatePipelineConfig.identifier) - - private val conversationServiceCandidatePipelineConfig = - conversationServiceCandidatePipelineConfigBuilder.build( - Seq(NonEmptyCandidatesGate(earlybirdCandidatePipelineScope)), - HomeConversationServiceCandidateDecorator(homeFeedbackActionInfoBuilder) - ) - - private val followingAdsCandidatePipelineConfig = - followingAdsCandidatePipelineBuilder.build(earlybirdCandidatePipelineScope) - - private val followingWhoToFollowCandidatePipelineConfig = - followingWhoToFollowCandidatePipelineConfigBuilder.build(earlybirdCandidatePipelineScope) - - private val flipPromptCandidatePipelineConfig = - flipPromptDependentCandidatePipelineConfigBuilder.build[FollowingQuery]( - supportedClientParam = Some(EnableFlipInjectionModuleCandidatePipelineParam) - ) - - override val candidatePipelines: Seq[CandidatePipelineConfig[FollowingQuery, _, _, _]] = - Seq(followingEarlybirdCandidatePipelineConfig) - - override val dependentCandidatePipelines: Seq[ - DependentCandidatePipelineConfig[FollowingQuery, _, _, _] - ] = Seq( - conversationServiceCandidatePipelineConfig, - followingAdsCandidatePipelineConfig, - followingWhoToFollowCandidatePipelineConfig, - flipPromptCandidatePipelineConfig, - editedTweetsCandidatePipelineConfig, - newTweetsPillCandidatePipelineConfig - ) - - override val failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = Map( - followingAdsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - followingWhoToFollowCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - flipPromptCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - editedTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - newTweetsPillCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - ) - - override val resultSelectors: Seq[Selector[FollowingQuery]] = Seq( - UpdateSortCandidates( - ordering = CandidatesUtil.reverseChronTweetsOrdering, - candidatePipeline = conversationServiceCandidatePipelineConfig.identifier - ), - DropMaxCandidates( - candidatePipeline = editedTweetsCandidatePipelineConfig.identifier, - maxSelectionsParam = MaxNumberReplaceInstructionsParam - ), - DropMaxCandidates( - candidatePipeline = conversationServiceCandidatePipelineConfig.identifier, - maxSelectionsParam = ServerMaxResultsParam - ), - DropModuleTooFewModuleItemResults( - candidatePipeline = followingWhoToFollowCandidatePipelineConfig.identifier, - minModuleItemsParam = StaticParam(WhoToFollowArmCandidatePipelineConfig.MinCandidatesSize) - ), - DropMaxModuleItemCandidates( - candidatePipeline = followingWhoToFollowCandidatePipelineConfig.identifier, - maxModuleItemsParam = StaticParam(WhoToFollowArmCandidatePipelineConfig.MaxCandidatesSize) - ), - InsertAppendResults(candidatePipeline = conversationServiceCandidatePipelineConfig.identifier), - InsertFixedPositionResults( - candidatePipeline = followingWhoToFollowCandidatePipelineConfig.identifier, - positionParam = WhoToFollowPositionParam - ), - InsertFixedPositionResults( - candidatePipeline = flipPromptCandidatePipelineConfig.identifier, - positionParam = FlipInlineInjectionModulePosition - ), - InsertAdResults( - surfaceAreaName = AdsInjectionSurfaceAreas.HomeTimeline, - adsInjector = adsInjector.forSurfaceArea(AdsInjectionSurfaceAreas.HomeTimeline), - adsCandidatePipeline = followingAdsCandidatePipelineConfig.identifier - ), - // This selector must come after the tweets are inserted into the results - UpdateNewTweetsPillDecoration( - pipelineScope = SpecificPipelines( - conversationServiceCandidatePipelineConfig.identifier, - newTweetsPillCandidatePipelineConfig.identifier - ), - stringCenter = stringCenterProvider.get(), - seeNewTweetsString = externalStrings.seeNewTweetsString, - tweetedString = externalStrings.tweetedString - ), - InsertAppendResults(candidatePipeline = editedTweetsCandidatePipelineConfig.identifier), - SelectConditionally( - selector = - InsertAppendResults(candidatePipeline = newTweetsPillCandidatePipelineConfig.identifier), - includeSelector = (_, _, results) => CandidatesUtil.containsType[TweetCandidate](results) - ), - UpdateHomeClientEventDetails( - candidatePipelines = Set(conversationServiceCandidatePipelineConfig.identifier) - ), - ) - - private val homeScribeClientEventSideEffect = HomeScribeClientEventSideEffect( - enableScribeClientEvents = enableScribeClientEvents, - logPipelinePublisher = clientEventsScribeEventPublisher, - injectedTweetsCandidatePipelineIdentifiers = - Seq(conversationServiceCandidatePipelineConfig.identifier), - adsCandidatePipelineIdentifier = Some(followingAdsCandidatePipelineConfig.identifier), - whoToFollowCandidatePipelineIdentifier = - Some(followingWhoToFollowCandidatePipelineConfig.identifier), - ) - - override val resultSideEffects: Seq[PipelineResultSideEffect[FollowingQuery, Timeline]] = Seq( - homeScribeClientEventSideEffect, - homeTimelineServedCandidatesSideEffect, - publishClientSentImpressionsEventBusSideEffect, - publishClientSentImpressionsManhattanSideEffect, - publishImpressionBloomFilterSideEffect, - truncateTimelinesPersistenceStoreSideEffect, - updateLastNonPollingTimeSideEffect, - updateTimelinesPersistenceStoreSideEffect, - ) - - override val domainMarshaller: DomainMarshaller[FollowingQuery, Timeline] = { - val instructionBuilders = Seq( - ReplaceEntryInstructionBuilder(ReplaceAllEntries), - AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder(), - ShowAlertInstructionBuilder(), - ShowCoverInstructionBuilder(), - ) - - val idSelector: PartialFunction[UniversalNoun[_], Long] = { - // exclude ads while determining tweet cursor values - case item: TweetItem if item.promotedMetadata.isEmpty => item.id - case module: TimelineModule - if module.items.headOption.exists(_.item.isInstanceOf[TweetItem]) => - module.items.last.item match { case item: TweetItem => item.id } - } - val topCursorBuilder = OrderedTopCursorBuilder(idSelector) - val bottomCursorBuilder = - OrderedBottomCursorBuilder(idSelector, GapIncludeInstruction.inverse()) - val gapCursorBuilder = OrderedGapCursorBuilder(idSelector, GapIncludeInstruction) - - val metadataBuilder = UrtMetadataBuilder( - title = None, - scribeConfigBuilder = Some( - StaticTimelineScribeConfigBuilder( - TimelineScribeConfig(page = Some("following"), section = None, entityToken = None))) - ) - - UrtDomainMarshaller( - instructionBuilders = instructionBuilders, - metadataBuilder = Some(metadataBuilder), - cursorBuilders = Seq(topCursorBuilder, bottomCursorBuilder, gapCursorBuilder) - ) - } - - override val transportMarshaller: TransportMarshaller[Timeline, urt.TimelineResponse] = - urtTransportMarshaller -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingProductPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingProductPipelineConfig.scala deleted file mode 100644 index 28c7cd6a0..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingProductPipelineConfig.scala +++ /dev/null @@ -1,131 +0,0 @@ -package com.twitter.home_mixer.product.following - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.marshaller.timelines.ChronologicalCursorUnmarshaller -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.home_mixer.model.request.FollowingProductContext -import com.twitter.home_mixer.model.request.HomeMixerRequest -import com.twitter.home_mixer.product.following.model.FollowingQuery -import com.twitter.home_mixer.product.following.param.FollowingParam.ServerMaxResultsParam -import com.twitter.home_mixer.product.following.param.FollowingParamConfig -import com.twitter.home_mixer.service.HomeMixerAccessPolicy.DefaultHomeMixerAccessPolicy -import com.twitter.home_mixer.service.HomeMixerAlertConfig.DefaultNotificationGroup -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer -import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.common.alert.EmptyResponseRateAlert -import com.twitter.product_mixer.core.functional_component.common.alert.LatencyAlert -import com.twitter.product_mixer.core.functional_component.common.alert.P99 -import com.twitter.product_mixer.core.functional_component.common.alert.SuccessRateAlert -import com.twitter.product_mixer.core.functional_component.common.alert.ThroughputAlert -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfAbove -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfBelow -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfLatencyAbove -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.Product -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor -import com.twitter.product_mixer.core.pipeline.PipelineConfig -import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest -import com.twitter.product_mixer.core.pipeline.pipeline_failure.MalformedCursor -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.product.ProductPipelineConfig -import com.twitter.product_mixer.core.product.ProductParamConfig -import com.twitter.product_mixer.core.util.SortIndexBuilder -import com.twitter.timelines.configapi.Params -import com.twitter.timelines.render.{thriftscala => urt} -import com.twitter.timelines.util.RequestCursorSerializer -import com.twitter.util.Time -import com.twitter.util.Try -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class FollowingProductPipelineConfig @Inject() ( - followingMixerPipelineConfig: FollowingMixerPipelineConfig, - followingParamConfig: FollowingParamConfig) - extends ProductPipelineConfig[HomeMixerRequest, FollowingQuery, urt.TimelineResponse] { - - override val identifier: ProductPipelineIdentifier = ProductPipelineIdentifier("Following") - - override val product: Product = FollowingProduct - override val paramConfig: ProductParamConfig = followingParamConfig - - override def pipelineQueryTransformer( - request: HomeMixerRequest, - params: Params - ): FollowingQuery = { - val context = request.productContext match { - case Some(context: FollowingProductContext) => context - case _ => throw PipelineFailure(BadRequest, "FollowingProductContext not found") - } - - val debugOptions = request.debugParams.flatMap(_.debugOptions) - - /** - * Unlike other clients, newly created tweets on Android have the sort index set to the current - * time instead of the top sort index + 1, so these tweets get stuck at the top of the timeline - * if subsequent timeline responses use the sort index from the previous response instead of - * the current time. - */ - val pipelineCursor = request.serializedRequestCursor.flatMap { cursor => - Try(UrtCursorSerializer.deserializeOrderedCursor(cursor)) - .getOrElse(ChronologicalCursorUnmarshaller(RequestCursorSerializer.deserialize(cursor))) - .map { - case UrtOrderedCursor(_, id, Some(GapCursor), gapBoundaryId) - if id.isEmpty || gapBoundaryId.isEmpty => - throw PipelineFailure(MalformedCursor, "Gap Cursor bounds not defined") - case topCursor @ UrtOrderedCursor(_, _, Some(TopCursor), _) => - val queryTime = debugOptions.flatMap(_.requestTimeOverride).getOrElse(Time.now) - topCursor.copy(initialSortIndex = SortIndexBuilder.timeToId(queryTime)) - case cursor => cursor - } - } - - FollowingQuery( - params = params, - clientContext = request.clientContext, - features = None, - pipelineCursor = pipelineCursor, - requestedMaxResults = Some(params(ServerMaxResultsParam)), - debugOptions = debugOptions, - deviceContext = context.deviceContext, - seenTweetIds = context.seenTweetIds, - dspClientContext = context.dspClientContext - ) - } - - override val pipelines: Seq[PipelineConfig] = Seq(followingMixerPipelineConfig) - - override def pipelineSelector( - query: FollowingQuery - ): ComponentIdentifier = followingMixerPipelineConfig.identifier - - override val alerts: Seq[Alert] = Seq( - SuccessRateAlert( - notificationGroup = DefaultNotificationGroup, - warnPredicate = TriggerIfBelow(99.9, 20, 30), - criticalPredicate = TriggerIfBelow(99.9, 30, 30), - ), - LatencyAlert( - notificationGroup = DefaultNotificationGroup, - percentile = P99, - warnPredicate = TriggerIfLatencyAbove(1100.millis, 15, 30), - criticalPredicate = TriggerIfLatencyAbove(1200.millis, 15, 30) - ), - ThroughputAlert( - notificationGroup = DefaultNotificationGroup, - warnPredicate = TriggerIfAbove(18000), - criticalPredicate = TriggerIfAbove(20000) - ), - EmptyResponseRateAlert( - notificationGroup = DefaultNotificationGroup, - warnPredicate = TriggerIfAbove(65), - criticalPredicate = TriggerIfAbove(80) - ) - ) - - override val debugAccessPolicies: Set[AccessPolicy] = DefaultHomeMixerAccessPolicy -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingWhoToFollowCandidatePipelineConfigBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingWhoToFollowCandidatePipelineConfigBuilder.scala deleted file mode 100644 index cd5b1795d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingWhoToFollowCandidatePipelineConfigBuilder.scala +++ /dev/null @@ -1,65 +0,0 @@ -package com.twitter.home_mixer.product.following - -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeWhoToFollowFeedbackActionInfoBuilder -import com.twitter.home_mixer.functional_component.gate.DismissFatigueGate -import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate -import com.twitter.home_mixer.model.HomeFeatures.DismissInfoFeature -import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature -import com.twitter.home_mixer.model.HomeFeatures.WhoToFollowExcludedUserIdsFeature -import com.twitter.home_mixer.product.following.model.FollowingQuery -import com.twitter.home_mixer.product.following.param.FollowingParam.EnableWhoToFollowCandidatePipelineParam -import com.twitter.home_mixer.product.following.param.FollowingParam.WhoToFollowDisplayLocationParam -import com.twitter.home_mixer.product.following.param.FollowingParam.WhoToFollowDisplayTypeIdParam -import com.twitter.home_mixer.product.following.param.FollowingParam.WhoToFollowMinInjectionIntervalParam -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ParamWhoToFollowModuleDisplayTypeBuilder -import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmCandidatePipelineConfig -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmDependentCandidatePipelineConfig -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmDependentCandidatePipelineConfigBuilder -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.configapi.StaticParam -import com.twitter.product_mixer.core.functional_component.gate.BaseGate -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelineservice.model.rich.EntityIdType -import com.twitter.timelineservice.suggests.thriftscala.SuggestType -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class FollowingWhoToFollowCandidatePipelineConfigBuilder @Inject() ( - whoToFollowArmDependentCandidatePipelineConfigBuilder: WhoToFollowArmDependentCandidatePipelineConfigBuilder, - homeWhoToFollowFeedbackActionInfoBuilder: HomeWhoToFollowFeedbackActionInfoBuilder) { - - def build( - requiredNonEmptyPipelines: CandidateScope - ): WhoToFollowArmDependentCandidatePipelineConfig[FollowingQuery] = { - val gates: Seq[BaseGate[PipelineQuery]] = Seq( - TimelinesPersistenceStoreLastInjectionGate( - WhoToFollowMinInjectionIntervalParam, - PersistenceEntriesFeature, - EntityIdType.WhoToFollow - ), - DismissFatigueGate(SuggestType.WhoToFollow, DismissInfoFeature), - NonEmptyCandidatesGate(requiredNonEmptyPipelines) - ) - - whoToFollowArmDependentCandidatePipelineConfigBuilder.build[FollowingQuery]( - identifier = WhoToFollowArmCandidatePipelineConfig.identifier, - supportedClientParam = Some(EnableWhoToFollowCandidatePipelineParam), - alerts = alerts, - gates = gates, - moduleDisplayTypeBuilder = - ParamWhoToFollowModuleDisplayTypeBuilder(WhoToFollowDisplayTypeIdParam), - feedbackActionInfoBuilder = Some(homeWhoToFollowFeedbackActionInfoBuilder), - displayLocationParam = StaticParam(WhoToFollowDisplayLocationParam.default), - excludedUserIdsFeature = Some(WhoToFollowExcludedUserIdsFeature), - profileUserIdFeature = None - ) - } - - private val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(70), - HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/BUILD.bazel deleted file mode 100644 index 70d5e5af9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/BUILD.bazel +++ /dev/null @@ -1,22 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "stringcenter/client", - "stringcenter/client/src/main/java", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/FollowingQuery.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/FollowingQuery.scala deleted file mode 100644 index c45c7cf68..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/FollowingQuery.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.home_mixer.product.following.model - -import com.twitter.adserver.thriftscala.HomeTimelineType -import com.twitter.adserver.thriftscala.TimelineRequestParams -import com.twitter.home_mixer.model.HomeAdsQuery -import com.twitter.dspbidder.commons.{thriftscala => dsp} -import com.twitter.home_mixer.model.request.DeviceContext -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.home_mixer.model.request.HasSeenTweetIds -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.onboarding.task.service.{thriftscala => ots} -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.transformer.HasFlipInjectionParams -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.request._ -import com.twitter.product_mixer.core.pipeline.HasPipelineCursor -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.Params - -case class FollowingQuery( - override val params: Params, - override val clientContext: ClientContext, - override val pipelineCursor: Option[UrtOrderedCursor], - override val requestedMaxResults: Option[Int], - override val debugOptions: Option[DebugOptions], - override val features: Option[FeatureMap], - override val deviceContext: Option[DeviceContext], - override val seenTweetIds: Option[Seq[Long]], - override val dspClientContext: Option[dsp.DspClientContext]) - extends PipelineQuery - with HasPipelineCursor[UrtOrderedCursor] - with HasDeviceContext - with HasSeenTweetIds - with HasFlipInjectionParams - with HomeAdsQuery { - override val product: Product = FollowingProduct - - override def withFeatureMap(features: FeatureMap): FollowingQuery = - copy(features = Some(features)) - - override val timelineRequestParams: Option[TimelineRequestParams] = - Some(TimelineRequestParams(homeTimelineType = Some(HomeTimelineType.HomeLatest))) - - // Fields below are used for FLIP Injection in Onboarding Task Service (OTS) - override val displayLocation: ots.DisplayLocation = ots.DisplayLocation.HomeLatestTimeline - override val rankingDisablerWithLatestControlsAvailable: Option[Boolean] = None - override val isEmptyState: Option[Boolean] = None - override val isFirstRequestAfterSignup: Option[Boolean] = None - override val isEndOfTimeline: Option[Boolean] = None - override val timelineId: Option[Long] = None -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/HomeMixerExternalStrings.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/HomeMixerExternalStrings.scala deleted file mode 100644 index 9c6faafa7..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/HomeMixerExternalStrings.scala +++ /dev/null @@ -1,87 +0,0 @@ -package com.twitter.home_mixer.product.following.model - -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.ExternalStringRegistry -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton - -@Singleton -class HomeMixerExternalStrings @Inject() ( - @ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry]) { - val seeNewTweetsString = - externalStringRegistryProvider.get().createProdString("SeeNewTweets") - val tweetedString = - externalStringRegistryProvider.get().createProdString("Tweeted") - val muteUserString = - externalStringRegistryProvider.get().createProdString("Feedback.muteUser") - val blockUserString = externalStringRegistryProvider.get().createProdString("Feedback.blockUser") - val unfollowUserString = - externalStringRegistryProvider.get().createProdString("Feedback.unfollowUser") - val unfollowUserConfirmationString = - externalStringRegistryProvider.get().createProdString("Feedback.unfollowUserConfirmation") - val reportTweetString = - externalStringRegistryProvider.get().createProdString("Feedback.reportTweet") - val dontLikeString = externalStringRegistryProvider.get().createProdString("Feedback.dontLike") - val dontLikeConfirmationString = - externalStringRegistryProvider.get().createProdString("Feedback.dontLikeConfirmation") - val showFewerTweetsString = - externalStringRegistryProvider.get().createProdString("Feedback.showFewerTweets") - val showFewerTweetsConfirmationString = - externalStringRegistryProvider.get().createProdString("Feedback.showFewerTweetsConfirmation") - val showFewerRetweetsString = - externalStringRegistryProvider.get().createProdString("Feedback.showFewerRetweets") - val showFewerRetweetsConfirmationString = - externalStringRegistryProvider.get().createProdString("Feedback.showFewerRetweetsConfirmation") - val notRelevantString = - externalStringRegistryProvider.get().createProdString("Feedback.notRelevant") - val notRelevantConfirmationString = - externalStringRegistryProvider.get().createProdString("Feedback.notRelevantConfirmation") - - val socialContextOneUserLikedString = - externalStringRegistryProvider.get().createProdString("SocialContext.oneUserLiked") - val socialContextTwoUsersLikedString = - externalStringRegistryProvider.get().createProdString("SocialContext.twoUsersLiked") - val socialContextMoreUsersLikedString = - externalStringRegistryProvider.get().createProdString("SocialContext.moreUsersLiked") - val socialContextLikedByTimelineTitle = - externalStringRegistryProvider.get().createProdString("SocialContext.likedByTimelineTitle") - - val socialContextOneUserFollowsString = - externalStringRegistryProvider.get().createProdString("SocialContext.oneUserFollows") - val socialContextTwoUsersFollowString = - externalStringRegistryProvider.get().createProdString("SocialContext.twoUsersFollow") - val socialContextMoreUsersFollowString = - externalStringRegistryProvider.get().createProdString("SocialContext.moreUsersFollow") - val socialContextFollowedByTimelineTitle = - externalStringRegistryProvider.get().createProdString("SocialContext.followedByTimelineTitle") - - val socialContextYouMightLikeString = - externalStringRegistryProvider.get().createProdString("SocialContext.youMightLike") - - val socialContextExtendedReply = - externalStringRegistryProvider.get().createProdString("SocialContext.extendedReply") - val socialContextReceivedReply = - externalStringRegistryProvider.get().createProdString("SocialContext.receivedReply") - - val socialContextPopularVideoString = - externalStringRegistryProvider.get().createProdString("SocialContext.popularVideo") - - val socialContextPopularInYourAreaString = - externalStringRegistryProvider.get().createProdString("PopgeoTweet.socialProof") - - val listToFollowModuleHeaderString = - externalStringRegistryProvider.get().createProdString("ListToFollowModule.header") - val listToFollowModuleFooterString = - externalStringRegistryProvider.get().createProdString("ListToFollowModule.footer") - val pinnedListsModuleHeaderString = - externalStringRegistryProvider.get().createProdString("PinnedListModule.header") - val pinnedListsModuleEmptyStateMessageString = - externalStringRegistryProvider.get().createProdString("PinnedListModule.emptyStateMessage") - - val ownedSubscribedListsModuleHeaderString = - externalStringRegistryProvider.get().createProdString("OwnedSubscribedListModule.header") - val ownedSubscribedListsModuleEmptyStateMessageString = - externalStringRegistryProvider - .get().createProdString("OwnedSubscribedListModule.emptyStateMessage") -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/BUILD.bazel deleted file mode 100644 index a56e3a1fd..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/BUILD.bazel +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "util/util-core/src/main/scala/com/twitter/conversions", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/FollowingParam.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/FollowingParam.scala deleted file mode 100644 index e43990507..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/FollowingParam.scala +++ /dev/null @@ -1,85 +0,0 @@ -package com.twitter.home_mixer.product.following.param - -import com.twitter.conversions.DurationOps._ -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.WhoToFollowModuleDisplayType -import com.twitter.timelines.configapi.DurationConversion -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSEnumParam -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.util.Duration - -object FollowingParam { - val SupportedClientFSName = "following_supported_client" - - object ServerMaxResultsParam - extends FSBoundedParam[Int]( - name = "following_server_max_results", - default = 100, - min = 1, - max = 500 - ) - - object EnableWhoToFollowCandidatePipelineParam - extends FSParam[Boolean]( - name = "following_enable_who_to_follow", - default = true - ) - - object EnableAdsCandidatePipelineParam - extends FSParam[Boolean]( - name = "following_enable_ads", - default = true - ) - - object EnableFlipInjectionModuleCandidatePipelineParam - extends FSParam[Boolean]( - name = "following_enable_flip_inline_injection_module", - default = true - ) - - object FlipInlineInjectionModulePosition - extends FSBoundedParam[Int]( - name = "following_flip_inline_injection_module_position", - default = 0, - min = 0, - max = 1000 - ) - - object WhoToFollowPositionParam - extends FSBoundedParam[Int]( - name = "following_who_to_follow_position", - default = 5, - min = 0, - max = 99 - ) - - object WhoToFollowMinInjectionIntervalParam - extends FSBoundedParam[Duration]( - "following_who_to_follow_min_injection_interval_in_minutes", - default = 1800.minutes, - min = 0.minutes, - max = 6000.minutes) - with HasDurationConversion { - override val durationConversion: DurationConversion = DurationConversion.FromMinutes - } - - object WhoToFollowDisplayTypeIdParam - extends FSEnumParam[WhoToFollowModuleDisplayType.type]( - name = "following_enable_who_to_follow_display_type_id", - default = WhoToFollowModuleDisplayType.Vertical, - enum = WhoToFollowModuleDisplayType - ) - - object WhoToFollowDisplayLocationParam - extends FSParam[String]( - name = "following_who_to_follow_display_location", - default = "timeline_reverse_chron" - ) - - object EnableFastAds - extends FSParam[Boolean]( - name = "following_enable_fast_ads", - default = true - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/FollowingParamConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/FollowingParamConfig.scala deleted file mode 100644 index df3b54801..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/FollowingParamConfig.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.home_mixer.product.following.param - -import com.twitter.home_mixer.param.decider.DeciderKey -import com.twitter.home_mixer.product.following.param.FollowingParam._ -import com.twitter.product_mixer.core.product.ProductParamConfig -import com.twitter.servo.decider.DeciderKeyName -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class FollowingParamConfig @Inject() () extends ProductParamConfig { - override val enabledDeciderKey: DeciderKeyName = DeciderKey.EnableFollowingProduct - override val supportedClientFSName: String = SupportedClientFSName - - override val booleanFSOverrides = - Seq( - EnableFlipInjectionModuleCandidatePipelineParam, - EnableWhoToFollowCandidatePipelineParam, - EnableAdsCandidatePipelineParam, - EnableFastAds, - ) - - override val boundedIntFSOverrides = Seq( - FlipInlineInjectionModulePosition, - WhoToFollowPositionParam, - ServerMaxResultsParam - ) - - override val stringFSOverrides = Seq(WhoToFollowDisplayLocationParam) - - override val boundedDurationFSOverrides = Seq(WhoToFollowMinInjectionIntervalParam) - - override val enumFSOverrides = Seq(WhoToFollowDisplayTypeIdParam) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/BUILD.bazel deleted file mode 100644 index 5f597c78b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/BUILD.bazel +++ /dev/null @@ -1,101 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "ads-injection/lib/src/main/scala/com/twitter/goldfinch/api", - "finagle/finagle-memcached/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/scorer", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/earlybird", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_scorer", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_service", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_tweetypie", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/async", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/param_gated", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_subscribe_module", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tweet_tlx", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", - "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/product_pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/item", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor", - "src/java/com/twitter/search/common/util/lang", - "src/scala/com/twitter/suggests/controller_data", - "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/search/common:constants-java", - "src/thrift/com/twitter/timelines/render:thrift-scala", - "src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java", - "src/thrift/com/twitter/timelineservice:thrift-scala", - "stitch/stitch-tweetypie", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", - "timelines/src/main/scala/com/twitter/timelines/injection/scribe", - "timelines/src/main/scala/com/twitter/timelines/model/candidate", - ], - exports = [ - "src/thrift/com/twitter/timelines/render:thrift-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsCandidatePipelineBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsCandidatePipelineBuilder.scala deleted file mode 100644 index 99ed0f584..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsCandidatePipelineBuilder.scala +++ /dev/null @@ -1,94 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedCandidateFeatureHydrator -import com.twitter.adserver.{thriftscala => ads} -import com.twitter.home_mixer.functional_component.decorator.builder.HomeAdsClientEventDetailsBuilder -import com.twitter.home_mixer.functional_component.gate.ExcludeSoftUserGate -import com.twitter.home_mixer.param.HomeGlobalParams -import com.twitter.home_mixer.param.HomeGlobalParams.EnableAdvertiserBrandSafetySettingsFeatureHydratorParam -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.home_mixer.product.for_you.param.ForYouParam.AdsNumOrganicItemsParam -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.candidate_source.ads.AdsProdThriftCandidateSource -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.ad.AdsCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder -import com.twitter.product_mixer.component_library.feature_hydrator.candidate.ads.AdvertiserBrandSafetySettingsFeatureHydrator -import com.twitter.product_mixer.component_library.model.candidate.ads.AdsCandidate -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsCandidatePipelineConfig -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsCandidatePipelineConfigBuilder -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.StaticAdsDisplayLocationBuilder -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.ValidAdImpressionIdFilter -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.gate.ParamNotGate -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.rtf.safety_level.TimelineHomePromotedHydrationSafetyLevel -import com.twitter.product_mixer.core.model.marshalling.response.urt.contextual_ref.TweetHydrationContext -import com.twitter.timelines.injection.scribe.InjectionScribeUtil -import com.twitter.timelineservice.suggests.{thriftscala => st} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ForYouAdsCandidatePipelineBuilder @Inject() ( - adsCandidatePipelineConfigBuilder: AdsCandidatePipelineConfigBuilder, - adsCandidateSource: AdsProdThriftCandidateSource, - advertiserBrandSafetySettingsFeatureHydrator: AdvertiserBrandSafetySettingsFeatureHydrator[ - ForYouQuery, - AdsCandidate - ]) { - - private val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ForYouAds") - - private val suggestType = st.SuggestType.Promoted - - private val clientEventInfoBuilder = ClientEventInfoBuilder( - component = InjectionScribeUtil.scribeComponent(suggestType).get, - detailsBuilder = Some(HomeAdsClientEventDetailsBuilder(Some(suggestType.name))) - ) - - private val contextualTweetRefBuilder = ContextualTweetRefBuilder( - TweetHydrationContext( - safetyLevelOverride = Some(TimelineHomePromotedHydrationSafetyLevel), - outerTweetContext = None - )) - - private val decorator = UrtItemCandidateDecorator( - AdsCandidateUrtItemBuilder( - tweetClientEventInfoBuilder = Some(clientEventInfoBuilder), - contextualTweetRefBuilder = Some(contextualTweetRefBuilder) - )) - - private val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), - HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() - ) - - def build( - organicCandidatePipelines: Option[CandidateScope] = None - ): AdsCandidatePipelineConfig[ForYouQuery] = - adsCandidatePipelineConfigBuilder.build[ForYouQuery]( - adsCandidateSource = adsCandidateSource, - identifier = identifier, - adsDisplayLocationBuilder = StaticAdsDisplayLocationBuilder(ads.DisplayLocation.TimelineHome), - estimateNumOrganicItems = _.params(AdsNumOrganicItemsParam).toShort, - gates = Seq( - ParamNotGate( - name = "AdsDisableInjectionBasedOnUserRole", - param = HomeGlobalParams.AdsDisableInjectionBasedOnUserRoleParam - ), - ExcludeSoftUserGate - ), - filters = Seq(ValidAdImpressionIdFilter), - postFilterFeatureHydration = Seq( - ParamGatedCandidateFeatureHydrator( - EnableAdvertiserBrandSafetySettingsFeatureHydratorParam, - advertiserBrandSafetySettingsFeatureHydrator - ) - ), - decorator = Some(decorator), - alerts = alerts, - urtRequest = Some(true), - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsDependentCandidatePipelineBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsDependentCandidatePipelineBuilder.scala deleted file mode 100644 index ef65d7062..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsDependentCandidatePipelineBuilder.scala +++ /dev/null @@ -1,111 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.adserver.{thriftscala => ads} -import com.twitter.home_mixer.functional_component.decorator.builder.HomeAdsClientEventDetailsBuilder -import com.twitter.home_mixer.functional_component.gate.ExcludeSoftUserGate -import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature -import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature -import com.twitter.home_mixer.param.HomeGlobalParams -import com.twitter.home_mixer.param.HomeGlobalParams.EnableAdvertiserBrandSafetySettingsFeatureHydratorParam -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.product_mixer.component_library.candidate_source.ads.AdsProdThriftCandidateSource -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.ad.AdsCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder -import com.twitter.product_mixer.component_library.feature_hydrator.candidate.ads.AdvertiserBrandSafetySettingsFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedCandidateFeatureHydrator -import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate -import com.twitter.product_mixer.component_library.model.candidate.ads.AdsCandidate -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsDependentCandidatePipelineConfig -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsDependentCandidatePipelineConfigBuilder -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.CountTruncatedItemCandidatesFromPipelines -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.StaticAdsDisplayLocationBuilder -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.TruncatedPipelineScopedOrganicItems -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.ValidAdImpressionIdFilter -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.gate.ParamNotGate -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.rtf.safety_level.TimelineHomePromotedHydrationSafetyLevel -import com.twitter.product_mixer.core.model.marshalling.response.urt.contextual_ref.TweetHydrationContext -import com.twitter.timelines.injection.scribe.InjectionScribeUtil -import com.twitter.timelineservice.suggests.{thriftscala => st} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ForYouAdsDependentCandidatePipelineBuilder @Inject() ( - adsCandidatePipelineConfigBuilder: AdsDependentCandidatePipelineConfigBuilder, - adsCandidateSource: AdsProdThriftCandidateSource, - advertiserBrandSafetySettingsFeatureHydrator: AdvertiserBrandSafetySettingsFeatureHydrator[ - ForYouQuery, - AdsCandidate - ]) { - - private val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ForYouAdsDependent") - - private val suggestType = st.SuggestType.Promoted - - private val MaxOrganicTweets = 35 - - private val clientEventInfoBuilder = ClientEventInfoBuilder( - component = InjectionScribeUtil.scribeComponent(suggestType).get, - detailsBuilder = Some(HomeAdsClientEventDetailsBuilder(Some(suggestType.name))) - ) - - private val contextualTweetRefBuilder = ContextualTweetRefBuilder( - TweetHydrationContext( - safetyLevelOverride = Some(TimelineHomePromotedHydrationSafetyLevel), - outerTweetContext = None - )) - - private val decorator = UrtItemCandidateDecorator( - AdsCandidateUrtItemBuilder( - tweetClientEventInfoBuilder = Some(clientEventInfoBuilder), - contextualTweetRefBuilder = Some(contextualTweetRefBuilder) - )) - - private val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), - HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() - ) - - def build( - organicCandidatePipelines: CandidateScope - ): AdsDependentCandidatePipelineConfig[ForYouQuery] = - adsCandidatePipelineConfigBuilder.build[ForYouQuery]( - adsCandidateSource = adsCandidateSource, - identifier = identifier, - adsDisplayLocationBuilder = StaticAdsDisplayLocationBuilder(ads.DisplayLocation.TimelineHome), - getOrganicItems = TruncatedPipelineScopedOrganicItems( - pipelines = organicCandidatePipelines, - textFeature = TweetTextFeature, - languageFeature = TweetLanguageFeature, - ordering = CandidatesUtil.scoreOrdering, - maxCount = MaxOrganicTweets - ), - countNumOrganicItems = - CountTruncatedItemCandidatesFromPipelines(organicCandidatePipelines, MaxOrganicTweets), - gates = Seq( - ParamNotGate( - name = "AdsDisableInjectionBasedOnUserRole", - param = HomeGlobalParams.AdsDisableInjectionBasedOnUserRoleParam - ), - ExcludeSoftUserGate, - NonEmptyCandidatesGate(organicCandidatePipelines) - ), - filters = Seq(ValidAdImpressionIdFilter), - postFilterFeatureHydration = Seq( - ParamGatedCandidateFeatureHydrator( - EnableAdvertiserBrandSafetySettingsFeatureHydratorParam, - advertiserBrandSafetySettingsFeatureHydrator - ) - ), - decorator = Some(decorator), - alerts = alerts, - urtRequest = Some(true) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouConversationServiceCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouConversationServiceCandidatePipelineConfig.scala deleted file mode 100644 index 494d3a584..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouConversationServiceCandidatePipelineConfig.scala +++ /dev/null @@ -1,138 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.home_mixer.candidate_pipeline.ConversationServiceResponseFeatureTransformer -import com.twitter.home_mixer.functional_component.decorator.HomeConversationServiceCandidateDecorator -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder -import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator -import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter -import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter -import com.twitter.home_mixer.functional_component.filter.PreviouslyServedTweetsFilter -import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter -import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature -import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource -import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSourceRequest -import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata -import com.twitter.product_mixer.component_library.filter.FeatureFilter -import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter -import com.twitter.product_mixer.component_library.gate.NoCandidatesGate -import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.gate.BaseGate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.functional_component.transformer.DependentCandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Candidate Pipeline Config that fetches Tweet ancestors from Conversation Service Candidate Source - */ -@Singleton -class ForYouConversationServiceCandidatePipelineConfig @Inject() ( - forYouScoredTweetsCandidatePipelineConfig: ForYouScoredTweetsCandidatePipelineConfig, - forYouTimelineScorerCandidatePipelineConfig: ForYouTimelineScorerCandidatePipelineConfig, - conversationServiceCandidateSource: ConversationServiceCandidateSource, - tweetypieFeatureHydrator: TweetypieFeatureHydrator, - namesFeatureHydrator: NamesFeatureHydrator, - invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter, - homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder) - extends DependentCandidatePipelineConfig[ - ForYouQuery, - ConversationServiceCandidateSourceRequest, - TweetWithConversationMetadata, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ForYouConversationService") - - override val gates: Seq[BaseGate[ForYouQuery]] = Seq( - NoCandidatesGate( - SpecificPipelines( - forYouTimelineScorerCandidatePipelineConfig.identifier, - forYouScoredTweetsCandidatePipelineConfig.identifier - ) - ), - NonEmptySeqFeatureGate(TimelineServiceTweetsFeature) - ) - - override val candidateSource: BaseCandidateSource[ - ConversationServiceCandidateSourceRequest, - TweetWithConversationMetadata - ] = conversationServiceCandidateSource - - override val queryTransformer: DependentCandidatePipelineQueryTransformer[ - ForYouQuery, - ConversationServiceCandidateSourceRequest - ] = { (query, candidates) => - val timelineServiceTweets = query.features - .map(_.getOrElse(TimelineServiceTweetsFeature, Seq.empty)).getOrElse(Seq.empty) - - val tweetsWithConversationMetadata = timelineServiceTweets.map { id => - TweetWithConversationMetadata( - tweetId = id, - userId = None, - sourceTweetId = None, - sourceUserId = None, - inReplyToTweetId = None, - conversationId = None, - ancestors = Seq.empty - ) - } - ConversationServiceCandidateSourceRequest(tweetsWithConversationMetadata) - } - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[TweetWithConversationMetadata] - ] = Seq(ConversationServiceResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - TweetWithConversationMetadata, - TweetCandidate - ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } - - override val preFilterFeatureHydrationPhase1: Seq[ - BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] - ] = Seq( - InNetworkFeatureHydrator, - tweetypieFeatureHydrator - ) - - override def filters: Seq[Filter[ForYouQuery, TweetCandidate]] = Seq( - PreviouslyServedTweetsFilter, - RetweetDeduplicationFilter, - FeatureFilter.fromFeature(FilterIdentifier("TweetypieHydrated"), IsHydratedFeature), - PredicateFeatureFilter.fromPredicate( - FilterIdentifier("QuotedTweetDropped"), - shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) } - ), - invalidSubscriptionTweetFilter, - InvalidConversationModuleFilter - ) - - override val postFilterFeatureHydration: Seq[ - BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] - ] = Seq(namesFeatureHydrator) - - override val decorator: Option[CandidateDecorator[ForYouQuery, TweetCandidate]] = - HomeConversationServiceCandidateDecorator(homeFeedbackActionInfoBuilder) - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), - HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouProductPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouProductPipelineConfig.scala deleted file mode 100644 index e1cf2161c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouProductPipelineConfig.scala +++ /dev/null @@ -1,142 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.marshaller.timelines.ChronologicalCursorUnmarshaller -import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.model.request.ForYouProductContext -import com.twitter.home_mixer.model.request.HomeMixerRequest -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnablePushToHomeMixerPipelineParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableScoredTweetsMixerPipelineParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.ServerMaxResultsParam -import com.twitter.home_mixer.product.for_you.param.ForYouParamConfig -import com.twitter.home_mixer.service.HomeMixerAccessPolicy.DefaultHomeMixerAccessPolicy -import com.twitter.home_mixer.service.HomeMixerAlertConfig.DefaultNotificationGroup -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer -import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.common.alert.EmptyResponseRateAlert -import com.twitter.product_mixer.core.functional_component.common.alert.LatencyAlert -import com.twitter.product_mixer.core.functional_component.common.alert.P99 -import com.twitter.product_mixer.core.functional_component.common.alert.SuccessRateAlert -import com.twitter.product_mixer.core.functional_component.common.alert.ThroughputAlert -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfAbove -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfBelow -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfLatencyAbove -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.Product -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor -import com.twitter.product_mixer.core.pipeline.PipelineConfig -import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.product.ProductPipelineConfig -import com.twitter.product_mixer.core.product.ProductParamConfig -import com.twitter.product_mixer.core.util.SortIndexBuilder -import com.twitter.timelines.configapi.Params -import com.twitter.timelines.render.{thriftscala => urt} -import com.twitter.timelines.util.RequestCursorSerializer -import com.twitter.util.Time -import com.twitter.util.Try -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ForYouProductPipelineConfig @Inject() ( - forYouTimelineScorerMixerPipelineConfig: ForYouTimelineScorerMixerPipelineConfig, - forYouScoredTweetsMixerPipelineConfig: ForYouScoredTweetsMixerPipelineConfig, - forYouPushToHomeMixerPipelineConfig: ForYouPushToHomeMixerPipelineConfig, - forYouParamConfig: ForYouParamConfig) - extends ProductPipelineConfig[HomeMixerRequest, ForYouQuery, urt.TimelineResponse] { - - override val identifier: ProductPipelineIdentifier = ProductPipelineIdentifier("ForYou") - - override val product: Product = ForYouProduct - - override val paramConfig: ProductParamConfig = forYouParamConfig - - override def pipelineQueryTransformer( - request: HomeMixerRequest, - params: Params - ): ForYouQuery = { - val context = request.productContext match { - case Some(context: ForYouProductContext) => context - case _ => throw PipelineFailure(BadRequest, "ForYouProductContext not found") - } - - val debugOptions = request.debugParams.flatMap(_.debugOptions) - - /** - * Unlike other clients, newly created tweets on Android have the sort index set to the current - * time instead of the top sort index + 1, so these tweets get stuck at the top of the timeline - * if subsequent timeline responses use the sort index from the previous response instead of - * the current time. - */ - val pipelineCursor = request.serializedRequestCursor.flatMap { cursor => - Try(UrtCursorSerializer.deserializeOrderedCursor(cursor)) - .getOrElse(ChronologicalCursorUnmarshaller(RequestCursorSerializer.deserialize(cursor))) - .map { - case topCursor @ UrtOrderedCursor(_, _, Some(TopCursor), _) => - val queryTime = debugOptions.flatMap(_.requestTimeOverride).getOrElse(Time.now) - topCursor.copy(initialSortIndex = SortIndexBuilder.timeToId(queryTime)) - case cursor => cursor - } - } - - ForYouQuery( - params = params, - clientContext = request.clientContext, - features = None, - pipelineCursor = pipelineCursor, - requestedMaxResults = Some(params(ServerMaxResultsParam)), - debugOptions = debugOptions, - deviceContext = context.deviceContext, - seenTweetIds = context.seenTweetIds, - dspClientContext = context.dspClientContext, - pushToHomeTweetId = context.pushToHomeTweetId - ) - } - - override val pipelines: Seq[PipelineConfig] = Seq( - forYouTimelineScorerMixerPipelineConfig, - forYouScoredTweetsMixerPipelineConfig, - forYouPushToHomeMixerPipelineConfig - ) - - override def pipelineSelector( - query: ForYouQuery - ): ComponentIdentifier = { - if (query.pushToHomeTweetId.isDefined && query.params(EnablePushToHomeMixerPipelineParam)) - forYouPushToHomeMixerPipelineConfig.identifier - else if (query.params(EnableScoredTweetsMixerPipelineParam)) - forYouScoredTweetsMixerPipelineConfig.identifier - else forYouTimelineScorerMixerPipelineConfig.identifier - } - - override val alerts: Seq[Alert] = Seq( - SuccessRateAlert( - notificationGroup = DefaultNotificationGroup, - warnPredicate = TriggerIfBelow(99.9, 20, 30), - criticalPredicate = TriggerIfBelow(99.9, 30, 30), - ), - LatencyAlert( - notificationGroup = DefaultNotificationGroup, - percentile = P99, - warnPredicate = TriggerIfLatencyAbove(2300.millis, 15, 30), - criticalPredicate = TriggerIfLatencyAbove(2800.millis, 15, 30) - ), - ThroughputAlert( - notificationGroup = DefaultNotificationGroup, - warnPredicate = TriggerIfAbove(70000), - criticalPredicate = TriggerIfAbove(80000) - ), - EmptyResponseRateAlert( - notificationGroup = DefaultNotificationGroup, - warnPredicate = TriggerIfAbove(2), - criticalPredicate = TriggerIfAbove(3) - ) - ) - - override val debugAccessPolicies: Set[AccessPolicy] = DefaultHomeMixerAccessPolicy -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPushToHomeMixerPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPushToHomeMixerPipelineConfig.scala deleted file mode 100644 index ac529f582..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPushToHomeMixerPipelineConfig.scala +++ /dev/null @@ -1,74 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.home_mixer.product.for_you.param.ForYouParam -import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.AddEntriesInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ClearCacheInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedBottomCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedTopCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ParamGatedIncludeInstruction -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.StaticTimelineScribeConfigBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtMetadataBuilder -import com.twitter.product_mixer.component_library.selector.InsertAppendResults -import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller -import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.UrtTransportMarshaller -import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig -import com.twitter.timelines.render.{thriftscala => urt} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ForYouPushToHomeMixerPipelineConfig @Inject() ( - forYouPushToHomeTweetCandidatePipelineConfig: ForYouPushToHomeTweetCandidatePipelineConfig, - urtTransportMarshaller: UrtTransportMarshaller) - extends MixerPipelineConfig[ForYouQuery, Timeline, urt.TimelineResponse] { - - override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("ForYouPushToHome") - - override val candidatePipelines: Seq[CandidatePipelineConfig[ForYouQuery, _, _, _]] = - Seq(forYouPushToHomeTweetCandidatePipelineConfig) - - override val resultSelectors: Seq[Selector[ForYouQuery]] = - Seq(InsertAppendResults(forYouPushToHomeTweetCandidatePipelineConfig.identifier)) - - override val domainMarshaller: DomainMarshaller[ForYouQuery, Timeline] = { - val instructionBuilders = Seq( - ClearCacheInstructionBuilder( - ParamGatedIncludeInstruction(ForYouParam.EnableClearCacheOnPushToHome)), - AddEntriesInstructionBuilder()) - - val idSelector: PartialFunction[UniversalNoun[_], Long] = { case item: TweetItem => item.id } - val topCursorBuilder = OrderedTopCursorBuilder(idSelector) - val bottomCursorBuilder = OrderedBottomCursorBuilder(idSelector) - - val metadataBuilder = UrtMetadataBuilder( - title = None, - scribeConfigBuilder = Some( - StaticTimelineScribeConfigBuilder( - TimelineScribeConfig( - page = Some("for_you_push_to_home"), - section = None, - entityToken = None) - ) - ) - ) - - UrtDomainMarshaller( - instructionBuilders = instructionBuilders, - metadataBuilder = Some(metadataBuilder), - cursorBuilders = Seq(topCursorBuilder, bottomCursorBuilder) - ) - } - - override val transportMarshaller: TransportMarshaller[Timeline, urt.TimelineResponse] = - urtTransportMarshaller -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPushToHomeTweetCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPushToHomeTweetCandidatePipelineConfig.scala deleted file mode 100644 index 49df4fdf8..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPushToHomeTweetCandidatePipelineConfig.scala +++ /dev/null @@ -1,81 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventInfoBuilder -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.product.for_you.functional_component.gate.PushToHomeRequestGate -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.functional_component.candidate_source.PassthroughCandidateSource -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.timelineservice.suggests.{thriftscala => st} - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ForYouPushToHomeTweetCandidatePipelineConfig @Inject() () - extends CandidatePipelineConfig[ - ForYouQuery, - ForYouQuery, - TweetCandidate, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ForYouPushToHomeTweet") - - override val gates: Seq[Gate[ForYouQuery]] = Seq(PushToHomeRequestGate) - - override val queryTransformer: CandidatePipelineQueryTransformer[ - ForYouQuery, - ForYouQuery - ] = identity - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[TweetCandidate] - ] = Seq(new CandidateFeatureTransformer[TweetCandidate] { - override def features: Set[Feature[_, _]] = Set(SuggestTypeFeature) - - override val identifier: TransformerIdentifier = - TransformerIdentifier("ForYouPushToHomeTweet") - - override def transform(input: TweetCandidate): FeatureMap = - FeatureMapBuilder().add(SuggestTypeFeature, Some(st.SuggestType.Magicrec)).build() - }) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - TweetCandidate, - TweetCandidate - ] = identity - - override val candidateSource: CandidateSource[ - ForYouQuery, - TweetCandidate - ] = PassthroughCandidateSource( - CandidateSourceIdentifier("PushToHomeTweet"), - { query => query.pushToHomeTweetId.toSeq.map(TweetCandidate(_)) } - ) - - override val decorator: Option[ - CandidateDecorator[ForYouQuery, TweetCandidate] - ] = { - val tweetItemBuilder = TweetCandidateUrtItemBuilder( - clientEventInfoBuilder = HomeClientEventInfoBuilder() - ) - Some(UrtItemCandidateDecorator(tweetItemBuilder)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsCandidatePipelineConfig.scala deleted file mode 100644 index d7716e190..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsCandidatePipelineConfig.scala +++ /dev/null @@ -1,155 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventInfoBuilder -import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder -import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeTweetSocialContextBuilder -import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator -import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter -import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter -import com.twitter.home_mixer.functional_component.gate.SupportedLanguagesGate -import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature -import com.twitter.home_mixer.model.HomeFeatures.IsNsfwFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature -import com.twitter.home_mixer.product.for_you.candidate_source.ScoredTweetWithConversationMetadata -import com.twitter.home_mixer.product.for_you.candidate_source.ScoredTweetsProductCandidateSource -import com.twitter.home_mixer.product.for_you.feature_hydrator.FocalTweetFeatureHydrator -import com.twitter.home_mixer.product.for_you.filter.SocialContextFilter -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableScoredTweetsCandidatePipelineParam -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ManualModuleId -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.StaticModuleDisplayTypeBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder -import com.twitter.product_mixer.component_library.filter.FeatureFilter -import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.VerticalConversation -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.timelines.configapi.decider.DeciderParam -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ForYouScoredTweetsCandidatePipelineConfig @Inject() ( - scoredTweetsProductCandidateSource: ScoredTweetsProductCandidateSource, - focalTweetFeatureHydrator: FocalTweetFeatureHydrator, - namesFeatureHydrator: NamesFeatureHydrator, - tweetypieFeatureHydrator: TweetypieFeatureHydrator, - invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter, - homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder, - homeTweetSocialContextBuilder: HomeTweetSocialContextBuilder) - extends CandidatePipelineConfig[ - ForYouQuery, - ForYouQuery, - ScoredTweetWithConversationMetadata, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ForYouScoredTweets") - - private val TweetypieHydratedFilterId = "TweetypieHydrated" - private val QuotedTweetDroppedFilterId = "QuotedTweetDropped" - private val OutOfNetworkNSFWFilterId = "OutOfNetworkNSFW" - private val ConversationModuleNamespace = EntryNamespace("home-conversation") - - override val gates: Seq[Gate[ForYouQuery]] = Seq(SupportedLanguagesGate) - - override val candidateSource: CandidateSource[ForYouQuery, ScoredTweetWithConversationMetadata] = - scoredTweetsProductCandidateSource - - override val enabledDeciderParam: Option[DeciderParam[Boolean]] = - Some(EnableScoredTweetsCandidatePipelineParam) - - override val queryTransformer: CandidatePipelineQueryTransformer[ForYouQuery, ForYouQuery] = - identity - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[ScoredTweetWithConversationMetadata] - ] = Seq(ForYouScoredTweetsResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - ScoredTweetWithConversationMetadata, - TweetCandidate - ] = { sourceResults => TweetCandidate(sourceResults.tweetId) } - - override val preFilterFeatureHydrationPhase1: Seq[ - BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] - ] = Seq(InNetworkFeatureHydrator, namesFeatureHydrator, tweetypieFeatureHydrator) - - override val filters: Seq[Filter[ForYouQuery, TweetCandidate]] = Seq( - FeatureFilter.fromFeature(FilterIdentifier(TweetypieHydratedFilterId), IsHydratedFeature), - PredicateFeatureFilter.fromPredicate( - FilterIdentifier(QuotedTweetDroppedFilterId), - shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) } - ), - PredicateFeatureFilter.fromPredicate( - FilterIdentifier(OutOfNetworkNSFWFilterId), - shouldKeepCandidate = { features => - features.getOrElse(InNetworkFeature, false) || - !features.getOrElse(IsNsfwFeature, false) - } - ), - SocialContextFilter, - invalidSubscriptionTweetFilter, - InvalidConversationModuleFilter - ) - - override val postFilterFeatureHydration: Seq[ - BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] - ] = Seq(focalTweetFeatureHydrator) - - override val decorator: Option[CandidateDecorator[ForYouQuery, TweetCandidate]] = { - val clientEventInfoBuilder = HomeClientEventInfoBuilder() - - val tweetItemBuilder = TweetCandidateUrtItemBuilder( - clientEventInfoBuilder = clientEventInfoBuilder, - socialContextBuilder = Some(homeTweetSocialContextBuilder), - timelinesScoreInfoBuilder = Some(HomeTimelinesScoreInfoBuilder), - feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder) - ) - - val tweetDecorator = UrtItemCandidateDecorator(tweetItemBuilder) - - val moduleBuilder = TimelineModuleBuilder( - entryNamespace = ConversationModuleNamespace, - clientEventInfoBuilder = clientEventInfoBuilder, - moduleIdGeneration = ManualModuleId(0L), - displayTypeBuilder = StaticModuleDisplayTypeBuilder(VerticalConversation), - metadataBuilder = Some(HomeConversationModuleMetadataBuilder()) - ) - - Some( - UrtMultipleModulesDecorator( - urtItemCandidateDecorator = tweetDecorator, - moduleBuilder = moduleBuilder, - groupByKey = (_, _, candidateFeatures) => - candidateFeatures.getOrElse(ConversationModuleFocalTweetIdFeature, None) - )) - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), - HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert(10, 20) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsMixerPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsMixerPipelineConfig.scala deleted file mode 100644 index 2137c5267..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsMixerPipelineConfig.scala +++ /dev/null @@ -1,376 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.clientapp.{thriftscala => ca} -import com.twitter.goldfinch.api.AdsInjectionSurfaceAreas -import com.twitter.home_mixer.candidate_pipeline.EditedTweetsCandidatePipelineConfig -import com.twitter.home_mixer.candidate_pipeline.NewTweetsPillCandidatePipelineConfig -import com.twitter.home_mixer.functional_component.decorator.urt.builder.AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder -import com.twitter.home_mixer.functional_component.feature_hydrator._ -import com.twitter.home_mixer.functional_component.selector.DebunchCandidates -import com.twitter.home_mixer.functional_component.selector.UpdateConversationModuleId -import com.twitter.home_mixer.functional_component.selector.UpdateHomeClientEventDetails -import com.twitter.home_mixer.functional_component.selector.UpdateNewTweetsPillDecoration -import com.twitter.home_mixer.functional_component.side_effect._ -import com.twitter.home_mixer.model.ClearCacheIncludeInstruction -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.param.HomeGlobalParams.MaxNumberReplaceInstructionsParam -import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeClientEventsFlag -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.home_mixer.product.for_you.feature_hydrator.TimelineServiceTweetsQueryFeatureHydrator -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.home_mixer.product.for_you.param.ForYouParam.ClearCacheOnPtr -import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableFlipInjectionModuleCandidatePipelineParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.FlipInlineInjectionModulePosition -import com.twitter.home_mixer.product.for_you.param.ForYouParam.ServerMaxResultsParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.TweetPreviewsPositionParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToFollowPositionParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToSubscribePositionParam -import com.twitter.home_mixer.product.for_you.side_effect.ServedCandidateFeatureKeysKafkaSideEffectBuilder -import com.twitter.home_mixer.product.for_you.side_effect.ServedCandidateKeysKafkaSideEffectBuilder -import com.twitter.home_mixer.product.for_you.side_effect.ServedStatsSideEffect -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.inject.annotations.Flag -import com.twitter.logpipeline.client.common.EventPublisher -import com.twitter.product_mixer.component_library.feature_hydrator.query.async.AsyncQueryFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.PreviewCreatorsQueryFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.FlipPromptCandidatePipelineConfigBuilder -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidatePipelineConfig -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidatePipelineConfig -import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ClearCacheInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedBottomCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedTopCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceAllEntries -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceEntryInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowAlertInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowCoverInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.StaticTimelineScribeConfigBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtMetadataBuilder -import com.twitter.product_mixer.component_library.selector.DropMaxCandidates -import com.twitter.product_mixer.component_library.selector.DropMaxModuleItemCandidates -import com.twitter.product_mixer.component_library.selector.DropModuleTooFewModuleItemResults -import com.twitter.product_mixer.component_library.selector.InsertAppendResults -import com.twitter.product_mixer.component_library.selector.InsertFixedPositionResults -import com.twitter.product_mixer.component_library.selector.SelectConditionally -import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates -import com.twitter.product_mixer.component_library.selector.UpdateSortModuleItemCandidates -import com.twitter.product_mixer.component_library.selector.ads.AdsInjector -import com.twitter.product_mixer.component_library.selector.ads.InsertAdResults -import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.configapi.StaticParam -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller -import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.UrtTransportMarshaller -import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.product_mixer.core.pipeline.FailOpenPolicy -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import com.twitter.timelines.render.{thriftscala => urt} -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton - -@Singleton -class ForYouScoredTweetsMixerPipelineConfig @Inject() ( - forYouAdsDependentCandidatePipelineBuilder: ForYouAdsDependentCandidatePipelineBuilder, - forYouConversationServiceCandidatePipelineConfig: ForYouConversationServiceCandidatePipelineConfig, - forYouPushToHomeTweetCandidatePipelineConfig: ForYouPushToHomeTweetCandidatePipelineConfig, - forYouScoredTweetsCandidatePipelineConfig: ForYouScoredTweetsCandidatePipelineConfig, - forYouWhoToFollowCandidatePipelineConfigBuilder: ForYouWhoToFollowCandidatePipelineConfigBuilder, - forYouWhoToSubscribeCandidatePipelineConfigBuilder: ForYouWhoToSubscribeCandidatePipelineConfigBuilder, - flipPromptCandidatePipelineConfigBuilder: FlipPromptCandidatePipelineConfigBuilder, - editedTweetsCandidatePipelineConfig: EditedTweetsCandidatePipelineConfig, - newTweetsPillCandidatePipelineConfig: NewTweetsPillCandidatePipelineConfig[ForYouQuery], - forYouTweetPreviewsCandidatePipelineConfig: ForYouTweetPreviewsCandidatePipelineConfig, - dismissInfoQueryFeatureHydrator: DismissInfoQueryFeatureHydrator, - gizmoduckUserQueryFeatureHydrator: GizmoduckUserQueryFeatureHydrator, - persistenceStoreQueryFeatureHydrator: PersistenceStoreQueryFeatureHydrator, - requestQueryFeatureHydrator: RequestQueryFeatureHydrator[ForYouQuery], - timelineServiceTweetsQueryFeatureHydrator: TimelineServiceTweetsQueryFeatureHydrator, - previewCreatorsQueryFeatureHydrator: PreviewCreatorsQueryFeatureHydrator, - sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, - adsInjector: AdsInjector, - servedCandidateKeysKafkaSideEffectBuilder: ServedCandidateKeysKafkaSideEffectBuilder, - servedCandidateFeatureKeysKafkaSideEffectBuilder: ServedCandidateFeatureKeysKafkaSideEffectBuilder, - updateTimelinesPersistenceStoreSideEffect: UpdateTimelinesPersistenceStoreSideEffect, - truncateTimelinesPersistenceStoreSideEffect: TruncateTimelinesPersistenceStoreSideEffect, - homeScribeServedCandidatesSideEffect: HomeScribeServedCandidatesSideEffect, - servedStatsSideEffect: ServedStatsSideEffect, - clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent], - externalStrings: HomeMixerExternalStrings, - @ProductScoped stringCenterProvider: Provider[StringCenter], - urtTransportMarshaller: UrtTransportMarshaller, - @Flag(ScribeClientEventsFlag) enableScribeClientEvents: Boolean) - extends MixerPipelineConfig[ForYouQuery, Timeline, urt.TimelineResponse] { - - override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("ForYouScoredTweets") - - private val MaxConsecutiveOutOfNetworkCandidates = 2 - - private val PushToHomeTweetPosition = 0 - - private val dependentCandidatesStep = MixerPipelineConfig.dependentCandidatePipelinesStep - - override val fetchQueryFeatures: Seq[QueryFeatureHydrator[ForYouQuery]] = Seq( - requestQueryFeatureHydrator, - persistenceStoreQueryFeatureHydrator, - timelineServiceTweetsQueryFeatureHydrator, - previewCreatorsQueryFeatureHydrator, - sgsFollowedUsersQueryFeatureHydrator, - AsyncQueryFeatureHydrator(dependentCandidatesStep, dismissInfoQueryFeatureHydrator), - AsyncQueryFeatureHydrator(dependentCandidatesStep, gizmoduckUserQueryFeatureHydrator), - ) - - private val scoredTweetsCandidatePipelineScope = - SpecificPipeline(forYouScoredTweetsCandidatePipelineConfig.identifier) - - private val forYouAdsCandidatePipelineConfig = forYouAdsDependentCandidatePipelineBuilder - .build(scoredTweetsCandidatePipelineScope) - - private val forYouWhoToFollowCandidatePipelineConfig = - forYouWhoToFollowCandidatePipelineConfigBuilder.build() - - private val forYouWhoToSubscribeCandidatePipelineConfig = - forYouWhoToSubscribeCandidatePipelineConfigBuilder.build() - - private val flipPromptCandidatePipelineConfig = - flipPromptCandidatePipelineConfigBuilder.build[ForYouQuery]( - supportedClientParam = Some(EnableFlipInjectionModuleCandidatePipelineParam) - ) - - override val candidatePipelines: Seq[CandidatePipelineConfig[ForYouQuery, _, _, _]] = Seq( - forYouScoredTweetsCandidatePipelineConfig, - forYouPushToHomeTweetCandidatePipelineConfig, - forYouWhoToFollowCandidatePipelineConfig, - forYouWhoToSubscribeCandidatePipelineConfig, - forYouTweetPreviewsCandidatePipelineConfig, - flipPromptCandidatePipelineConfig - ) - - override val dependentCandidatePipelines: Seq[ - DependentCandidatePipelineConfig[ForYouQuery, _, _, _] - ] = Seq( - forYouAdsCandidatePipelineConfig, - forYouConversationServiceCandidatePipelineConfig, - editedTweetsCandidatePipelineConfig, - newTweetsPillCandidatePipelineConfig - ) - - override val failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = Map( - forYouScoredTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - forYouAdsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - forYouWhoToFollowCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - forYouWhoToSubscribeCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - forYouTweetPreviewsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - flipPromptCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - editedTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - newTweetsPillCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - ) - - override val resultSelectors: Seq[Selector[ForYouQuery]] = Seq( - UpdateSortCandidates( - ordering = CandidatesUtil.reverseChronTweetsOrdering, - candidatePipeline = forYouConversationServiceCandidatePipelineConfig.identifier - ), - UpdateSortCandidates( - ordering = CandidatesUtil.scoreOrdering, - candidatePipeline = forYouScoredTweetsCandidatePipelineConfig.identifier - ), - UpdateSortModuleItemCandidates( - candidatePipeline = forYouScoredTweetsCandidatePipelineConfig.identifier, - ordering = CandidatesUtil.conversationModuleTweetsOrdering - ), - DebunchCandidates( - pipelineScope = SpecificPipeline(forYouScoredTweetsCandidatePipelineConfig.identifier), - mustDebunch = { - case item: ItemCandidateWithDetails => - !item.features.getOrElse(InNetworkFeature, false) - case module: ModuleCandidateWithDetails => - !module.candidates.last.features.getOrElse(InNetworkFeature, false) - }, - maxBunchSize = MaxConsecutiveOutOfNetworkCandidates - ), - UpdateConversationModuleId( - pipelineScope = SpecificPipeline(forYouScoredTweetsCandidatePipelineConfig.identifier) - ), - DropMaxCandidates( - candidatePipeline = forYouConversationServiceCandidatePipelineConfig.identifier, - maxSelectionsParam = ServerMaxResultsParam - ), - DropMaxCandidates( - candidatePipeline = forYouScoredTweetsCandidatePipelineConfig.identifier, - maxSelectionsParam = ServerMaxResultsParam - ), - DropMaxCandidates( - candidatePipeline = editedTweetsCandidatePipelineConfig.identifier, - maxSelectionsParam = MaxNumberReplaceInstructionsParam - ), - DropMaxModuleItemCandidates( - candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, - maxModuleItemsParam = StaticParam(WhoToFollowCandidatePipelineConfig.MaxCandidatesSize) - ), - DropMaxModuleItemCandidates( - candidatePipeline = forYouWhoToSubscribeCandidatePipelineConfig.identifier, - maxModuleItemsParam = StaticParam(WhoToSubscribeCandidatePipelineConfig.MaxCandidatesSize) - ), - // The Conversation Service pipeline will only run if the Scored Tweets pipeline returned nothing - InsertAppendResults( - candidatePipeline = forYouConversationServiceCandidatePipelineConfig.identifier - ), - InsertAppendResults( - candidatePipeline = forYouScoredTweetsCandidatePipelineConfig.identifier - ), - InsertFixedPositionResults( - candidatePipeline = forYouTweetPreviewsCandidatePipelineConfig.identifier, - positionParam = TweetPreviewsPositionParam - ), - InsertFixedPositionResults( - candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, - positionParam = WhoToFollowPositionParam - ), - InsertFixedPositionResults( - candidatePipeline = forYouWhoToSubscribeCandidatePipelineConfig.identifier, - positionParam = WhoToSubscribePositionParam - ), - InsertFixedPositionResults( - candidatePipeline = flipPromptCandidatePipelineConfig.identifier, - positionParam = FlipInlineInjectionModulePosition - ), - // Insert Push To Home Tweet at top of Timeline - InsertFixedPositionResults( - candidatePipeline = forYouPushToHomeTweetCandidatePipelineConfig.identifier, - positionParam = StaticParam(PushToHomeTweetPosition) - ), - InsertAdResults( - surfaceAreaName = AdsInjectionSurfaceAreas.HomeTimeline, - adsInjector = adsInjector.forSurfaceArea(AdsInjectionSurfaceAreas.HomeTimeline), - adsCandidatePipeline = forYouAdsCandidatePipelineConfig.identifier - ), - // This selector must come after the tweets are inserted into the results - DropModuleTooFewModuleItemResults( - candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, - minModuleItemsParam = StaticParam(WhoToFollowCandidatePipelineConfig.MinCandidatesSize) - ), - DropModuleTooFewModuleItemResults( - candidatePipeline = forYouWhoToSubscribeCandidatePipelineConfig.identifier, - minModuleItemsParam = StaticParam(WhoToSubscribeCandidatePipelineConfig.MinCandidatesSize) - ), - UpdateNewTweetsPillDecoration( - pipelineScope = SpecificPipelines( - forYouConversationServiceCandidatePipelineConfig.identifier, - forYouScoredTweetsCandidatePipelineConfig.identifier, - newTweetsPillCandidatePipelineConfig.identifier - ), - stringCenter = stringCenterProvider.get(), - seeNewTweetsString = externalStrings.seeNewTweetsString, - tweetedString = externalStrings.tweetedString - ), - InsertAppendResults(candidatePipeline = editedTweetsCandidatePipelineConfig.identifier), - SelectConditionally( - selector = - InsertAppendResults(candidatePipeline = newTweetsPillCandidatePipelineConfig.identifier), - includeSelector = (_, _, results) => CandidatesUtil.containsType[TweetCandidate](results) - ), - UpdateHomeClientEventDetails( - candidatePipelines = Set( - forYouConversationServiceCandidatePipelineConfig.identifier, - forYouScoredTweetsCandidatePipelineConfig.identifier - ) - ), - ) - - private val servedCandidateKeysKafkaSideEffect = - servedCandidateKeysKafkaSideEffectBuilder.build( - Set(forYouScoredTweetsCandidatePipelineConfig.identifier)) - - private val servedCandidateFeatureKeysKafkaSideEffect = - servedCandidateFeatureKeysKafkaSideEffectBuilder.build( - Set(forYouScoredTweetsCandidatePipelineConfig.identifier)) - - private val homeScribeClientEventSideEffect = HomeScribeClientEventSideEffect( - enableScribeClientEvents = enableScribeClientEvents, - logPipelinePublisher = clientEventsScribeEventPublisher, - injectedTweetsCandidatePipelineIdentifiers = Seq( - forYouScoredTweetsCandidatePipelineConfig.identifier, - forYouConversationServiceCandidatePipelineConfig.identifier - ), - adsCandidatePipelineIdentifier = Some(forYouAdsCandidatePipelineConfig.identifier), - whoToFollowCandidatePipelineIdentifier = Some( - forYouWhoToFollowCandidatePipelineConfig.identifier - ), - whoToSubscribeCandidatePipelineIdentifier = - Some(forYouWhoToSubscribeCandidatePipelineConfig.identifier) - ) - - override val resultSideEffects: Seq[PipelineResultSideEffect[ForYouQuery, Timeline]] = Seq( - servedCandidateKeysKafkaSideEffect, - servedCandidateFeatureKeysKafkaSideEffect, - updateTimelinesPersistenceStoreSideEffect, - truncateTimelinesPersistenceStoreSideEffect, - homeScribeClientEventSideEffect, - homeScribeServedCandidatesSideEffect, - servedStatsSideEffect - ) - - override val domainMarshaller: DomainMarshaller[ForYouQuery, Timeline] = { - val instructionBuilders = Seq( - ClearCacheInstructionBuilder( - ClearCacheIncludeInstruction( - ClearCacheOnPtr.EnableParam, - ClearCacheOnPtr.MinEntriesParam, - ) - ), - ReplaceEntryInstructionBuilder(ReplaceAllEntries), - // excludes alert, cover, and replace candidates - AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder(), - ShowAlertInstructionBuilder(), - ShowCoverInstructionBuilder(), - ) - - val idSelector: PartialFunction[UniversalNoun[_], Long] = { - // exclude ads while determining tweet cursor values - case item: TweetItem if item.promotedMetadata.isEmpty => item.id - case module: TimelineModule - if module.items.headOption.exists(_.item.isInstanceOf[TweetItem]) => - module.items.last.item match { case item: TweetItem => item.id } - } - val topCursorBuilder = OrderedTopCursorBuilder(idSelector) - val bottomCursorBuilder = OrderedBottomCursorBuilder(idSelector) - - val metadataBuilder = UrtMetadataBuilder( - title = None, - scribeConfigBuilder = Some( - StaticTimelineScribeConfigBuilder( - TimelineScribeConfig( - page = Some("for_you_scored_tweets"), - section = None, - entityToken = None))) - ) - - UrtDomainMarshaller( - instructionBuilders = instructionBuilders, - metadataBuilder = Some(metadataBuilder), - cursorBuilders = Seq(topCursorBuilder, bottomCursorBuilder) - ) - } - - override val transportMarshaller: TransportMarshaller[Timeline, urt.TimelineResponse] = - urtTransportMarshaller -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsResponseFeatureTransformer.scala deleted file mode 100644 index eafc09352..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsResponseFeatureTransformer.scala +++ /dev/null @@ -1,103 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.home_mixer.model.HomeFeatures._ -import com.twitter.home_mixer.product.for_you.candidate_source.ScoredTweetWithConversationMetadata -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecWithEducationTopicContextFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType -import com.twitter.timelines.render.{thriftscala => tl} -import com.twitter.timelineservice.suggests.{thriftscala => tls} - -object ForYouScoredTweetsResponseFeatureTransformer - extends CandidateFeatureTransformer[ScoredTweetWithConversationMetadata] { - - override val identifier: TransformerIdentifier = - TransformerIdentifier("ForYouScoredTweetsResponse") - - override val features: Set[Feature[_, _]] = Set( - AncestorsFeature, - AuthorIdFeature, - AuthorIsBlueVerifiedFeature, - AuthorIsCreatorFeature, - AuthorIsGoldVerifiedFeature, - AuthorIsGrayVerifiedFeature, - AuthorIsLegacyVerifiedFeature, - ConversationModuleFocalTweetIdFeature, - ConversationModuleIdFeature, - DirectedAtUserIdFeature, - ExclusiveConversationAuthorIdFeature, - FullScoringSucceededFeature, - FavoritedByUserIdsFeature, - FollowedByUserIdsFeature, - InNetworkFeature, - InReplyToTweetIdFeature, - InReplyToUserIdFeature, - IsAncestorCandidateFeature, - IsReadFromCacheFeature, - IsRetweetFeature, - PerspectiveFilteredLikedByUserIdsFeature, - QuotedTweetIdFeature, - QuotedUserIdFeature, - SGSValidFollowedByUserIdsFeature, - SGSValidLikedByUserIdsFeature, - ScoreFeature, - SourceTweetIdFeature, - SourceUserIdFeature, - StreamToKafkaFeature, - SuggestTypeFeature, - TopicContextFunctionalityTypeFeature, - TopicIdSocialContextFeature - ) - - override def transform(input: ScoredTweetWithConversationMetadata): FeatureMap = - FeatureMapBuilder() - .add(AncestorsFeature, input.ancestors.getOrElse(Seq.empty)) - .add(AuthorIdFeature, Some(input.authorId)) - .add(AuthorIsBlueVerifiedFeature, input.authorIsBlueVerified.getOrElse(false)) - .add(AuthorIsGoldVerifiedFeature, input.authorIsGoldVerified.getOrElse(false)) - .add(AuthorIsGrayVerifiedFeature, input.authorIsGrayVerified.getOrElse(false)) - .add(AuthorIsLegacyVerifiedFeature, input.authorIsLegacyVerified.getOrElse(false)) - .add(AuthorIsCreatorFeature, input.authorIsCreator.getOrElse(false)) - .add(ConversationModuleIdFeature, input.conversationId) - .add(ConversationModuleFocalTweetIdFeature, input.conversationFocalTweetId) - .add(DirectedAtUserIdFeature, input.directedAtUserId) - .add(ExclusiveConversationAuthorIdFeature, input.exclusiveConversationAuthorId) - .add(SGSValidLikedByUserIdsFeature, input.sgsValidLikedByUserIds.getOrElse(Seq.empty)) - .add(SGSValidFollowedByUserIdsFeature, input.sgsValidFollowedByUserIds.getOrElse(Seq.empty)) - .add(FavoritedByUserIdsFeature, input.sgsValidLikedByUserIds.getOrElse(Seq.empty)) - .add(FollowedByUserIdsFeature, input.sgsValidFollowedByUserIds.getOrElse(Seq.empty)) - .add(FullScoringSucceededFeature, true) - .add(InNetworkFeature, input.inNetwork.getOrElse(true)) - .add(InReplyToTweetIdFeature, input.inReplyToTweetId) - .add(InReplyToUserIdFeature, input.inReplyToUserId) - .add(IsAncestorCandidateFeature, input.conversationFocalTweetId.exists(_ != input.tweetId)) - .add(IsReadFromCacheFeature, input.isReadFromCache.getOrElse(false)) - .add(IsRetweetFeature, input.sourceTweetId.isDefined) - .add( - PerspectiveFilteredLikedByUserIdsFeature, - input.perspectiveFilteredLikedByUserIds.getOrElse(Seq.empty)) - .add(QuotedTweetIdFeature, input.quotedTweetId) - .add(QuotedUserIdFeature, input.quotedUserId) - .add(ScoreFeature, input.score) - .add(SourceTweetIdFeature, input.sourceTweetId) - .add(SourceUserIdFeature, input.sourceUserId) - .add(StreamToKafkaFeature, input.streamToKafka.getOrElse(false)) - .add(SuggestTypeFeature, input.suggestType.orElse(Some(tls.SuggestType.Undefined))) - .add( - TopicContextFunctionalityTypeFeature, - input.topicFunctionalityType.collect { - case tl.TopicContextFunctionalityType.Basic => BasicTopicContextFunctionalityType - case tl.TopicContextFunctionalityType.Recommendation => - RecommendationTopicContextFunctionalityType - case tl.TopicContextFunctionalityType.RecWithEducation => - RecWithEducationTopicContextFunctionalityType - } - ) - .add(TopicIdSocialContextFeature, input.topicId) - .build() -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerCandidatePipelineConfig.scala deleted file mode 100644 index bb93a4110..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerCandidatePipelineConfig.scala +++ /dev/null @@ -1,241 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventInfoBuilder -import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder -import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeTweetSocialContextBuilder -import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.PerspectiveFilteredSocialContextFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.SGSValidSocialContextFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator -import com.twitter.home_mixer.functional_component.filter.FeedbackFatigueFilter -import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter -import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter -import com.twitter.home_mixer.functional_component.filter.RejectTweetFromViewerFilter -import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter -import com.twitter.home_mixer.functional_component.scorer.FeedbackFatigueScorer -import com.twitter.home_mixer.functional_component.scorer.OONTweetScalingScorer -import com.twitter.home_mixer.marshaller.timelines.DeviceContextMarshaller -import com.twitter.home_mixer.marshaller.timelines.TimelineServiceCursorMarshaller -import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature -import com.twitter.home_mixer.model.HomeFeatures.IsNsfwFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature -import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature -import com.twitter.home_mixer.model.request.DeviceContext -import com.twitter.home_mixer.product.for_you.feature_hydrator.FocalTweetFeatureHydrator -import com.twitter.home_mixer.product.for_you.filter.SocialContextFilter -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableTimelineScorerCandidatePipelineParam -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.candidate_source.timeline_scorer.ScoredTweetCandidateWithFocalTweet -import com.twitter.product_mixer.component_library.candidate_source.timeline_scorer.TimelineScorerCandidateSource -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ManualModuleId -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.StaticModuleDisplayTypeBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder -import com.twitter.product_mixer.component_library.filter.FeatureFilter -import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.VerticalConversation -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.model.candidate.CandidateTweetSourceId -import com.twitter.timelines.service.{thriftscala => tst} -import com.twitter.timelinescorer.{thriftscala => t} -import com.twitter.timelineservice.{thriftscala => tlst} -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Candidate Pipeline Config that fetches tweets from the Timeline Scorer Candidate Source - */ -@Singleton -class ForYouTimelineScorerCandidatePipelineConfig @Inject() ( - timelineScorerCandidateSource: TimelineScorerCandidateSource, - deviceContextMarshaller: DeviceContextMarshaller, - tweetypieFeatureHydrator: TweetypieFeatureHydrator, - sgsValidSocialContextFeatureHydrator: SGSValidSocialContextFeatureHydrator, - perspectiveFilteredSocialContextFeatureHydrator: PerspectiveFilteredSocialContextFeatureHydrator, - namesFeatureHydrator: NamesFeatureHydrator, - focalTweetFeatureHydrator: FocalTweetFeatureHydrator, - invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter, - homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder, - homeTweetSocialContextBuilder: HomeTweetSocialContextBuilder) - extends CandidatePipelineConfig[ - ForYouQuery, - t.ScoredTweetsRequest, - ScoredTweetCandidateWithFocalTweet, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ForYouTimelineScorerTweets") - - private val TweetypieHydratedFilterId = "TweetypieHydrated" - private val QuotedTweetDroppedFilterId = "QuotedTweetDropped" - private val OutOfNetworkNSFWFilterId = "OutOfNetworkNSFW" - private val ConversationModuleNamespace = EntryNamespace("home-conversation") - - override val supportedClientParam: Option[FSParam[Boolean]] = - Some(EnableTimelineScorerCandidatePipelineParam) - - override val candidateSource: BaseCandidateSource[ - t.ScoredTweetsRequest, - ScoredTweetCandidateWithFocalTweet - ] = timelineScorerCandidateSource - - override val queryTransformer: CandidatePipelineQueryTransformer[ - ForYouQuery, - t.ScoredTweetsRequest - ] = { query => - val deviceContext = query.deviceContext.getOrElse(DeviceContext.Empty) - - val scoredTweetsRequestContext = t.v1.ScoredTweetsRequestContext( - contextualUserId = query.clientContext.userId, - timelineId = query.clientContext.userId.map(tlst.TimelineId(tlst.TimelineType.Home, _, None)), - deviceContext = Some(deviceContextMarshaller(deviceContext, query.clientContext)), - seenTweetIds = query.seenTweetIds, - contextualUserContext = Some(tst.ContextualUserContext(query.clientContext.userRoles)), - timelineRequestCursor = query.pipelineCursor.flatMap(TimelineServiceCursorMarshaller(_)) - ) - - val candidateTweetSourceIds = Seq( - CandidateTweetSourceId.RecycledTweet, - CandidateTweetSourceId.OrganicTweet, - CandidateTweetSourceId.AncestorsOnlyOrganicTweet, - CandidateTweetSourceId.BackfillOrganicTweet, - CandidateTweetSourceId.CroonTweet, - CandidateTweetSourceId.RecommendedTweet, - CandidateTweetSourceId.FrsTweet, - CandidateTweetSourceId.ListTweet, - CandidateTweetSourceId.RecommendedTrendTweet, - CandidateTweetSourceId.PopularTopicTweet - ) - - val timelineServiceTweets = - query.features.map(_.getOrElse(TimelineServiceTweetsFeature, Seq.empty)).getOrElse(Seq.empty) - - val timelineEntries = timelineServiceTweets.map { id => - tlst.TimelineEntry.Tweet(tlst.Tweet(statusId = id, sortIndex = id)) - } - - t.ScoredTweetsRequest.V1( - t.v1.ScoredTweetsRequest( - scoredTweetsRequestContext = Some(scoredTweetsRequestContext), - candidateTweetSourceIds = - Some(candidateTweetSourceIds.flatMap(CandidateTweetSourceId.toThrift)), - maxResultsCount = query.requestedMaxResults, - organicTimeline = Some( - tlst.Timeline( - timelineId = tlst.TimelineId( - timelineType = tlst.TimelineType.Home, - id = query.getRequiredUserId, - canonicalTimelineId = None), - entries = timelineEntries, - modules = tlst.TimelineModules() - )) - ) - ) - } - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[ScoredTweetCandidateWithFocalTweet] - ] = Seq(ForYouTimelineScorerResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - ScoredTweetCandidateWithFocalTweet, - TweetCandidate - ] = { candidateWithFocalTweetId => - TweetCandidate(id = candidateWithFocalTweetId.candidate.tweetId) - } - - override val preFilterFeatureHydrationPhase1: Seq[ - BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] - ] = Seq( - namesFeatureHydrator, - tweetypieFeatureHydrator, - InNetworkFeatureHydrator, - sgsValidSocialContextFeatureHydrator, - perspectiveFilteredSocialContextFeatureHydrator, - ) - - override def filters: Seq[Filter[ForYouQuery, TweetCandidate]] = Seq( - RetweetDeduplicationFilter, - FeatureFilter.fromFeature(FilterIdentifier(TweetypieHydratedFilterId), IsHydratedFeature), - PredicateFeatureFilter.fromPredicate( - FilterIdentifier(QuotedTweetDroppedFilterId), - shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) } - ), - PredicateFeatureFilter.fromPredicate( - FilterIdentifier(OutOfNetworkNSFWFilterId), - shouldKeepCandidate = { features => - features.getOrElse(InNetworkFeature, false) || - !features.getOrElse(IsNsfwFeature, false) - } - ), - FeedbackFatigueFilter, - RejectTweetFromViewerFilter, - SocialContextFilter, - invalidSubscriptionTweetFilter, - InvalidConversationModuleFilter - ) - - override val postFilterFeatureHydration: Seq[ - BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] - ] = Seq(focalTweetFeatureHydrator) - - override val scorers: Seq[Scorer[ForYouQuery, TweetCandidate]] = - Seq(OONTweetScalingScorer, FeedbackFatigueScorer) - - override val decorator: Option[CandidateDecorator[ForYouQuery, TweetCandidate]] = { - val clientEventInfoBuilder = HomeClientEventInfoBuilder() - - val tweetItemBuilder = TweetCandidateUrtItemBuilder( - clientEventInfoBuilder = clientEventInfoBuilder, - socialContextBuilder = Some(homeTweetSocialContextBuilder), - timelinesScoreInfoBuilder = Some(HomeTimelinesScoreInfoBuilder), - feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder) - ) - - val tweetDecorator = UrtItemCandidateDecorator(tweetItemBuilder) - - val moduleBuilder = TimelineModuleBuilder( - entryNamespace = ConversationModuleNamespace, - clientEventInfoBuilder = clientEventInfoBuilder, - moduleIdGeneration = ManualModuleId(0L), - displayTypeBuilder = StaticModuleDisplayTypeBuilder(VerticalConversation), - metadataBuilder = Some(HomeConversationModuleMetadataBuilder()) - ) - - Some( - UrtMultipleModulesDecorator( - urtItemCandidateDecorator = tweetDecorator, - moduleBuilder = moduleBuilder, - groupByKey = (_, _, candidateFeatures) => - candidateFeatures.getOrElse(ConversationModuleFocalTweetIdFeature, None) - )) - } - - override val alerts: Seq[Alert] = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), - HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert(10, 20) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerMixerPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerMixerPipelineConfig.scala deleted file mode 100644 index 441e5796d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerMixerPipelineConfig.scala +++ /dev/null @@ -1,403 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.clientapp.{thriftscala => ca} -import com.twitter.goldfinch.api.AdsInjectionSurfaceAreas -import com.twitter.home_mixer.candidate_pipeline.EditedTweetsCandidatePipelineConfig -import com.twitter.home_mixer.candidate_pipeline.NewTweetsPillCandidatePipelineConfig -import com.twitter.home_mixer.functional_component.decorator.urt.builder.AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder -import com.twitter.home_mixer.functional_component.feature_hydrator.FeedbackHistoryQueryFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator._ -import com.twitter.home_mixer.functional_component.selector.DebunchCandidates -import com.twitter.home_mixer.functional_component.selector.UpdateConversationModuleId -import com.twitter.home_mixer.functional_component.selector.UpdateHomeClientEventDetails -import com.twitter.home_mixer.functional_component.selector.UpdateNewTweetsPillDecoration -import com.twitter.home_mixer.functional_component.side_effect._ -import com.twitter.home_mixer.model.ClearCacheIncludeInstruction -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.param.HomeGlobalParams.EnableImpressionBloomFilter -import com.twitter.home_mixer.param.HomeGlobalParams.MaxNumberReplaceInstructionsParam -import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeClientEventsFlag -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.home_mixer.product.for_you.feature_hydrator.TimelineServiceTweetsQueryFeatureHydrator -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.home_mixer.product.for_you.param.ForYouParam.ClearCacheOnPtr -import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableFlipInjectionModuleCandidatePipelineParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.FlipInlineInjectionModulePosition -import com.twitter.home_mixer.product.for_you.param.ForYouParam.ServerMaxResultsParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.TweetPreviewsPositionParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToFollowPositionParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToSubscribePositionParam -import com.twitter.home_mixer.product.for_you.side_effect.ServedCandidateFeatureKeysKafkaSideEffectBuilder -import com.twitter.home_mixer.product.for_you.side_effect.ServedCandidateKeysKafkaSideEffectBuilder -import com.twitter.home_mixer.product.for_you.side_effect.ServedStatsSideEffect -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.inject.annotations.Flag -import com.twitter.logpipeline.client.common.EventPublisher -import com.twitter.product_mixer.component_library.feature_hydrator.query.async.AsyncQueryFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweetsQueryFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.AsyncParamGatedQueryFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.PreviewCreatorsQueryFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.FlipPromptCandidatePipelineConfigBuilder -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmCandidatePipelineConfig -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidatePipelineConfig -import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ClearCacheInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedBottomCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedTopCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceAllEntries -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceEntryInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowAlertInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowCoverInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.StaticTimelineScribeConfigBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtMetadataBuilder -import com.twitter.product_mixer.component_library.selector.DropMaxCandidates -import com.twitter.product_mixer.component_library.selector.DropMaxModuleItemCandidates -import com.twitter.product_mixer.component_library.selector.DropModuleTooFewModuleItemResults -import com.twitter.product_mixer.component_library.selector.InsertAppendResults -import com.twitter.product_mixer.component_library.selector.InsertFixedPositionResults -import com.twitter.product_mixer.component_library.selector.SelectConditionally -import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates -import com.twitter.product_mixer.component_library.selector.UpdateSortModuleItemCandidates -import com.twitter.product_mixer.component_library.selector.ads.AdsInjector -import com.twitter.product_mixer.component_library.selector.ads.InsertAdResults -import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.configapi.StaticParam -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller -import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.UrtTransportMarshaller -import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.product_mixer.core.pipeline.FailOpenPolicy -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import com.twitter.timelines.render.{thriftscala => urt} -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton - -@Singleton -class ForYouTimelineScorerMixerPipelineConfig @Inject() ( - forYouTimelineScorerCandidatePipelineConfig: ForYouTimelineScorerCandidatePipelineConfig, - forYouPushToHomeTweetCandidatePipelineConfig: ForYouPushToHomeTweetCandidatePipelineConfig, - forYouConversationServiceCandidatePipelineConfig: ForYouConversationServiceCandidatePipelineConfig, - forYouAdsDependentCandidatePipelineBuilder: ForYouAdsDependentCandidatePipelineBuilder, - forYouWhoToFollowCandidatePipelineConfigBuilder: ForYouWhoToFollowCandidatePipelineConfigBuilder, - forYouWhoToSubscribeCandidatePipelineConfigBuilder: ForYouWhoToSubscribeCandidatePipelineConfigBuilder, - flipPromptCandidatePipelineConfigBuilder: FlipPromptCandidatePipelineConfigBuilder, - editedTweetsCandidatePipelineConfig: EditedTweetsCandidatePipelineConfig, - newTweetsPillCandidatePipelineConfig: NewTweetsPillCandidatePipelineConfig[ForYouQuery], - forYouTweetPreviewsCandidatePipelineConfig: ForYouTweetPreviewsCandidatePipelineConfig, - dismissInfoQueryFeatureHydrator: DismissInfoQueryFeatureHydrator, - gizmoduckUserQueryFeatureHydrator: GizmoduckUserQueryFeatureHydrator, - impressionBloomFilterQueryFeatureHydrator: ImpressionBloomFilterQueryFeatureHydrator[ForYouQuery], - manhattanTweetImpressionsQueryFeatureHydrator: TweetImpressionsQueryFeatureHydrator[ForYouQuery], - memcacheTweetImpressionsQueryFeatureHydrator: ImpressedTweetsQueryFeatureHydrator, - persistenceStoreQueryFeatureHydrator: PersistenceStoreQueryFeatureHydrator, - requestQueryFeatureHydrator: RequestQueryFeatureHydrator[ForYouQuery], - timelineServiceTweetsQueryFeatureHydrator: TimelineServiceTweetsQueryFeatureHydrator, - lastNonPollingTimeQueryFeatureHydrator: LastNonPollingTimeQueryFeatureHydrator, - feedbackHistoryQueryFeatureHydrator: FeedbackHistoryQueryFeatureHydrator, - previewCreatorsQueryFeatureHydrator: PreviewCreatorsQueryFeatureHydrator, - sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, - adsInjector: AdsInjector, - servedCandidateKeysKafkaSideEffectBuilder: ServedCandidateKeysKafkaSideEffectBuilder, - servedCandidateFeatureKeysKafkaSideEffectBuilder: ServedCandidateFeatureKeysKafkaSideEffectBuilder, - updateLastNonPollingTimeSideEffect: UpdateLastNonPollingTimeSideEffect[ForYouQuery, Timeline], - publishClientSentImpressionsEventBusSideEffect: PublishClientSentImpressionsEventBusSideEffect, - publishClientSentImpressionsManhattanSideEffect: PublishClientSentImpressionsManhattanSideEffect, - publishImpressionBloomFilterSideEffect: PublishImpressionBloomFilterSideEffect, - updateTimelinesPersistenceStoreSideEffect: UpdateTimelinesPersistenceStoreSideEffect, - truncateTimelinesPersistenceStoreSideEffect: TruncateTimelinesPersistenceStoreSideEffect, - homeScribeServedCandidatesSideEffect: HomeScribeServedCandidatesSideEffect, - servedStatsSideEffect: ServedStatsSideEffect, - clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent], - externalStrings: HomeMixerExternalStrings, - @ProductScoped stringCenterProvider: Provider[StringCenter], - urtTransportMarshaller: UrtTransportMarshaller, - @Flag(ScribeClientEventsFlag) enableScribeClientEvents: Boolean) - extends MixerPipelineConfig[ForYouQuery, Timeline, urt.TimelineResponse] { - - override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("ForYouTimelineScorer") - - private val MaxConsecutiveOutOfNetworkCandidates = 2 - - private val PushToHomeTweetPosition = 0 - - private val dependentCandidatesStep = MixerPipelineConfig.dependentCandidatePipelinesStep - private val resultSelectorsStep = MixerPipelineConfig.resultSelectorsStep - - override def fetchQueryFeatures: Seq[QueryFeatureHydrator[ForYouQuery]] = Seq( - requestQueryFeatureHydrator, - persistenceStoreQueryFeatureHydrator, - timelineServiceTweetsQueryFeatureHydrator, - feedbackHistoryQueryFeatureHydrator, - previewCreatorsQueryFeatureHydrator, - sgsFollowedUsersQueryFeatureHydrator, - AsyncQueryFeatureHydrator(dependentCandidatesStep, dismissInfoQueryFeatureHydrator), - AsyncQueryFeatureHydrator(dependentCandidatesStep, gizmoduckUserQueryFeatureHydrator), - AsyncQueryFeatureHydrator(dependentCandidatesStep, lastNonPollingTimeQueryFeatureHydrator), - AsyncParamGatedQueryFeatureHydrator( - EnableImpressionBloomFilter, - resultSelectorsStep, - impressionBloomFilterQueryFeatureHydrator), - AsyncQueryFeatureHydrator(resultSelectorsStep, manhattanTweetImpressionsQueryFeatureHydrator), - AsyncQueryFeatureHydrator(resultSelectorsStep, memcacheTweetImpressionsQueryFeatureHydrator) - ) - - private val timelineScorerCandidatePipelineScope = - SpecificPipeline(forYouTimelineScorerCandidatePipelineConfig.identifier) - - private val forYouAdsCandidatePipelineConfig = forYouAdsDependentCandidatePipelineBuilder - .build(timelineScorerCandidatePipelineScope) - - private val forYouWhoToFollowCandidatePipelineConfig = - forYouWhoToFollowCandidatePipelineConfigBuilder.build() - - private val forYouWhoToSubscribeCandidatePipelineConfig = - forYouWhoToSubscribeCandidatePipelineConfigBuilder.build() - - private val flipPromptCandidatePipelineConfig = - flipPromptCandidatePipelineConfigBuilder.build[ForYouQuery]( - supportedClientParam = Some(EnableFlipInjectionModuleCandidatePipelineParam) - ) - - override val candidatePipelines: Seq[CandidatePipelineConfig[ForYouQuery, _, _, _]] = Seq( - forYouTimelineScorerCandidatePipelineConfig, - forYouPushToHomeTweetCandidatePipelineConfig, - forYouWhoToFollowCandidatePipelineConfig, - forYouWhoToSubscribeCandidatePipelineConfig, - forYouTweetPreviewsCandidatePipelineConfig, - flipPromptCandidatePipelineConfig - ) - - override val dependentCandidatePipelines: Seq[ - DependentCandidatePipelineConfig[ForYouQuery, _, _, _] - ] = Seq( - forYouAdsCandidatePipelineConfig, - forYouConversationServiceCandidatePipelineConfig, - editedTweetsCandidatePipelineConfig, - newTweetsPillCandidatePipelineConfig - ) - - override val failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = Map( - forYouTimelineScorerCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - forYouAdsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - forYouWhoToFollowCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - forYouWhoToSubscribeCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - forYouTweetPreviewsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - flipPromptCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - editedTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - newTweetsPillCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - ) - - override val resultSelectors: Seq[Selector[ForYouQuery]] = Seq( - UpdateSortCandidates( - ordering = CandidatesUtil.reverseChronTweetsOrdering, - candidatePipeline = forYouConversationServiceCandidatePipelineConfig.identifier - ), - UpdateSortCandidates( - ordering = CandidatesUtil.scoreOrdering, - candidatePipeline = forYouTimelineScorerCandidatePipelineConfig.identifier - ), - UpdateSortModuleItemCandidates( - candidatePipeline = forYouTimelineScorerCandidatePipelineConfig.identifier, - ordering = CandidatesUtil.conversationModuleTweetsOrdering - ), - DebunchCandidates( - pipelineScope = SpecificPipeline(forYouTimelineScorerCandidatePipelineConfig.identifier), - mustDebunch = { - case item: ItemCandidateWithDetails => - !item.features.getOrElse(InNetworkFeature, false) - case module: ModuleCandidateWithDetails => - !module.candidates.last.features.getOrElse(InNetworkFeature, false) - }, - maxBunchSize = MaxConsecutiveOutOfNetworkCandidates - ), - UpdateConversationModuleId( - pipelineScope = SpecificPipeline(forYouTimelineScorerCandidatePipelineConfig.identifier) - ), - DropMaxCandidates( - candidatePipeline = forYouConversationServiceCandidatePipelineConfig.identifier, - maxSelectionsParam = ServerMaxResultsParam - ), - DropMaxCandidates( - candidatePipeline = forYouTimelineScorerCandidatePipelineConfig.identifier, - maxSelectionsParam = ServerMaxResultsParam - ), - DropMaxCandidates( - candidatePipeline = editedTweetsCandidatePipelineConfig.identifier, - maxSelectionsParam = MaxNumberReplaceInstructionsParam - ), - DropMaxModuleItemCandidates( - candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, - maxModuleItemsParam = StaticParam(WhoToFollowArmCandidatePipelineConfig.MaxCandidatesSize) - ), - DropModuleTooFewModuleItemResults( - candidatePipeline = forYouWhoToSubscribeCandidatePipelineConfig.identifier, - minModuleItemsParam = StaticParam(WhoToSubscribeCandidatePipelineConfig.MinCandidatesSize) - ), - DropMaxModuleItemCandidates( - candidatePipeline = forYouWhoToSubscribeCandidatePipelineConfig.identifier, - maxModuleItemsParam = StaticParam(WhoToSubscribeCandidatePipelineConfig.MaxCandidatesSize) - ), - // Add Conversation Service tweets to results only if the scored pipeline doesn't return any - SelectConditionally( - selector = InsertAppendResults( - candidatePipeline = forYouConversationServiceCandidatePipelineConfig.identifier), - includeSelector = (_, candidates, _) => - !candidates.exists(candidate => - forYouTimelineScorerCandidatePipelineConfig.identifier == candidate.source) - ), - InsertAppendResults(candidatePipeline = forYouTimelineScorerCandidatePipelineConfig.identifier), - InsertFixedPositionResults( - candidatePipeline = forYouTweetPreviewsCandidatePipelineConfig.identifier, - positionParam = TweetPreviewsPositionParam - ), - InsertFixedPositionResults( - candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, - positionParam = WhoToFollowPositionParam - ), - InsertFixedPositionResults( - candidatePipeline = forYouWhoToSubscribeCandidatePipelineConfig.identifier, - positionParam = WhoToSubscribePositionParam - ), - InsertFixedPositionResults( - candidatePipeline = flipPromptCandidatePipelineConfig.identifier, - positionParam = FlipInlineInjectionModulePosition - ), - // Insert Push To Home Tweet at top of Timeline - InsertFixedPositionResults( - candidatePipeline = forYouPushToHomeTweetCandidatePipelineConfig.identifier, - positionParam = StaticParam(PushToHomeTweetPosition) - ), - InsertAdResults( - surfaceAreaName = AdsInjectionSurfaceAreas.HomeTimeline, - adsInjector = adsInjector.forSurfaceArea(AdsInjectionSurfaceAreas.HomeTimeline), - adsCandidatePipeline = forYouAdsCandidatePipelineConfig.identifier - ), - // This selector must come after the tweets are inserted into the results - DropModuleTooFewModuleItemResults( - candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, - minModuleItemsParam = StaticParam(WhoToFollowArmCandidatePipelineConfig.MinCandidatesSize) - ), - UpdateNewTweetsPillDecoration( - pipelineScope = SpecificPipelines( - forYouConversationServiceCandidatePipelineConfig.identifier, - forYouTimelineScorerCandidatePipelineConfig.identifier, - newTweetsPillCandidatePipelineConfig.identifier - ), - stringCenter = stringCenterProvider.get(), - seeNewTweetsString = externalStrings.seeNewTweetsString, - tweetedString = externalStrings.tweetedString - ), - InsertAppendResults(candidatePipeline = editedTweetsCandidatePipelineConfig.identifier), - SelectConditionally( - selector = - InsertAppendResults(candidatePipeline = newTweetsPillCandidatePipelineConfig.identifier), - includeSelector = (_, _, results) => CandidatesUtil.containsType[TweetCandidate](results) - ), - UpdateHomeClientEventDetails( - candidatePipelines = Set( - forYouConversationServiceCandidatePipelineConfig.identifier, - forYouTimelineScorerCandidatePipelineConfig.identifier - ) - ), - ) - - private val servedCandidateKeysKafkaSideEffect = - servedCandidateKeysKafkaSideEffectBuilder.build( - Set(forYouTimelineScorerCandidatePipelineConfig.identifier)) - - private val servedCandidateFeatureKeysKafkaSideEffect = - servedCandidateFeatureKeysKafkaSideEffectBuilder.build( - Set(forYouTimelineScorerCandidatePipelineConfig.identifier)) - - private val homeScribeClientEventSideEffect = HomeScribeClientEventSideEffect( - enableScribeClientEvents = enableScribeClientEvents, - logPipelinePublisher = clientEventsScribeEventPublisher, - injectedTweetsCandidatePipelineIdentifiers = Seq( - forYouTimelineScorerCandidatePipelineConfig.identifier, - forYouConversationServiceCandidatePipelineConfig.identifier - ), - adsCandidatePipelineIdentifier = Some(forYouAdsCandidatePipelineConfig.identifier), - whoToFollowCandidatePipelineIdentifier = - Some(forYouWhoToFollowCandidatePipelineConfig.identifier), - whoToSubscribeCandidatePipelineIdentifier = - Some(forYouWhoToSubscribeCandidatePipelineConfig.identifier) - ) - - override val resultSideEffects: Seq[PipelineResultSideEffect[ForYouQuery, Timeline]] = Seq( - homeScribeClientEventSideEffect, - homeScribeServedCandidatesSideEffect, - publishClientSentImpressionsEventBusSideEffect, - publishClientSentImpressionsManhattanSideEffect, - publishImpressionBloomFilterSideEffect, - servedCandidateKeysKafkaSideEffect, - servedCandidateFeatureKeysKafkaSideEffect, - servedStatsSideEffect, - truncateTimelinesPersistenceStoreSideEffect, - updateLastNonPollingTimeSideEffect, - updateTimelinesPersistenceStoreSideEffect - ) - - override val domainMarshaller: DomainMarshaller[ForYouQuery, Timeline] = { - val instructionBuilders = Seq( - ClearCacheInstructionBuilder( - ClearCacheIncludeInstruction( - ClearCacheOnPtr.EnableParam, - ClearCacheOnPtr.MinEntriesParam - ) - ), - ReplaceEntryInstructionBuilder(ReplaceAllEntries), - // excludes alert, cover, and replace candidates - AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder(), - ShowAlertInstructionBuilder(), - ShowCoverInstructionBuilder(), - ) - - val idSelector: PartialFunction[UniversalNoun[_], Long] = { - // exclude ads while determining tweet cursor values - case item: TweetItem if item.promotedMetadata.isEmpty => item.id - case module: TimelineModule - if module.items.headOption.exists(_.item.isInstanceOf[TweetItem]) => - module.items.last.item match { case item: TweetItem => item.id } - } - val topCursorBuilder = OrderedTopCursorBuilder(idSelector) - val bottomCursorBuilder = OrderedBottomCursorBuilder(idSelector) - - val metadataBuilder = UrtMetadataBuilder( - title = None, - scribeConfigBuilder = Some( - StaticTimelineScribeConfigBuilder( - TimelineScribeConfig( - page = Some("for_you_timeline_scorer"), - section = None, - entityToken = None))) - ) - - UrtDomainMarshaller( - instructionBuilders = instructionBuilders, - metadataBuilder = Some(metadataBuilder), - cursorBuilders = Seq(topCursorBuilder, bottomCursorBuilder) - ) - } - - override val transportMarshaller: TransportMarshaller[Timeline, urt.TimelineResponse] = - urtTransportMarshaller -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerResponseFeatureTransformer.scala deleted file mode 100644 index 236ca9a26..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerResponseFeatureTransformer.scala +++ /dev/null @@ -1,199 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.tweetconvosvc.tweet_ancestor.{thriftscala => ta} -import com.twitter.home_mixer.model.HomeFeatures._ -import com.twitter.mediaservices.commons.tweetmedia.{thriftscala => mt} -import com.twitter.product_mixer.component_library.candidate_source.timeline_scorer.ScoredTweetCandidateWithFocalTweet -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecWithEducationTopicContextFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType -import com.twitter.search.common.constants.thriftjava.ThriftLanguage -import com.twitter.search.common.util.lang.ThriftLanguageUtil -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.timelinemixer.injection.model.candidate.AudioSpaceMetaData -import com.twitter.timelines.conversation_features.{thriftscala => cvt} -import com.twitter.timelinescorer.common.scoredtweetcandidate.{thriftscala => stc} -import com.twitter.timelineservice.suggests.{thriftscala => tls} - -object ForYouTimelineScorerResponseFeatureTransformer - extends CandidateFeatureTransformer[ScoredTweetCandidateWithFocalTweet] { - - override val identifier: TransformerIdentifier = - TransformerIdentifier("ForYouTimelineScorerResponse") - - override val features: Set[Feature[_, _]] = Set( - AncestorsFeature, - AudioSpaceMetaDataFeature, - AuthorIdFeature, - AuthorIsBlueVerifiedFeature, - AuthorIsCreatorFeature, - AuthorIsGoldVerifiedFeature, - AuthorIsGrayVerifiedFeature, - AuthorIsLegacyVerifiedFeature, - AuthoredByContextualUserFeature, - CandidateSourceIdFeature, - ConversationFeature, - ConversationModuleFocalTweetIdFeature, - ConversationModuleIdFeature, - DirectedAtUserIdFeature, - EarlybirdFeature, - EntityTokenFeature, - ExclusiveConversationAuthorIdFeature, - FavoritedByUserIdsFeature, - FollowedByUserIdsFeature, - TopicIdSocialContextFeature, - TopicContextFunctionalityTypeFeature, - FromInNetworkSourceFeature, - FullScoringSucceededFeature, - HasDisplayedTextFeature, - InNetworkFeature, - InReplyToTweetIdFeature, - IsAncestorCandidateFeature, - IsExtendedReplyFeature, - IsRandomTweetFeature, - IsReadFromCacheFeature, - IsRetweetFeature, - IsRetweetedReplyFeature, - NonSelfFavoritedByUserIdsFeature, - NumImagesFeature, - OriginalTweetCreationTimeFromSnowflakeFeature, - PredictionRequestIdFeature, - QuotedTweetIdFeature, - ScoreFeature, - SimclustersTweetTopKClustersWithScoresFeature, - SourceTweetIdFeature, - SourceUserIdFeature, - StreamToKafkaFeature, - SuggestTypeFeature, - TweetLanguageFeature, - VideoDurationMsFeature, - ) - - // Convert language code to ISO 639-3 format - private def getLanguageISOFormatByValue(languageCodeValue: Int): String = - ThriftLanguageUtil.getLanguageCodeOf(ThriftLanguage.findByValue(languageCodeValue)) - - override def transform( - candidateWithFocalTweet: ScoredTweetCandidateWithFocalTweet - ): FeatureMap = { - val candidate: stc.v1.ScoredTweetCandidate = candidateWithFocalTweet.candidate - val focalTweetId = candidateWithFocalTweet.focalTweetIdOpt - - val originalTweetId = candidate.sourceTweetId.getOrElse(candidate.tweetId) - val tweetFeatures = candidate.tweetFeaturesMap.flatMap(_.get(originalTweetId)) - val earlybirdFeatures = tweetFeatures.flatMap(_.recapFeatures.flatMap(_.tweetFeatures)) - val directedAtUserIsInFirstDegree = - earlybirdFeatures.flatMap(_.directedAtUserIdIsInFirstDegree) - val isReply = candidate.inReplyToTweetId.nonEmpty - val isRetweet = candidate.isRetweet.getOrElse(false) - val isInNetwork = candidate.isInNetwork.getOrElse(true) - val conversationFeatures = candidate.conversationFeatures.flatMap { - case cvt.ConversationFeatures.V1(candidate) => Some(candidate) - case _ => None - } - val numImages = candidate.mediaMetaData - .map( - _.count(mediaEntity => - mediaEntity.mediaInfo.exists(_.isInstanceOf[mt.MediaInfo.ImageInfo]) || - mediaEntity.mediaInfo.isEmpty)) - val hasImage = earlybirdFeatures.exists(_.hasImage) - val hasVideo = earlybirdFeatures.exists(_.hasVideo) - val hasCard = earlybirdFeatures.exists(_.hasCard) - val hasQuote = earlybirdFeatures.exists(_.hasQuote.contains(true)) - val hasDisplayedText = earlybirdFeatures.exists(_.tweetLength.exists(length => { - val numMedia = Seq(hasVideo, (hasImage || hasCard), hasQuote).count(b => b) - val tcoLengthsPlusSpaces = 23 * numMedia + (if (numMedia > 0) numMedia - 1 else 0) - length > tcoLengthsPlusSpaces - })) - val suggestType = candidate.overrideSuggestType.orElse(Some(tls.SuggestType.Undefined)) - - val topicSocialProofMetadataOpt = candidate.entityData.flatMap(_.topicSocialProofMetadata) - val topicIdSocialContextOpt = topicSocialProofMetadataOpt.map(_.topicId) - val topicContextFunctionalityTypeOpt = - topicSocialProofMetadataOpt.map(_.topicContextFunctionalityType).collect { - case stc.v1.TopicContextFunctionalityType.Basic => BasicTopicContextFunctionalityType - case stc.v1.TopicContextFunctionalityType.Recommendation => - RecommendationTopicContextFunctionalityType - case stc.v1.TopicContextFunctionalityType.RecWithEducation => - RecWithEducationTopicContextFunctionalityType - } - - FeatureMapBuilder() - .add( - AncestorsFeature, - candidate.ancestors - .getOrElse(Seq.empty) - .map(ancestor => ta.TweetAncestor(ancestor.tweetId, ancestor.userId.getOrElse(0L)))) - .add( - AudioSpaceMetaDataFeature, - candidate.audioSpaceMetaDatalist.map(_.head).map(AudioSpaceMetaData.fromThrift)) - .add(AuthorIdFeature, Some(candidate.authorId)) - .add(AuthorIsBlueVerifiedFeature, candidate.authorIsBlueVerified.getOrElse(false)) - .add( - AuthorIsCreatorFeature, - candidate.authorIsCreator.getOrElse(false) - ) - .add(AuthorIsGoldVerifiedFeature, candidate.authorIsGoldVerified.getOrElse(false)) - .add(AuthorIsGrayVerifiedFeature, candidate.authorIsGrayVerified.getOrElse(false)) - .add(AuthorIsLegacyVerifiedFeature, candidate.authorIsLegacyVerified.getOrElse(false)) - .add( - AuthoredByContextualUserFeature, - candidate.viewerId.contains(candidate.authorId) || - candidate.viewerId.exists(candidate.sourceUserId.contains)) - .add(CandidateSourceIdFeature, candidate.candidateTweetSourceId) - .add(ConversationFeature, conversationFeatures) - .add(ConversationModuleIdFeature, candidate.conversationId) - .add(ConversationModuleFocalTweetIdFeature, focalTweetId) - .add(DirectedAtUserIdFeature, candidate.directedAtUserId) - .add(EarlybirdFeature, earlybirdFeatures) - // This is temporary, will need to be updated with the encoded string. - .add(EntityTokenFeature, Some("test_EntityTokenForYou")) - .add(ExclusiveConversationAuthorIdFeature, candidate.exclusiveConversationAuthorId) - .add(FavoritedByUserIdsFeature, candidate.favoritedByUserIds.getOrElse(Seq.empty)) - .add(FollowedByUserIdsFeature, candidate.followedByUserIds.getOrElse(Seq.empty)) - .add(TopicIdSocialContextFeature, topicIdSocialContextOpt) - .add(TopicContextFunctionalityTypeFeature, topicContextFunctionalityTypeOpt) - .add(FullScoringSucceededFeature, candidate.fullScoringSucceeded.getOrElse(false)) - .add(HasDisplayedTextFeature, hasDisplayedText) - .add(InNetworkFeature, candidate.isInNetwork.getOrElse(true)) - .add(InReplyToTweetIdFeature, candidate.inReplyToTweetId) - .add(IsAncestorCandidateFeature, candidate.isAncestorCandidate.getOrElse(false)) - .add( - IsExtendedReplyFeature, - isInNetwork && isReply && !isRetweet && directedAtUserIsInFirstDegree.contains(false)) - .add(FromInNetworkSourceFeature, candidate.isInNetwork.getOrElse(true)) - .add(IsRandomTweetFeature, candidate.isRandomTweet.getOrElse(false)) - .add(IsReadFromCacheFeature, candidate.isReadFromCache.getOrElse(false)) - .add(IsRetweetFeature, candidate.isRetweet.getOrElse(false)) - .add(IsRetweetedReplyFeature, isReply && isRetweet) - .add( - NonSelfFavoritedByUserIdsFeature, - candidate.favoritedByUserIds.getOrElse(Seq.empty).filterNot(_ == candidate.authorId)) - .add(NumImagesFeature, numImages) - .add( - OriginalTweetCreationTimeFromSnowflakeFeature, - SnowflakeId.timeFromIdOpt(originalTweetId)) - .add(PredictionRequestIdFeature, candidate.predictionRequestId) - .add(ScoreFeature, Some(candidate.score)) - .add( - SimclustersTweetTopKClustersWithScoresFeature, - candidate.simclustersTweetTopKClustersWithScores.map(_.toMap).getOrElse(Map.empty)) - .add( - StreamToKafkaFeature, - candidate.predictionRequestId.nonEmpty && candidate.fullScoringSucceeded.getOrElse(false)) - .add(SourceTweetIdFeature, candidate.sourceTweetId) - .add(SourceUserIdFeature, candidate.sourceUserId) - .add(SuggestTypeFeature, suggestType) - .add(QuotedTweetIdFeature, candidate.quotedTweetId) - .add( - TweetLanguageFeature, - earlybirdFeatures.flatMap(_.language.map(_.value)).map(getLanguageISOFormatByValue)) - .add(VideoDurationMsFeature, earlybirdFeatures.flatMap(_.videoDurationMs)) - .build() - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTweetPreviewsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTweetPreviewsCandidatePipelineConfig.scala deleted file mode 100644 index c60278bfb..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTweetPreviewsCandidatePipelineConfig.scala +++ /dev/null @@ -1,143 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.home_mixer.functional_component.candidate_source.EarlybirdCandidateSource -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder -import com.twitter.home_mixer.functional_component.filter.DropMaxCandidatesFilter -import com.twitter.home_mixer.functional_component.filter.PreviouslyServedTweetPreviewsFilter -import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate -import com.twitter.home_mixer.model.HomeFeatures.AuthorEnabledPreviewsFeature -import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature -import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature -import com.twitter.home_mixer.product.for_you.feature_hydrator.AuthorEnabledPreviewsFeatureHydrator -import com.twitter.home_mixer.product.for_you.feature_hydrator.TweetPreviewTweetypieCandidateFeatureHydrator -import com.twitter.home_mixer.product.for_you.filter.TweetPreviewTextFilter -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableTweetPreviewsCandidatePipelineParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.TweetPreviewsMaxCandidatesParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.TweetPreviewsMinInjectionIntervalParam -import com.twitter.home_mixer.product.for_you.query_transformer.TweetPreviewsQueryTransformer -import com.twitter.home_mixer.product.for_you.response_transformer.TweetPreviewResponseFeatureTransformer -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.PreviewCreatorsFeature -import com.twitter.product_mixer.component_library.filter.FeatureFilter -import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSourceWithExtractedFeatures -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.rtf.safety_level.TimelineHomeTweetPreviewHydrationSafetyLevel -import com.twitter.product_mixer.core.model.marshalling.response.urt.contextual_ref.TweetHydrationContext -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.search.earlybird.{thriftscala => eb} -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.injection.scribe.InjectionScribeUtil -import com.twitter.timelineservice.model.rich.EntityIdType -import com.twitter.timelineservice.suggests.{thriftscala => st} - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ForYouTweetPreviewsCandidatePipelineConfig @Inject() ( - earlybirdCandidateSource: EarlybirdCandidateSource, - authorEnabledPreviewsFeatureHydrator: AuthorEnabledPreviewsFeatureHydrator, - tweetPreviewsQueryTransformer: TweetPreviewsQueryTransformer, - tweetPreviewTweetypieCandidateFeatureHydrator: TweetPreviewTweetypieCandidateFeatureHydrator, - homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder) - extends CandidatePipelineConfig[ - ForYouQuery, - eb.EarlybirdRequest, - eb.ThriftSearchResult, - TweetCandidate - ] { - - val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ForYouTweetPreviews") - - override val supportedClientParam: Option[FSParam[Boolean]] = - Some(EnableTweetPreviewsCandidatePipelineParam) - - override val gates: Seq[Gate[ForYouQuery]] = { - Seq( - TimelinesPersistenceStoreLastInjectionGate( - TweetPreviewsMinInjectionIntervalParam, - PersistenceEntriesFeature, - EntityIdType.TweetPreview - ), - NonEmptySeqFeatureGate(PreviewCreatorsFeature) - ) - } - - override val queryTransformer: CandidatePipelineQueryTransformer[ - PipelineQuery, - eb.EarlybirdRequest - ] = tweetPreviewsQueryTransformer - - override val candidateSource: CandidateSourceWithExtractedFeatures[ - eb.EarlybirdRequest, - eb.ThriftSearchResult - ] = earlybirdCandidateSource - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[eb.ThriftSearchResult] - ] = Seq(TweetPreviewResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - eb.ThriftSearchResult, - TweetCandidate - ] = { tweet => TweetCandidate(tweet.id) } - - override val preFilterFeatureHydrationPhase1: Seq[ - BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] - ] = Seq( - authorEnabledPreviewsFeatureHydrator, - tweetPreviewTweetypieCandidateFeatureHydrator, - ) - - override val filters: Seq[ - Filter[ForYouQuery, TweetCandidate] - ] = Seq( - PreviouslyServedTweetPreviewsFilter, - FeatureFilter - .fromFeature(FilterIdentifier("TweetPreviewVisibilityFiltering"), IsHydratedFeature), - FeatureFilter - .fromFeature(FilterIdentifier("AuthorEnabledPreviews"), AuthorEnabledPreviewsFeature), - TweetPreviewTextFilter, - DropMaxCandidatesFilter(TweetPreviewsMaxCandidatesParam) - ) - - override val decorator: Option[CandidateDecorator[PipelineQuery, TweetCandidate]] = { - val component = InjectionScribeUtil.scribeComponent(st.SuggestType.TweetPreview).get - val clientEventInfoBuilder = ClientEventInfoBuilder[PipelineQuery, TweetCandidate](component) - - val tweetItemBuilder = TweetCandidateUrtItemBuilder[PipelineQuery, TweetCandidate]( - clientEventInfoBuilder = clientEventInfoBuilder, - contextualTweetRefBuilder = Some( - ContextualTweetRefBuilder( - TweetHydrationContext( - safetyLevelOverride = Some(TimelineHomeTweetPreviewHydrationSafetyLevel), - outerTweetContext = None - ) - ) - ), - feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder), - ) - - Some(UrtItemCandidateDecorator(tweetItemBuilder)) - } - - override val alerts: Seq[Alert] = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(95)) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToFollowCandidatePipelineConfigBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToFollowCandidatePipelineConfigBuilder.scala deleted file mode 100644 index 856f1fa6b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToFollowCandidatePipelineConfigBuilder.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeWhoToFollowFeedbackActionInfoBuilder -import com.twitter.home_mixer.functional_component.gate.DismissFatigueGate -import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate -import com.twitter.home_mixer.model.HomeFeatures.DismissInfoFeature -import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature -import com.twitter.home_mixer.model.HomeFeatures.WhoToFollowExcludedUserIdsFeature -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableWhoToFollowCandidatePipelineParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToFollowDisplayLocationParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToFollowDisplayTypeIdParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToFollowMinInjectionIntervalParam -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ParamWhoToFollowModuleDisplayTypeBuilder -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmCandidatePipelineConfig -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmCandidatePipelineConfigBuilder -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.timelineservice.model.rich.EntityIdType -import com.twitter.timelineservice.suggests.thriftscala.SuggestType -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ForYouWhoToFollowCandidatePipelineConfigBuilder @Inject() ( - whoToFollowArmCandidatePipelineConfigBuilder: WhoToFollowArmCandidatePipelineConfigBuilder, - homeWhoToFollowFeedbackActionInfoBuilder: HomeWhoToFollowFeedbackActionInfoBuilder) { - - def build(): WhoToFollowArmCandidatePipelineConfig[ForYouQuery] = { - val gates: Seq[Gate[ForYouQuery]] = Seq( - TimelinesPersistenceStoreLastInjectionGate( - WhoToFollowMinInjectionIntervalParam, - PersistenceEntriesFeature, - EntityIdType.WhoToFollow - ), - DismissFatigueGate(SuggestType.WhoToFollow, DismissInfoFeature) - ) - - whoToFollowArmCandidatePipelineConfigBuilder.build[ForYouQuery]( - identifier = WhoToFollowArmCandidatePipelineConfig.identifier, - supportedClientParam = Some(EnableWhoToFollowCandidatePipelineParam), - alerts = alerts, - gates = gates, - moduleDisplayTypeBuilder = - ParamWhoToFollowModuleDisplayTypeBuilder(WhoToFollowDisplayTypeIdParam), - feedbackActionInfoBuilder = Some(homeWhoToFollowFeedbackActionInfoBuilder), - excludedUserIdsFeature = Some(WhoToFollowExcludedUserIdsFeature), - displayLocationParam = WhoToFollowDisplayLocationParam - ) - } - - private val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(70), - HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToSubscribeCandidatePipelineConfigBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToSubscribeCandidatePipelineConfigBuilder.scala deleted file mode 100644 index bd4437b7c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToSubscribeCandidatePipelineConfigBuilder.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeWhoToSubscribeFeedbackActionInfoBuilder -import com.twitter.home_mixer.functional_component.gate.DismissFatigueGate -import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate -import com.twitter.home_mixer.model.HomeFeatures.DismissInfoFeature -import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableWhoToSubscribeCandidatePipelineParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToSubscribeDisplayTypeIdParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToSubscribeMinInjectionIntervalParam -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ParamWhoToFollowModuleDisplayTypeBuilder -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidatePipelineConfig -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidatePipelineConfigBuilder -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.timelineservice.model.rich.EntityIdType -import com.twitter.timelineservice.suggests.thriftscala.SuggestType -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ForYouWhoToSubscribeCandidatePipelineConfigBuilder @Inject() ( - whoToSubscribeCandidatePipelineConfigBuilder: WhoToSubscribeCandidatePipelineConfigBuilder, - homeWhoToSubscribeFeedbackActionInfoBuilder: HomeWhoToSubscribeFeedbackActionInfoBuilder) { - - def build(): WhoToSubscribeCandidatePipelineConfig[ForYouQuery] = { - val gates: Seq[Gate[ForYouQuery]] = Seq( - TimelinesPersistenceStoreLastInjectionGate( - WhoToSubscribeMinInjectionIntervalParam, - PersistenceEntriesFeature, - EntityIdType.WhoToSubscribe - ), - DismissFatigueGate(SuggestType.WhoToSubscribe, DismissInfoFeature) - ) - - whoToSubscribeCandidatePipelineConfigBuilder.build[ForYouQuery]( - identifier = WhoToSubscribeCandidatePipelineConfig.identifier, - supportedClientParam = Some(EnableWhoToSubscribeCandidatePipelineParam), - alerts = alerts, - gates = gates, - moduleDisplayTypeBuilder = - ParamWhoToFollowModuleDisplayTypeBuilder(WhoToSubscribeDisplayTypeIdParam), - feedbackActionInfoBuilder = Some(homeWhoToSubscribeFeedbackActionInfoBuilder) - ) - } - - private val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(70), - HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/BUILD.bazel deleted file mode 100644 index 540823800..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/BUILD.bazel +++ /dev/null @@ -1,24 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/product_pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/thrift/com/twitter/search:earlybird-scala", - "stitch/stitch-timelineservice/src/main/scala", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/ScoredTweetsProductCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/ScoredTweetsProductCandidateSource.scala deleted file mode 100644 index d1daeb93e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/ScoredTweetsProductCandidateSource.scala +++ /dev/null @@ -1,174 +0,0 @@ -package com.twitter.home_mixer.product.for_you.candidate_source - -import com.google.inject.Provider -import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature -import com.twitter.home_mixer.model.request.HomeMixerRequest -import com.twitter.home_mixer.model.request.ScoredTweetsProduct -import com.twitter.home_mixer.model.request.ScoredTweetsProductContext -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.home_mixer.{thriftscala => t} -import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer -import com.twitter.product_mixer.core.functional_component.candidate_source.product_pipeline.ProductPipelineCandidateSource -import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry -import com.twitter.timelines.render.{thriftscala => tl} -import com.twitter.timelineservice.suggests.{thriftscala => st} -import com.twitter.tweetconvosvc.tweet_ancestor.{thriftscala => ta} -import javax.inject.Inject -import javax.inject.Singleton - -/** - * [[ScoredTweetWithConversationMetadata]] - **/ -case class ScoredTweetWithConversationMetadata( - tweetId: Long, - authorId: Long, - score: Option[Double] = None, - suggestType: Option[st.SuggestType] = None, - sourceTweetId: Option[Long] = None, - sourceUserId: Option[Long] = None, - quotedTweetId: Option[Long] = None, - quotedUserId: Option[Long] = None, - inReplyToTweetId: Option[Long] = None, - inReplyToUserId: Option[Long] = None, - directedAtUserId: Option[Long] = None, - inNetwork: Option[Boolean] = None, - sgsValidLikedByUserIds: Option[Seq[Long]] = None, - sgsValidFollowedByUserIds: Option[Seq[Long]] = None, - ancestors: Option[Seq[ta.TweetAncestor]] = None, - topicId: Option[Long] = None, - topicFunctionalityType: Option[tl.TopicContextFunctionalityType] = None, - conversationId: Option[Long] = None, - conversationFocalTweetId: Option[Long] = None, - isReadFromCache: Option[Boolean] = None, - streamToKafka: Option[Boolean] = None, - exclusiveConversationAuthorId: Option[Long] = None, - authorIsBlueVerified: Option[Boolean] = None, - authorIsGoldVerified: Option[Boolean] = None, - authorIsGrayVerified: Option[Boolean] = None, - authorIsLegacyVerified: Option[Boolean] = None, - authorIsCreator: Option[Boolean] = None, - perspectiveFilteredLikedByUserIds: Option[Seq[Long]] = None) - -@Singleton -class ScoredTweetsProductCandidateSource @Inject() ( - override val productPipelineRegistry: Provider[ProductPipelineRegistry], - override val paramsBuilder: Provider[ParamsBuilder]) - extends ProductPipelineCandidateSource[ - ForYouQuery, - HomeMixerRequest, - t.ScoredTweetsResponse, - ScoredTweetWithConversationMetadata - ] { - - override val identifier: CandidateSourceIdentifier = - CandidateSourceIdentifier("ScoredTweetsProduct") - - private val MaxModuleSize = 3 - private val MaxAncestorsInConversation = 2 - - override def pipelineRequestTransformer(productPipelineQuery: ForYouQuery): HomeMixerRequest = { - HomeMixerRequest( - clientContext = productPipelineQuery.clientContext, - product = ScoredTweetsProduct, - productContext = Some( - ScoredTweetsProductContext( - productPipelineQuery.deviceContext, - productPipelineQuery.seenTweetIds, - productPipelineQuery.features.map(_.getOrElse(ServedTweetIdsFeature, Seq.empty)), - productPipelineQuery.features.map(_.getOrElse(TimelineServiceTweetsFeature, Seq.empty)) - )), - serializedRequestCursor = - productPipelineQuery.pipelineCursor.map(UrtCursorSerializer.serializeCursor), - maxResults = productPipelineQuery.requestedMaxResults, - debugParams = None, - homeRequestParam = false - ) - } - - override def productPipelineResultTransformer( - productPipelineResult: t.ScoredTweetsResponse - ): Seq[ScoredTweetWithConversationMetadata] = { - val scoredTweets = productPipelineResult.scoredTweets.flatMap { focalTweet => - val parentTweets = focalTweet.ancestors.getOrElse(Seq.empty).sortBy(-_.tweetId) - val (intermediates, root) = parentTweets.splitAt(parentTweets.size - 1) - val truncatedIntermediates = - intermediates.take(MaxModuleSize - MaxAncestorsInConversation).reverse - val rootScoredTweet: Seq[ScoredTweetWithConversationMetadata] = root.map { ancestor => - ScoredTweetWithConversationMetadata( - tweetId = ancestor.tweetId, - authorId = ancestor.userId, - suggestType = focalTweet.suggestType, - conversationId = Some(ancestor.tweetId), - conversationFocalTweetId = Some(focalTweet.tweetId), - exclusiveConversationAuthorId = focalTweet.exclusiveConversationAuthorId - ) - } - val conversationId = rootScoredTweet.headOption.map(_.tweetId) - - val tweetsToParents = - if (parentTweets.nonEmpty) parentTweets.zip(parentTweets.tail).toMap - else Map.empty[ta.TweetAncestor, ta.TweetAncestor] - - val intermediateScoredTweets = truncatedIntermediates.map { ancestor => - ScoredTweetWithConversationMetadata( - tweetId = ancestor.tweetId, - authorId = ancestor.userId, - suggestType = focalTweet.suggestType, - inReplyToTweetId = tweetsToParents.get(ancestor).map(_.tweetId), - conversationId = conversationId, - conversationFocalTweetId = Some(focalTweet.tweetId), - exclusiveConversationAuthorId = focalTweet.exclusiveConversationAuthorId - ) - } - val parentScoredTweets = rootScoredTweet ++ intermediateScoredTweets - - val conversationFocalTweetId = - if (parentScoredTweets.nonEmpty) Some(focalTweet.tweetId) else None - - val focalScoredTweet = ScoredTweetWithConversationMetadata( - tweetId = focalTweet.tweetId, - authorId = focalTweet.authorId, - score = focalTweet.score, - suggestType = focalTweet.suggestType, - sourceTweetId = focalTweet.sourceTweetId, - sourceUserId = focalTweet.sourceUserId, - quotedTweetId = focalTweet.quotedTweetId, - quotedUserId = focalTweet.quotedUserId, - inReplyToTweetId = parentScoredTweets.lastOption.map(_.tweetId), - inReplyToUserId = focalTweet.inReplyToUserId, - directedAtUserId = focalTweet.directedAtUserId, - inNetwork = focalTweet.inNetwork, - sgsValidLikedByUserIds = focalTweet.sgsValidLikedByUserIds, - sgsValidFollowedByUserIds = focalTweet.sgsValidFollowedByUserIds, - topicId = focalTweet.topicId, - topicFunctionalityType = focalTweet.topicFunctionalityType, - ancestors = focalTweet.ancestors, - conversationId = conversationId, - conversationFocalTweetId = conversationFocalTweetId, - isReadFromCache = focalTweet.isReadFromCache, - streamToKafka = focalTweet.streamToKafka, - exclusiveConversationAuthorId = focalTweet.exclusiveConversationAuthorId, - authorIsBlueVerified = focalTweet.authorMetadata.map(_.blueVerified), - authorIsGoldVerified = focalTweet.authorMetadata.map(_.goldVerified), - authorIsGrayVerified = focalTweet.authorMetadata.map(_.grayVerified), - authorIsLegacyVerified = focalTweet.authorMetadata.map(_.legacyVerified), - authorIsCreator = focalTweet.authorMetadata.map(_.creator), - perspectiveFilteredLikedByUserIds = focalTweet.perspectiveFilteredLikedByUserIds - ) - - parentScoredTweets :+ focalScoredTweet - } - - val dedupedTweets = scoredTweets.groupBy(_.tweetId).map { - case (_, duplicateAncestors) => duplicateAncestors.maxBy(_.score.getOrElse(0.0)) - } - - // Sort by tweet id to prevent issues with future assumptions of the root being the first - // tweet and the focal being the last tweet in a module. The tweets as a whole do not need - // to be sorted overall, only the relative order within modules must be kept. - dedupedTweets.toSeq.sortBy(_.tweetId) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/AuthorEnabledPreviewsFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/AuthorEnabledPreviewsFeatureHydrator.scala deleted file mode 100644 index f93a5041f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/AuthorEnabledPreviewsFeatureHydrator.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.home_mixer.product.for_you.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorEnabledPreviewsFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.strato.generated.client.audiencerewards.audienceRewardsService.GetCreatorPreferencesOnUserClientColumn - -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Hydrates the `AuthorEnabledPreviews` feature for tweets authored by creators by querying the - * `GetCreatorPreferences` Strato column. This feature corresponds to the `previews_enabled` field of that column. - * Given a tweet from a creator, this feature indicates whether that creator has enabled previews - * on their profile. - */ -@Singleton -class AuthorEnabledPreviewsFeatureHydrator @Inject() ( - getCreatorPreferencesOnUserClientColumn: GetCreatorPreferencesOnUserClientColumn) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("AuthorEnabledPreviews") - - override val features: Set[Feature[_, _]] = Set(AuthorEnabledPreviewsFeature) - - private val fetcher = getCreatorPreferencesOnUserClientColumn.fetcher - - private val DefaultAuthorEnabledPreviewsValue = true - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - val candidateAuthors = candidates - .map(_.features.getOrElse(AuthorIdFeature, None)) - .toSet - .flatten - - // Build a map of creator -> authorEnabledPreviews, then use it to populate candidate features - val authorIdToFeatureStitch = Stitch.collect { - candidateAuthors - .map { author => - val isAuthorEnabledPreviews = fetcher.fetch(author).map { - _.v.map(_.previewsEnabled).getOrElse(DefaultAuthorEnabledPreviewsValue) - } - (author, isAuthorEnabledPreviews) - }.toMap - } - - authorIdToFeatureStitch.map { authorIdToFeatureMap => - candidates.map { - _.features.getOrElse(AuthorIdFeature, None) match { - case Some(authorId) => FeatureMapBuilder() - .add(AuthorEnabledPreviewsFeature, authorIdToFeatureMap(authorId)) - .build() - case _ => FeatureMapBuilder() - .add(AuthorEnabledPreviewsFeature, DefaultAuthorEnabledPreviewsValue) - .build() - } - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/BUILD.bazel deleted file mode 100644 index 87a260b7e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/BUILD.bazel +++ /dev/null @@ -1,28 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "audience-rewards/thrift/src/main/thrift/common:thrift-scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_tweetypie", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "stitch/stitch-core", - "stitch/stitch-socialgraph", - "stitch/stitch-timelineservice", - "strato/config/columns/audiencerewards/audienceRewardsService:audienceRewardsService-strato-client", - "strato/src/main/scala/com/twitter/strato/client", - "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/feedback", - "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", - "util/util-core", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/FocalTweetFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/FocalTweetFeatureHydrator.scala deleted file mode 100644 index 0270a81ab..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/FocalTweetFeatureHydrator.scala +++ /dev/null @@ -1,84 +0,0 @@ -package com.twitter.home_mixer.product.for_you.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleIdFeature -import com.twitter.home_mixer.model.HomeFeatures.FocalTweetAuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.FocalTweetRealNamesFeature -import com.twitter.home_mixer.model.HomeFeatures.FocalTweetScreenNamesFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.RealNamesFeature -import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Social context for convo modules is hydrated on the root Tweet but needs info about the focal - * Tweet (e.g. author) to render the banner. This hydrator copies focal Tweet data into the root. - */ -@Singleton -class FocalTweetFeatureHydrator @Inject() () - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FocalTweet") - - override val features: Set[Feature[_, _]] = Set( - FocalTweetAuthorIdFeature, - FocalTweetInNetworkFeature, - FocalTweetRealNamesFeature, - FocalTweetScreenNamesFeature - ) - - private val DefaultFeatureMap = FeatureMapBuilder() - .add(FocalTweetAuthorIdFeature, None) - .add(FocalTweetInNetworkFeature, None) - .add(FocalTweetRealNamesFeature, None) - .add(FocalTweetScreenNamesFeature, None) - .build() - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - // Build a map of all the focal tweets to their corresponding features - val focalTweetIdToFeatureMap = candidates.flatMap { candidate => - val focalTweetId = candidate.features.getOrElse(ConversationModuleFocalTweetIdFeature, None) - if (focalTweetId.contains(candidate.candidate.id)) { - Some(candidate.candidate.id -> candidate.features) - } else None - }.toMap - - val updatedFeatureMap = candidates.map { candidate => - val focalTweetId = candidate.features.getOrElse(ConversationModuleFocalTweetIdFeature, None) - val conversationId = candidate.features.getOrElse(ConversationModuleIdFeature, None) - - // Check if the candidate is a root tweet and ensure its focal tweet's features are available - if (conversationId.contains(candidate.candidate.id) - && focalTweetId.exists(focalTweetIdToFeatureMap.contains)) { - val featureMap = focalTweetIdToFeatureMap.get(focalTweetId.get).get - FeatureMapBuilder() - .add(FocalTweetAuthorIdFeature, featureMap.getOrElse(AuthorIdFeature, None)) - .add(FocalTweetInNetworkFeature, Some(featureMap.getOrElse(InNetworkFeature, true))) - .add( - FocalTweetRealNamesFeature, - Some(featureMap.getOrElse(RealNamesFeature, Map.empty[Long, String]))) - .add( - FocalTweetScreenNamesFeature, - Some(featureMap.getOrElse(ScreenNamesFeature, Map.empty[Long, String]))) - .build() - } else DefaultFeatureMap - } - - Stitch.value(updatedFeatureMap) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TimelineServiceTweetsQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TimelineServiceTweetsQueryFeatureHydrator.scala deleted file mode 100644 index e0ae2207e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TimelineServiceTweetsQueryFeatureHydrator.scala +++ /dev/null @@ -1,62 +0,0 @@ -package com.twitter.home_mixer.product.for_you.feature_hydrator - -import com.twitter.home_mixer.marshaller.timelines.DeviceContextMarshaller -import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature -import com.twitter.home_mixer.model.request.DeviceContext -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.stitch.timelineservice.TimelineService -import com.twitter.timelineservice.{thriftscala => t} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class TimelineServiceTweetsQueryFeatureHydrator @Inject() ( - timelineService: TimelineService, - deviceContextMarshaller: DeviceContextMarshaller) - extends QueryFeatureHydrator[PipelineQuery with HasDeviceContext] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("TimelineServiceTweets") - - override val features: Set[Feature[_, _]] = Set(TimelineServiceTweetsFeature) - - private val MaxTimelineServiceTweets = 200 - - override def hydrate(query: PipelineQuery with HasDeviceContext): Stitch[FeatureMap] = { - val deviceContext = query.deviceContext.getOrElse(DeviceContext.Empty) - - val timelineQueryOptions = t.TimelineQueryOptions( - contextualUserId = query.clientContext.userId, - deviceContext = Some(deviceContextMarshaller(deviceContext, query.clientContext)) - ) - - val timelineServiceQuery = t.TimelineQuery( - timelineType = t.TimelineType.Home, - timelineId = query.getRequiredUserId, - maxCount = MaxTimelineServiceTweets.toShort, - cursor2 = None, - options = Some(timelineQueryOptions), - timelineId2 = query.clientContext.userId.map(t.TimelineId(t.TimelineType.Home, _, None)), - ) - - timelineService.getTimeline(timelineServiceQuery).map { timeline => - val tweets = timeline.entries.collect { - case t.TimelineEntry.Tweet(tweet) => tweet.statusId - } - - FeatureMapBuilder().add(TimelineServiceTweetsFeature, tweets).build() - } - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.7) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetPreviewTweetypieCandidateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetPreviewTweetypieCandidateFeatureHydrator.scala deleted file mode 100644 index 07f3ae0e9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetPreviewTweetypieCandidateFeatureHydrator.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.home_mixer.product.for_you.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature -import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature -import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.spam.rtf.{thriftscala => rtf} -import com.twitter.stitch.Stitch -import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient} -import com.twitter.tweetypie.{thriftscala => TP} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class TweetPreviewTweetypieCandidateFeatureHydrator @Inject() ( - tweetypieStitchClient: TweetypieStitchClient) - extends CandidateFeatureHydrator[PipelineQuery, BaseTweetCandidate] { - - private val CoreTweetFields: Set[TP.TweetInclude] = Set[TP.TweetInclude]( - TP.TweetInclude.TweetFieldId(TP.Tweet.IdField.id), - TP.TweetInclude.TweetFieldId(TP.Tweet.CoreDataField.id) - ) - - private val DefaultFeatureMap = FeatureMapBuilder() - .add(TweetTextFeature, None) - .add(IsHydratedFeature, false) - .add(AuthorIdFeature, None) - .build() - - override val features: Set[Feature[_, _]] = - Set(TweetTextFeature, IsHydratedFeature, AuthorIdFeature) - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("TweetPreviewTweetypie") - - override def apply( - query: PipelineQuery, - candidate: BaseTweetCandidate, - existingFeatures: FeatureMap - ): Stitch[FeatureMap] = { - tweetypieStitchClient - .getTweetFields( - tweetId = candidate.id, - options = TP.GetTweetFieldsOptions( - tweetIncludes = CoreTweetFields, - includeRetweetedTweet = false, - includeQuotedTweet = false, - visibilityPolicy = TP.TweetVisibilityPolicy.UserVisible, - safetyLevel = Some(rtf.SafetyLevel.TimelineHomeTweetPreview), - forUserId = query.getOptionalUserId - ) - ).map { - case TP.GetTweetFieldsResult(_, TP.TweetFieldsResultState.Found(found), quoteOpt, _) => - val tweetText = found.tweet.coreData.map(_.text) - FeatureMapBuilder() - .add(TweetTextFeature, tweetText) - .add(IsHydratedFeature, true) - .add(AuthorIdFeature, found.tweet.coreData.map(_.userId)) - .build() - // If no tweet result found, return default features - case _ => - DefaultFeatureMap - } - - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/BUILD.bazel deleted file mode 100644 index 0a875371d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/BUILD.bazel +++ /dev/null @@ -1,15 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter", - "stitch/stitch-core", - "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/SocialContextFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/SocialContextFilter.scala deleted file mode 100644 index 5b007969b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/SocialContextFilter.scala +++ /dev/null @@ -1,76 +0,0 @@ -package com.twitter.home_mixer.product.for_you.filter - -import com.twitter.home_mixer.model.HomeFeatures._ -import com.twitter.home_mixer.product.for_you.param.ForYouParam -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelineservice.suggests.{thriftscala => st} - -object SocialContextFilter extends Filter[PipelineQuery, TweetCandidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("SocialContext") - - // Tweets from candidate sources which don't need generic like/follow/topic proof - private val AllowedSources: Set[st.SuggestType] = Set( - st.SuggestType.RankedListTweet, - st.SuggestType.RecommendedTrendTweet, - st.SuggestType.MediaTweet - ) - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - val enableIsVerifiedAuthorFilter = - query.params(ForYouParam.EnableVerifiedAuthorSocialContextBypassParam) - - val enableTopicSocialContextFilter = - query.params(ForYouParam.EnableTopicSocialContextFilterParam) - - val validTweetIds = candidates - .filter { candidate => - candidate.features.getOrElse(InNetworkFeature, true) || - candidate.features.getOrElse(SuggestTypeFeature, None).exists(AllowedSources.contains) || - candidate.features.getOrElse(ConversationModuleFocalTweetIdFeature, None).isDefined || - (enableIsVerifiedAuthorFilter && isVerifiedAuthor(candidate.features)) || - hasLikedBySocialContext(candidate.features) || - hasFollowedBySocialContext(candidate.features) || - (enableTopicSocialContextFilter && hasTopicSocialContext(candidate.features)) - }.map(_.candidate.id).toSet - - val (kept, removed) = - candidates.map(_.candidate).partition(candidate => validTweetIds.contains(candidate.id)) - - Stitch.value(FilterResult(kept = kept, removed = removed)) - } - - private def isVerifiedAuthor(candidateFeatures: FeatureMap): Boolean = { - candidateFeatures.getOrElse(AuthorIsBlueVerifiedFeature, false) || - candidateFeatures.getOrElse(AuthorIsGoldVerifiedFeature, false) || - candidateFeatures.getOrElse(AuthorIsGrayVerifiedFeature, false) || - candidateFeatures.getOrElse(AuthorIsLegacyVerifiedFeature, false) - } - - private def hasLikedBySocialContext(candidateFeatures: FeatureMap): Boolean = - candidateFeatures - .getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty) - .exists( - candidateFeatures - .getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Seq.empty) - .toSet.contains - ) - - private def hasFollowedBySocialContext(candidateFeatures: FeatureMap): Boolean = - candidateFeatures.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty).nonEmpty - - private def hasTopicSocialContext(candidateFeatures: FeatureMap): Boolean = { - candidateFeatures.getOrElse(TopicIdSocialContextFeature, None).isDefined && - candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None).isDefined - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/TweetPreviewTextFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/TweetPreviewTextFilter.scala deleted file mode 100644 index 61125fcd3..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/TweetPreviewTextFilter.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.twitter.home_mixer.product.for_you.filter - -import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -object TweetPreviewTextFilter extends Filter[PipelineQuery, TweetCandidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("TweetPreviewText") - - private val PreviewTextLength = 50 - private val MinTweetLength = PreviewTextLength * 2 - private val MaxNewlines = 2 - private val HttpPrefix = "http://" - private val HttpsPrefix = "https://" - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - - val (kept, removed) = candidates - .partition { candidate => - val text = candidate.features.get(TweetTextFeature).getOrElse("") - - text.length > MinTweetLength && - text.take(PreviewTextLength).count(_ == '\n') <= MaxNewlines && - !(text.startsWith(HttpPrefix) || text.startsWith(HttpsPrefix)) - } - - val filterResult = FilterResult( - kept = kept.map(_.candidate), - removed = removed.map(_.candidate) - ) - - Stitch.value(filterResult) - } - -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate/BUILD.bazel deleted file mode 100644 index 9a1d186dd..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate/PushToHomeRequestGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate/PushToHomeRequestGate.scala deleted file mode 100644 index 4c9d81021..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate/PushToHomeRequestGate.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.home_mixer.product.for_you.functional_component.gate - -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier -import com.twitter.stitch.Stitch - -/** - * Continues when the request is a Push-To-Home notification request - */ -object PushToHomeRequestGate extends Gate[ForYouQuery] { - override val identifier: GateIdentifier = GateIdentifier("PushToHomeRequest") - - override def shouldContinue(query: ForYouQuery): Stitch[Boolean] = - Stitch.value(query.pushToHomeTweetId.isDefined) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/BUILD.bazel deleted file mode 100644 index bcf9519f5..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/BUILD.bazel +++ /dev/null @@ -1,21 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/ForYouQuery.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/ForYouQuery.scala deleted file mode 100644 index dda427350..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/ForYouQuery.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.home_mixer.product.for_you.model - -import com.twitter.adserver.thriftscala.HomeTimelineType -import com.twitter.adserver.thriftscala.TimelineRequestParams -import com.twitter.dspbidder.commons.{thriftscala => dsp} -import com.twitter.home_mixer.model.HomeAdsQuery -import com.twitter.home_mixer.model.request.DeviceContext -import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.home_mixer.model.request.HasSeenTweetIds -import com.twitter.onboarding.task.service.{thriftscala => ots} -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.transformer.HasFlipInjectionParams -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.request._ -import com.twitter.product_mixer.core.pipeline.HasPipelineCursor -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.Params - -case class ForYouQuery( - override val params: Params, - override val clientContext: ClientContext, - override val pipelineCursor: Option[UrtOrderedCursor], - override val requestedMaxResults: Option[Int], - override val debugOptions: Option[DebugOptions], - override val features: Option[FeatureMap], - override val deviceContext: Option[DeviceContext], - override val seenTweetIds: Option[Seq[Long]], - override val dspClientContext: Option[dsp.DspClientContext], - pushToHomeTweetId: Option[Long]) - extends PipelineQuery - with HasPipelineCursor[UrtOrderedCursor] - with HasDeviceContext - with HasSeenTweetIds - with HasFlipInjectionParams - with HomeAdsQuery { - override val product: Product = ForYouProduct - - override def withFeatureMap(features: FeatureMap): ForYouQuery = - copy(features = Some(features)) - - override val timelineRequestParams: Option[TimelineRequestParams] = - Some(TimelineRequestParams(homeTimelineType = Some(HomeTimelineType.Home))) - - // Fields below are used for FLIP Injection in Onboarding Task Service (OTS) - override val displayLocation: ots.DisplayLocation = ots.DisplayLocation.HomeTimeline - override val rankingDisablerWithLatestControlsAvailable: Option[Boolean] = None - override val isEmptyState: Option[Boolean] = None - override val isFirstRequestAfterSignup: Option[Boolean] = None - override val isEndOfTimeline: Option[Boolean] = None - override val timelineId: Option[Long] = None -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/ForYouTweetsResponse.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/ForYouTweetsResponse.scala deleted file mode 100644 index 68b0d6736..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/ForYouTweetsResponse.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.home_mixer.product.for_you.model - -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling - -case class ForYouTweetsResponse(tweetCandidates: Seq[Long]) extends HasMarshalling diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/BUILD.bazel deleted file mode 100644 index a56e3a1fd..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/BUILD.bazel +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "util/util-core/src/main/scala/com/twitter/conversions", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParam.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParam.scala deleted file mode 100644 index 5d117199f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParam.scala +++ /dev/null @@ -1,215 +0,0 @@ -package com.twitter.home_mixer.product.for_you.param - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.param.decider.DeciderKey -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.WhoToFollowModuleDisplayType -import com.twitter.timelines.configapi.DurationConversion -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSEnumParam -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.timelines.configapi.decider.BooleanDeciderParam -import com.twitter.util.Duration - -object ForYouParam { - val SupportedClientFSName = "for_you_supported_client" - - object EnableTopicSocialContextFilterParam - extends FSParam[Boolean]( - name = "for_you_enable_topic_social_context_filter", - default = true - ) - - object EnableVerifiedAuthorSocialContextBypassParam - extends FSParam[Boolean]( - name = "for_you_enable_verified_author_social_context_bypass", - default = true - ) - - object EnableTimelineScorerCandidatePipelineParam - extends FSParam[Boolean]( - name = "for_you_enable_timeline_scorer_candidate_pipeline", - default = false - ) - - object EnableScoredTweetsCandidatePipelineParam - extends BooleanDeciderParam(DeciderKey.EnableForYouScoredTweetsCandidatePipeline) - - object EnableWhoToFollowCandidatePipelineParam - extends FSParam[Boolean]( - name = "for_you_enable_who_to_follow", - default = true - ) - - object EnableWhoToSubscribeCandidatePipelineParam - extends FSParam[Boolean]( - name = "for_you_enable_who_to_subscribe", - default = true - ) - - object EnableTweetPreviewsCandidatePipelineParam - extends FSParam[Boolean]( - name = "for_you_enable_tweet_previews_candidate_pipeline", - default = true - ) - - object EnablePushToHomeMixerPipelineParam - extends FSParam[Boolean]( - name = "for_you_enable_push_to_home_mixer_pipeline", - default = false - ) - - object EnableScoredTweetsMixerPipelineParam - extends FSParam[Boolean]( - name = "for_you_enable_scored_tweets_mixer_pipeline", - default = true - ) - - object ServerMaxResultsParam - extends FSBoundedParam[Int]( - name = "for_you_server_max_results", - default = 35, - min = 1, - max = 500 - ) - - object AdsNumOrganicItemsParam - extends FSBoundedParam[Int]( - name = "for_you_ads_num_organic_items", - default = 35, - min = 1, - max = 100 - ) - - object WhoToFollowPositionParam - extends FSBoundedParam[Int]( - name = "for_you_who_to_follow_position", - default = 5, - min = 0, - max = 99 - ) - - object WhoToFollowMinInjectionIntervalParam - extends FSBoundedParam[Duration]( - "for_you_who_to_follow_min_injection_interval_in_minutes", - default = 1800.minutes, - min = 0.minutes, - max = 6000.minutes) - with HasDurationConversion { - override val durationConversion: DurationConversion = DurationConversion.FromMinutes - } - - object WhoToFollowDisplayTypeIdParam - extends FSEnumParam[WhoToFollowModuleDisplayType.type]( - name = "for_you_enable_who_to_follow_display_type_id", - default = WhoToFollowModuleDisplayType.Vertical, - enum = WhoToFollowModuleDisplayType - ) - - object WhoToFollowDisplayLocationParam - extends FSParam[String]( - name = "for_you_who_to_follow_display_location", - default = "timeline" - ) - - object WhoToSubscribePositionParam - extends FSBoundedParam[Int]( - name = "for_you_who_to_subscribe_position", - default = 7, - min = 0, - max = 99 - ) - - object WhoToSubscribeMinInjectionIntervalParam - extends FSBoundedParam[Duration]( - "for_you_who_to_subscribe_min_injection_interval_in_minutes", - default = 1800.minutes, - min = 0.minutes, - max = 6000.minutes) - with HasDurationConversion { - override val durationConversion: DurationConversion = DurationConversion.FromMinutes - } - - object WhoToSubscribeDisplayTypeIdParam - extends FSEnumParam[WhoToFollowModuleDisplayType.type]( - name = "for_you_enable_who_to_subscribe_display_type_id", - default = WhoToFollowModuleDisplayType.Vertical, - enum = WhoToFollowModuleDisplayType - ) - - object TweetPreviewsPositionParam - extends FSBoundedParam[Int]( - name = "for_you_tweet_previews_position", - default = 3, - min = 0, - max = 99 - ) - - object TweetPreviewsMinInjectionIntervalParam - extends FSBoundedParam[Duration]( - "for_you_tweet_previews_min_injection_interval_in_minutes", - default = 2.hours, - min = 0.minutes, - max = 600.minutes) - with HasDurationConversion { - override val durationConversion: DurationConversion = DurationConversion.FromMinutes - } - - object TweetPreviewsMaxCandidatesParam - extends FSBoundedParam[Int]( - name = "for_you_tweet_previews_max_candidates", - default = 1, - min = 0, - // NOTE: previews are injected at a fixed position, so max candidates = 1 - // to avoid bunching of previews. - max = 1 - ) - - object EnableFlipInjectionModuleCandidatePipelineParam - extends FSParam[Boolean]( - name = "for_you_enable_flip_inline_injection_module", - default = true - ) - - object FlipInlineInjectionModulePosition - extends FSBoundedParam[Int]( - name = "for_you_flip_inline_injection_module_position", - default = 0, - min = 0, - max = 1000 - ) - - object ClearCacheOnPtr { - object EnableParam - extends FSParam[Boolean]( - name = "for_you_clear_cache_ptr_enable", - default = false - ) - - case object MinEntriesParam - extends FSBoundedParam[Int]( - name = "for_you_clear_cache_ptr_min_entries", - default = 10, - min = 0, - max = 35 - ) - } - - object EnableClearCacheOnPushToHome - extends FSParam[Boolean]( - name = "for_you_enable_clear_cache_push_to_home", - default = false - ) - - object EnableServedCandidateKafkaPublishingParam - extends FSParam[Boolean]( - name = "for_you_enable_served_candidate_kafka_publishing", - default = true - ) - - object ExperimentStatsParam - extends FSParam[String]( - name = "for_you_experiment_stats", - default = "" - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParamConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParamConfig.scala deleted file mode 100644 index 001ee57ad..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParamConfig.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.home_mixer.product.for_you.param - -import com.twitter.home_mixer.param.decider.DeciderKey -import com.twitter.home_mixer.product.for_you.param.ForYouParam._ -import com.twitter.product_mixer.core.product.ProductParamConfig -import com.twitter.servo.decider.DeciderKeyName -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ForYouParamConfig @Inject() () extends ProductParamConfig { - override val enabledDeciderKey: DeciderKeyName = DeciderKey.EnableForYouProduct - override val supportedClientFSName: String = SupportedClientFSName - - override val booleanDeciderOverrides = Seq( - EnableScoredTweetsCandidatePipelineParam - ) - - override val booleanFSOverrides = Seq( - ClearCacheOnPtr.EnableParam, - EnableFlipInjectionModuleCandidatePipelineParam, - EnablePushToHomeMixerPipelineParam, - EnableScoredTweetsMixerPipelineParam, - EnableServedCandidateKafkaPublishingParam, - EnableTimelineScorerCandidatePipelineParam, - EnableTopicSocialContextFilterParam, - EnableVerifiedAuthorSocialContextBypassParam, - EnableWhoToFollowCandidatePipelineParam, - EnableWhoToSubscribeCandidatePipelineParam, - EnableTweetPreviewsCandidatePipelineParam, - EnableClearCacheOnPushToHome - ) - - override val boundedIntFSOverrides = Seq( - AdsNumOrganicItemsParam, - ClearCacheOnPtr.MinEntriesParam, - FlipInlineInjectionModulePosition, - ServerMaxResultsParam, - WhoToFollowPositionParam, - WhoToSubscribePositionParam, - TweetPreviewsPositionParam, - TweetPreviewsMaxCandidatesParam - ) - - override val stringFSOverrides = Seq( - WhoToFollowDisplayLocationParam, - ExperimentStatsParam - ) - - override val boundedDurationFSOverrides = Seq( - WhoToFollowMinInjectionIntervalParam, - WhoToSubscribeMinInjectionIntervalParam, - TweetPreviewsMinInjectionIntervalParam - ) - - override val enumFSOverrides = Seq( - WhoToFollowDisplayTypeIdParam, - WhoToSubscribeDisplayTypeIdParam - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/BUILD.bazel deleted file mode 100644 index 17e99c46c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/BUILD.bazel +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "src/java/com/twitter/search/common/schema/earlybird", - "src/java/com/twitter/search/queryparser/query:core-query-nodes", - "src/java/com/twitter/search/queryparser/query/search:search-query-nodes", - "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/socialgraph:thrift-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/TweetPreviewsQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/TweetPreviewsQueryTransformer.scala deleted file mode 100644 index 41940b823..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/TweetPreviewsQueryTransformer.scala +++ /dev/null @@ -1,92 +0,0 @@ -package com.twitter.home_mixer.product.for_you.query_transformer - -import com.twitter.conversions.DurationOps.richDurationFromInt -import com.twitter.finagle.thrift.ClientId -import com.twitter.finagle.tracing.Trace -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.PreviewCreatorsFeature -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.search.common.ranking.{thriftscala => scr} -import com.twitter.search.common.schema.earlybird.EarlybirdFieldConstants.EarlybirdFieldConstant -import com.twitter.search.earlybird.{thriftscala => t} -import com.twitter.search.queryparser.query.Conjunction -import com.twitter.search.queryparser.query.Query -import com.twitter.search.queryparser.query.search.SearchOperator -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class TweetPreviewsQueryTransformer @Inject() (clientId: ClientId) - extends CandidatePipelineQueryTransformer[PipelineQuery, t.EarlybirdRequest] { - - private val MaxPreviewTweets = 200 - private val EarlybirdRelevanceTensorflowModel = "timelines_rectweet_replica" - private val SinceDuration = 7.days - - val MetadataOptions = t.ThriftSearchResultMetadataOptions( - getReferencedTweetAuthorId = true, - getFromUserId = true - ) - - override def transform(query: PipelineQuery): t.EarlybirdRequest = { - val candidatePreviewCreatorIds = - query.features.map(_.get(PreviewCreatorsFeature)).getOrElse(Seq.empty) - - val searchQuery = new Conjunction( - // Include subscriber only (aka exclusive) Tweets - new SearchOperator.Builder() - .setType(SearchOperator.Type.FILTER) - .addOperand(EarlybirdFieldConstant.EXCLUSIVE_FILTER_TERM) - .build(), - // Include only original Tweets - new SearchOperator.Builder() - .setType(SearchOperator.Type.FILTER) - .addOperand(EarlybirdFieldConstant.NATIVE_RETWEETS_FILTER_TERM) - .setOccur(Query.Occur.MUST_NOT) - .build(), - new SearchOperator.Builder() - .setType(SearchOperator.Type.FILTER) - .addOperand(EarlybirdFieldConstant.REPLIES_FILTER_TERM) - .setOccur(Query.Occur.MUST_NOT) - .build(), - new SearchOperator.Builder() - .setType(SearchOperator.Type.FILTER) - .addOperand(EarlybirdFieldConstant.QUOTE_FILTER_TERM) - .setOccur(Query.Occur.MUST_NOT) - .build(), - new SearchOperator(SearchOperator.Type.SINCE_TIME, SinceDuration.ago.inSeconds.toString) - ) - - t.EarlybirdRequest( - searchQuery = t.ThriftSearchQuery( - serializedQuery = Some(searchQuery.serialize), - fromUserIDFilter64 = Some(candidatePreviewCreatorIds), - numResults = MaxPreviewTweets, - rankingMode = t.ThriftSearchRankingMode.Relevance, - relevanceOptions = Some( - t.ThriftSearchRelevanceOptions( - filterDups = true, - keepDupWithHigherScore = true, - proximityScoring = true, - maxConsecutiveSameUser = Some(5), - rankingParams = Some( - scr.ThriftRankingParams( - `type` = Some(scr.ThriftScoringFunctionType.TensorflowBased), - selectedTensorflowModel = Some(EarlybirdRelevanceTensorflowModel), - minScore = -1.0e100, - applyBoosts = false, - ) - ), - ), - ), - resultMetadataOptions = Some(MetadataOptions), - searcherId = query.getOptionalUserId, - ), - getOlderResults = Some(true), // needed for archive access to older tweets - clientRequestID = Some(s"${Trace.id.traceId}"), - followedUserIds = Some(candidatePreviewCreatorIds.toSeq), - numResultsToReturnAtRoot = Some(MaxPreviewTweets), - clientId = Some(clientId.name), - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/BUILD.bazel deleted file mode 100644 index cf629b665..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "src/java/com/twitter/search/common/schema/earlybird", - "src/thrift/com/twitter/search:earlybird-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/TweetPreviewResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/TweetPreviewResponseFeatureTransformer.scala deleted file mode 100644 index 8b783db38..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/TweetPreviewResponseFeatureTransformer.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.home_mixer.product.for_you.response_transformer - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsTweetPreviewFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.timelineservice.suggests.{thriftscala => st} -import com.twitter.search.earlybird.{thriftscala => eb} - -object TweetPreviewResponseFeatureTransformer - extends CandidateFeatureTransformer[eb.ThriftSearchResult] { - - override val identifier: TransformerIdentifier = - TransformerIdentifier("TweetPreviewResponse") - - override val features: Set[Feature[_, _]] = - Set(AuthorIdFeature, IsTweetPreviewFeature, SuggestTypeFeature) - - def transform( - input: eb.ThriftSearchResult - ): FeatureMap = { - FeatureMapBuilder() - .add(IsTweetPreviewFeature, true) - .add(SuggestTypeFeature, Some(st.SuggestType.TweetPreview)) - .add(AuthorIdFeature, input.metadata.map(_.fromUserId)) - .build() - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/scorer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/scorer/BUILD.bazel deleted file mode 100644 index c320c8838..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/scorer/BUILD.bazel +++ /dev/null @@ -1,10 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "timelineservice/common:model", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/BUILD.bazel deleted file mode 100644 index 90faf3d45..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/BUILD.bazel +++ /dev/null @@ -1,24 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "src/scala/com/twitter/timelines/prediction/common/adapters", - "src/scala/com/twitter/timelines/prediction/features/common", - "src/thrift/com/twitter/timelines/served_candidates_logging:served_candidates_logging-scala", - "src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java", - "timelines/ml:kafka", - "timelines/ml/cont_train/common/client/src/main/scala/com/twitter/timelines/ml/cont_train/common/client/kafka", - "timelines/ml/cont_train/common/domain/src/main/scala/com/twitter/timelines/ml/cont_train/common/domain/non_scalding", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffect.scala deleted file mode 100644 index bfa7dd52e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffect.scala +++ /dev/null @@ -1,112 +0,0 @@ -package com.twitter.home_mixer.product.for_you.side_effect - -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.IsReadFromCacheFeature -import com.twitter.home_mixer.model.HomeFeatures.PredictionRequestIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ServedIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableServedCandidateKafkaPublishingParam -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.side_effect.KafkaPublishingSideEffect -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.identifier -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.ml.cont_train.common.domain.non_scalding.ServedCandidateFeatureKeysAdapter -import com.twitter.timelines.ml.cont_train.common.domain.non_scalding.ServedCandidateFeatureKeysFields -import com.twitter.timelines.ml.kafka.serde.CandidateFeatureKeySerde -import com.twitter.timelines.ml.kafka.serde.TBaseSerde -import com.twitter.timelines.served_candidates_logging.{thriftscala => sc} -import com.twitter.timelines.suggests.common.poly_data_record.{thriftjava => pldr} -import com.twitter.timelineservice.suggests.{thriftscala => tls} -import org.apache.kafka.clients.producer.ProducerRecord -import org.apache.kafka.common.serialization.Serializer -import scala.collection.JavaConverters._ - -/** - * Pipeline side-effect that publishes candidate keys to a Kafka topic. - */ -class ServedCandidateFeatureKeysKafkaSideEffect( - topic: String, - sourceIdentifiers: Set[identifier.CandidatePipelineIdentifier]) - extends KafkaPublishingSideEffect[ - sc.CandidateFeatureKey, - pldr.PolyDataRecord, - PipelineQuery, - Timeline - ] - with PipelineResultSideEffect.Conditionally[PipelineQuery, Timeline] { - - import ServedCandidateKafkaSideEffect._ - - override val identifier: SideEffectIdentifier = SideEffectIdentifier("ServedCandidateFeatureKeys") - - override def onlyIf( - query: PipelineQuery, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: Timeline - ): Boolean = query.params.getBoolean(EnableServedCandidateKafkaPublishingParam) - - override val bootstrapServer: String = "/s/kafka/timeline:kafka-tls" - - override val keySerde: Serializer[sc.CandidateFeatureKey] = - CandidateFeatureKeySerde().serializer() - - override val valueSerde: Serializer[pldr.PolyDataRecord] = - TBaseSerde.Thrift[pldr.PolyDataRecord]().serializer - - override val clientId: String = "home_mixer_served_candidate_feature_keys_producer" - - override def buildRecords( - query: PipelineQuery, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: Timeline - ): Seq[ProducerRecord[sc.CandidateFeatureKey, pldr.PolyDataRecord]] = { - val servedRequestIdOpt = - query.features.getOrElse(FeatureMap.empty).getOrElse(ServedRequestIdFeature, None) - - extractCandidates(query, selectedCandidates, sourceIdentifiers).map { candidate => - val isReadFromCache = candidate.features.getOrElse(IsReadFromCacheFeature, false) - val servedId = candidate.features.get(ServedIdFeature).get - - val key = sc.CandidateFeatureKey( - tweetId = candidate.candidateIdLong, - viewerId = query.getRequiredUserId, - servedId = servedId) - - val record = - ServedCandidateFeatureKeysAdapter - .adaptToDataRecords( - ServedCandidateFeatureKeysFields( - candidateTweetSourceId = candidate.features - .getOrElse(CandidateSourceIdFeature, None).map(_.value.toLong).getOrElse(2L), - predictionRequestId = - candidate.features.getOrElse(PredictionRequestIdFeature, None).get, - servedRequestIdOpt = if (isReadFromCache) servedRequestIdOpt else None, - servedId = servedId, - injectionModuleName = candidate.getClass.getSimpleName, - viewerFollowsOriginalAuthor = - Some(candidate.features.getOrElse(InNetworkFeature, true)), - suggestType = candidate.features - .getOrElse(SuggestTypeFeature, None).getOrElse(tls.SuggestType.RankedOrganicTweet), - finalPositionIndex = Some(candidate.sourcePosition), - isReadFromCache = isReadFromCache - )).asScala.head - - new ProducerRecord(topic, key, pldr.PolyDataRecord.dataRecord(record)) - } - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(98.5) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffectBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffectBuilder.scala deleted file mode 100644 index 5e4164276..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffectBuilder.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.home_mixer.product.for_you.side_effect - -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class ServedCandidateFeatureKeysKafkaSideEffectBuilder @Inject() ( - injectedServiceIdentifier: ServiceIdentifier) { - def build( - sourceIdentifiers: Set[CandidatePipelineIdentifier] - ): ServedCandidateFeatureKeysKafkaSideEffect = { - val topic = injectedServiceIdentifier.environment.toLowerCase match { - case "prod" => "tq_ct_served_candidate_feature_keys" - case _ => "tq_ct_served_candidate_feature_keys_staging" - } - new ServedCandidateFeatureKeysKafkaSideEffect(topic, sourceIdentifiers) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKafkaSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKafkaSideEffect.scala deleted file mode 100644 index 7da751764..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKafkaSideEffect.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.home_mixer.product.for_you.side_effect - -import com.twitter.home_mixer.model.HomeFeatures.IsReadFromCacheFeature -import com.twitter.home_mixer.model.HomeFeatures.PredictionRequestIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ServedIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature -import com.twitter.home_mixer.model.HomeFeatures.StreamToKafkaFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object ServedCandidateKafkaSideEffect { - - def extractCandidates( - query: PipelineQuery, - selectedCandidates: Seq[CandidateWithDetails], - sourceIdentifiers: Set[CandidatePipelineIdentifier] - ): Seq[ItemCandidateWithDetails] = { - val servedRequestIdOpt = - query.features.getOrElse(FeatureMap.empty).getOrElse(ServedRequestIdFeature, None) - - selectedCandidates.iterator - .filter(candidate => sourceIdentifiers.contains(candidate.source)) - .flatMap { - case item: ItemCandidateWithDetails => Seq(item) - case module: ModuleCandidateWithDetails => module.candidates - } - .filter(candidate => candidate.features.getOrElse(StreamToKafkaFeature, false)) - .map { candidate => - val servedId = - if (candidate.features.getOrElse(IsReadFromCacheFeature, false) && - servedRequestIdOpt.nonEmpty) - servedRequestIdOpt - else - candidate.features.getOrElse(PredictionRequestIdFeature, None) - - candidate.copy(features = candidate.features + (ServedIdFeature, servedId)) - }.toSeq - // deduplicate by (tweetId, userId, servedId) - .groupBy { candidate => - ( - candidate.candidateIdLong, - query.getRequiredUserId, - candidate.features.getOrElse(ServedIdFeature, None)) - }.values.map(_.head).toSeq - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKeysKafkaSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKeysKafkaSideEffect.scala deleted file mode 100644 index aa1247fbb..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKeysKafkaSideEffect.scala +++ /dev/null @@ -1,111 +0,0 @@ -package com.twitter.home_mixer.product.for_you.side_effect - -import com.twitter.home_mixer.model.HomeFeatures.IsReadFromCacheFeature -import com.twitter.home_mixer.model.HomeFeatures.PredictionRequestIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ServedIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature -import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableServedCandidateKafkaPublishingParam -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.util.SRichDataRecord -import com.twitter.product_mixer.component_library.side_effect.KafkaPublishingSideEffect -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.ml.cont_train.common.domain.non_scalding.DataRecordLoggingRelatedFeatures.tlmServedKeysFeatureContext -import com.twitter.timelines.ml.kafka.serde.ServedCandidateKeySerde -import com.twitter.timelines.ml.kafka.serde.TBaseSerde -import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures -import com.twitter.timelines.served_candidates_logging.{thriftscala => sc} -import com.twitter.timelines.suggests.common.poly_data_record.{thriftjava => pldr} -import com.twitter.util.Time -import org.apache.kafka.clients.producer.ProducerRecord -import org.apache.kafka.common.serialization.Serializer - -/** - * Pipeline side-effect that publishes candidate keys to a Kafka topic. - */ -class ServedCandidateKeysKafkaSideEffect( - topic: String, - sourceIdentifiers: Set[CandidatePipelineIdentifier]) - extends KafkaPublishingSideEffect[ - sc.ServedCandidateKey, - pldr.PolyDataRecord, - PipelineQuery, - Timeline - ] - with PipelineResultSideEffect.Conditionally[PipelineQuery, Timeline] { - - import ServedCandidateKafkaSideEffect._ - - override val identifier: SideEffectIdentifier = SideEffectIdentifier("ServedCandidateKeys") - - override def onlyIf( - query: PipelineQuery, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: Timeline - ): Boolean = query.params.getBoolean(EnableServedCandidateKafkaPublishingParam) - - override val bootstrapServer: String = "/s/kafka/timeline:kafka-tls" - - override val keySerde: Serializer[sc.ServedCandidateKey] = ServedCandidateKeySerde.serializer() - - override val valueSerde: Serializer[pldr.PolyDataRecord] = - TBaseSerde.Thrift[pldr.PolyDataRecord]().serializer - - override val clientId: String = "home_mixer_served_candidate_keys_producer" - - override def buildRecords( - query: PipelineQuery, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: Timeline - ): Seq[ProducerRecord[sc.ServedCandidateKey, pldr.PolyDataRecord]] = { - val servedTimestamp = Time.now.inMilliseconds - val servedRequestIdOpt = - query.features.getOrElse(FeatureMap.empty).getOrElse(ServedRequestIdFeature, None) - - extractCandidates(query, selectedCandidates, sourceIdentifiers).collect { - // Only publish non-cached tweets to the ServedCandidateKey topic - case candidate if !candidate.features.getOrElse(IsReadFromCacheFeature, false) => - val key = sc.ServedCandidateKey( - tweetId = candidate.candidateIdLong, - viewerId = query.getRequiredUserId, - servedId = -1L - ) - - val record = SRichDataRecord(new DataRecord, tlmServedKeysFeatureContext) - record.setFeatureValueFromOption( - TimelinesSharedFeatures.PREDICTION_REQUEST_ID, - candidate.features.getOrElse(PredictionRequestIdFeature, None) - ) - record - .setFeatureValueFromOption(TimelinesSharedFeatures.SERVED_REQUEST_ID, servedRequestIdOpt) - record.setFeatureValueFromOption( - TimelinesSharedFeatures.SERVED_ID, - candidate.features.getOrElse(ServedIdFeature, None) - ) - record.setFeatureValueFromOption( - TimelinesSharedFeatures.INJECTION_TYPE, - record.getFeatureValueOpt(TimelinesSharedFeatures.INJECTION_TYPE)) - record.setFeatureValue( - TimelinesSharedFeatures.SERVED_TIMESTAMP, - servedTimestamp - ) - record.record.dropUnknownFeatures() - - new ProducerRecord(topic, key, pldr.PolyDataRecord.dataRecord(record.getRecord)) - } - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(98.5) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKeysKafkaSideEffectBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKeysKafkaSideEffectBuilder.scala deleted file mode 100644 index 31ee389b9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKeysKafkaSideEffectBuilder.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.home_mixer.product.for_you.side_effect - -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class ServedCandidateKeysKafkaSideEffectBuilder @Inject() ( - injectedServiceIdentifier: ServiceIdentifier) { - def build( - sourceIdentifiers: Set[CandidatePipelineIdentifier] - ): ServedCandidateKeysKafkaSideEffect = { - val topic = injectedServiceIdentifier.environment.toLowerCase match { - case "prod" => "tq_ct_served_candidate_keys" - case _ => "tq_ct_served_candidate_keys_staging" - } - new ServedCandidateKeysKafkaSideEffect(topic, sourceIdentifiers) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedStatsSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedStatsSideEffect.scala deleted file mode 100644 index be7c2a533..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedStatsSideEffect.scala +++ /dev/null @@ -1,90 +0,0 @@ -package com.twitter.home_mixer.product.for_you.side_effect - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.product.for_you.param.ForYouParam.ExperimentStatsParam -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ServedStatsSideEffect @Inject() (statsReceiver: StatsReceiver) - extends PipelineResultSideEffect[PipelineQuery, Timeline] { - - override val identifier: SideEffectIdentifier = SideEffectIdentifier("ServedStats") - - private val baseStatsReceiver = statsReceiver.scope(identifier.toString) - private val suggestTypeStatsReceiver = baseStatsReceiver.scope("SuggestType") - private val responseSizeStatsReceiver = baseStatsReceiver.scope("ResponseSize") - private val contentBalanceStatsReceiver = baseStatsReceiver.scope("ContentBalance") - - private val inNetworkStatsReceiver = contentBalanceStatsReceiver.scope("InNetwork") - private val outOfNetworkStatsReceiver = contentBalanceStatsReceiver.scope("OutOfNetwork") - private val replyStatsReceiver = contentBalanceStatsReceiver.scope("Reply") - private val originalStatsReceiver = contentBalanceStatsReceiver.scope("Original") - - private val emptyStatsReceiver = responseSizeStatsReceiver.scope("Empty") - private val lessThan5StatsReceiver = responseSizeStatsReceiver.scope("LessThan5") - private val lessThan10StatsReceiver = responseSizeStatsReceiver.scope("LessThan10") - - override def apply( - inputs: PipelineResultSideEffect.Inputs[PipelineQuery, Timeline] - ): Stitch[Unit] = { - val tweetCandidates = CandidatesUtil - .getItemCandidates(inputs.selectedCandidates).filter(_.isCandidateType[TweetCandidate]()) - - val expBucket = inputs.query.params(ExperimentStatsParam) - - recordSuggestTypeStats(tweetCandidates, expBucket) - recordContentBalanceStats(tweetCandidates, expBucket) - recordResponseSizeStats(tweetCandidates, expBucket) - Stitch.Unit - } - - def recordSuggestTypeStats( - candidates: Seq[ItemCandidateWithDetails], - expBucket: String - ): Unit = { - candidates.groupBy(getSuggestType).foreach { - case (suggestType, suggestTypeCandidates) => - suggestTypeStatsReceiver - .scope(expBucket).counter(suggestType).incr(suggestTypeCandidates.size) - } - } - - def recordContentBalanceStats( - candidates: Seq[ItemCandidateWithDetails], - expBucket: String - ): Unit = { - val (in, oon) = candidates.partition(_.features.getOrElse(InNetworkFeature, true)) - inNetworkStatsReceiver.counter(expBucket).incr(in.size) - outOfNetworkStatsReceiver.counter(expBucket).incr(oon.size) - - val (reply, original) = - candidates.partition(_.features.getOrElse(InReplyToTweetIdFeature, None).isDefined) - replyStatsReceiver.counter(expBucket).incr(reply.size) - originalStatsReceiver.counter(expBucket).incr(original.size) - } - - def recordResponseSizeStats( - candidates: Seq[ItemCandidateWithDetails], - expBucket: String - ): Unit = { - if (candidates.size == 0) emptyStatsReceiver.counter(expBucket).incr() - if (candidates.size < 5) lessThan5StatsReceiver.counter(expBucket).incr() - if (candidates.size < 10) lessThan10StatsReceiver.counter(expBucket).incr() - } - - private def getSuggestType(candidate: CandidateWithDetails): String = - candidate.features.getOrElse(SuggestTypeFeature, None).map(_.name).getOrElse("None") -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BUILD.bazel deleted file mode 100644 index 3a52b15dc..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BUILD.bazel +++ /dev/null @@ -1,53 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", - "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/item", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", - "src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala", - "src/thrift/com/twitter/search:blender-scala", - "src/thrift/com/twitter/timelines/render:thrift-scala", - ], - exports = [ - "src/thrift/com/twitter/timelines/render:thrift-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BlenderUsersCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BlenderUsersCandidatePipelineConfig.scala deleted file mode 100644 index e0919ae9d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BlenderUsersCandidatePipelineConfig.scala +++ /dev/null @@ -1,68 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users - -import com.twitter.home_mixer.product.list_recommended_users.candidate_source.BlenderUsersCandidateSource -import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.IsGizmoduckValidUserFeatureHydrator -import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.IsSGSValidUserFeatureHydrator -import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.RecentListMembersFeature -import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersQuery -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.user.UserCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder -import com.twitter.product_mixer.component_library.gate.EmptySeqFeatureGate -import com.twitter.product_mixer.component_library.model.candidate.UserCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.search.blender.thriftscala.ThriftBlenderRequest -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class BlenderUsersCandidatePipelineConfig @Inject() ( - blenderUsersCandidateSource: BlenderUsersCandidateSource, - isGizmoduckValidUserFeatureHydrator: IsGizmoduckValidUserFeatureHydrator, - isSGSValidUserFeatureHydrator: IsSGSValidUserFeatureHydrator) - extends CandidatePipelineConfig[ - ListRecommendedUsersQuery, - ThriftBlenderRequest, - Long, - UserCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("BlenderUsers") - - override val gates: Seq[Gate[ListRecommendedUsersQuery]] = - Seq(EmptySeqFeatureGate(RecentListMembersFeature)) - - override val queryTransformer: CandidatePipelineQueryTransformer[ - ListRecommendedUsersQuery, - ThriftBlenderRequest - ] = BlenderUsersCandidatePipelineQueryTransformer - - override val candidateSource: BaseCandidateSource[ThriftBlenderRequest, Long] = - blenderUsersCandidateSource - - override val resultTransformer: CandidatePipelineResultsTransformer[ - Long, - UserCandidate - ] = { candidate => UserCandidate(id = candidate) } - - override val preFilterFeatureHydrationPhase1: Seq[ - BaseCandidateFeatureHydrator[ListRecommendedUsersQuery, UserCandidate, _] - ] = Seq( - isGizmoduckValidUserFeatureHydrator, - isSGSValidUserFeatureHydrator - ) - - override val decorator: Option[CandidateDecorator[ListRecommendedUsersQuery, UserCandidate]] = { - val clientEventInfoBuilder = ClientEventInfoBuilder("user") - val userItemBuilder = UserCandidateUrtItemBuilder(clientEventInfoBuilder) - Some(UrtItemCandidateDecorator(userItemBuilder)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BlenderUsersCandidatePipelineQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BlenderUsersCandidatePipelineQueryTransformer.scala deleted file mode 100644 index 03844fd38..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BlenderUsersCandidatePipelineQueryTransformer.scala +++ /dev/null @@ -1,49 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users - -import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersQuery -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.search.adaptive.adaptive_results.thriftscala.ResultType -import com.twitter.search.blender.adaptive_search.thriftscala.AdaptiveSearchRequest -import com.twitter.search.blender.thriftscala.ThriftBlenderRequest -import com.twitter.search.blender.thriftscala.ThriftBlenderTweetypieOptions -import com.twitter.search.blender.thriftscala.ThriftBlenderWorkflowID -import com.twitter.search.common.constants.thriftscala.ThriftQuerySource -import com.twitter.spam.rtf.thriftscala.SafetyLevel - -object BlenderUsersCandidatePipelineQueryTransformer - extends CandidatePipelineQueryTransformer[ListRecommendedUsersQuery, ThriftBlenderRequest] { - - override val identifier: TransformerIdentifier = TransformerIdentifier("BlenderUsers") - - /** - * This is a user-defined descriptor used by Blender to track the source of traffic, and it - * is different from a client id, which is set during Finagle client construction. - */ - private val ClientAppName = "timelinemixer.list_recommended_users" - - override def transform(query: ListRecommendedUsersQuery): ThriftBlenderRequest = { - - ThriftBlenderRequest( - workflowID = Some(ThriftBlenderWorkflowID.AdaptiveSearch), - userID = Some(query.getRequiredUserId), // perspectival - uiLang = query.clientContext.languageCode, // perspectival - clientAppName = Some(ClientAppName), - adaptiveSearchRequest = Some( - AdaptiveSearchRequest( - rawQuery = query.listName, - numResults = 40, - getPromotedContent = false, - resultFilter = Some(ResultType.User), - ) - ), - querySource = Some(ThriftQuerySource.TypedQuery), - getCorrections = true, - tweetypieOptions = Some( - ThriftBlenderTweetypieOptions( - safetyLevel = Some(SafetyLevel.Recommendations) - ) - ) - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListMemberBasedUsersCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListMemberBasedUsersCandidatePipelineConfig.scala deleted file mode 100644 index 756c58d72..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListMemberBasedUsersCandidatePipelineConfig.scala +++ /dev/null @@ -1,98 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users - -import com.twitter.hermit.candidate.{thriftscala => t} -import com.twitter.home_mixer.product.list_recommended_users.candidate_source.SimilarityBasedUsersCandidateSource -import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.IsGizmoduckValidUserFeatureHydrator -import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.IsListMemberFeatureHydrator -import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.IsSGSValidUserFeatureHydrator -import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.RecentListMembersFeature -import com.twitter.home_mixer.product.list_recommended_users.filter.DropMaxCandidatesByAggregatedScoreFilter -import com.twitter.home_mixer.product.list_recommended_users.filter.PreviouslyServedUsersFilter -import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.IsListMemberFeature -import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersQuery -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.user.UserCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder -import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter -import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate -import com.twitter.product_mixer.component_library.model.candidate.UserCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ListMemberBasedUsersCandidatePipelineConfig @Inject() ( - similarityBasedUsersCandidateSource: SimilarityBasedUsersCandidateSource, - isGizmoduckValidUserFeatureHydrator: IsGizmoduckValidUserFeatureHydrator, - isListMemberFeatureHydrator: IsListMemberFeatureHydrator, - isSGSValidUserFeatureHydrator: IsSGSValidUserFeatureHydrator) - extends CandidatePipelineConfig[ - ListRecommendedUsersQuery, - Seq[Long], - t.Candidate, - UserCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ListMemberBasedUsers") - - override val gates: Seq[Gate[ListRecommendedUsersQuery]] = - Seq(NonEmptySeqFeatureGate(RecentListMembersFeature)) - - override val queryTransformer: CandidatePipelineQueryTransformer[ListRecommendedUsersQuery, Seq[ - Long - ]] = { query => - query.features.map(_.getOrElse(RecentListMembersFeature, Seq.empty)).getOrElse(Seq.empty) - } - - override val candidateSource: BaseCandidateSource[Seq[Long], t.Candidate] = - similarityBasedUsersCandidateSource - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[t.Candidate] - ] = Seq(ListMemberBasedUsersResponseFeatureTransfromer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - t.Candidate, - UserCandidate - ] = { candidate => - UserCandidate(id = candidate.userId) - } - - override val preFilterFeatureHydrationPhase1: Seq[ - BaseCandidateFeatureHydrator[ListRecommendedUsersQuery, UserCandidate, _] - ] = Seq(isListMemberFeatureHydrator) - - override val filters: Seq[Filter[ListRecommendedUsersQuery, UserCandidate]] = - Seq( - PreviouslyServedUsersFilter, - PredicateFeatureFilter.fromPredicate( - FilterIdentifier("IsListMember"), - shouldKeepCandidate = { features => !features.getOrElse(IsListMemberFeature, false) } - ), - DropMaxCandidatesByAggregatedScoreFilter - ) - - override val postFilterFeatureHydration: Seq[ - BaseCandidateFeatureHydrator[ListRecommendedUsersQuery, UserCandidate, _] - ] = Seq( - isGizmoduckValidUserFeatureHydrator, - isSGSValidUserFeatureHydrator - ) - - override val decorator: Option[CandidateDecorator[ListRecommendedUsersQuery, UserCandidate]] = { - val clientEventInfoBuilder = ClientEventInfoBuilder("user") - val userItemBuilder = UserCandidateUrtItemBuilder(clientEventInfoBuilder) - Some(UrtItemCandidateDecorator(userItemBuilder)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListMemberBasedUsersResponseFeatureTransfromer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListMemberBasedUsersResponseFeatureTransfromer.scala deleted file mode 100644 index 7153d1f06..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListMemberBasedUsersResponseFeatureTransfromer.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users - -import com.twitter.hermit.candidate.{thriftscala => t} -import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.ScoreFeature -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier - -object ListMemberBasedUsersResponseFeatureTransfromer - extends CandidateFeatureTransformer[t.Candidate] { - - override val identifier: TransformerIdentifier = TransformerIdentifier("ListMemberBasedUsers") - - override val features: Set[Feature[_, _]] = Set(ScoreFeature) - - override def transform(candidate: t.Candidate): FeatureMap = FeatureMapBuilder() - .add(ScoreFeature, candidate.score) - .build() -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListRecommendedUsersMixerPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListRecommendedUsersMixerPipelineConfig.scala deleted file mode 100644 index 8c72aefec..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListRecommendedUsersMixerPipelineConfig.scala +++ /dev/null @@ -1,111 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users - -import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.RecentListMembersQueryFeatureHydrator -import com.twitter.home_mixer.product.list_recommended_users.gate.ViewerIsListOwnerGate -import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.IsGizmoduckValidUserFeature -import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.IsSGSValidUserFeature -import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersQuery -import com.twitter.home_mixer.product.list_recommended_users.param.ListRecommendedUsersParam.ExcludedIdsMaxLengthParam -import com.twitter.home_mixer.product.list_recommended_users.param.ListRecommendedUsersParam.ServerMaxResultsParam -import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.AddEntriesWithReplaceInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceAllEntries -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceEntryInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.StaticTimelineScribeConfigBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UnorderedExcludeIdsBottomCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtMetadataBuilder -import com.twitter.product_mixer.component_library.selector.DropFilteredCandidates -import com.twitter.product_mixer.component_library.selector.DropMaxCandidates -import com.twitter.product_mixer.component_library.selector.InsertAppendResults -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller -import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.UrtTransportMarshaller -import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig -import com.twitter.timelines.render.{thriftscala => urt} - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ListRecommendedUsersMixerPipelineConfig @Inject() ( - listMemberBasedUsersCandidatePipelineConfig: ListMemberBasedUsersCandidatePipelineConfig, - blenderUsersCandidatePipelineConfig: BlenderUsersCandidatePipelineConfig, - viewerIsListOwnerGate: ViewerIsListOwnerGate, - recentListMembersQueryFeatureHydrator: RecentListMembersQueryFeatureHydrator, - urtTransportMarshaller: UrtTransportMarshaller) - extends MixerPipelineConfig[ListRecommendedUsersQuery, Timeline, urt.TimelineResponse] { - - override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("ListRecommendedUsers") - - override val gates = Seq(viewerIsListOwnerGate) - - override val fetchQueryFeatures: Seq[QueryFeatureHydrator[ListRecommendedUsersQuery]] = - Seq(recentListMembersQueryFeatureHydrator) - - override val candidatePipelines: Seq[ - CandidatePipelineConfig[ListRecommendedUsersQuery, _, _, _] - ] = Seq( - listMemberBasedUsersCandidatePipelineConfig, - blenderUsersCandidatePipelineConfig - ) - - private val candidatePipelineIdentifiers = Set( - listMemberBasedUsersCandidatePipelineConfig.identifier, - blenderUsersCandidatePipelineConfig.identifier - ) - - override val resultSelectors: Seq[Selector[ListRecommendedUsersQuery]] = Seq( - DropFilteredCandidates( - candidatePipelines = candidatePipelineIdentifiers, - filter = candidate => - candidate.features.getOrElse(IsSGSValidUserFeature, false) && - candidate.features.getOrElse(IsGizmoduckValidUserFeature, false) - ), - DropMaxCandidates( - candidatePipelines = candidatePipelineIdentifiers, - maxSelectionsParam = ServerMaxResultsParam), - InsertAppendResults(candidatePipelineIdentifiers) - ) - - override val domainMarshaller: DomainMarshaller[ListRecommendedUsersQuery, Timeline] = { - val instructionBuilders = Seq( - ReplaceEntryInstructionBuilder(ReplaceAllEntries), - AddEntriesWithReplaceInstructionBuilder() - ) - - val metadataBuilder = UrtMetadataBuilder( - title = None, - scribeConfigBuilder = Some( - StaticTimelineScribeConfigBuilder( - TimelineScribeConfig( - page = Some("list_recommended_users"), - section = None, - entityToken = None))) - ) - - val excludeIdsSelector: PartialFunction[UniversalNoun[_], Long] = { - case item: UserItem => item.id - } - - val cursorBuilder = UnorderedExcludeIdsBottomCursorBuilder( - excludedIdsMaxLengthParam = ExcludedIdsMaxLengthParam, - excludeIdsSelector = excludeIdsSelector) - - UrtDomainMarshaller( - instructionBuilders = instructionBuilders, - metadataBuilder = Some(metadataBuilder), - cursorBuilders = Seq(cursorBuilder) - ) - } - - override val transportMarshaller: TransportMarshaller[Timeline, urt.TimelineResponse] = - urtTransportMarshaller -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListRecommendedUsersProductPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListRecommendedUsersProductPipelineConfig.scala deleted file mode 100644 index f0c2ac70e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListRecommendedUsersProductPipelineConfig.scala +++ /dev/null @@ -1,102 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.marshaller.timelines.RecommendedUsersCursorUnmarshaller -import com.twitter.home_mixer.model.request.HomeMixerRequest -import com.twitter.home_mixer.model.request.ListRecommendedUsersProduct -import com.twitter.home_mixer.model.request.ListRecommendedUsersProductContext -import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersQuery -import com.twitter.home_mixer.product.list_recommended_users.param.ListRecommendedUsersParam.ServerMaxResultsParam -import com.twitter.home_mixer.product.list_recommended_users.param.ListRecommendedUsersParamConfig -import com.twitter.home_mixer.service.HomeMixerAccessPolicy.DefaultHomeMixerAccessPolicy -import com.twitter.home_mixer.service.HomeMixerAlertConfig.DefaultNotificationGroup -import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer -import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfBelow -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfLatencyAbove -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.common.alert.LatencyAlert -import com.twitter.product_mixer.core.functional_component.common.alert.P99 -import com.twitter.product_mixer.core.functional_component.common.alert.SuccessRateAlert -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.request -import com.twitter.product_mixer.core.pipeline.PipelineConfig -import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.product.ProductPipelineConfig -import com.twitter.product_mixer.core.product.ProductParamConfig -import com.twitter.timelines.configapi.Params -import com.twitter.timelines.render.{thriftscala => urt} -import com.twitter.timelines.util.RequestCursorSerializer -import com.twitter.util.Try - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ListRecommendedUsersProductPipelineConfig @Inject() ( - listRecommendedUsersMixerPipelineConfig: ListRecommendedUsersMixerPipelineConfig, - listRecommendedUsersParamConfig: ListRecommendedUsersParamConfig) - extends ProductPipelineConfig[ - HomeMixerRequest, - ListRecommendedUsersQuery, - urt.TimelineResponse - ] { - - override val identifier: ProductPipelineIdentifier = - ProductPipelineIdentifier("ListRecommendedUsers") - override val product: request.Product = ListRecommendedUsersProduct - override val paramConfig: ProductParamConfig = listRecommendedUsersParamConfig - - override def pipelineQueryTransformer( - request: HomeMixerRequest, - params: Params - ): ListRecommendedUsersQuery = { - val context = request.productContext match { - case Some(context: ListRecommendedUsersProductContext) => context - case _ => throw PipelineFailure(BadRequest, "ListRecommendedUsersProductContext not found") - } - - val debugOptions = request.debugParams.flatMap(_.debugOptions) - - val pipelineCursor = request.serializedRequestCursor.flatMap { cursor => - Try(UrtCursorSerializer.deserializeUnorderedExcludeIdsCursor(cursor)) - .getOrElse(RecommendedUsersCursorUnmarshaller(RequestCursorSerializer.deserialize(cursor))) - } - - ListRecommendedUsersQuery( - listId = context.listId, - params = params, - clientContext = request.clientContext, - features = None, - pipelineCursor = pipelineCursor, - requestedMaxResults = Some(params(ServerMaxResultsParam)), - debugOptions = debugOptions, - selectedUserIds = context.selectedUserIds, - excludedUserIds = context.excludedUserIds, - listName = context.listName - ) - } - - override def pipelines: Seq[PipelineConfig] = Seq(listRecommendedUsersMixerPipelineConfig) - - override def pipelineSelector(query: ListRecommendedUsersQuery): ComponentIdentifier = - listRecommendedUsersMixerPipelineConfig.identifier - - override val alerts: Seq[Alert] = Seq( - SuccessRateAlert( - notificationGroup = DefaultNotificationGroup, - warnPredicate = TriggerIfBelow(99.9, 20, 30), - criticalPredicate = TriggerIfBelow(99.9, 30, 30), - ), - LatencyAlert( - notificationGroup = DefaultNotificationGroup, - percentile = P99, - warnPredicate = TriggerIfLatencyAbove(1000.millis, 15, 30), - criticalPredicate = TriggerIfLatencyAbove(1500.millis, 15, 30) - ) - ) - - override val debugAccessPolicies: Set[AccessPolicy] = DefaultHomeMixerAccessPolicy -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/BUILD.bazel deleted file mode 100644 index e1fa1de1c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala", - "src/thrift/com/twitter/search:blender-scala", - "strato/config/columns/recommendations/similarity:similarity-strato-client", - "strato/src/main/scala/com/twitter/strato/client", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/BlenderUsersCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/BlenderUsersCandidateSource.scala deleted file mode 100644 index 3b7ac39fc..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/BlenderUsersCandidateSource.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users.candidate_source - -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.search.adaptive.adaptive_results.thriftscala.AdaptiveSearchResultData -import com.twitter.search.adaptive.adaptive_results.thriftscala.Result -import com.twitter.search.adaptive.adaptive_results.thriftscala.ResultData -import com.twitter.search.blender.adaptive_search.thriftscala.AdaptiveSearchResponse -import com.twitter.search.blender.adaptive_search.thriftscala.Container -import com.twitter.search.blender.thriftscala.BlenderService -import com.twitter.search.blender.thriftscala.ThriftBlenderRequest -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class BlenderUsersCandidateSource @Inject() ( - blenderClient: BlenderService.MethodPerEndpoint) - extends CandidateSource[ThriftBlenderRequest, Long] { - - override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("BlenderUsers") - - override def apply(request: ThriftBlenderRequest): Stitch[Seq[Long]] = { - Stitch.callFuture( - blenderClient.serveV2(request).map { response => - val userIdsOpt = - response.adaptiveSearchResponse.map(extractUserIdsFromAdaptiveSearchResponse) - userIdsOpt.getOrElse(Seq.empty) - } - ) - } - - private def extractUserIdsFromAdaptiveSearchResponse( - response: AdaptiveSearchResponse - ): Seq[Long] = { - response match { - case AdaptiveSearchResponse(Some(Seq(Container(Some(results), _))), _, _) => - results.map(_.data).collect { - case AdaptiveSearchResultData.Result(Result(ResultData.User(user), _)) => - user.id - } - case _ => Seq.empty - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/SimilarityBasedUsersCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/SimilarityBasedUsersCandidateSource.scala deleted file mode 100644 index f5428d032..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/SimilarityBasedUsersCandidateSource.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users.candidate_source - -import com.twitter.hermit.candidate.{thriftscala => t} -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher -import com.twitter.strato.generated.client.recommendations.similarity.SimilarUsersBySimsOnUserClientColumn - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class SimilarityBasedUsersCandidateSource @Inject() ( - similarUsersBySimsOnUserClientColumn: SimilarUsersBySimsOnUserClientColumn) - extends CandidateSource[Seq[Long], t.Candidate] { - - override val identifier: CandidateSourceIdentifier = - CandidateSourceIdentifier("SimilarityBasedUsers") - - private val fetcher: Fetcher[Long, Unit, t.Candidates] = - similarUsersBySimsOnUserClientColumn.fetcher - - private val MaxCandidatesToKeep = 4000 - - override def apply(request: Seq[Long]): Stitch[Seq[t.Candidate]] = { - Stitch - .collect { - request.map { userId => - fetcher - .fetch(userId, Unit).map { result => - result.v.map(_.candidates).getOrElse(Seq.empty) - }.map { candidates => - val sortedCandidates = candidates.sortBy(-_.score) - sortedCandidates.take(MaxCandidatesToKeep) - } - } - }.map(_.flatten) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/BUILD.bazel deleted file mode 100644 index bdd2de735..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/BUILD.bazel +++ /dev/null @@ -1,15 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "src/thrift/com/twitter/gizmoduck:thrift-scala", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "stitch/stitch-gizmoduck", - "stitch/stitch-socialgraph", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsGizmoduckValidUserFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsGizmoduckValidUserFeatureHydrator.scala deleted file mode 100644 index 80a41adf0..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsGizmoduckValidUserFeatureHydrator.scala +++ /dev/null @@ -1,66 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users.feature_hydrator - -import com.twitter.gizmoduck.{thriftscala => gt} -import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.IsGizmoduckValidUserFeature -import com.twitter.product_mixer.component_library.model.candidate.UserCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.spam.rtf.{thriftscala => rtf} -import com.twitter.stitch.Stitch -import com.twitter.stitch.gizmoduck.Gizmoduck -import com.twitter.util.Return - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class IsGizmoduckValidUserFeatureHydrator @Inject() (gizmoduck: Gizmoduck) - extends BulkCandidateFeatureHydrator[PipelineQuery, UserCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("IsGizmoduckValidUser") - - override val features: Set[Feature[_, _]] = Set(IsGizmoduckValidUserFeature) - - private val queryFields: Set[gt.QueryFields] = Set(gt.QueryFields.Safety) - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[UserCandidate]] - ): Stitch[Seq[FeatureMap]] = { - val context = gt.LookupContext( - forUserId = query.getOptionalUserId, - includeProtected = true, - safetyLevel = Some(rtf.SafetyLevel.Recommendations) - ) - val userIds = candidates.map(_.candidate.id) - - Stitch - .collectToTry( - userIds.map(userId => gizmoduck.getUserById(userId, queryFields, context))).map { - userResults => - val idToUserSafetyMap = userResults - .collect { - case Return(user) => user - }.map(user => user.id -> user.safety).toMap - - candidates.map { candidate => - val safety = idToUserSafetyMap.getOrElse(candidate.candidate.id, None) - val isValidUser = safety.isDefined && - !safety.exists(_.deactivated) && - !safety.exists(_.suspended) && - !safety.exists(_.isProtected) && - !safety.flatMap(_.offboarded).getOrElse(false) - - FeatureMapBuilder() - .add(IsGizmoduckValidUserFeature, isValidUser) - .build() - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsListMemberFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsListMemberFeatureHydrator.scala deleted file mode 100644 index c9e529de5..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsListMemberFeatureHydrator.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users.feature_hydrator - -import com.twitter.home_mixer.model.request.HasListId -import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.IsListMemberFeature -import com.twitter.product_mixer.component_library.model.candidate.UserCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.socialgraph.{thriftscala => sg} -import com.twitter.stitch.Stitch -import com.twitter.stitch.socialgraph.SocialGraph - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class IsListMemberFeatureHydrator @Inject() (socialGraph: SocialGraph) - extends BulkCandidateFeatureHydrator[PipelineQuery with HasListId, UserCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("IsListMember") - - override val features: Set[Feature[_, _]] = Set(IsListMemberFeature) - - override def apply( - query: PipelineQuery with HasListId, - candidates: Seq[CandidateWithFeatures[UserCandidate]] - ): Stitch[Seq[FeatureMap]] = { - val userIds = candidates.map(_.candidate.id) - val request = sg.IdsRequest( - relationships = Seq( - sg.SrcRelationship( - source = query.listId, - relationshipType = sg.RelationshipType.ListHasMember, - hasRelationship = true, - targets = Some(userIds))), - pageRequest = Some(sg.PageRequest(selectAll = Some(true))) - ) - - socialGraph.ids(request).map(_.ids).map { listMembers => - val listMembersSet = listMembers.toSet - candidates.map { candidate => - FeatureMapBuilder() - .add(IsListMemberFeature, listMembersSet.contains(candidate.candidate.id)) - .build() - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsSGSValidUserFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsSGSValidUserFeatureHydrator.scala deleted file mode 100644 index de70d45d8..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsSGSValidUserFeatureHydrator.scala +++ /dev/null @@ -1,65 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users.feature_hydrator - -import com.twitter.home_mixer.model.request.HasListId -import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.IsSGSValidUserFeature -import com.twitter.product_mixer.component_library.model.candidate.UserCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.socialgraph.{thriftscala => sg} -import com.twitter.stitch.Stitch -import com.twitter.stitch.socialgraph.SocialGraph - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class IsSGSValidUserFeatureHydrator @Inject() (socialGraph: SocialGraph) - extends BulkCandidateFeatureHydrator[PipelineQuery with HasListId, UserCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("IsSGSValidUser") - - override def features: Set[Feature[_, _]] = Set(IsSGSValidUserFeature) - - override def apply( - query: PipelineQuery with HasListId, - candidates: Seq[CandidateWithFeatures[UserCandidate]] - ): Stitch[Seq[FeatureMap]] = { - val sourceId = query.getRequiredUserId - val targetUserIds = candidates.map(_.candidate.id) - val request = sg.IdsRequest( - relationships = Seq( - sg.SrcRelationship( - source = sourceId, - relationshipType = sg.RelationshipType.Blocking, - hasRelationship = true, - targets = Some(targetUserIds)), - sg.SrcRelationship( - source = sourceId, - relationshipType = sg.RelationshipType.BlockedBy, - hasRelationship = true, - targets = Some(targetUserIds)), - sg.SrcRelationship( - source = sourceId, - relationshipType = sg.RelationshipType.Muting, - hasRelationship = true, - targets = Some(targetUserIds)) - ), - pageRequest = Some(sg.PageRequest(selectAll = Some(true))), - context = Some(sg.LookupContext(performUnion = Some(true))) - ) - - socialGraph.ids(request).map(_.ids).map(_.toSet).map { hasRelationshipUserIds => - candidates.map { candidate => - FeatureMapBuilder() - .add(IsSGSValidUserFeature, !hasRelationshipUserIds.contains(candidate.candidate.id)) - .build() - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/RecentListMembersQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/RecentListMembersQueryFeatureHydrator.scala deleted file mode 100644 index 81a5ce504..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/RecentListMembersQueryFeatureHydrator.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users.feature_hydrator - -import com.twitter.home_mixer.model.request.HasListId -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.socialgraph.{thriftscala => sg} -import com.twitter.stitch.Stitch -import com.twitter.stitch.socialgraph.SocialGraph - -import javax.inject.Inject -import javax.inject.Singleton - -case object RecentListMembersFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] { - override val defaultValue: Seq[Long] = Seq.empty -} - -@Singleton -class RecentListMembersQueryFeatureHydrator @Inject() (socialGraph: SocialGraph) - extends QueryFeatureHydrator[PipelineQuery with HasListId] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("RecentListMembers") - - override val features: Set[Feature[_, _]] = Set(RecentListMembersFeature) - - private val MaxRecentMembers = 10 - - override def hydrate(query: PipelineQuery with HasListId): Stitch[FeatureMap] = { - val request = sg.IdsRequest( - relationships = Seq(sg - .SrcRelationship(query.listId, sg.RelationshipType.ListHasMember, hasRelationship = true)), - pageRequest = Some(sg.PageRequest(selectAll = Some(true), count = Some(MaxRecentMembers))) - ) - socialGraph.ids(request).map(_.ids).map { listMembers => - FeatureMapBuilder().add(RecentListMembersFeature, listMembers).build() - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/BUILD.bazel deleted file mode 100644 index 72a5f251f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter", - ], - exports = [], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/DropMaxCandidatesByAggregatedScoreFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/DropMaxCandidatesByAggregatedScoreFilter.scala deleted file mode 100644 index a68c37450..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/DropMaxCandidatesByAggregatedScoreFilter.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users.filter - -import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.ScoreFeature -import com.twitter.product_mixer.component_library.model.candidate.UserCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -object DropMaxCandidatesByAggregatedScoreFilter extends Filter[PipelineQuery, UserCandidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("DropMaxCandidatesByAggregatedScore") - - private val MaxSimilarUserCandidates = 150 - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[UserCandidate]] - ): Stitch[FilterResult[UserCandidate]] = { - val userIdToAggregatedScoreMap = candidates - .groupBy(_.candidate.id) - .map { - case (userId, candidates) => - val aggregatedScore = candidates.map(_.features.getOrElse(ScoreFeature, 0.0)).sum - (userId, aggregatedScore) - } - - val sortedCandidates = candidates.sortBy(candidate => - -userIdToAggregatedScoreMap.getOrElse(candidate.candidate.id, 0.0)) - - val (kept, removed) = sortedCandidates.map(_.candidate).splitAt(MaxSimilarUserCandidates) - - Stitch.value(FilterResult(kept, removed)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/PreviouslyServedUsersFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/PreviouslyServedUsersFilter.scala deleted file mode 100644 index ac8c3107d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/PreviouslyServedUsersFilter.scala +++ /dev/null @@ -1,35 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users.filter - -import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.RecentListMembersFeature -import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersQuery -import com.twitter.product_mixer.component_library.model.candidate.UserCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.stitch.Stitch - -object PreviouslyServedUsersFilter extends Filter[ListRecommendedUsersQuery, UserCandidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("PreviouslyServedUsers") - - override def apply( - query: ListRecommendedUsersQuery, - candidates: Seq[CandidateWithFeatures[UserCandidate]] - ): Stitch[FilterResult[UserCandidate]] = { - - val recentListMembers = query.features.map(_.getOrElse(RecentListMembersFeature, Seq.empty)) - - val servedUserIds = query.pipelineCursor.map(_.excludedIds) - - val excludedUserIds = (recentListMembers.getOrElse(Seq.empty) ++ - query.selectedUserIds.getOrElse(Seq.empty) ++ - query.excludedUserIds.getOrElse(Seq.empty) ++ - servedUserIds.getOrElse(Seq.empty)).toSet - - val (removed, kept) = - candidates.map(_.candidate).partition(candidate => excludedUserIds.contains(candidate.id)) - - Stitch.value(FilterResult(kept = kept, removed = removed)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate/BUILD.bazel deleted file mode 100644 index f8a7083b9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate/BUILD.bazel +++ /dev/null @@ -1,12 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "stitch/stitch-socialgraph", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate/ViewerIsListOwnerGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate/ViewerIsListOwnerGate.scala deleted file mode 100644 index e7ab3084e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate/ViewerIsListOwnerGate.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users.gate - -import com.twitter.home_mixer.model.request.HasListId -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.socialgraph.{thriftscala => sg} -import com.twitter.stitch.Stitch -import com.twitter.stitch.socialgraph.SocialGraph - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class ViewerIsListOwnerGate @Inject() (socialGraph: SocialGraph) - extends Gate[PipelineQuery with HasListId] { - - override val identifier: GateIdentifier = GateIdentifier("ViewerIsListOwner") - - private val relationship = sg.Relationship(relationshipType = sg.RelationshipType.ListOwning) - - override def shouldContinue(query: PipelineQuery with HasListId): Stitch[Boolean] = { - val request = sg.ExistsRequest( - source = query.getRequiredUserId, - target = query.listId, - relationships = Seq(relationship)) - socialGraph.exists(request).map(_.exists) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/BUILD.bazel deleted file mode 100644 index 78302f75b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/BUILD.bazel +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListRecommendedUsersFeatures.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListRecommendedUsersFeatures.scala deleted file mode 100644 index baed8171b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListRecommendedUsersFeatures.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users.model - -import com.twitter.product_mixer.component_library.model.candidate.UserCandidate -import com.twitter.product_mixer.core.feature.Feature - -object ListRecommendedUsersFeatures { - // Candidate features - object IsGizmoduckValidUserFeature extends Feature[UserCandidate, Boolean] - object IsListMemberFeature extends Feature[UserCandidate, Boolean] - object IsSGSValidUserFeature extends Feature[UserCandidate, Boolean] - object ScoreFeature extends Feature[UserCandidate, Double] -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListRecommendedUsersQuery.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListRecommendedUsersQuery.scala deleted file mode 100644 index 15b16c25b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListRecommendedUsersQuery.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users.model - -import com.twitter.home_mixer.model.request.HasListId -import com.twitter.home_mixer.model.request.ListRecommendedUsersProduct -import com.twitter.product_mixer.component_library.model.cursor.UrtUnorderedExcludeIdsCursor -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.request._ -import com.twitter.product_mixer.core.pipeline.HasPipelineCursor -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.Params - -case class ListRecommendedUsersQuery( - override val listId: Long, - override val params: Params, - override val clientContext: ClientContext, - override val pipelineCursor: Option[UrtUnorderedExcludeIdsCursor], - override val requestedMaxResults: Option[Int], - override val debugOptions: Option[DebugOptions], - override val features: Option[FeatureMap], - selectedUserIds: Option[Seq[Long]], - excludedUserIds: Option[Seq[Long]], - listName: Option[String]) - extends PipelineQuery - with HasPipelineCursor[UrtUnorderedExcludeIdsCursor] - with HasListId { - - override val product: Product = ListRecommendedUsersProduct - - override def withFeatureMap(features: FeatureMap): ListRecommendedUsersQuery = - copy(features = Some(features)) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/BUILD.bazel deleted file mode 100644 index 18867ad9e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/BUILD.bazel +++ /dev/null @@ -1,12 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/ListRecommendedUsersParam.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/ListRecommendedUsersParam.scala deleted file mode 100644 index 3822a1987..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/ListRecommendedUsersParam.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users.param - -import com.twitter.timelines.configapi.FSBoundedParam - -object ListRecommendedUsersParam { - val SupportedClientFSName = "list_recommended_users_supported_client" - - object ServerMaxResultsParam - extends FSBoundedParam[Int]( - name = "list_recommended_users_server_max_results", - default = 10, - min = 1, - max = 500 - ) - - object ExcludedIdsMaxLengthParam - extends FSBoundedParam[Int]( - name = "list_recommended_users_excluded_ids_max_length", - default = 2000, - min = 0, - max = 5000 - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/ListRecommendedUsersParamConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/ListRecommendedUsersParamConfig.scala deleted file mode 100644 index b162ef755..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/ListRecommendedUsersParamConfig.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users.param - -import com.twitter.home_mixer.param.decider.DeciderKey -import com.twitter.home_mixer.product.list_recommended_users.param.ListRecommendedUsersParam.ExcludedIdsMaxLengthParam -import com.twitter.home_mixer.product.list_recommended_users.param.ListRecommendedUsersParam.ServerMaxResultsParam -import com.twitter.home_mixer.product.list_recommended_users.param.ListRecommendedUsersParam.SupportedClientFSName -import com.twitter.product_mixer.core.product.ProductParamConfig -import com.twitter.servo.decider.DeciderKeyName - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ListRecommendedUsersParamConfig @Inject() () extends ProductParamConfig { - override val enabledDeciderKey: DeciderKeyName = DeciderKey.EnableListRecommendedUsersProduct - override val supportedClientFSName: String = SupportedClientFSName - - override val boundedIntFSOverrides = Seq( - ServerMaxResultsParam, - ExcludedIdsMaxLengthParam - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/BUILD.bazel deleted file mode 100644 index a7d22148a..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/BUILD.bazel +++ /dev/null @@ -1,61 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "ads-injection/lib/src/main/scala/com/twitter/goldfinch/api", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_service", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", - "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/item", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", - "src/thrift/com/twitter/timelines/render:thrift-scala", - "timelines/src/main/scala/com/twitter/timelines/injection/scribe", - ], - exports = [ - "src/thrift/com/twitter/timelines/render:thrift-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsAdsCandidatePipelineBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsAdsCandidatePipelineBuilder.scala deleted file mode 100644 index 9e5a4e541..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsAdsCandidatePipelineBuilder.scala +++ /dev/null @@ -1,93 +0,0 @@ -package com.twitter.home_mixer.product.list_tweets - -import com.twitter.adserver.{thriftscala => ads} -import com.twitter.home_mixer.functional_component.decorator.builder.HomeAdsClientEventDetailsBuilder -import com.twitter.home_mixer.functional_component.gate.ExcludeSoftUserGate -import com.twitter.home_mixer.param.HomeGlobalParams -import com.twitter.home_mixer.param.HomeGlobalParams.EnableAdvertiserBrandSafetySettingsFeatureHydratorParam -import com.twitter.home_mixer.product.list_tweets.model.ListTweetsQuery -import com.twitter.home_mixer.product.list_tweets.param.ListTweetsParam.EnableAdsCandidatePipelineParam -import com.twitter.product_mixer.component_library.candidate_source.ads.AdsProdThriftCandidateSource -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.ad.AdsCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder -import com.twitter.product_mixer.component_library.feature_hydrator.candidate.ads.AdvertiserBrandSafetySettingsFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedCandidateFeatureHydrator -import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate -import com.twitter.product_mixer.component_library.model.candidate.ads.AdsCandidate -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsDependentCandidatePipelineConfig -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsDependentCandidatePipelineConfigBuilder -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.CountCandidatesFromPipelines -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.StaticAdsDisplayLocationBuilder -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.ValidAdImpressionIdFilter -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.gate.ParamNotGate -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.rtf.safety_level.TimelineHomePromotedHydrationSafetyLevel -import com.twitter.product_mixer.core.model.marshalling.response.urt.contextual_ref.TweetHydrationContext -import com.twitter.timelines.injection.scribe.InjectionScribeUtil -import com.twitter.timelineservice.suggests.{thriftscala => st} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ListTweetsAdsCandidatePipelineBuilder @Inject() ( - adsCandidatePipelineConfigBuilder: AdsDependentCandidatePipelineConfigBuilder, - adsCandidateSource: AdsProdThriftCandidateSource, - advertiserBrandSafetySettingsFeatureHydrator: AdvertiserBrandSafetySettingsFeatureHydrator[ - ListTweetsQuery, - AdsCandidate - ]) { - - private val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ListTweetsAds") - - private val suggestType = st.SuggestType.Promoted - - private val clientEventInfoBuilder = ClientEventInfoBuilder( - component = InjectionScribeUtil.scribeComponent(suggestType).get, - detailsBuilder = Some(HomeAdsClientEventDetailsBuilder(Some(suggestType.name))) - ) - - private val contextualTweetRefBuilder = ContextualTweetRefBuilder( - TweetHydrationContext( - safetyLevelOverride = Some(TimelineHomePromotedHydrationSafetyLevel), - outerTweetContext = None - )) - - private val decorator = UrtItemCandidateDecorator( - AdsCandidateUrtItemBuilder( - tweetClientEventInfoBuilder = Some(clientEventInfoBuilder), - contextualTweetRefBuilder = Some(contextualTweetRefBuilder) - ) - ) - - def build( - organicCandidatePipelines: CandidateScope - ): AdsDependentCandidatePipelineConfig[ListTweetsQuery] = - adsCandidatePipelineConfigBuilder.build[ListTweetsQuery]( - adsCandidateSource = adsCandidateSource, - identifier = identifier, - adsDisplayLocationBuilder = - StaticAdsDisplayLocationBuilder(ads.DisplayLocation.TimelineHomeReverseChron), - countNumOrganicItems = CountCandidatesFromPipelines(organicCandidatePipelines), - supportedClientParam = Some(EnableAdsCandidatePipelineParam), - gates = Seq( - ParamNotGate( - name = "AdsDisableInjectionBasedOnUserRole", - param = HomeGlobalParams.AdsDisableInjectionBasedOnUserRoleParam - ), - ExcludeSoftUserGate, - NonEmptyCandidatesGate(organicCandidatePipelines) - ), - filters = Seq(ValidAdImpressionIdFilter), - postFilterFeatureHydration = Seq( - ParamGatedCandidateFeatureHydrator( - EnableAdvertiserBrandSafetySettingsFeatureHydratorParam, - advertiserBrandSafetySettingsFeatureHydrator - ) - ), - decorator = Some(decorator), - urtRequest = Some(true), - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsMixerPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsMixerPipelineConfig.scala deleted file mode 100644 index 2d7dd3b31..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsMixerPipelineConfig.scala +++ /dev/null @@ -1,162 +0,0 @@ -package com.twitter.home_mixer.product.list_tweets - -import com.twitter.clientapp.{thriftscala => ca} -import com.twitter.goldfinch.api.AdsInjectionSurfaceAreas -import com.twitter.home_mixer.candidate_pipeline.ConversationServiceCandidatePipelineConfigBuilder -import com.twitter.home_mixer.functional_component.feature_hydrator.RequestQueryFeatureHydrator -import com.twitter.home_mixer.functional_component.side_effect.HomeScribeClientEventSideEffect -import com.twitter.home_mixer.model.GapIncludeInstruction -import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeClientEventsFlag -import com.twitter.home_mixer.product.list_tweets.decorator.ListConversationServiceCandidateDecorator -import com.twitter.home_mixer.product.list_tweets.model.ListTweetsQuery -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.inject.annotations.Flag -import com.twitter.logpipeline.client.common.EventPublisher -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator -import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate -import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.AddEntriesWithReplaceAndShowAlertInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedBottomCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedGapCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedTopCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceAllEntries -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceEntryInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowAlertInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.StaticTimelineScribeConfigBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtMetadataBuilder -import com.twitter.product_mixer.component_library.selector.InsertAppendResults -import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates -import com.twitter.product_mixer.component_library.selector.ads.AdsInjector -import com.twitter.product_mixer.component_library.selector.ads.InsertAdResults -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller -import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.UrtTransportMarshaller -import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.product_mixer.core.pipeline.FailOpenPolicy -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig -import com.twitter.timelines.render.{thriftscala => urt} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ListTweetsMixerPipelineConfig @Inject() ( - listTweetsTimelineServiceCandidatePipelineConfig: ListTweetsTimelineServiceCandidatePipelineConfig, - conversationServiceCandidatePipelineConfigBuilder: ConversationServiceCandidatePipelineConfigBuilder[ - ListTweetsQuery - ], - listTweetsAdsCandidatePipelineBuilder: ListTweetsAdsCandidatePipelineBuilder, - requestQueryFeatureHydrator: RequestQueryFeatureHydrator[ListTweetsQuery], - sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, - adsInjector: AdsInjector, - clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent], - urtTransportMarshaller: UrtTransportMarshaller, - @Flag(ScribeClientEventsFlag) enableScribeClientEvents: Boolean) - extends MixerPipelineConfig[ListTweetsQuery, Timeline, urt.TimelineResponse] { - - override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("ListTweets") - - private val conversationServiceCandidatePipelineConfig = - conversationServiceCandidatePipelineConfigBuilder.build( - Seq( - NonEmptyCandidatesGate( - SpecificPipelines(listTweetsTimelineServiceCandidatePipelineConfig.identifier)) - ), - ListConversationServiceCandidateDecorator() - ) - - private val listTweetsAdsCandidatePipelineConfig = listTweetsAdsCandidatePipelineBuilder.build( - SpecificPipelines(listTweetsTimelineServiceCandidatePipelineConfig.identifier) - ) - - override val candidatePipelines: Seq[CandidatePipelineConfig[ListTweetsQuery, _, _, _]] = - Seq(listTweetsTimelineServiceCandidatePipelineConfig) - - override val dependentCandidatePipelines: Seq[ - DependentCandidatePipelineConfig[ListTweetsQuery, _, _, _] - ] = - Seq(conversationServiceCandidatePipelineConfig, listTweetsAdsCandidatePipelineConfig) - - override val failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = Map( - conversationServiceCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - listTweetsAdsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always) - - override val resultSelectors: Seq[Selector[ListTweetsQuery]] = Seq( - UpdateSortCandidates( - ordering = CandidatesUtil.reverseChronTweetsOrdering, - candidatePipeline = conversationServiceCandidatePipelineConfig.identifier - ), - InsertAppendResults(candidatePipeline = conversationServiceCandidatePipelineConfig.identifier), - InsertAdResults( - surfaceAreaName = AdsInjectionSurfaceAreas.HomeTimeline, - adsInjector = adsInjector.forSurfaceArea(AdsInjectionSurfaceAreas.HomeTimeline), - adsCandidatePipeline = listTweetsAdsCandidatePipelineConfig.identifier - ), - ) - - override val fetchQueryFeatures: Seq[QueryFeatureHydrator[ListTweetsQuery]] = Seq( - requestQueryFeatureHydrator, - sgsFollowedUsersQueryFeatureHydrator - ) - - private val homeScribeClientEventSideEffect = HomeScribeClientEventSideEffect( - enableScribeClientEvents = enableScribeClientEvents, - logPipelinePublisher = clientEventsScribeEventPublisher, - injectedTweetsCandidatePipelineIdentifiers = - Seq(conversationServiceCandidatePipelineConfig.identifier), - adsCandidatePipelineIdentifier = Some(listTweetsAdsCandidatePipelineConfig.identifier), - ) - - override val resultSideEffects: Seq[PipelineResultSideEffect[ListTweetsQuery, Timeline]] = - Seq(homeScribeClientEventSideEffect) - - override val domainMarshaller: DomainMarshaller[ListTweetsQuery, Timeline] = { - val instructionBuilders = Seq( - ReplaceEntryInstructionBuilder(ReplaceAllEntries), - AddEntriesWithReplaceAndShowAlertInstructionBuilder(), - ShowAlertInstructionBuilder() - ) - - val idSelector: PartialFunction[UniversalNoun[_], Long] = { - // exclude ads while determining tweet cursor values - case item: TweetItem if item.promotedMetadata.isEmpty => item.id - case module: TimelineModule - if module.items.headOption.exists(_.item.isInstanceOf[TweetItem]) => - module.items.last.item match { - case item: TweetItem => item.id - } - } - - val topCursorBuilder = OrderedTopCursorBuilder(idSelector) - val bottomCursorBuilder = - OrderedBottomCursorBuilder(idSelector, GapIncludeInstruction.inverse()) - val gapCursorBuilder = OrderedGapCursorBuilder(idSelector, GapIncludeInstruction) - - val metadataBuilder = UrtMetadataBuilder( - title = None, - scribeConfigBuilder = Some( - StaticTimelineScribeConfigBuilder( - TimelineScribeConfig(page = Some("list_tweets"), section = None, entityToken = None))) - ) - - UrtDomainMarshaller( - instructionBuilders = instructionBuilders, - metadataBuilder = Some(metadataBuilder), - cursorBuilders = Seq(topCursorBuilder, bottomCursorBuilder, gapCursorBuilder) - ) - } - - override val transportMarshaller: TransportMarshaller[Timeline, urt.TimelineResponse] = - urtTransportMarshaller -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsProductPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsProductPipelineConfig.scala deleted file mode 100644 index b4b5b1924..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsProductPipelineConfig.scala +++ /dev/null @@ -1,130 +0,0 @@ -package com.twitter.home_mixer.product.list_tweets - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.marshaller.timelines.ChronologicalCursorUnmarshaller -import com.twitter.home_mixer.model.request.HomeMixerRequest -import com.twitter.home_mixer.model.request.ListTweetsProduct -import com.twitter.home_mixer.model.request.ListTweetsProductContext -import com.twitter.home_mixer.product.list_tweets.model.ListTweetsQuery -import com.twitter.home_mixer.product.list_tweets.param.ListTweetsParam.ServerMaxResultsParam -import com.twitter.home_mixer.product.list_tweets.param.ListTweetsParamConfig -import com.twitter.home_mixer.service.HomeMixerAccessPolicy.DefaultHomeMixerAccessPolicy -import com.twitter.home_mixer.service.HomeMixerAlertConfig.DefaultNotificationGroup -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer -import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.common.alert.EmptyResponseRateAlert -import com.twitter.product_mixer.core.functional_component.common.alert.LatencyAlert -import com.twitter.product_mixer.core.functional_component.common.alert.P99 -import com.twitter.product_mixer.core.functional_component.common.alert.SuccessRateAlert -import com.twitter.product_mixer.core.functional_component.common.alert.ThroughputAlert -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfAbove -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfBelow -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfLatencyAbove -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.request -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor -import com.twitter.product_mixer.core.pipeline.PipelineConfig -import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest -import com.twitter.product_mixer.core.pipeline.pipeline_failure.MalformedCursor -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.product.ProductPipelineConfig -import com.twitter.product_mixer.core.product.ProductParamConfig -import com.twitter.product_mixer.core.util.SortIndexBuilder -import com.twitter.timelines.configapi.Params -import com.twitter.timelines.render.{thriftscala => urt} -import com.twitter.timelines.util.RequestCursorSerializer -import com.twitter.util.Time -import com.twitter.util.Try -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ListTweetsProductPipelineConfig @Inject() ( - listTweetsMixerPipelineConfig: ListTweetsMixerPipelineConfig, - listTweetsParamConfig: ListTweetsParamConfig) - extends ProductPipelineConfig[HomeMixerRequest, ListTweetsQuery, urt.TimelineResponse] { - - override val identifier: ProductPipelineIdentifier = ProductPipelineIdentifier("ListTweets") - override val product: request.Product = ListTweetsProduct - override val paramConfig: ProductParamConfig = listTweetsParamConfig - override val denyLoggedOutUsers: Boolean = false - - override def pipelineQueryTransformer( - request: HomeMixerRequest, - params: Params - ): ListTweetsQuery = { - val context = request.productContext match { - case Some(context: ListTweetsProductContext) => context - case _ => throw PipelineFailure(BadRequest, "ListTweetsProductContext not found") - } - - val debugOptions = request.debugParams.flatMap(_.debugOptions) - - /** - * Unlike other clients, newly created tweets on Android have the sort index set to the current - * time instead of the top sort index + 1, so these tweets get stuck at the top of the timeline - * if subsequent timeline responses use the sort index from the previous response instead of - * the current time. - */ - val pipelineCursor = request.serializedRequestCursor.flatMap { cursor => - Try(UrtCursorSerializer.deserializeOrderedCursor(cursor)) - .getOrElse(ChronologicalCursorUnmarshaller(RequestCursorSerializer.deserialize(cursor))) - .map { - case UrtOrderedCursor(_, id, Some(GapCursor), gapBoundaryId) - if id.isEmpty || gapBoundaryId.isEmpty => - throw PipelineFailure(MalformedCursor, "Gap Cursor bounds not defined") - case topCursor @ UrtOrderedCursor(_, _, Some(TopCursor), _) => - val queryTime = debugOptions.flatMap(_.requestTimeOverride).getOrElse(Time.now) - topCursor.copy(initialSortIndex = SortIndexBuilder.timeToId(queryTime)) - case cursor => cursor - } - } - - ListTweetsQuery( - params = params, - clientContext = request.clientContext, - features = None, - pipelineCursor = pipelineCursor, - requestedMaxResults = Some(params(ServerMaxResultsParam)), - debugOptions = debugOptions, - listId = context.listId, - deviceContext = context.deviceContext, - dspClientContext = context.dspClientContext - ) - } - - override def pipelines: Seq[PipelineConfig] = Seq(listTweetsMixerPipelineConfig) - - override def pipelineSelector(query: ListTweetsQuery): ComponentIdentifier = - listTweetsMixerPipelineConfig.identifier - - override val alerts: Seq[Alert] = Seq( - SuccessRateAlert( - notificationGroup = DefaultNotificationGroup, - warnPredicate = TriggerIfBelow(99.9, 20, 30), - criticalPredicate = TriggerIfBelow(99.9, 30, 30), - ), - LatencyAlert( - notificationGroup = DefaultNotificationGroup, - percentile = P99, - warnPredicate = TriggerIfLatencyAbove(300.millis, 15, 30), - criticalPredicate = TriggerIfLatencyAbove(400.millis, 15, 30) - ), - ThroughputAlert( - notificationGroup = DefaultNotificationGroup, - warnPredicate = TriggerIfAbove(3000), - criticalPredicate = TriggerIfAbove(4000) - ), - EmptyResponseRateAlert( - notificationGroup = DefaultNotificationGroup, - warnPredicate = TriggerIfAbove(65), - criticalPredicate = TriggerIfAbove(80) - ) - ) - - override val debugAccessPolicies: Set[AccessPolicy] = DefaultHomeMixerAccessPolicy -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsTimelineServiceCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsTimelineServiceCandidatePipelineConfig.scala deleted file mode 100644 index 1af95e873..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsTimelineServiceCandidatePipelineConfig.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.twitter.home_mixer.product.list_tweets - -import com.twitter.home_mixer.candidate_pipeline.TimelineServiceResponseFeatureTransformer -import com.twitter.home_mixer.marshaller.timelines.TimelineServiceCursorMarshaller -import com.twitter.home_mixer.product.list_tweets.model.ListTweetsQuery -import com.twitter.home_mixer.product.list_tweets.param.ListTweetsParam.ServerMaxResultsParam -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.candidate_source.timeline_service.TimelineServiceTweetCandidateSource -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.timelineservice.{thriftscala => t} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ListTweetsTimelineServiceCandidatePipelineConfig @Inject() ( - timelineServiceTweetCandidateSource: TimelineServiceTweetCandidateSource) - extends CandidatePipelineConfig[ListTweetsQuery, t.TimelineQuery, t.Tweet, TweetCandidate] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ListTweetsTimelineServiceTweets") - - override val queryTransformer: CandidatePipelineQueryTransformer[ - ListTweetsQuery, - t.TimelineQuery - ] = { query => - val timelineQueryOptions = t.TimelineQueryOptions( - contextualUserId = query.clientContext.userId, - ) - - t.TimelineQuery( - timelineType = t.TimelineType.List, - timelineId = query.listId, - maxCount = query.maxResults(ServerMaxResultsParam).toShort, - cursor2 = query.pipelineCursor.flatMap(TimelineServiceCursorMarshaller(_)), - options = Some(timelineQueryOptions), - timelineId2 = Some(t.TimelineId(t.TimelineType.List, query.listId, None)) - ) - } - - override def candidateSource: BaseCandidateSource[t.TimelineQuery, t.Tweet] = - timelineServiceTweetCandidateSource - - override val resultTransformer: CandidatePipelineResultsTransformer[t.Tweet, TweetCandidate] = { - sourceResult => TweetCandidate(id = sourceResult.statusId) - } - - override val featuresFromCandidateSourceTransformers: Seq[CandidateFeatureTransformer[t.Tweet]] = - Seq(TimelineServiceResponseFeatureTransformer) - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.7) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/BUILD.bazel deleted file mode 100644 index 4258ffc36..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/BUILD.bazel +++ /dev/null @@ -1,15 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/builder", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "timelines/src/main/scala/com/twitter/timelines/injection/scribe", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/ListConversationServiceCandidateDecorator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/ListConversationServiceCandidateDecorator.scala deleted file mode 100644 index a02b628e8..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/ListConversationServiceCandidateDecorator.scala +++ /dev/null @@ -1,49 +0,0 @@ -package com.twitter.home_mixer.product.list_tweets.decorator - -import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder -import com.twitter.home_mixer.functional_component.decorator.builder.ListClientEventDetailsBuilder -import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.StaticModuleDisplayTypeBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.VerticalConversation -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.injection.scribe.InjectionScribeUtil -import com.twitter.timelineservice.suggests.{thriftscala => st} - -object ListConversationServiceCandidateDecorator { - - private val ConversationModuleNamespace = EntryNamespace("list-conversation") - - def apply(): Some[UrtMultipleModulesDecorator[PipelineQuery, TweetCandidate, Long]] = { - val suggestType = st.SuggestType.OrganicListTweet - val component = InjectionScribeUtil.scribeComponent(suggestType).get - val clientEventInfoBuilder = ClientEventInfoBuilder( - component = component, - detailsBuilder = Some(ListClientEventDetailsBuilder(st.SuggestType.OrganicListTweet)) - ) - val tweetItemBuilder = TweetCandidateUrtItemBuilder( - clientEventInfoBuilder = clientEventInfoBuilder - ) - - val moduleBuilder = TimelineModuleBuilder( - entryNamespace = ConversationModuleNamespace, - clientEventInfoBuilder = clientEventInfoBuilder, - displayTypeBuilder = StaticModuleDisplayTypeBuilder(VerticalConversation), - metadataBuilder = Some(HomeConversationModuleMetadataBuilder()) - ) - - Some( - UrtMultipleModulesDecorator( - urtItemCandidateDecorator = UrtItemCandidateDecorator(tweetItemBuilder), - moduleBuilder = moduleBuilder, - groupByKey = (_, _, candidateFeatures) => - candidateFeatures.getOrElse(ConversationModuleFocalTweetIdFeature, None) - )) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/builder/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/builder/BUILD.bazel deleted file mode 100644 index 3ed81a196..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/builder/BUILD.bazel +++ /dev/null @@ -1,10 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/model/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/model/BUILD.bazel deleted file mode 100644 index fac4ba949..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/model/BUILD.bazel +++ /dev/null @@ -1,19 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/model/ListTweetsQuery.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/model/ListTweetsQuery.scala deleted file mode 100644 index ce8c73c0a..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/model/ListTweetsQuery.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.home_mixer.product.list_tweets.model - -import com.twitter.adserver.thriftscala.HomeTimelineType -import com.twitter.adserver.thriftscala.TimelineRequestParams -import com.twitter.dspbidder.commons.{thriftscala => dsp} -import com.twitter.home_mixer.model.HomeAdsQuery -import com.twitter.home_mixer.model.request.DeviceContext -import com.twitter.home_mixer.model.request.HasListId -import com.twitter.home_mixer.model.request.ListTweetsProduct -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.request._ -import com.twitter.product_mixer.core.pipeline.HasPipelineCursor -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.Params - -case class ListTweetsQuery( - override val params: Params, - override val clientContext: ClientContext, - override val pipelineCursor: Option[UrtOrderedCursor], - override val requestedMaxResults: Option[Int], - override val debugOptions: Option[DebugOptions], - override val features: Option[FeatureMap], - override val listId: Long, - override val deviceContext: Option[DeviceContext], - override val dspClientContext: Option[dsp.DspClientContext]) - extends PipelineQuery - with HasPipelineCursor[UrtOrderedCursor] - with HasListId - with HomeAdsQuery { - override val product: Product = ListTweetsProduct - - override def withFeatureMap(features: FeatureMap): ListTweetsQuery = - copy(features = Some(features)) - - override val timelineRequestParams: Option[TimelineRequestParams] = - Some(TimelineRequestParams(homeTimelineType = Some(HomeTimelineType.HomeLatest))) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/BUILD.bazel deleted file mode 100644 index 18867ad9e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/BUILD.bazel +++ /dev/null @@ -1,12 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/ListTweetsParam.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/ListTweetsParam.scala deleted file mode 100644 index 827d29f10..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/ListTweetsParam.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.home_mixer.product.list_tweets.param - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam - -object ListTweetsParam { - val SupportedClientFSName = "list_tweets_supported_client" - - object EnableAdsCandidatePipelineParam - extends FSParam[Boolean]( - name = "list_tweets_enable_ads", - default = false - ) - - object ServerMaxResultsParam - extends FSBoundedParam[Int]( - name = "list_tweets_server_max_results", - default = 100, - min = 1, - max = 500 - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/ListTweetsParamConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/ListTweetsParamConfig.scala deleted file mode 100644 index 01575db92..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/ListTweetsParamConfig.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.home_mixer.product.list_tweets.param - -import com.twitter.home_mixer.param.decider.DeciderKey -import com.twitter.home_mixer.product.list_tweets.param.ListTweetsParam.EnableAdsCandidatePipelineParam -import com.twitter.home_mixer.product.list_tweets.param.ListTweetsParam.ServerMaxResultsParam -import com.twitter.home_mixer.product.list_tweets.param.ListTweetsParam.SupportedClientFSName -import com.twitter.product_mixer.core.product.ProductParamConfig -import com.twitter.servo.decider.DeciderKeyName -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ListTweetsParamConfig @Inject() () extends ProductParamConfig { - override val enabledDeciderKey: DeciderKeyName = DeciderKey.EnableListTweetsProduct - override val supportedClientFSName: String = SupportedClientFSName - - override val booleanFSOverrides = - Seq(EnableAdsCandidatePipelineParam) - - override val boundedIntFSOverrides = Seq( - ServerMaxResultsParam - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/BUILD.bazel deleted file mode 100644 index 4bbbec94d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/BUILD.bazel +++ /dev/null @@ -1,40 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "explore/explore-ranker/thrift/src/main/thrift:thrift-scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/selector", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/async", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/param_gated", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsProductPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsProductPipelineConfig.scala deleted file mode 100644 index 27278db0c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsProductPipelineConfig.scala +++ /dev/null @@ -1,74 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets - -import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature -import com.twitter.home_mixer.model.request.HomeMixerRequest -import com.twitter.home_mixer.model.request.ScoredTweetsProduct -import com.twitter.home_mixer.model.request.ScoredTweetsProductContext -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.ServerMaxResultsParam -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParamConfig -import com.twitter.home_mixer.service.HomeMixerAccessPolicy.DefaultHomeMixerAccessPolicy -import com.twitter.home_mixer.{thriftscala => t} -import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.Product -import com.twitter.product_mixer.core.pipeline.PipelineConfig -import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.product.ProductPipelineConfig -import com.twitter.product_mixer.core.product.ProductParamConfig -import com.twitter.timelines.configapi.Params -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ScoredTweetsProductPipelineConfig @Inject() ( - scoredTweetsRecommendationPipelineConfig: ScoredTweetsRecommendationPipelineConfig, - scoredTweetsParamConfig: ScoredTweetsParamConfig) - extends ProductPipelineConfig[HomeMixerRequest, ScoredTweetsQuery, t.ScoredTweets] { - - override val identifier: ProductPipelineIdentifier = ProductPipelineIdentifier("ScoredTweets") - - override val product: Product = ScoredTweetsProduct - - override val paramConfig: ProductParamConfig = scoredTweetsParamConfig - - override def pipelineQueryTransformer( - request: HomeMixerRequest, - params: Params - ): ScoredTweetsQuery = { - val context = request.productContext match { - case Some(context: ScoredTweetsProductContext) => context - case _ => throw PipelineFailure(BadRequest, "ScoredTweetsProductContext not found") - } - - val featureMap = FeatureMapBuilder() - .add(ServedTweetIdsFeature, context.servedTweetIds.getOrElse(Seq.empty)) - .add(TimelineServiceTweetsFeature, context.backfillTweetIds.getOrElse(Seq.empty)) - .build() - - ScoredTweetsQuery( - params = params, - clientContext = request.clientContext, - pipelineCursor = - request.serializedRequestCursor.flatMap(UrtCursorSerializer.deserializeOrderedCursor), - requestedMaxResults = Some(params(ServerMaxResultsParam)), - debugOptions = request.debugParams.flatMap(_.debugOptions), - features = Some(featureMap), - deviceContext = context.deviceContext, - seenTweetIds = context.seenTweetIds, - qualityFactorStatus = None - ) - } - - override val pipelines: Seq[PipelineConfig] = Seq(scoredTweetsRecommendationPipelineConfig) - - override def pipelineSelector(query: ScoredTweetsQuery): ComponentIdentifier = - scoredTweetsRecommendationPipelineConfig.identifier - - override val debugAccessPolicies: Set[AccessPolicy] = DefaultHomeMixerAccessPolicy -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsRecommendationPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsRecommendationPipelineConfig.scala deleted file mode 100644 index e3d50040b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsRecommendationPipelineConfig.scala +++ /dev/null @@ -1,340 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.functional_component.feature_hydrator.FeedbackHistoryQueryFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.ImpressionBloomFilterQueryFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.RealGraphInNetworkScoresQueryFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.RequestQueryFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.TweetImpressionsQueryFeatureHydrator -import com.twitter.home_mixer.functional_component.filter.FeedbackFatigueFilter -import com.twitter.home_mixer.functional_component.filter.PreviouslySeenTweetsFilter -import com.twitter.home_mixer.functional_component.filter.PreviouslyServedTweetsFilter -import com.twitter.home_mixer.functional_component.filter.RejectTweetFromViewerFilter -import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter -import com.twitter.home_mixer.functional_component.side_effect.PublishClientSentImpressionsEventBusSideEffect -import com.twitter.home_mixer.functional_component.side_effect.PublishClientSentImpressionsManhattanSideEffect -import com.twitter.home_mixer.functional_component.side_effect.PublishImpressionBloomFilterSideEffect -import com.twitter.home_mixer.functional_component.side_effect.UpdateLastNonPollingTimeSideEffect -import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsSupportAccountReplyFeature -import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature -import com.twitter.home_mixer.param.HomeGlobalParams.EnableImpressionBloomFilter -import com.twitter.home_mixer.param.HomeMixerFlagName.TargetFetchLatency -import com.twitter.home_mixer.param.HomeMixerFlagName.TargetScoringLatency -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.CachedScoredTweetsCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsBackfillCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsFrsCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsInNetworkCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsListsCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsPopularVideosCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsTweetMixerCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsUtegCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.CachedScoredTweetsQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.ListIdsQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RealGraphQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RealTimeInteractionGraphUserVertexQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RequestTimeQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TwhinUserEngagementQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TwhinUserFollowQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.UserLanguagesFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.UserStateQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.PartAAggregateQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.PartBAggregateQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates.UserEngagementRealTimeAggregatesFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.filter.DuplicateConversationTweetsFilter -import com.twitter.home_mixer.product.scored_tweets.filter.OutOfNetworkCompetitorFilter -import com.twitter.home_mixer.product.scored_tweets.filter.OutOfNetworkCompetitorURLFilter -import com.twitter.home_mixer.product.scored_tweets.filter.ScoredTweetsSocialContextFilter -import com.twitter.home_mixer.product.scored_tweets.marshaller.ScoredTweetsResponseDomainMarshaller -import com.twitter.home_mixer.product.scored_tweets.marshaller.ScoredTweetsResponseTransportMarshaller -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.MaxInNetworkResultsParam -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.MaxOutOfNetworkResultsParam -import com.twitter.home_mixer.product.scored_tweets.scoring_pipeline.ScoredTweetsHeuristicScoringPipelineConfig -import com.twitter.home_mixer.product.scored_tweets.scoring_pipeline.ScoredTweetsModelScoringPipelineConfig -import com.twitter.home_mixer.product.scored_tweets.selector.KeepBestOutOfNetworkCandidatePerAuthorPerSuggestType -import com.twitter.home_mixer.product.scored_tweets.side_effect.CachedScoredTweetsSideEffect -import com.twitter.home_mixer.product.scored_tweets.side_effect.ScribeScoredCandidatesSideEffect -import com.twitter.home_mixer.product.scored_tweets.side_effect.ScribeServedCommonFeaturesAndCandidateFeaturesSideEffect -import com.twitter.home_mixer.{thriftscala => t} -import com.twitter.inject.annotations.Flag -import com.twitter.product_mixer.component_library.feature_hydrator.query.async.AsyncQueryFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweetsQueryFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.ParamGatedQueryFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator -import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.component_library.selector.DropDuplicateCandidates -import com.twitter.product_mixer.component_library.selector.DropFilteredMaxCandidates -import com.twitter.product_mixer.component_library.selector.DropMaxCandidates -import com.twitter.product_mixer.component_library.selector.IdAndClassDuplicationKey -import com.twitter.product_mixer.component_library.selector.InsertAppendResults -import com.twitter.product_mixer.component_library.selector.PickFirstCandidateMerger -import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates -import com.twitter.product_mixer.component_library.selector.sorter.FeatureValueSorter -import com.twitter.product_mixer.core.functional_component.common.AllExceptPipelines -import com.twitter.product_mixer.core.functional_component.common.AllPipelines -import com.twitter.product_mixer.core.functional_component.configapi.StaticParam -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller -import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.model.common.identifier.RecommendationPipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.pipeline.FailOpenPolicy -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.recommendation.RecommendationPipelineConfig -import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipelineConfig -import com.twitter.product_mixer.core.quality_factor.BoundsWithDefault -import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactorConfig -import com.twitter.product_mixer.core.quality_factor.QualityFactorConfig -import com.twitter.util.Duration - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ScoredTweetsRecommendationPipelineConfig @Inject() ( - scoredTweetsInNetworkCandidatePipelineConfig: ScoredTweetsInNetworkCandidatePipelineConfig, - scoredTweetsUtegCandidatePipelineConfig: ScoredTweetsUtegCandidatePipelineConfig, - scoredTweetsTweetMixerCandidatePipelineConfig: ScoredTweetsTweetMixerCandidatePipelineConfig, - scoredTweetsFrsCandidatePipelineConfig: ScoredTweetsFrsCandidatePipelineConfig, - scoredTweetsListsCandidatePipelineConfig: ScoredTweetsListsCandidatePipelineConfig, - scoredTweetsPopularVideosCandidatePipelineConfig: ScoredTweetsPopularVideosCandidatePipelineConfig, - scoredTweetsBackfillCandidatePipelineConfig: ScoredTweetsBackfillCandidatePipelineConfig, - cachedScoredTweetsCandidatePipelineConfig: CachedScoredTweetsCandidatePipelineConfig, - requestQueryFeatureHydrator: RequestQueryFeatureHydrator[ScoredTweetsQuery], - requestTimeQueryFeatureHydrator: RequestTimeQueryFeatureHydrator, - realTimeInteractionGraphUserVertexQueryFeatureHydrator: RealTimeInteractionGraphUserVertexQueryFeatureHydrator, - userStateQueryFeatureHydrator: UserStateQueryFeatureHydrator, - userEngagementRealTimeAggregatesFeatureHydrator: UserEngagementRealTimeAggregatesFeatureHydrator, - twhinUserEngagementQueryFeatureHydrator: TwhinUserEngagementQueryFeatureHydrator, - twhinUserFollowQueryFeatureHydrator: TwhinUserFollowQueryFeatureHydrator, - cachedScoredTweetsQueryFeatureHydrator: CachedScoredTweetsQueryFeatureHydrator, - sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, - scoredTweetsModelScoringPipelineConfig: ScoredTweetsModelScoringPipelineConfig, - impressionBloomFilterQueryFeatureHydrator: ImpressionBloomFilterQueryFeatureHydrator[ - ScoredTweetsQuery - ], - manhattanTweetImpressionsQueryFeatureHydrator: TweetImpressionsQueryFeatureHydrator[ - ScoredTweetsQuery - ], - memcacheTweetImpressionsQueryFeatureHydrator: ImpressedTweetsQueryFeatureHydrator, - listIdsQueryFeatureHydrator: ListIdsQueryFeatureHydrator, - feedbackHistoryQueryFeatureHydrator: FeedbackHistoryQueryFeatureHydrator, - publishClientSentImpressionsEventBusSideEffect: PublishClientSentImpressionsEventBusSideEffect, - publishClientSentImpressionsManhattanSideEffect: PublishClientSentImpressionsManhattanSideEffect, - publishImpressionBloomFilterSideEffect: PublishImpressionBloomFilterSideEffect, - realGraphInNetworkScoresQueryFeatureHydrator: RealGraphInNetworkScoresQueryFeatureHydrator, - realGraphQueryFeatureHydrator: RealGraphQueryFeatureHydrator, - userLanguagesFeatureHydrator: UserLanguagesFeatureHydrator, - partAAggregateQueryFeatureHydrator: PartAAggregateQueryFeatureHydrator, - partBAggregateQueryFeatureHydrator: PartBAggregateQueryFeatureHydrator, - cachedScoredTweetsSideEffect: CachedScoredTweetsSideEffect, - scribeScoredCandidatesSideEffect: ScribeScoredCandidatesSideEffect, - scribeServedCommonFeaturesAndCandidateFeaturesSideEffect: ScribeServedCommonFeaturesAndCandidateFeaturesSideEffect, - updateLastNonPollingTimeSideEffect: UpdateLastNonPollingTimeSideEffect[ - ScoredTweetsQuery, - ScoredTweetsResponse - ], - @Flag(TargetFetchLatency) targetFetchLatency: Duration, - @Flag(TargetScoringLatency) targetScoringLatency: Duration) - extends RecommendationPipelineConfig[ - ScoredTweetsQuery, - TweetCandidate, - ScoredTweetsResponse, - t.ScoredTweetsResponse - ] { - - override val identifier: RecommendationPipelineIdentifier = - RecommendationPipelineIdentifier("ScoredTweets") - - private val SubscriptionReplyFilterId = "SubscriptionReply" - private val MaxBackfillTweets = 50 - - private val scoringStep = RecommendationPipelineConfig.scoringPipelinesStep - - override val fetchQueryFeatures: Seq[QueryFeatureHydrator[ScoredTweetsQuery]] = Seq( - requestQueryFeatureHydrator, - realGraphInNetworkScoresQueryFeatureHydrator, - cachedScoredTweetsQueryFeatureHydrator, - sgsFollowedUsersQueryFeatureHydrator, - ParamGatedQueryFeatureHydrator( - EnableImpressionBloomFilter, - impressionBloomFilterQueryFeatureHydrator - ), - manhattanTweetImpressionsQueryFeatureHydrator, - memcacheTweetImpressionsQueryFeatureHydrator, - listIdsQueryFeatureHydrator, - userStateQueryFeatureHydrator, - AsyncQueryFeatureHydrator(scoringStep, feedbackHistoryQueryFeatureHydrator), - AsyncQueryFeatureHydrator(scoringStep, realGraphQueryFeatureHydrator), - AsyncQueryFeatureHydrator(scoringStep, requestTimeQueryFeatureHydrator), - AsyncQueryFeatureHydrator(scoringStep, userLanguagesFeatureHydrator), - AsyncQueryFeatureHydrator(scoringStep, userEngagementRealTimeAggregatesFeatureHydrator), - AsyncQueryFeatureHydrator(scoringStep, realTimeInteractionGraphUserVertexQueryFeatureHydrator), - AsyncQueryFeatureHydrator(scoringStep, twhinUserFollowQueryFeatureHydrator), - AsyncQueryFeatureHydrator(scoringStep, twhinUserEngagementQueryFeatureHydrator), - AsyncQueryFeatureHydrator(scoringStep, partAAggregateQueryFeatureHydrator), - AsyncQueryFeatureHydrator(scoringStep, partBAggregateQueryFeatureHydrator), - ) - - override val candidatePipelines: Seq[ - CandidatePipelineConfig[ScoredTweetsQuery, _, _, TweetCandidate] - ] = Seq( - cachedScoredTweetsCandidatePipelineConfig, - scoredTweetsInNetworkCandidatePipelineConfig, - scoredTweetsUtegCandidatePipelineConfig, - scoredTweetsTweetMixerCandidatePipelineConfig, - scoredTweetsFrsCandidatePipelineConfig, - scoredTweetsListsCandidatePipelineConfig, - scoredTweetsPopularVideosCandidatePipelineConfig, - scoredTweetsBackfillCandidatePipelineConfig - ) - - override val postCandidatePipelinesSelectors: Seq[Selector[ScoredTweetsQuery]] = Seq( - DropDuplicateCandidates( - pipelineScope = AllPipelines, - duplicationKey = IdAndClassDuplicationKey, - mergeStrategy = PickFirstCandidateMerger - ), - InsertAppendResults(AllPipelines) - ) - - override val globalFilters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq( - // sort these to have the "cheaper" filters run first - RejectTweetFromViewerFilter, - RetweetDeduplicationFilter, - PreviouslySeenTweetsFilter, - PreviouslyServedTweetsFilter, - PredicateFeatureFilter.fromPredicate( - FilterIdentifier(SubscriptionReplyFilterId), - shouldKeepCandidate = { features => - features.getOrElse(InReplyToTweetIdFeature, None).isEmpty || - features.getOrElse(ExclusiveConversationAuthorIdFeature, None).isEmpty - } - ), - FeedbackFatigueFilter - ) - - override val candidatePipelineFailOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = - Map( - cachedScoredTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - scoredTweetsInNetworkCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - scoredTweetsUtegCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - scoredTweetsTweetMixerCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - scoredTweetsFrsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - scoredTweetsListsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - scoredTweetsPopularVideosCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - scoredTweetsBackfillCandidatePipelineConfig.identifier -> FailOpenPolicy.Always - ) - - override val scoringPipelineFailOpenPolicies: Map[ScoringPipelineIdentifier, FailOpenPolicy] = - Map( - ScoredTweetsHeuristicScoringPipelineConfig.identifier -> FailOpenPolicy.Always - ) - - private val candidatePipelineQualityFactorConfig = LinearLatencyQualityFactorConfig( - qualityFactorBounds = BoundsWithDefault(minInclusive = 0.1, maxInclusive = 1.0, default = 0.95), - initialDelay = 60.seconds, - targetLatency = targetFetchLatency, - targetLatencyPercentile = 95.0, - delta = 0.00125 - ) - - private val scoringPipelineQualityFactorConfig = - candidatePipelineQualityFactorConfig.copy(targetLatency = targetScoringLatency) - - override val qualityFactorConfigs: Map[ComponentIdentifier, QualityFactorConfig] = Map( - // candidate pipelines - scoredTweetsInNetworkCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, - scoredTweetsUtegCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, - scoredTweetsTweetMixerCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, - scoredTweetsFrsCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, - scoredTweetsListsCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, - scoredTweetsPopularVideosCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, - scoredTweetsBackfillCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, - // scoring pipelines - scoredTweetsModelScoringPipelineConfig.identifier -> scoringPipelineQualityFactorConfig, - ) - - override val scoringPipelines: Seq[ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate]] = - Seq( - // scoring pipeline - run on non-cached candidates only since cached ones are already scored - scoredTweetsModelScoringPipelineConfig, - // re-scoring pipeline - run on all candidates since these are request specific - ScoredTweetsHeuristicScoringPipelineConfig - ) - - override val postScoringFilters = Seq( - ScoredTweetsSocialContextFilter, - OutOfNetworkCompetitorFilter, - OutOfNetworkCompetitorURLFilter, - DuplicateConversationTweetsFilter, - PredicateFeatureFilter.fromPredicate( - FilterIdentifier("IsSupportAccountReply"), - shouldKeepCandidate = { features => - !features.getOrElse(IsSupportAccountReplyFeature, false) - }) - ) - - override val resultSelectors: Seq[Selector[ScoredTweetsQuery]] = Seq( - KeepBestOutOfNetworkCandidatePerAuthorPerSuggestType(AllPipelines), - UpdateSortCandidates(AllPipelines, FeatureValueSorter.descending(ScoreFeature)), - DropFilteredMaxCandidates( - pipelineScope = - AllExceptPipelines(Set(scoredTweetsBackfillCandidatePipelineConfig.identifier)), - filter = { - case ItemCandidateWithDetails(_, _, features) => - features.getOrElse(InNetworkFeature, false) - case _ => false - }, - maxSelectionsParam = MaxInNetworkResultsParam - ), - DropFilteredMaxCandidates( - pipelineScope = AllPipelines, - filter = { - case ItemCandidateWithDetails(_, _, features) => - !features.getOrElse(InNetworkFeature, false) - case _ => false - }, - maxSelectionsParam = MaxOutOfNetworkResultsParam - ), - DropMaxCandidates( - candidatePipeline = scoredTweetsBackfillCandidatePipelineConfig.identifier, - maxSelectionsParam = StaticParam(MaxBackfillTweets) - ), - InsertAppendResults(AllPipelines) - ) - - override val resultSideEffects: Seq[ - PipelineResultSideEffect[ScoredTweetsQuery, ScoredTweetsResponse] - ] = Seq( - cachedScoredTweetsSideEffect, - publishClientSentImpressionsEventBusSideEffect, - publishClientSentImpressionsManhattanSideEffect, - publishImpressionBloomFilterSideEffect, - scribeScoredCandidatesSideEffect, - scribeServedCommonFeaturesAndCandidateFeaturesSideEffect, - updateLastNonPollingTimeSideEffect - ) - - override val domainMarshaller: DomainMarshaller[ - ScoredTweetsQuery, - ScoredTweetsResponse - ] = ScoredTweetsResponseDomainMarshaller - - override val transportMarshaller: TransportMarshaller[ - ScoredTweetsResponse, - t.ScoredTweetsResponse - ] = ScoredTweetsResponseTransportMarshaller -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/BUILD.bazel deleted file mode 100644 index c5bb294a1..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/BUILD.bazel +++ /dev/null @@ -1,37 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "explore/explore-ranker/thrift/src/main/thrift:thrift-scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/explore_ranker", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweet_mixer", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "src/thrift/com/twitter/timelineranker:thrift-scala", - "tweet-mixer/thrift/src/main/thrift:thrift-scala", - ], - exports = [ - "src/thrift/com/twitter/timelineranker:thrift-scala", - "tweet-mixer/thrift/src/main/thrift:thrift-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/CachedScoredTweetsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/CachedScoredTweetsCandidatePipelineConfig.scala deleted file mode 100644 index 65a42cbee..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/CachedScoredTweetsCandidatePipelineConfig.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline - -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.CachedScoredTweetsCandidatePipelineConfig._ -import com.twitter.home_mixer.product.scored_tweets.candidate_source.CachedScoredTweetsCandidateSource -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.response_transformer.CachedScoredTweetsResponseFeatureTransformer -import com.twitter.home_mixer.{thriftscala => hmt} -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Candidate Pipeline Config that fetches tweets from Scored Tweets Cache. - */ -@Singleton -class CachedScoredTweetsCandidatePipelineConfig @Inject() ( - cachedScoredTweetsCandidateSource: CachedScoredTweetsCandidateSource) - extends CandidatePipelineConfig[ - ScoredTweetsQuery, - ScoredTweetsQuery, - hmt.ScoredTweet, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = Identifier - - override val queryTransformer: CandidatePipelineQueryTransformer[ - ScoredTweetsQuery, - ScoredTweetsQuery - ] = identity - - override val candidateSource: BaseCandidateSource[ScoredTweetsQuery, hmt.ScoredTweet] = - cachedScoredTweetsCandidateSource - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[hmt.ScoredTweet] - ] = Seq(CachedScoredTweetsResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - hmt.ScoredTweet, - TweetCandidate - ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } -} - -object CachedScoredTweetsCandidatePipelineConfig { - val Identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("CachedScoredTweets") -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsBackfillCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsBackfillCandidatePipelineConfig.scala deleted file mode 100644 index c34031c78..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsBackfillCandidatePipelineConfig.scala +++ /dev/null @@ -1,94 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline - -import com.twitter.home_mixer.functional_component.filter.ReplyFilter -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TweetypieStaticEntitiesFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate -import com.twitter.home_mixer.product.scored_tweets.gate.MinTimeSinceLastRequestGate -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidatePipeline -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableBackfillCandidatePipelineParam -import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsBackfillResponseFeatureTransformer -import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter -import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.functional_component.candidate_source.PassthroughCandidateSource -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.decider.DeciderParam -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ScoredTweetsBackfillCandidatePipelineConfig @Inject() ( - tweetypieStaticEntitiesHydrator: TweetypieStaticEntitiesFeatureHydrator) - extends CandidatePipelineConfig[ - ScoredTweetsQuery, - ScoredTweetsQuery, - Long, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ScoredTweetsBackfill") - - private val HasAuthorFilterId = "HasAuthor" - - override val enabledDeciderParam: Option[DeciderParam[Boolean]] = - Some(CandidatePipeline.EnableBackfillParam) - - override val supportedClientParam: Option[FSParam[Boolean]] = - Some(EnableBackfillCandidatePipelineParam) - - override val gates: Seq[Gate[ScoredTweetsQuery]] = - Seq( - MinTimeSinceLastRequestGate, - NonEmptySeqFeatureGate(TimelineServiceTweetsFeature), - MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) - ) - - override val queryTransformer: CandidatePipelineQueryTransformer[ - ScoredTweetsQuery, - ScoredTweetsQuery - ] = identity - - override def candidateSource: CandidateSource[ScoredTweetsQuery, Long] = - PassthroughCandidateSource( - identifier = CandidateSourceIdentifier("ScoredTweetsBackfill"), - candidateExtractor = { query => - query.features.map(_.getOrElse(TimelineServiceTweetsFeature, Seq.empty)).toSeq.flatten - } - ) - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[Long] - ] = Seq(ScoredTweetsBackfillResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[Long, TweetCandidate] = { - sourceResult => TweetCandidate(id = sourceResult) - } - - override val preFilterFeatureHydrationPhase1: Seq[ - BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] - ] = Seq(tweetypieStaticEntitiesHydrator) - - override val filters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq( - ReplyFilter, - PredicateFeatureFilter.fromPredicate( - FilterIdentifier(HasAuthorFilterId), - shouldKeepCandidate = _.getOrElse(AuthorIdFeature, None).isDefined - ) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsFrsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsFrsCandidatePipelineConfig.scala deleted file mode 100644 index f25a839b7..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsFrsCandidatePipelineConfig.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline - -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.FrsSeedUsersQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidatePipeline -import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerFrsQueryTransformer -import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsFrsResponseFeatureTransformer -import com.twitter.product_mixer.component_library.candidate_source.timeline_ranker.TimelineRankerRecapCandidateSource -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.timelineranker.{thriftscala => tlr} -import com.twitter.timelines.configapi.decider.DeciderParam -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Candidate Pipeline Config that takes user recommendations from Follow Recommendation Service (FRS) - * and makes a TimelineRanker->Earlybird query for tweet candidates from those users. - * Additionally, the candidate pipeline hydrates followedByUserIds so that followed-by social proof - * can be used. - */ -@Singleton -class ScoredTweetsFrsCandidatePipelineConfig @Inject() ( - timelineRankerRecapCandidateSource: TimelineRankerRecapCandidateSource, - frsSeedUsersQueryFeatureHydrator: FrsSeedUsersQueryFeatureHydrator) - extends CandidatePipelineConfig[ - ScoredTweetsQuery, - tlr.RecapQuery, - tlr.CandidateTweet, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ScoredTweetsFrs") - - override val enabledDeciderParam: Option[DeciderParam[Boolean]] = - Some(CandidatePipeline.EnableFrsParam) - - override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( - MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) - ) - - override val queryFeatureHydration: Seq[ - BaseQueryFeatureHydrator[ScoredTweetsQuery, _] - ] = Seq(frsSeedUsersQueryFeatureHydrator) - - override val candidateSource: BaseCandidateSource[tlr.RecapQuery, tlr.CandidateTweet] = - timelineRankerRecapCandidateSource - - override val queryTransformer: CandidatePipelineQueryTransformer[ - ScoredTweetsQuery, - tlr.RecapQuery - ] = TimelineRankerFrsQueryTransformer(identifier) - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[tlr.CandidateTweet] - ] = Seq(ScoredTweetsFrsResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - tlr.CandidateTweet, - TweetCandidate - ] = { candidate => TweetCandidate(candidate.tweet.get.id) } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsInNetworkCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsInNetworkCandidatePipelineConfig.scala deleted file mode 100644 index 63bd1add8..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsInNetworkCandidatePipelineConfig.scala +++ /dev/null @@ -1,82 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline - -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.IsExtendedReplyFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.ReplyFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RetweetSourceTweetFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.filter.RetweetSourceTweetRemovingFilter -import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidatePipeline -import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerInNetworkQueryTransformer -import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsInNetworkResponseFeatureTransformer -import com.twitter.product_mixer.component_library.candidate_source.timeline_ranker.TimelineRankerInNetworkCandidateSource -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.timelineranker.{thriftscala => t} -import com.twitter.timelines.configapi.decider.DeciderParam -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Candidate Pipeline Config to fetch in-network tweets from Timeline Ranker's Recycled source - */ -@Singleton -class ScoredTweetsInNetworkCandidatePipelineConfig @Inject() ( - timelineRankerInNetworkCandidateSource: TimelineRankerInNetworkCandidateSource, - replyFeatureHydrator: ReplyFeatureHydrator) - extends CandidatePipelineConfig[ - ScoredTweetsQuery, - t.RecapQuery, - t.CandidateTweet, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ScoredTweetsInNetwork") - - override val enabledDeciderParam: Option[DeciderParam[Boolean]] = - Some(CandidatePipeline.EnableInNetworkParam) - - override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( - MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) - ) - - override val candidateSource: BaseCandidateSource[t.RecapQuery, t.CandidateTweet] = - timelineRankerInNetworkCandidateSource - - override val queryTransformer: CandidatePipelineQueryTransformer[ - ScoredTweetsQuery, - t.RecapQuery - ] = TimelineRankerInNetworkQueryTransformer(identifier) - - override val preFilterFeatureHydrationPhase1: Seq[ - BaseCandidateFeatureHydrator[PipelineQuery, TweetCandidate, _] - ] = Seq(RetweetSourceTweetFeatureHydrator) - - override def filters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq( - RetweetSourceTweetRemovingFilter - ) - - override val postFilterFeatureHydration: Seq[ - BaseCandidateFeatureHydrator[PipelineQuery, TweetCandidate, _] - ] = Seq(IsExtendedReplyFeatureHydrator, replyFeatureHydrator) - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[t.CandidateTweet] - ] = Seq(ScoredTweetsInNetworkResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - t.CandidateTweet, - TweetCandidate - ] = { sourceResult => TweetCandidate(id = sourceResult.tweet.get.id) } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsListsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsListsCandidatePipelineConfig.scala deleted file mode 100644 index 1161a8278..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsListsCandidatePipelineConfig.scala +++ /dev/null @@ -1,86 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline - -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TweetypieStaticEntitiesFeatureHydrator -import com.twitter.home_mixer.functional_component.filter.ReplyFilter -import com.twitter.home_mixer.functional_component.filter.RetweetFilter -import com.twitter.home_mixer.product.scored_tweets.candidate_source.ListsCandidateSource -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.ListIdsFeature -import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidatePipeline -import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsListsResponseFeatureTransformer -import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.timelines.configapi.decider.DeciderParam -import com.twitter.timelineservice.{thriftscala => t} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ScoredTweetsListsCandidatePipelineConfig @Inject() ( - listsCandidateSource: ListsCandidateSource, - tweetypieStaticEntitiesHydrator: TweetypieStaticEntitiesFeatureHydrator) - extends CandidatePipelineConfig[ - ScoredTweetsQuery, - Seq[t.TimelineQuery], - t.Tweet, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ScoredTweetsLists") - - private val MaxTweetsToFetchPerList = 20 - - override val enabledDeciderParam: Option[DeciderParam[Boolean]] = - Some(CandidatePipeline.EnableListsParam) - - override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( - NonEmptySeqFeatureGate(ListIdsFeature), - MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) - ) - - override val queryTransformer: CandidatePipelineQueryTransformer[ - ScoredTweetsQuery, - Seq[t.TimelineQuery] - ] = { query => - val listIds = query.features.map(_.get(ListIdsFeature)).get - listIds.map { listId => - t.TimelineQuery( - timelineType = t.TimelineType.List, - timelineId = listId, - maxCount = MaxTweetsToFetchPerList.toShort, - options = Some(t.TimelineQueryOptions(query.clientContext.userId)), - timelineId2 = Some(t.TimelineId(t.TimelineType.List, listId, None)) - ) - } - } - - override def candidateSource: CandidateSource[Seq[t.TimelineQuery], t.Tweet] = - listsCandidateSource - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[t.Tweet] - ] = Seq(ScoredTweetsListsResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[t.Tweet, TweetCandidate] = { - sourceResult => TweetCandidate(id = sourceResult.statusId) - } - - override val preFilterFeatureHydrationPhase1: Seq[ - BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] - ] = Seq(tweetypieStaticEntitiesHydrator) - - override val filters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = - Seq(ReplyFilter, RetweetFilter) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsPopularVideosCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsPopularVideosCandidatePipelineConfig.scala deleted file mode 100644 index c8b28b0ff..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsPopularVideosCandidatePipelineConfig.scala +++ /dev/null @@ -1,82 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline - -import com.twitter.explore_ranker.{thriftscala => ert} -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TweetypieStaticEntitiesFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidatePipeline -import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsPopularVideosResponseFeatureTransformer -import com.twitter.home_mixer.util.CachedScoredTweetsHelper -import com.twitter.product_mixer.component_library.candidate_source.explore_ranker.ExploreRankerImmersiveRecsCandidateSource -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.marshaller.request.ClientContextMarshaller -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.timelines.configapi.decider.DeciderParam -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ScoredTweetsPopularVideosCandidatePipelineConfig @Inject() ( - exploreRankerCandidateSource: ExploreRankerImmersiveRecsCandidateSource, - tweetypieStaticEntitiesFeatureHydrator: TweetypieStaticEntitiesFeatureHydrator) - extends CandidatePipelineConfig[ - ScoredTweetsQuery, - ert.ExploreRankerRequest, - ert.ExploreTweetRecommendation, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ScoredTweetsPopularVideos") - - private val MaxTweetsToFetch = 40 - - override val enabledDeciderParam: Option[DeciderParam[Boolean]] = - Some(CandidatePipeline.EnablePopularVideosParam) - - override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( - MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) - ) - - override val queryTransformer: CandidatePipelineQueryTransformer[ - ScoredTweetsQuery, - ert.ExploreRankerRequest - ] = { query => - val excludedTweetIds = query.features.map( - CachedScoredTweetsHelper.tweetImpressionsAndCachedScoredTweets(_, identifier)) - - ert.ExploreRankerRequest( - clientContext = ClientContextMarshaller(query.clientContext), - product = ert.Product.HomeTimelineVideoInline, - productContext = Some( - ert.ProductContext.HomeTimelineVideoInline(ert.HomeTimelineVideoInline(excludedTweetIds))), - maxResults = Some(MaxTweetsToFetch) - ) - } - - override def candidateSource: BaseCandidateSource[ - ert.ExploreRankerRequest, - ert.ExploreTweetRecommendation - ] = exploreRankerCandidateSource - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[ert.ExploreTweetRecommendation] - ] = Seq(ScoredTweetsPopularVideosResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - ert.ExploreTweetRecommendation, - TweetCandidate - ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } - - override val preFilterFeatureHydrationPhase1: Seq[ - BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] - ] = Seq(tweetypieStaticEntitiesFeatureHydrator) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsTweetMixerCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsTweetMixerCandidatePipelineConfig.scala deleted file mode 100644 index df16e3f11..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsTweetMixerCandidatePipelineConfig.scala +++ /dev/null @@ -1,100 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline - -import com.twitter.tweet_mixer.{thriftscala => t} -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TweetypieStaticEntitiesFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidatePipeline -import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsTweetMixerResponseFeatureTransformer -import com.twitter.home_mixer.util.CachedScoredTweetsHelper -import com.twitter.product_mixer.component_library.candidate_source.tweet_mixer.TweetMixerCandidateSource -import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.marshaller.request.ClientContextMarshaller -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.timelines.configapi.decider.DeciderParam - -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Candidate Pipeline Config that fetches tweets from TweetMixer. - */ -@Singleton -class ScoredTweetsTweetMixerCandidatePipelineConfig @Inject() ( - tweetMixerCandidateSource: TweetMixerCandidateSource, - tweetypieStaticEntitiesFeatureHydrator: TweetypieStaticEntitiesFeatureHydrator) - extends CandidatePipelineConfig[ - ScoredTweetsQuery, - t.TweetMixerRequest, - t.TweetResult, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ScoredTweetsTweetMixer") - - val HasAuthorFilterId = "HasAuthor" - - override val enabledDeciderParam: Option[DeciderParam[Boolean]] = - Some(CandidatePipeline.EnableTweetMixerParam) - - override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( - MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam), - ) - - override val candidateSource: BaseCandidateSource[t.TweetMixerRequest, t.TweetResult] = - tweetMixerCandidateSource - - private val MaxTweetsToFetch = 400 - - override val queryTransformer: CandidatePipelineQueryTransformer[ - ScoredTweetsQuery, - t.TweetMixerRequest - ] = { query => - val maxCount = (query.getQualityFactorCurrentValue(identifier) * MaxTweetsToFetch).toInt - - val excludedTweetIds = query.features.map( - CachedScoredTweetsHelper.tweetImpressionsAndCachedScoredTweets(_, identifier)) - - t.TweetMixerRequest( - clientContext = ClientContextMarshaller(query.clientContext), - product = t.Product.HomeRecommendedTweets, - productContext = Some( - t.ProductContext.HomeRecommendedTweetsProductContext( - t.HomeRecommendedTweetsProductContext(excludedTweetIds = excludedTweetIds.map(_.toSet)))), - maxResults = Some(maxCount) - ) - } - - override val preFilterFeatureHydrationPhase1: Seq[ - BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] - ] = Seq(tweetypieStaticEntitiesFeatureHydrator) - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[t.TweetResult] - ] = Seq(ScoredTweetsTweetMixerResponseFeatureTransformer) - - override val filters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq( - PredicateFeatureFilter.fromPredicate( - FilterIdentifier(HasAuthorFilterId), - shouldKeepCandidate = _.getOrElse(AuthorIdFeature, None).isDefined - ) - ) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - t.TweetResult, - TweetCandidate - ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsUtegCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsUtegCandidatePipelineConfig.scala deleted file mode 100644 index 9178d0789..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsUtegCandidatePipelineConfig.scala +++ /dev/null @@ -1,62 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline - -import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidatePipeline -import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerUtegQueryTransformer -import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsUtegResponseFeatureTransformer -import com.twitter.product_mixer.component_library.candidate_source.timeline_ranker.TimelineRankerUtegCandidateSource -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.timelineranker.{thriftscala => t} -import com.twitter.timelines.configapi.decider.DeciderParam -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Candidate Pipeline Config that fetches tweets from the Timeline Ranker UTEG Candidate Source - */ -@Singleton -class ScoredTweetsUtegCandidatePipelineConfig @Inject() ( - timelineRankerUtegCandidateSource: TimelineRankerUtegCandidateSource) - extends CandidatePipelineConfig[ - ScoredTweetsQuery, - t.UtegLikedByTweetsQuery, - t.CandidateTweet, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ScoredTweetsUteg") - - override val enabledDeciderParam: Option[DeciderParam[Boolean]] = - Some(CandidatePipeline.EnableUtegParam) - - override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( - MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) - ) - - override val candidateSource: BaseCandidateSource[t.UtegLikedByTweetsQuery, t.CandidateTweet] = - timelineRankerUtegCandidateSource - - override val queryTransformer: CandidatePipelineQueryTransformer[ - ScoredTweetsQuery, - t.UtegLikedByTweetsQuery - ] = TimelineRankerUtegQueryTransformer(identifier) - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[t.CandidateTweet] - ] = Seq(ScoredTweetsUtegResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - t.CandidateTweet, - TweetCandidate - ] = { sourceResult => TweetCandidate(id = sourceResult.tweet.get.id) } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/BUILD.bazel deleted file mode 100644 index 618c4a081..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/BUILD.bazel +++ /dev/null @@ -1,21 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/earlybird", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", - "src/thrift/com/twitter/search:earlybird-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdFrsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdFrsCandidatePipelineConfig.scala deleted file mode 100644 index 0d59bf3ca..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdFrsCandidatePipelineConfig.scala +++ /dev/null @@ -1,69 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.earlybird - -import com.twitter.finagle.thrift.ClientId -import com.twitter.home_mixer.functional_component.candidate_source.EarlybirdCandidateSource -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.FrsSeedUsersQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets -import com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird.EarlybirdFrsQueryTransformer -import com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird.ScoredTweetsEarlybirdFrsResponseFeatureTransformer -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.search.earlybird.{thriftscala => eb} -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Candidate Pipeline Config that fetches tweets from the earlybird FRS Candidate Source - */ -@Singleton -class ScoredTweetsEarlybirdFrsCandidatePipelineConfig @Inject() ( - earlybirdCandidateSource: EarlybirdCandidateSource, - frsSeedUsersQueryFeatureHydrator: FrsSeedUsersQueryFeatureHydrator, - clientId: ClientId) - extends CandidatePipelineConfig[ - ScoredTweetsQuery, - eb.EarlybirdRequest, - eb.ThriftSearchResult, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ScoredTweetsEarlybirdFrs") - - override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( - MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) - ) - - override val queryFeatureHydration: Seq[ - BaseQueryFeatureHydrator[ScoredTweetsQuery, _] - ] = Seq(frsSeedUsersQueryFeatureHydrator) - - override val candidateSource: BaseCandidateSource[eb.EarlybirdRequest, eb.ThriftSearchResult] = - earlybirdCandidateSource - - override val queryTransformer: CandidatePipelineQueryTransformer[ - ScoredTweetsQuery, - eb.EarlybirdRequest - ] = EarlybirdFrsQueryTransformer(identifier, clientId = Some(clientId.name)) - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[eb.ThriftSearchResult] - ] = Seq(ScoredTweetsEarlybirdFrsResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - eb.ThriftSearchResult, - TweetCandidate - ] = { sourceResult => TweetCandidate(id = sourceResult.id) } - - override def filters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq.empty -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig.scala deleted file mode 100644 index acb26ce42..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.earlybird - -import com.twitter.finagle.thrift.ClientId -import com.twitter.home_mixer.functional_component.candidate_source.EarlybirdCandidateSource -import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets -import com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird.EarlybirdInNetworkQueryTransformer -import com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird.ScoredTweetsEarlybirdInNetworkResponseFeatureTransformer -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.search.earlybird.{thriftscala => eb} -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Candidate Pipeline Config that fetches tweets from the earlybird InNetwork Candidate Source - */ -@Singleton -class ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig @Inject() ( - earlybirdCandidateSource: EarlybirdCandidateSource, - clientId: ClientId) - extends CandidatePipelineConfig[ - ScoredTweetsQuery, - eb.EarlybirdRequest, - eb.ThriftSearchResult, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ScoredTweetsEarlybirdInNetwork") - - override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( - MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) - ) - - override val candidateSource: BaseCandidateSource[eb.EarlybirdRequest, eb.ThriftSearchResult] = - earlybirdCandidateSource - - override val queryTransformer: CandidatePipelineQueryTransformer[ - ScoredTweetsQuery, - eb.EarlybirdRequest - ] = EarlybirdInNetworkQueryTransformer(identifier, clientId = Some(clientId.name)) - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[eb.ThriftSearchResult] - ] = Seq(ScoredTweetsEarlybirdInNetworkResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - eb.ThriftSearchResult, - TweetCandidate - ] = { sourceResult => TweetCandidate(id = sourceResult.id) } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/BUILD.bazel deleted file mode 100644 index 4461d8460..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "src/thrift/com/twitter/timelineservice:thrift-scala", - "stitch/stitch-timelineservice", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/CachedScoredTweetsCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/CachedScoredTweetsCandidateSource.scala deleted file mode 100644 index a5f556e7c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/CachedScoredTweetsCandidateSource.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.candidate_source - -import com.twitter.home_mixer.util.CachedScoredTweetsHelper -import com.twitter.home_mixer.{thriftscala => hmt} -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class CachedScoredTweetsCandidateSource @Inject() () - extends CandidateSource[PipelineQuery, hmt.ScoredTweet] { - - override val identifier: CandidateSourceIdentifier = - CandidateSourceIdentifier("CachedScoredTweets") - - override def apply(request: PipelineQuery): Stitch[Seq[hmt.ScoredTweet]] = { - Stitch.value( - request.features.map(CachedScoredTweetsHelper.unseenCachedScoredTweets).getOrElse(Seq.empty)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/ListsCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/ListsCandidateSource.scala deleted file mode 100644 index 395ac7da8..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/ListsCandidateSource.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.candidate_source - -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import com.twitter.stitch.timelineservice.TimelineService -import com.twitter.timelineservice.{thriftscala => tls} - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ListsCandidateSource @Inject() (timelineService: TimelineService) - extends CandidateSource[Seq[tls.TimelineQuery], tls.Tweet] { - - override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("Lists") - - override def apply(requests: Seq[tls.TimelineQuery]): Stitch[Seq[tls.Tweet]] = { - val timelines = Stitch.traverse(requests) { request => timelineService.getTimeline(request) } - - timelines.map { - _.flatMap { - _.entries.collect { case tls.TimelineEntry.Tweet(tweet) => tweet } - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AncestorFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AncestorFeatureHydrator.scala deleted file mode 100644 index 9c5e8f3ab..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AncestorFeatureHydrator.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch -import com.twitter.tweetconvosvc.tweet_ancestor.{thriftscala => ta} -import com.twitter.tweetconvosvc.{thriftscala => tcs} -import com.twitter.util.Future -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class AncestorFeatureHydrator @Inject() ( - conversationServiceClient: tcs.ConversationService.MethodPerEndpoint) - extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Ancestor") - - override val features: Set[Feature[_, _]] = Set(AncestorsFeature) - - private val DefaultFeatureMap = FeatureMapBuilder().add(AncestorsFeature, Seq.empty).build() - - override def apply( - query: PipelineQuery, - candidate: TweetCandidate, - existingFeatures: FeatureMap - ): Stitch[FeatureMap] = OffloadFuturePools.offloadFuture { - if (existingFeatures.getOrElse(InReplyToTweetIdFeature, None).isDefined) { - val ancestorsRequest = tcs.GetAncestorsRequest(Seq(candidate.id)) - conversationServiceClient.getAncestors(ancestorsRequest).map { getAncestorsResponse => - val ancestors = getAncestorsResponse.ancestors.headOption - .collect { - case tcs.TweetAncestorsResult.TweetAncestors(ancestorsResult) - if ancestorsResult.nonEmpty => - ancestorsResult.head.ancestors ++ getTruncatedRootTweet(ancestorsResult.head) - }.getOrElse(Seq.empty) - - FeatureMapBuilder().add(AncestorsFeature, ancestors).build() - } - } else Future.value(DefaultFeatureMap) - } - - private def getTruncatedRootTweet( - ancestors: ta.TweetAncestors, - ): Option[ta.TweetAncestor] = { - ancestors.conversationRootAuthorId.collect { - case rootAuthorId - if ancestors.state == ta.ReplyState.Partial && - ancestors.ancestors.last.tweetId != ancestors.conversationId => - ta.TweetAncestor(ancestors.conversationId, rootAuthorId) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AuthorFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AuthorFeatureHydrator.scala deleted file mode 100644 index c8dd6a934..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AuthorFeatureHydrator.scala +++ /dev/null @@ -1,86 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.AuthorFeatureRepository -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.author_features.AuthorFeaturesAdapter -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.home_mixer.util.ObservedKeyValueResultHandler -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.servo.repository.KeyValueRepository -import com.twitter.servo.repository.KeyValueResult -import com.twitter.stitch.Stitch -import com.twitter.timelines.author_features.v1.{thriftjava => af} -import com.twitter.util.Future -import com.twitter.util.Try -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton -import scala.collection.JavaConverters._ - -object AuthorFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class AuthorFeatureHydrator @Inject() ( - @Named(AuthorFeatureRepository) client: KeyValueRepository[Seq[Long], Long, af.AuthorFeatures], - override val statsReceiver: StatsReceiver) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] - with ObservedKeyValueResultHandler { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("AuthorFeature") - - override val features: Set[Feature[_, _]] = Set(AuthorFeature) - - override val statScope: String = identifier.toString - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { - val possiblyAuthorIds = extractKeys(candidates) - val authorIds = possiblyAuthorIds.flatten - - val response: Future[KeyValueResult[Long, af.AuthorFeatures]] = - if (authorIds.nonEmpty) client(authorIds) - else Future.value(KeyValueResult.empty) - - response.map { result => - possiblyAuthorIds.map { possiblyAuthorId => - val value = observedGet(key = possiblyAuthorId, keyValueResult = result) - val transformedValue = postTransformer(value) - - FeatureMapBuilder().add(AuthorFeature, transformedValue).build() - } - } - } - - private def postTransformer(authorFeatures: Try[Option[af.AuthorFeatures]]): Try[DataRecord] = { - authorFeatures.map { - _.map { features => AuthorFeaturesAdapter.adaptToDataRecords(features).asScala.head } - .getOrElse(new DataRecord()) - } - } - - private def extractKeys( - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Seq[Option[Long]] = { - candidates.map { candidate => - CandidatesUtil.getOriginalAuthorId(candidate.features) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AuthorIsCreatorFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AuthorIsCreatorFeatureHydrator.scala deleted file mode 100644 index cc66983cc..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AuthorIsCreatorFeatureHydrator.scala +++ /dev/null @@ -1,79 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorIsCreatorFeature -import com.twitter.home_mixer.util.MissingKeyException -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch -import com.twitter.strato.generated.client.audiencerewards.audienceRewardsService.GetSuperFollowEligibilityOnUserClientColumn -import com.twitter.util.Throw -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class AuthorIsCreatorFeatureHydrator @Inject() ( - getSuperFollowEligibilityOnUserClientColumn: GetSuperFollowEligibilityOnUserClientColumn, - statsReceiver: StatsReceiver) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("AuthorIsCreator") - - override val features: Set[Feature[_, _]] = - Set(AuthorIsCreatorFeature) - - private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) - private val keyFoundCounter = scopedStatsReceiver.counter("key/found") - private val keyFailureCounter = scopedStatsReceiver.counter("key/failure") - - private val MissingKeyFeatureMap = FeatureMapBuilder() - .add(AuthorIsCreatorFeature, Throw(MissingKeyException)) - .build() - - private val DefaultFeatureMap = FeatureMapBuilder() - .add(AuthorIsCreatorFeature, false) - .build() - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - OffloadFuturePools.offloadStitch { - val authorIds = candidates.flatMap(_.features.getOrElse(AuthorIdFeature, None)).distinct - Stitch - .collect { - authorIds.map { authorId => - getSuperFollowEligibilityOnUserClientColumn.fetcher - .fetch(authorId) - .map { authorId -> _.v } - } - }.map { authorIdsToIsCreator => - val authorIdsToIsCreatorMap = authorIdsToIsCreator.toMap - candidates.map { candidate => - candidate.features.getOrElse(AuthorIdFeature, None) match { - case Some(authorId) => - authorIdsToIsCreatorMap.get(authorId) match { - case Some(response) => - keyFoundCounter.incr() - FeatureMapBuilder() - .add(AuthorIsCreatorFeature, response.getOrElse(false)).build() - case _ => - keyFailureCounter.incr() - DefaultFeatureMap - } - case _ => MissingKeyFeatureMap - } - } - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/BUILD.bazel deleted file mode 100644 index 58a566de8..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/BUILD.bazel +++ /dev/null @@ -1,70 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/recommendations", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "representation-scorer/server/src/main/scala/com/twitter/representationscorer/common", - "servo/repo/src/main/scala", - "servo/util/src/main/scala", - "snowflake/src/main/scala/com/twitter/snowflake/id", - "src/java/com/twitter/ml/api/constant", - "src/scala/com/twitter/ml/api/util", - "src/scala/com/twitter/timelines/prediction/adapters/conversation_features", - "src/scala/com/twitter/timelines/prediction/adapters/real_graph", - "src/scala/com/twitter/timelines/prediction/adapters/realtime_interaction_graph", - "src/scala/com/twitter/timelines/prediction/adapters/twistly", - "src/scala/com/twitter/timelines/prediction/adapters/two_hop_features", - "src/scala/com/twitter/timelines/prediction/common/adapters", - "src/scala/com/twitter/timelines/prediction/common/util", - "src/scala/com/twitter/timelines/prediction/features/common", - "src/scala/com/twitter/timelines/prediction/features/realtime_interaction_graph", - "src/scala/com/twitter/timelines/prediction/features/time_features", - "src/thrift/com/twitter/ml/api:data-java", - "src/thrift/com/twitter/onboarding/relevance/features:features-java", - "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", - "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/timelineranker:thrift-scala", - "src/thrift/com/twitter/timelines/author_features:thrift-java", - "stitch/stitch-gizmoduck", - "stitch/stitch-socialgraph", - "stitch/stitch-tweetypie", - "strato/config/columns/audiencerewards/audienceRewardsService:getSuperFollowEligibility-strato-client", - "strato/config/columns/ml/featureStore:featureStore-strato-client", - "timelines/src/main/scala/com/twitter/timelines/clients/strato/topics", - "timelines/src/main/scala/com/twitter/timelines/clients/strato/twistly", - "timelines/src/main/scala/com/twitter/timelines/common/model", - "timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils", - "timelines/src/main/scala/com/twitter/timelines/model/candidate", - "timelines/src/main/scala/com/twitter/timelines/model/types", - "topic-social-proof/server/src/main/thrift:thrift-scala", - "topiclisting/topiclisting-core/src/main/scala/com/twitter/topiclisting", - "tweetconvosvc/thrift/src/main/thrift:thrift-scala", - "user_session_store/src/main/scala/com/twitter/user_session_store", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/CachedScoredTweetsQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/CachedScoredTweetsQueryFeatureHydrator.scala deleted file mode 100644 index 873f043be..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/CachedScoredTweetsQueryFeatureHydrator.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures._ -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets -import com.twitter.home_mixer.{thriftscala => hmt} -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.cache.TtlCache -import com.twitter.stitch.Stitch -import com.twitter.util.Return -import com.twitter.util.Throw -import com.twitter.util.Time - -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Fetch scored Tweets from cache and exclude the expired ones - */ -@Singleton -case class CachedScoredTweetsQueryFeatureHydrator @Inject() ( - scoredTweetsCache: TtlCache[Long, hmt.ScoredTweetsResponse]) - extends QueryFeatureHydrator[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("CachedScoredTweets") - - override val features: Set[Feature[_, _]] = Set(CachedScoredTweetsFeature) - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - val userId = query.getRequiredUserId - val tweetScoreTtl = query.params(CachedScoredTweets.TTLParam) - - Stitch.callFuture(scoredTweetsCache.get(Seq(userId))).map { keyValueResult => - keyValueResult(userId) match { - case Return(cachedCandidatesOpt) => - val cachedScoredTweets = cachedCandidatesOpt.map(_.scoredTweets).getOrElse(Seq.empty) - val nonExpiredTweets = cachedScoredTweets.filter { tweet => - tweet.lastScoredTimestampMs.exists(Time.fromMilliseconds(_).untilNow < tweetScoreTtl) - } - FeatureMapBuilder().add(CachedScoredTweetsFeature, nonExpiredTweets).build() - case Throw(exception) => throw exception - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/EarlybirdFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/EarlybirdFeatureHydrator.scala deleted file mode 100644 index cc3c2aeeb..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/EarlybirdFeatureHydrator.scala +++ /dev/null @@ -1,151 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.earlybird.EarlybirdAdapter -import com.twitter.home_mixer.model.HomeFeatures.DeviceLanguageFeature -import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature -import com.twitter.home_mixer.model.HomeFeatures.EarlybirdSearchResultFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature -import com.twitter.home_mixer.model.HomeFeatures.UserScreenNameFeature -import com.twitter.home_mixer.param.HomeMixerInjectionNames.EarlybirdRepository -import com.twitter.home_mixer.util.ObservedKeyValueResultHandler -import com.twitter.home_mixer.util.earlybird.EarlybirdResponseUtil -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.search.earlybird.{thriftscala => eb} -import com.twitter.servo.keyvalue.KeyValueResult -import com.twitter.servo.repository.KeyValueRepository -import com.twitter.stitch.Stitch -import com.twitter.util.Return -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton -import scala.collection.JavaConverters._ - -object EarlybirdDataRecordFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class EarlybirdFeatureHydrator @Inject() ( - @Named(EarlybirdRepository) client: KeyValueRepository[ - (Seq[Long], Long), - Long, - eb.ThriftSearchResult - ], - override val statsReceiver: StatsReceiver) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] - with ObservedKeyValueResultHandler { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Earlybird") - - override val features: Set[Feature[_, _]] = Set( - EarlybirdDataRecordFeature, - EarlybirdFeature, - EarlybirdSearchResultFeature, - TweetUrlsFeature - ) - - override val statScope: String = identifier.toString - - private val scopedStatsReceiver = statsReceiver.scope(statScope) - private val originalKeyFoundCounter = scopedStatsReceiver.counter("originalKey/found") - private val originalKeyLossCounter = scopedStatsReceiver.counter("originalKey/loss") - - private val ebSearchResultNotExistPredicate: CandidateWithFeatures[TweetCandidate] => Boolean = - candidate => candidate.features.getOrElse(EarlybirdSearchResultFeature, None).isEmpty - private val ebFeaturesNotExistPredicate: CandidateWithFeatures[TweetCandidate] => Boolean = - candidate => candidate.features.getOrElse(EarlybirdFeature, None).isEmpty - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { - val candidatesToHydrate = candidates.filter { candidate => - val isEmpty = - ebFeaturesNotExistPredicate(candidate) && ebSearchResultNotExistPredicate(candidate) - if (isEmpty) originalKeyLossCounter.incr() else originalKeyFoundCounter.incr() - isEmpty - } - - client((candidatesToHydrate.map(_.candidate.id), query.getRequiredUserId)) - .map(handleResponse(query, candidates, _, candidatesToHydrate)) - } - - private def handleResponse( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]], - results: KeyValueResult[Long, eb.ThriftSearchResult], - candidatesToHydrate: Seq[CandidateWithFeatures[TweetCandidate]] - ): Seq[FeatureMap] = { - val queryFeatureMap = query.features.getOrElse(FeatureMap.empty) - val userLanguages = queryFeatureMap.getOrElse(UserLanguagesFeature, Seq.empty) - val uiLanguageCode = queryFeatureMap.getOrElse(DeviceLanguageFeature, None) - val screenName = queryFeatureMap.getOrElse(UserScreenNameFeature, None) - val followedUserIds = queryFeatureMap.getOrElse(SGSFollowedUsersFeature, Seq.empty).toSet - - val searchResults = candidatesToHydrate - .map { candidate => - observedGet(Some(candidate.candidate.id), results) - }.collect { - case Return(Some(value)) => value - } - - val allSearchResults = searchResults ++ - candidates.filter(!ebSearchResultNotExistPredicate(_)).flatMap { candidate => - candidate.features - .getOrElse(EarlybirdSearchResultFeature, None) - } - val idToSearchResults = allSearchResults.map(sr => sr.id -> sr).toMap - val tweetIdToEbFeatures = EarlybirdResponseUtil.getTweetThriftFeaturesByTweetId( - searcherUserId = query.getRequiredUserId, - screenName = screenName, - userLanguages = userLanguages, - uiLanguageCode = uiLanguageCode, - followedUserIds = followedUserIds, - mutuallyFollowingUserIds = Set.empty, - searchResults = allSearchResults, - sourceTweetSearchResults = Seq.empty, - ) - - candidates.map { candidate => - val transformedEbFeatures = tweetIdToEbFeatures.get(candidate.candidate.id) - val earlybirdFeatures = - if (transformedEbFeatures.nonEmpty) transformedEbFeatures - else candidate.features.getOrElse(EarlybirdFeature, None) - - val candidateIsRetweet = candidate.features.getOrElse(IsRetweetFeature, false) - val sourceTweetEbFeatures = - candidate.features.getOrElse(SourceTweetEarlybirdFeature, None) - - val originalTweetEbFeatures = - if (candidateIsRetweet && sourceTweetEbFeatures.nonEmpty) - sourceTweetEbFeatures - else earlybirdFeatures - - val earlybirdDataRecord = - EarlybirdAdapter.adaptToDataRecords(originalTweetEbFeatures).asScala.head - - FeatureMapBuilder() - .add(EarlybirdFeature, earlybirdFeatures) - .add(EarlybirdDataRecordFeature, earlybirdDataRecord) - .add(EarlybirdSearchResultFeature, idToSearchResults.get(candidate.candidate.id)) - .add(TweetUrlsFeature, earlybirdFeatures.flatMap(_.urlsList).getOrElse(Seq.empty)) - .build() - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/FrsSeedUsersQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/FrsSeedUsersQueryFeatureHydrator.scala deleted file mode 100644 index 2508067b0..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/FrsSeedUsersQueryFeatureHydrator.scala +++ /dev/null @@ -1,64 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.follow_recommendations.{thriftscala => frs} -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.product_mixer.component_library.candidate_source.recommendations.UserFollowRecommendationsCandidateSource -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyView -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -object FrsSeedUserIdsFeature extends Feature[TweetCandidate, Option[Seq[Long]]] -object FrsUserToFollowedByUserIdsFeature extends Feature[TweetCandidate, Map[Long, Seq[Long]]] - -@Singleton -case class FrsSeedUsersQueryFeatureHydrator @Inject() ( - userFollowRecommendationsCandidateSource: UserFollowRecommendationsCandidateSource) - extends QueryFeatureHydrator[ScoredTweetsQuery] { - - private val maxUsersToFetch = 100 - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FrsSeedUsers") - - override def features: Set[Feature[_, _]] = Set( - FrsSeedUserIdsFeature, - FrsUserToFollowedByUserIdsFeature - ) - - override def hydrate(query: ScoredTweetsQuery): Stitch[FeatureMap] = { - val frsRequest = frs.RecommendationRequest( - clientContext = frs.ClientContext(query.getOptionalUserId), - displayLocation = frs.DisplayLocation.HomeTimelineTweetRecs, - maxResults = Some(maxUsersToFetch) - ) - - userFollowRecommendationsCandidateSource(StratoKeyView(frsRequest, Unit)) - .map { userRecommendations: Seq[frs.UserRecommendation] => - val seedUserIds = userRecommendations.map(_.userId) - val seedUserIdsSet = seedUserIds.toSet - - val userToFollowedByUserIds: Map[Long, Seq[Long]] = userRecommendations.flatMap { - userRecommendation => - if (seedUserIdsSet.contains(userRecommendation.userId)) { - val followProof = - userRecommendation.reason.flatMap(_.accountProof).flatMap(_.followProof) - val followedByUserIds = followProof.map(_.userIds).getOrElse(Seq.empty) - Some(userRecommendation.userId -> followedByUserIds) - } else { - None - } - }.toMap - - FeatureMapBuilder() - .add(FrsSeedUserIdsFeature, Some(seedUserIds)) - .add(FrsUserToFollowedByUserIdsFeature, userToFollowedByUserIds) - .build() - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/GizmoduckAuthorFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/GizmoduckAuthorFeatureHydrator.scala deleted file mode 100644 index 7c96ef474..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/GizmoduckAuthorFeatureHydrator.scala +++ /dev/null @@ -1,83 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.conversions.DurationOps._ -import com.twitter.ads.entities.db.{thriftscala => ae} -import com.twitter.gizmoduck.{thriftscala => gt} -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorIsProtectedFeature -import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.IsSupportAccountReplyFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch -import com.twitter.snowflake.id.SnowflakeId -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class GizmoduckAuthorFeatureHydrator @Inject() (gizmoduck: gt.UserService.MethodPerEndpoint) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("GizmoduckAuthor") - - override val features: Set[Feature[_, _]] = - Set(AuthorIsBlueVerifiedFeature, AuthorIsProtectedFeature, IsSupportAccountReplyFeature) - - private val queryFields: Set[gt.QueryFields] = - Set(gt.QueryFields.AdvertiserAccount, gt.QueryFields.Profile, gt.QueryFields.Safety) - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { - val authorIds = candidates.flatMap(_.features.getOrElse(AuthorIdFeature, None)) - - val response = gizmoduck.get( - userIds = authorIds.distinct, - queryFields = queryFields, - context = gt.LookupContext() - ) - - response.map { hydratedAuthors => - val userMetadataMap = hydratedAuthors - .collect { - case userResult if userResult.user.isDefined => - val user = userResult.user.get - val blueVerified = user.safety.flatMap(_.isBlueVerified).getOrElse(false) - val isProtected = user.safety.exists(_.isProtected) - (user.id, (blueVerified, isProtected)) - }.toMap.withDefaultValue((false, false)) - - candidates.map { candidate => - val authorId = candidate.features.get(AuthorIdFeature).get - val (isBlueVerified, isProtected) = userMetadataMap(authorId) - - // Some accounts run promotions on Twitter and send replies automatically. - // We assume that a reply that took more than one minute is not an auto-reply. - // If time difference doesn't exist, this means that one of the tweets was - // not snowflake and therefore much older, and therefore OK as an extended reply. - val timeDifference = candidate.features.getOrElse(InReplyToTweetIdFeature, None).map { - SnowflakeId.timeFromId(candidate.candidate.id) - SnowflakeId.timeFromId(_) - } - val isAutoReply = timeDifference.exists(_ < 1.minute) - - FeatureMapBuilder() - .add(AuthorIsBlueVerifiedFeature, isBlueVerified) - .add(AuthorIsProtectedFeature, isProtected) - .add(IsSupportAccountReplyFeature, isAutoReply) - .build() - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/GraphTwoHopFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/GraphTwoHopFeatureHydrator.scala deleted file mode 100644 index 03d70445e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/GraphTwoHopFeatureHydrator.scala +++ /dev/null @@ -1,97 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.graph_feature_service.{thriftscala => gfs} -import com.twitter.home_mixer.model.HomeFeatures.FollowedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.param.HomeMixerInjectionNames.GraphTwoHopRepository -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.home_mixer.util.ObservedKeyValueResultHandler -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.servo.repository.KeyValueRepository -import com.twitter.stitch.Stitch -import com.twitter.timelines.prediction.adapters.two_hop_features.TwoHopFeaturesAdapter -import com.twitter.util.Try -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton -import scala.collection.JavaConverters._ - -object GraphTwoHopFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class GraphTwoHopFeatureHydrator @Inject() ( - @Named(GraphTwoHopRepository) client: KeyValueRepository[(Seq[Long], Long), Long, Seq[ - gfs.IntersectionValue - ]], - override val statsReceiver: StatsReceiver) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] - with ObservedKeyValueResultHandler { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("GraphTwoHop") - - override val features: Set[Feature[_, _]] = Set(GraphTwoHopFeature, FollowedByUserIdsFeature) - - override val statScope: String = identifier.toString - - private val twoHopFeaturesAdapter = new TwoHopFeaturesAdapter - - private val FollowFeatureType = gfs.FeatureType(gfs.EdgeType.Following, gfs.EdgeType.FollowedBy) - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { - // Apply filters to in network candidates for retweets only. - val (inNetworkCandidates, oonCandidates) = candidates.partition { candidate => - candidate.features.getOrElse(FromInNetworkSourceFeature, false) - } - - val inNetworkCandidatesToHydrate = - inNetworkCandidates.filter(_.features.getOrElse(IsRetweetFeature, false)) - - val candidatesToHydrate = (inNetworkCandidatesToHydrate ++ oonCandidates) - .flatMap(candidate => CandidatesUtil.getOriginalAuthorId(candidate.features)).distinct - - val response = client((candidatesToHydrate, query.getRequiredUserId)) - - response.map { result => - candidates.map { candidate => - val originalAuthorId = CandidatesUtil.getOriginalAuthorId(candidate.features) - - val value = observedGet(key = originalAuthorId, keyValueResult = result) - val transformedValue = postTransformer(value) - val followedByUserIds = value.toOption - .flatMap(getFollowedByUserIds(_)) - .getOrElse(Seq.empty) - - FeatureMapBuilder() - .add(GraphTwoHopFeature, transformedValue) - .add(FollowedByUserIdsFeature, followedByUserIds) - .build() - } - } - } - - private def getFollowedByUserIds(input: Option[Seq[gfs.IntersectionValue]]): Option[Seq[Long]] = - input.map(_.filter(_.featureType == FollowFeatureType).flatMap(_.intersectionIds).flatten) - - private def postTransformer(input: Try[Option[Seq[gfs.IntersectionValue]]]): Try[DataRecord] = - input.map(twoHopFeaturesAdapter.adaptToDataRecords(_).asScala.head) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/IsExtendedReplyFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/IsExtendedReplyFeatureHydrator.scala deleted file mode 100644 index fb114e55c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/IsExtendedReplyFeatureHydrator.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsExtendedReplyFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch - -object IsExtendedReplyFeatureHydrator - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("IsExtendedReply") - - override def features: Set[Feature[_, _]] = Set(IsExtendedReplyFeature) - - private val TrueFeatureMap = FeatureMapBuilder().add(IsExtendedReplyFeature, true).build() - private val FalseFeatureMap = FeatureMapBuilder().add(IsExtendedReplyFeature, false).build() - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { - val followedUsers = - query.features.map(_.get(SGSFollowedUsersFeature)).getOrElse(Seq.empty).toSet - - candidates.map { candidate => - val features = candidate.features - val isExtendedReply = features.getOrElse(InReplyToTweetIdFeature, None).nonEmpty && - !features.getOrElse(IsRetweetFeature, false) && - features.getOrElse(InReplyToUserIdFeature, None).exists(!followedUsers.contains(_)) - - if (isExtendedReply) TrueFeatureMap else FalseFeatureMap - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ListIdsQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ListIdsQueryFeatureHydrator.scala deleted file mode 100644 index db60d8866..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ListIdsQueryFeatureHydrator.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.socialgraph.{thriftscala => sg} -import com.twitter.stitch.Stitch -import com.twitter.stitch.socialgraph.SocialGraph -import javax.inject.Inject -import javax.inject.Singleton - -case object ListIdsFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] { - override val defaultValue: Seq[Long] = Seq.empty -} - -@Singleton -class ListIdsQueryFeatureHydrator @Inject() (socialGraph: SocialGraph) - extends QueryFeatureHydrator[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ListIds") - - override val features: Set[Feature[_, _]] = Set(ListIdsFeature) - - private val MaxListsToFetch = 20 - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - val userId = query.getRequiredUserId - - val ownedSubscribedRequest = sg.IdsRequest( - relationships = Seq( - sg.SrcRelationship(userId, sg.RelationshipType.ListIsSubscriber, hasRelationship = true), - sg.SrcRelationship(userId, sg.RelationshipType.ListOwning, hasRelationship = true) - ), - pageRequest = Some(sg.PageRequest(selectAll = Some(false), count = Some(MaxListsToFetch))), - context = Some( - sg.LookupContext( - includeInactive = false, - performUnion = Some(true), - includeAll = Some(false) - ) - ) - ) - - socialGraph.ids(ownedSubscribedRequest).map { response => - FeatureMapBuilder().add(ListIdsFeature, response.ids).build() - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/MetricCenterUserCountingFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/MetricCenterUserCountingFeatureHydrator.scala deleted file mode 100644 index bc51aabb9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/MetricCenterUserCountingFeatureHydrator.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.param.HomeMixerInjectionNames.MetricCenterUserCountingFeatureRepository -import com.twitter.home_mixer.util.ObservedKeyValueResultHandler -import com.twitter.onboarding.relevance.features.{thriftjava => rf} -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.servo.keyvalue.KeyValueResult -import com.twitter.servo.repository.KeyValueRepository -import com.twitter.stitch.Stitch -import com.twitter.util.Future -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -object MetricCenterUserCountingFeature - extends Feature[TweetCandidate, Option[rf.MCUserCountingFeatures]] - -@Singleton -class MetricCenterUserCountingFeatureHydrator @Inject() ( - @Named(MetricCenterUserCountingFeatureRepository) client: KeyValueRepository[Seq[ - Long - ], Long, rf.MCUserCountingFeatures], - override val statsReceiver: StatsReceiver) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] - with ObservedKeyValueResultHandler { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("MetricCenterUserCounting") - - override val features: Set[Feature[_, _]] = Set(MetricCenterUserCountingFeature) - - override val statScope: String = identifier.toString - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { - val possiblyAuthorIds = extractKeys(candidates) - val userIds = possiblyAuthorIds.flatten - - val response: Future[KeyValueResult[Long, rf.MCUserCountingFeatures]] = - if (userIds.isEmpty) Future.value(KeyValueResult.empty) else client(userIds) - - response.map { result => - possiblyAuthorIds.map { possiblyAuthorId => - val value = observedGet(key = possiblyAuthorId, keyValueResult = result) - FeatureMapBuilder().add(MetricCenterUserCountingFeature, value).build() - } - } - } - - private def extractKeys( - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Seq[Option[Long]] = { - candidates.map { candidate => - candidate.features - .getTry(AuthorIdFeature) - .toOption - .flatten - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphQueryFeatureHydrator.scala deleted file mode 100644 index a48d3742e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphQueryFeatureHydrator.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealGraphFeatureRepository -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.repository.Repository -import com.twitter.timelines.real_graph.{thriftscala => rg} -import com.twitter.stitch.Stitch -import com.twitter.timelines.model.UserId -import com.twitter.timelines.real_graph.v1.thriftscala.RealGraphEdgeFeatures -import com.twitter.user_session_store.{thriftscala => uss} - -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -object RealGraphFeatures extends Feature[PipelineQuery, Option[Map[UserId, RealGraphEdgeFeatures]]] - -@Singleton -class RealGraphQueryFeatureHydrator @Inject() ( - @Named(RealGraphFeatureRepository) repository: Repository[Long, Option[uss.UserSession]]) - extends QueryFeatureHydrator[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("RealGraphFeatures") - - override val features: Set[Feature[_, _]] = Set(RealGraphFeatures) - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - Stitch.callFuture { - repository(query.getRequiredUserId).map { userSession => - val realGraphFeaturesMap = userSession.flatMap { userSession => - userSession.realGraphFeatures.collect { - case rg.RealGraphFeatures.V1(realGraphFeatures) => - val edgeFeatures = realGraphFeatures.edgeFeatures ++ realGraphFeatures.oonEdgeFeatures - edgeFeatures.map { edge => edge.destId -> edge }.toMap - } - } - - FeatureMapBuilder().add(RealGraphFeatures, realGraphFeaturesMap).build() - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphViewerAuthorFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphViewerAuthorFeatureHydrator.scala deleted file mode 100644 index 6048213aa..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphViewerAuthorFeatureHydrator.scala +++ /dev/null @@ -1,123 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RealGraphViewerAuthorFeatureHydrator.getCombinedRealGraphFeatures -import com.twitter.home_mixer.util.MissingKeyException -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch -import com.twitter.timelines.prediction.adapters.real_graph.RealGraphEdgeFeaturesCombineAdapter -import com.twitter.timelines.prediction.adapters.real_graph.RealGraphFeaturesAdapter -import com.twitter.timelines.real_graph.v1.{thriftscala => v1} -import com.twitter.timelines.real_graph.{thriftscala => rg} -import com.twitter.util.Throw -import javax.inject.Inject -import javax.inject.Singleton -import scala.collection.JavaConverters._ - -object RealGraphViewerAuthorDataRecordFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -object RealGraphViewerAuthorsDataRecordFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class RealGraphViewerAuthorFeatureHydrator @Inject() () - extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("RealGraphViewerAuthor") - - override val features: Set[Feature[_, _]] = - Set(RealGraphViewerAuthorDataRecordFeature, RealGraphViewerAuthorsDataRecordFeature) - - private val realGraphEdgeFeaturesAdapter = new RealGraphFeaturesAdapter - private val realGraphEdgeFeaturesCombineAdapter = - new RealGraphEdgeFeaturesCombineAdapter(prefix = "authors.realgraph") - - private val MissingKeyFeatureMap = FeatureMapBuilder() - .add(RealGraphViewerAuthorDataRecordFeature, Throw(MissingKeyException)) - .add(RealGraphViewerAuthorsDataRecordFeature, Throw(MissingKeyException)) - .build() - - override def apply( - query: PipelineQuery, - candidate: TweetCandidate, - existingFeatures: FeatureMap - ): Stitch[FeatureMap] = OffloadFuturePools.offload { - val viewerId = query.getRequiredUserId - val realGraphFeatures = query.features - .flatMap(_.getOrElse(RealGraphFeatures, None)) - .getOrElse(Map.empty[Long, v1.RealGraphEdgeFeatures]) - - existingFeatures.getOrElse(AuthorIdFeature, None) match { - case Some(authorId) => - val realGraphAuthorFeatures = - getRealGraphViewerAuthorFeatures(viewerId, authorId, realGraphFeatures) - val realGraphAuthorDataRecord = realGraphEdgeFeaturesAdapter - .adaptToDataRecords(realGraphAuthorFeatures).asScala.headOption.getOrElse(new DataRecord) - - val combinedRealGraphFeaturesDataRecord = for { - inReplyToAuthorId <- existingFeatures.getOrElse(InReplyToUserIdFeature, None) - } yield { - val combinedRealGraphFeatures = - getCombinedRealGraphFeatures(Seq(authorId, inReplyToAuthorId), realGraphFeatures) - realGraphEdgeFeaturesCombineAdapter - .adaptToDataRecords(Some(combinedRealGraphFeatures)).asScala.headOption - .getOrElse(new DataRecord) - } - - FeatureMapBuilder() - .add(RealGraphViewerAuthorDataRecordFeature, realGraphAuthorDataRecord) - .add( - RealGraphViewerAuthorsDataRecordFeature, - combinedRealGraphFeaturesDataRecord.getOrElse(new DataRecord)) - .build() - case _ => MissingKeyFeatureMap - } - } - - private def getRealGraphViewerAuthorFeatures( - viewerId: Long, - authorId: Long, - realGraphEdgeFeaturesMap: Map[Long, v1.RealGraphEdgeFeatures] - ): rg.UserRealGraphFeatures = { - realGraphEdgeFeaturesMap.get(authorId) match { - case Some(realGraphEdgeFeatures) => - rg.UserRealGraphFeatures( - srcId = viewerId, - features = rg.RealGraphFeatures.V1( - v1.RealGraphFeatures(edgeFeatures = Seq(realGraphEdgeFeatures)))) - case _ => - rg.UserRealGraphFeatures( - srcId = viewerId, - features = rg.RealGraphFeatures.V1(v1.RealGraphFeatures(edgeFeatures = Seq.empty))) - } - } -} - -object RealGraphViewerAuthorFeatureHydrator { - def getCombinedRealGraphFeatures( - userIds: Seq[Long], - realGraphEdgeFeaturesMap: Map[Long, v1.RealGraphEdgeFeatures] - ): rg.RealGraphFeatures = { - val edgeFeatures = userIds.flatMap(realGraphEdgeFeaturesMap.get) - rg.RealGraphFeatures.V1(v1.RealGraphFeatures(edgeFeatures = edgeFeatures)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphViewerRelatedUsersFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphViewerRelatedUsersFeatureHydrator.scala deleted file mode 100644 index 47e2431cf..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphViewerRelatedUsersFeatureHydrator.scala +++ /dev/null @@ -1,73 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.MentionUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch -import com.twitter.timelines.prediction.adapters.real_graph.RealGraphEdgeFeaturesCombineAdapter -import com.twitter.timelines.real_graph.v1.{thriftscala => v1} -import javax.inject.Inject -import javax.inject.Singleton -import scala.collection.JavaConverters._ - -object RealGraphViewerRelatedUsersDataRecordFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class RealGraphViewerRelatedUsersFeatureHydrator @Inject() () - extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("RealGraphViewerRelatedUsers") - - override val features: Set[Feature[_, _]] = Set(RealGraphViewerRelatedUsersDataRecordFeature) - - private val RealGraphEdgeFeaturesCombineAdapter = new RealGraphEdgeFeaturesCombineAdapter - - override def apply( - query: PipelineQuery, - candidate: TweetCandidate, - existingFeatures: FeatureMap - ): Stitch[FeatureMap] = OffloadFuturePools.offload { - val realGraphQueryFeatures = query.features - .flatMap(_.getOrElse(RealGraphFeatures, None)) - .getOrElse(Map.empty[Long, v1.RealGraphEdgeFeatures]) - - val allRelatedUserIds = getRelatedUserIds(existingFeatures) - val realGraphFeatures = RealGraphViewerAuthorFeatureHydrator.getCombinedRealGraphFeatures( - allRelatedUserIds, - realGraphQueryFeatures - ) - val realGraphFeaturesDataRecord = RealGraphEdgeFeaturesCombineAdapter - .adaptToDataRecords(Some(realGraphFeatures)).asScala.headOption - .getOrElse(new DataRecord) - - FeatureMapBuilder() - .add(RealGraphViewerRelatedUsersDataRecordFeature, realGraphFeaturesDataRecord) - .build() - } - - private def getRelatedUserIds(features: FeatureMap): Seq[Long] = { - (CandidatesUtil.getEngagerUserIds(features) ++ - features.getOrElse(AuthorIdFeature, None) ++ - features.getOrElse(MentionUserIdFeature, Seq.empty) ++ - features.getOrElse(SourceUserIdFeature, None) ++ - features.getOrElse(DirectedAtUserIdFeature, None)).distinct - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealTimeInteractionGraphEdgeFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealTimeInteractionGraphEdgeFeatureHydrator.scala deleted file mode 100644 index f690ff0e0..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealTimeInteractionGraphEdgeFeatureHydrator.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch -import com.twitter.timelines.prediction.adapters.realtime_interaction_graph.RealTimeInteractionGraphFeaturesAdapter -import com.twitter.timelines.prediction.features.realtime_interaction_graph.RealTimeInteractionGraphEdgeFeatures -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Singleton -import scala.collection.JavaConverters._ - -object RealTimeInteractionGraphEdgeFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class RealTimeInteractionGraphEdgeFeatureHydrator @Inject() () - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("RealTimeInteractionGraphEdge") - - override val features: Set[Feature[_, _]] = Set(RealTimeInteractionGraphEdgeFeature) - - private val realTimeInteractionGraphFeaturesAdapter = new RealTimeInteractionGraphFeaturesAdapter - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { - val userVertex = - query.features.flatMap(_.getOrElse(RealTimeInteractionGraphUserVertexQueryFeature, None)) - val realTimeInteractionGraphFeaturesMap = - userVertex.map(RealTimeInteractionGraphEdgeFeatures(_, Time.now)) - - candidates.map { candidate => - val feature = candidate.features.getOrElse(AuthorIdFeature, None).flatMap { authorId => - realTimeInteractionGraphFeaturesMap.flatMap(_.get(authorId)) - } - - val dataRecordFeature = - realTimeInteractionGraphFeaturesAdapter.adaptToDataRecords(feature).asScala.head - - FeatureMapBuilder().add(RealTimeInteractionGraphEdgeFeature, dataRecordFeature).build() - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealTimeInteractionGraphUserVertexQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealTimeInteractionGraphUserVertexQueryFeatureHydrator.scala deleted file mode 100644 index 60d82cc28..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealTimeInteractionGraphUserVertexQueryFeatureHydrator.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.google.inject.name.Named -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealTimeInteractionGraphUserVertexCache -import com.twitter.home_mixer.util.ObservedKeyValueResultHandler -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.cache.ReadCache -import com.twitter.stitch.Stitch -import com.twitter.wtf.real_time_interaction_graph.{thriftscala => ig} -import javax.inject.Inject -import javax.inject.Singleton - -object RealTimeInteractionGraphUserVertexQueryFeature - extends Feature[PipelineQuery, Option[ig.UserVertex]] - -@Singleton -class RealTimeInteractionGraphUserVertexQueryFeatureHydrator @Inject() ( - @Named(RealTimeInteractionGraphUserVertexCache) client: ReadCache[Long, ig.UserVertex], - override val statsReceiver: StatsReceiver) - extends QueryFeatureHydrator[PipelineQuery] - with ObservedKeyValueResultHandler { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("RealTimeInteractionGraphUserVertex") - - override val features: Set[Feature[_, _]] = Set(RealTimeInteractionGraphUserVertexQueryFeature) - - override val statScope: String = identifier.toString - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - val userId = query.getRequiredUserId - - Stitch.callFuture( - client.get(Seq(userId)).map { results => - val feature = observedGet(key = Some(userId), keyValueResult = results) - FeatureMapBuilder() - .add(RealTimeInteractionGraphUserVertexQueryFeature, feature) - .build() - } - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ReplyFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ReplyFeatureHydrator.scala deleted file mode 100644 index 80d857a26..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ReplyFeatureHydrator.scala +++ /dev/null @@ -1,242 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.model.ContentFeatures -import com.twitter.home_mixer.model.HomeFeatures._ -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.content.InReplyToContentFeatureAdapter -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.earlybird.InReplyToEarlybirdAdapter -import com.twitter.home_mixer.util.ReplyRetweetUtil -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.search.common.features.thriftscala.ThriftTweetFeatures -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.stitch.Stitch -import com.twitter.timelines.conversation_features.v1.thriftscala.ConversationFeatures -import com.twitter.timelines.conversation_features.{thriftscala => cf} -import com.twitter.timelines.prediction.adapters.conversation_features.ConversationFeaturesAdapter -import com.twitter.util.Duration -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Singleton -import scala.collection.JavaConverters._ - -object InReplyToTweetHydratedEarlybirdFeature - extends Feature[TweetCandidate, Option[ThriftTweetFeatures]] - -object ConversationDataRecordFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -object InReplyToEarlybirdDataRecordFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -object InReplyToTweetypieContentDataRecordFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -/** - * The purpose of this hydrator is to - * 1) hydrate simple features into replies and their ancestor tweets - * 2) keep both the normal replies and ancestor source candidates, but hydrate into the candidates - * features useful for predicting the quality of the replies and source ancestor tweets. - */ -@Singleton -class ReplyFeatureHydrator @Inject() (statsReceiver: StatsReceiver) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ReplyTweet") - - override val features: Set[Feature[_, _]] = Set( - ConversationDataRecordFeature, - InReplyToTweetHydratedEarlybirdFeature, - InReplyToEarlybirdDataRecordFeature, - InReplyToTweetypieContentDataRecordFeature - ) - - private val defaulDataRecord: DataRecord = new DataRecord() - - private val DefaultFeatureMap = FeatureMapBuilder() - .add(ConversationDataRecordFeature, defaulDataRecord) - .add(InReplyToTweetHydratedEarlybirdFeature, None) - .add(InReplyToEarlybirdDataRecordFeature, defaulDataRecord) - .add(InReplyToTweetypieContentDataRecordFeature, defaulDataRecord) - .build() - - private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) - private val hydratedReplyCounter = scopedStatsReceiver.counter("hydratedReply") - private val hydratedAncestorCounter = scopedStatsReceiver.counter("hydratedAncestor") - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { - val replyToInReplyToTweetMap = - ReplyRetweetUtil.replyTweetIdToInReplyToTweetMap(candidates) - val candidatesWithRepliesHydrated = candidates.map { candidate => - replyToInReplyToTweetMap - .get(candidate.candidate.id).map { inReplyToTweet => - hydratedReplyCounter.incr() - hydratedReplyCandidate(candidate, inReplyToTweet) - }.getOrElse((candidate, None, None)) - } - - /** - * Update ancestor tweets with descendant replies and hydrate simple features from one of - * the descendants. - */ - val ancestorTweetToDescendantRepliesMap = - ReplyRetweetUtil.ancestorTweetIdToDescendantRepliesMap(candidates) - val candidatesWithRepliesAndAncestorTweetsHydrated = candidatesWithRepliesHydrated.map { - case ( - maybeAncestorTweetCandidate, - updatedReplyConversationFeatures, - inReplyToTweetEarlyBirdFeature) => - ancestorTweetToDescendantRepliesMap - .get(maybeAncestorTweetCandidate.candidate.id) - .map { descendantReplies => - hydratedAncestorCounter.incr() - val (ancestorTweetCandidate, updatedConversationFeatures): ( - CandidateWithFeatures[TweetCandidate], - Option[ConversationFeatures] - ) = - hydrateAncestorTweetCandidate( - maybeAncestorTweetCandidate, - descendantReplies, - updatedReplyConversationFeatures) - (ancestorTweetCandidate, inReplyToTweetEarlyBirdFeature, updatedConversationFeatures) - } - .getOrElse( - ( - maybeAncestorTweetCandidate, - inReplyToTweetEarlyBirdFeature, - updatedReplyConversationFeatures)) - } - - candidatesWithRepliesAndAncestorTweetsHydrated.map { - case (candidate, inReplyToTweetEarlyBirdFeature, updatedConversationFeatures) => - val conversationDataRecordFeature = updatedConversationFeatures - .map(f => ConversationFeaturesAdapter.adaptToDataRecord(cf.ConversationFeatures.V1(f))) - .getOrElse(defaulDataRecord) - - val inReplyToEarlybirdDataRecord = - InReplyToEarlybirdAdapter - .adaptToDataRecords(inReplyToTweetEarlyBirdFeature).asScala.head - val inReplyToContentDataRecord = InReplyToContentFeatureAdapter - .adaptToDataRecords( - inReplyToTweetEarlyBirdFeature.map(ContentFeatures.fromThrift)).asScala.head - - FeatureMapBuilder() - .add(ConversationDataRecordFeature, conversationDataRecordFeature) - .add(InReplyToTweetHydratedEarlybirdFeature, inReplyToTweetEarlyBirdFeature) - .add(InReplyToEarlybirdDataRecordFeature, inReplyToEarlybirdDataRecord) - .add(InReplyToTweetypieContentDataRecordFeature, inReplyToContentDataRecord) - .build() - case _ => DefaultFeatureMap - } - } - - private def hydratedReplyCandidate( - replyCandidate: CandidateWithFeatures[TweetCandidate], - inReplyToTweetCandidate: CandidateWithFeatures[TweetCandidate] - ): ( - CandidateWithFeatures[TweetCandidate], - Option[ConversationFeatures], - Option[ThriftTweetFeatures] - ) = { - val tweetedAfterInReplyToTweetInSecs = - ( - originalTweetAgeFromSnowflake(inReplyToTweetCandidate), - originalTweetAgeFromSnowflake(replyCandidate)) match { - case (Some(inReplyToTweetAge), Some(replyTweetAge)) => - Some((inReplyToTweetAge - replyTweetAge).inSeconds.toLong) - case _ => None - } - - val existingConversationFeatures = Some( - replyCandidate.features - .getOrElse(ConversationFeature, None).getOrElse(ConversationFeatures())) - - val updatedConversationFeatures = existingConversationFeatures match { - case Some(v1) => - Some( - v1.copy( - tweetedAfterInReplyToTweetInSecs = tweetedAfterInReplyToTweetInSecs, - isSelfReply = Some( - replyCandidate.features.getOrElse( - AuthorIdFeature, - None) == inReplyToTweetCandidate.features.getOrElse(AuthorIdFeature, None)) - ) - ) - case _ => None - } - - // Note: if inReplyToTweet is a retweet, we need to read early bird feature from the merged - // early bird feature field from RetweetSourceTweetFeatureHydrator class. - // But if inReplyToTweet is a reply, we return its early bird feature directly - val inReplyToTweetThriftTweetFeaturesOpt = { - if (inReplyToTweetCandidate.features.getOrElse(IsRetweetFeature, false)) { - inReplyToTweetCandidate.features.getOrElse(SourceTweetEarlybirdFeature, None) - } else { - inReplyToTweetCandidate.features.getOrElse(EarlybirdFeature, None) - } - } - - (replyCandidate, updatedConversationFeatures, inReplyToTweetThriftTweetFeaturesOpt) - } - - private def hydrateAncestorTweetCandidate( - ancestorTweetCandidate: CandidateWithFeatures[TweetCandidate], - descendantReplies: Seq[CandidateWithFeatures[TweetCandidate]], - updatedReplyConversationFeatures: Option[ConversationFeatures] - ): (CandidateWithFeatures[TweetCandidate], Option[ConversationFeatures]) = { - // Ancestor could be a reply. For example, in thread: tweetA -> tweetB -> tweetC, - // tweetB is a reply and ancestor at the same time. Hence, tweetB's conversation feature - // will be updated by hydratedReplyCandidate and hydrateAncestorTweetCandidate functions. - val existingConversationFeatures = - if (updatedReplyConversationFeatures.nonEmpty) - updatedReplyConversationFeatures - else - Some( - ancestorTweetCandidate.features - .getOrElse(ConversationFeature, None).getOrElse(ConversationFeatures())) - - val updatedConversationFeatures = existingConversationFeatures match { - case Some(v1) => - Some( - v1.copy( - hasDescendantReplyCandidate = Some(true), - hasInNetworkDescendantReply = - Some(descendantReplies.exists(_.features.getOrElse(InNetworkFeature, false))) - )) - case _ => None - } - (ancestorTweetCandidate, updatedConversationFeatures) - } - - private def originalTweetAgeFromSnowflake( - candidate: CandidateWithFeatures[TweetCandidate] - ): Option[Duration] = { - SnowflakeId - .timeFromIdOpt( - candidate.features - .getOrElse(SourceTweetIdFeature, None).getOrElse(candidate.candidate.id)) - .map(Time.now - _) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RequestTimeQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RequestTimeQueryFeatureHydrator.scala deleted file mode 100644 index fd9656d8c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RequestTimeQueryFeatureHydrator.scala +++ /dev/null @@ -1,122 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.model.HomeFeatures.FollowingLastNonPollingTimeFeature -import com.twitter.home_mixer.model.HomeFeatures.LastNonPollingTimeFeature -import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.util.FDsl._ -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.stitch.Stitch -import com.twitter.timelines.prediction.features.time_features.AccountAgeInterval -import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.ACCOUNT_AGE_INTERVAL -import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.IS_12_MONTH_NEW_USER -import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.IS_30_DAY_NEW_USER -import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.TIME_BETWEEN_NON_POLLING_REQUESTS_AVG -import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.TIME_SINCE_LAST_NON_POLLING_REQUEST -import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.TIME_SINCE_VIEWER_ACCOUNT_CREATION_SECS -import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.USER_ID_IS_SNOWFLAKE_ID -import com.twitter.user_session_store.ReadRequest -import com.twitter.user_session_store.ReadWriteUserSessionStore -import com.twitter.user_session_store.UserSessionDataset -import com.twitter.user_session_store.UserSessionDataset.UserSessionDataset -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Singleton - -object RequestTimeDataRecordFeature - extends DataRecordInAFeature[PipelineQuery] - with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -case class RequestTimeQueryFeatureHydrator @Inject() ( - userSessionStore: ReadWriteUserSessionStore) - extends QueryFeatureHydrator[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("RequestTime") - - override val features: Set[Feature[_, _]] = Set( - FollowingLastNonPollingTimeFeature, - LastNonPollingTimeFeature, - NonPollingTimesFeature, - RequestTimeDataRecordFeature - ) - - private val datasets: Set[UserSessionDataset] = Set(UserSessionDataset.NonPollingTimes) - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - userSessionStore - .read(ReadRequest(query.getRequiredUserId, datasets)) - .map { userSession => - val nonPollingTimestamps = userSession.flatMap(_.nonPollingTimestamps) - - val lastNonPollingTime = nonPollingTimestamps - .flatMap(_.nonPollingTimestampsMs.headOption) - .map(Time.fromMilliseconds) - - val followingLastNonPollingTime = nonPollingTimestamps - .flatMap(_.mostRecentHomeLatestNonPollingTimestampMs) - .map(Time.fromMilliseconds) - - val nonPollingTimes = nonPollingTimestamps - .map(_.nonPollingTimestampsMs) - .getOrElse(Seq.empty) - - val requestTimeDataRecord = getRequestTimeDataRecord(query, nonPollingTimes) - - FeatureMapBuilder() - .add(FollowingLastNonPollingTimeFeature, followingLastNonPollingTime) - .add(LastNonPollingTimeFeature, lastNonPollingTime) - .add(NonPollingTimesFeature, nonPollingTimes) - .add(RequestTimeDataRecordFeature, requestTimeDataRecord) - .build() - } - } - - def getRequestTimeDataRecord(query: PipelineQuery, nonPollingTimes: Seq[Long]): DataRecord = { - val requestTimeMs = query.queryTime.inMillis - val accountAge = SnowflakeId.timeFromIdOpt(query.getRequiredUserId) - val timeSinceAccountCreation = accountAge.map(query.queryTime.since) - val timeSinceEarliestNonPollingRequest = - nonPollingTimes.lastOption.map(requestTimeMs - _) - val timeSinceLastNonPollingRequest = - nonPollingTimes.headOption.map(requestTimeMs - _) - - new DataRecord() - .setFeatureValue(USER_ID_IS_SNOWFLAKE_ID, accountAge.isDefined) - .setFeatureValue( - IS_30_DAY_NEW_USER, - timeSinceAccountCreation.map(_ < 30.days).getOrElse(false) - ) - .setFeatureValue( - IS_12_MONTH_NEW_USER, - timeSinceAccountCreation.map(_ < 365.days).getOrElse(false) - ) - .setFeatureValueFromOption( - ACCOUNT_AGE_INTERVAL, - timeSinceAccountCreation.flatMap(AccountAgeInterval.fromDuration).map(_.id.toLong) - ) - .setFeatureValueFromOption( - TIME_SINCE_VIEWER_ACCOUNT_CREATION_SECS, - timeSinceAccountCreation.map(_.inSeconds.toDouble) - ) - .setFeatureValueFromOption( - TIME_BETWEEN_NON_POLLING_REQUESTS_AVG, - timeSinceEarliestNonPollingRequest.map(_.toDouble / math.max(1.0, nonPollingTimes.size)) - ) - .setFeatureValueFromOption( - TIME_SINCE_LAST_NON_POLLING_REQUEST, - timeSinceLastNonPollingRequest.map(_.toDouble) - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RetweetSourceTweetFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RetweetSourceTweetFeatureHydrator.scala deleted file mode 100644 index 33cbef884..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RetweetSourceTweetFeatureHydrator.scala +++ /dev/null @@ -1,76 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures._ -import com.twitter.product_mixer.component_library.candidate_source.timeline_ranker.TimelineRankerInNetworkSourceTweetsByTweetIdMapFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.search.common.features.thriftscala.ThriftTweetFeatures -import com.twitter.stitch.Stitch -import com.twitter.timelineranker.thriftscala.CandidateTweet - -object SourceTweetEarlybirdFeature extends Feature[TweetCandidate, Option[ThriftTweetFeatures]] - -/** - * Feature Hydrator that bulk hydrates source tweets' features to retweet candidates - */ -object RetweetSourceTweetFeatureHydrator - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("RetweetSourceTweet") - - override val features: Set[Feature[_, _]] = Set( - SourceTweetEarlybirdFeature, - ) - - private val DefaultFeatureMap = FeatureMapBuilder() - .add(SourceTweetEarlybirdFeature, None) - .build() - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - val sourceTweetsByTweetId: Option[Map[Long, CandidateTweet]] = { - query.features.map( - _.getOrElse( - TimelineRankerInNetworkSourceTweetsByTweetIdMapFeature, - Map.empty[Long, CandidateTweet])) - } - - /** - * Return DefaultFeatureMap (no-op to candidate) when it is unfeasible to hydrate the - * source tweet's feature to the current candidate: early bird does not return source - * tweets info / candidate is not a retweet / sourceTweetId is not found - */ - Stitch.value { - if (sourceTweetsByTweetId.exists(_.nonEmpty)) { - candidates.map { candidate => - val candidateIsRetweet = candidate.features.getOrElse(IsRetweetFeature, false) - val sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None) - if (!candidateIsRetweet || sourceTweetId.isEmpty) { - DefaultFeatureMap - } else { - val sourceTweet = sourceTweetsByTweetId.flatMap(_.get(sourceTweetId.get)) - if (sourceTweet.nonEmpty) { - val source = sourceTweet.get - FeatureMapBuilder() - .add(SourceTweetEarlybirdFeature, source.features) - .build() - } else { - DefaultFeatureMap - } - } - } - } else { - candidates.map(_ => DefaultFeatureMap) - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SimClustersEngagementSimilarityFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SimClustersEngagementSimilarityFeatureHydrator.scala deleted file mode 100644 index f66966f00..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SimClustersEngagementSimilarityFeatureHydrator.scala +++ /dev/null @@ -1,75 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.Conditionally -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch -import com.twitter.timelines.clients.strato.twistly.SimClustersRecentEngagementSimilarityClient -import com.twitter.timelines.configapi.decider.BooleanDeciderParam -import com.twitter.timelines.prediction.adapters.twistly.SimClustersRecentEngagementSimilarityFeaturesAdapter -import javax.inject.Inject -import javax.inject.Singleton - -object SimClustersEngagementSimilarityFeature - extends DataRecordInAFeature[PipelineQuery] - with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class SimClustersEngagementSimilarityFeatureHydrator @Inject() ( - simClustersEngagementSimilarityClient: SimClustersRecentEngagementSimilarityClient) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] - with Conditionally[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("SimClustersEngagementSimilarity") - - override val features: Set[Feature[_, _]] = Set(SimClustersEngagementSimilarityFeature) - - private val simClustersRecentEngagementSimilarityFeaturesAdapter = - new SimClustersRecentEngagementSimilarityFeaturesAdapter - - override def onlyIf(query: PipelineQuery): Boolean = { - val param: BooleanDeciderParam = - ScoredTweetsParam.EnableSimClustersSimilarityFeatureHydrationDeciderParam - query.params.apply(param) - } - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { - val tweetToCandidates = candidates.map(candidate => candidate.candidate.id -> candidate).toMap - val tweetIds = tweetToCandidates.keySet.toSeq - val userId = query.getRequiredUserId - val userTweetEdges = tweetIds.map(tweetId => (userId, tweetId)) - simClustersEngagementSimilarityClient - .getSimClustersRecentEngagementSimilarityScores(userTweetEdges).map { - simClustersRecentEngagementSimilarityScoresMap => - candidates.map { candidate => - val similarityFeatureOpt = simClustersRecentEngagementSimilarityScoresMap - .get(userId -> candidate.candidate.id).flatten - val dataRecordOpt = similarityFeatureOpt.map { similarityFeature => - simClustersRecentEngagementSimilarityFeaturesAdapter - .adaptToDataRecords(similarityFeature) - .get(0) - } - FeatureMapBuilder() - .add(SimClustersEngagementSimilarityFeature, dataRecordOpt.getOrElse(new DataRecord)) - .build() - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SimClustersUserTweetScoresHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SimClustersUserTweetScoresHydrator.scala deleted file mode 100644 index c938f34c5..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SimClustersUserTweetScoresHydrator.scala +++ /dev/null @@ -1,86 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.dal.personal_data.{thriftjava => pd} -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.datarecord.DataRecordOptionalFeature -import com.twitter.product_mixer.core.feature.datarecord.DoubleDataRecordCompatible -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch -import com.twitter.strato.catalog.Fetch -import com.twitter.strato.generated.client.ml.featureStore.SimClustersUserInterestedInTweetEmbeddingDotProduct20M145K2020OnUserTweetEdgeClientColumn -import javax.inject.Inject -import javax.inject.Singleton - -object SimClustersUserInterestedInTweetEmbeddingDataRecordFeature - extends DataRecordOptionalFeature[TweetCandidate, Double] - with DoubleDataRecordCompatible { - override val featureName: String = - "user-tweet.recommendations.sim_clusters_scores.user_interested_in_tweet_embedding_dot_product_20m_145k_2020" - override val personalDataTypes: Set[pd.PersonalDataType] = - Set(pd.PersonalDataType.InferredInterests) -} - -@Singleton -class SimClustersUserTweetScoresHydrator @Inject() ( - simClustersColumn: SimClustersUserInterestedInTweetEmbeddingDotProduct20M145K2020OnUserTweetEdgeClientColumn, - statsReceiver: StatsReceiver) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("SimClustersUserTweetScores") - - override val features: Set[Feature[_, _]] = Set( - SimClustersUserInterestedInTweetEmbeddingDataRecordFeature) - - private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) - private val keyFoundCounter = scopedStatsReceiver.counter("key/found") - private val keyLossCounter = scopedStatsReceiver.counter("key/loss") - private val keyFailureCounter = scopedStatsReceiver.counter("key/failure") - private val keySkipCounter = scopedStatsReceiver.counter("key/skip") - - private val DefaultFeatureMap = FeatureMapBuilder() - .add(SimClustersUserInterestedInTweetEmbeddingDataRecordFeature, None) - .build() - private val MinFavToHydrate = 9 - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { - Stitch.run { - Stitch.collect { - candidates.map { candidate => - val ebFeatures = candidate.features.getOrElse(EarlybirdFeature, None) - val favCount = ebFeatures.flatMap(_.favCountV2).getOrElse(0) - - if (ebFeatures.isEmpty || favCount >= MinFavToHydrate) { - simClustersColumn.fetcher - .fetch((query.getRequiredUserId, candidate.candidate.id), Unit) - .map { - case Fetch.Result(response, _) => - if (response.nonEmpty) keyFoundCounter.incr() else keyLossCounter.incr() - FeatureMapBuilder() - .add(SimClustersUserInterestedInTweetEmbeddingDataRecordFeature, response) - .build() - case _ => - keyFailureCounter.incr() - DefaultFeatureMap - } - } else { - keySkipCounter.incr() - Stitch.value(DefaultFeatureMap) - } - } - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TSPInferredTopicFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TSPInferredTopicFeatureHydrator.scala deleted file mode 100644 index f6047730f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TSPInferredTopicFeatureHydrator.scala +++ /dev/null @@ -1,148 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.contentrecommender.{thriftscala => cr} -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature -import com.twitter.home_mixer.model.HomeFeatures.TSPMetricTagFeature -import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature -import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.inferred_topic.InferredTopicAdapter -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContextFunctionalityType -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch -import com.twitter.timelines.clients.strato.topics.TopicSocialProofClient -import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => sid} -import com.twitter.topiclisting.TopicListingViewerContext -import com.twitter.tsp.{thriftscala => tsp} -import javax.inject.Inject -import javax.inject.Singleton -import scala.collection.JavaConverters._ - -object TSPInferredTopicFeature extends Feature[TweetCandidate, Map[Long, Double]] -object TSPInferredTopicDataRecordFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class TSPInferredTopicFeatureHydrator @Inject() ( - topicSocialProofClient: TopicSocialProofClient) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TSPInferredTopic") - - override val features: Set[Feature[_, _]] = Set( - TSPInferredTopicFeature, - TSPInferredTopicDataRecordFeature, - TopicIdSocialContextFeature, - TopicContextFunctionalityTypeFeature - ) - - private val topK = 3 - - private val SourcesToSetSocialProof: Set[sid.CandidateTweetSourceId] = - Set(sid.CandidateTweetSourceId.Simcluster) - - private val DefaultFeatureMap = FeatureMapBuilder() - .add(TSPInferredTopicFeature, Map.empty[Long, Double]) - .add(TSPInferredTopicDataRecordFeature, new DataRecord()) - .add(TopicIdSocialContextFeature, None) - .add(TopicContextFunctionalityTypeFeature, None) - .build() - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { - val tags = candidates.collect { - case candidate if candidate.features.getTry(TSPMetricTagFeature).isReturn => - candidate.candidate.id -> candidate.features - .getOrElse(TSPMetricTagFeature, Set.empty[tsp.MetricTag]) - }.toMap - - val topicSocialProofRequest = tsp.TopicSocialProofRequest( - userId = query.getRequiredUserId, - tweetIds = candidates.map(_.candidate.id).toSet, - displayLocation = cr.DisplayLocation.HomeTimeline, - topicListingSetting = tsp.TopicListingSetting.Followable, - context = TopicListingViewerContext.fromClientContext(query.clientContext).toThrift, - bypassModes = None, - // Only TweetMixer source has this data. Convert the TweetMixer metric tag to tsp metric tag. - tags = if (tags.isEmpty) None else Some(tags) - ) - - topicSocialProofClient - .getTopicTweetSocialProofResponse(topicSocialProofRequest) - .map { - case Some(response) => - handleResponse(response, candidates) - case _ => candidates.map { _ => DefaultFeatureMap } - } - } - - private def handleResponse( - response: tsp.TopicSocialProofResponse, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Seq[FeatureMap] = { - candidates.map { candidate => - val topicWithScores = response.socialProofs.getOrElse(candidate.candidate.id, Seq.empty) - if (topicWithScores.nonEmpty) { - val (socialProofId, socialProofFunctionalityType) = - if (candidate.features - .getOrElse(CandidateSourceIdFeature, None) - .exists(SourcesToSetSocialProof.contains)) { - getSocialProof(topicWithScores) - } else (None, None) - - val inferredTopicFeatures = - topicWithScores.sortBy(-_.score).take(topK).map(a => (a.topicId, a.score)).toMap - - val inferredTopicDataRecord = - InferredTopicAdapter.adaptToDataRecords(inferredTopicFeatures).asScala.head - - FeatureMapBuilder() - .add(TSPInferredTopicFeature, inferredTopicFeatures) - .add(TSPInferredTopicDataRecordFeature, inferredTopicDataRecord) - .add(TopicIdSocialContextFeature, socialProofId) - .add(TopicContextFunctionalityTypeFeature, socialProofFunctionalityType) - .build() - } else DefaultFeatureMap - } - } - - private def getSocialProof( - topicWithScores: Seq[tsp.TopicWithScore] - ): (Option[Long], Option[TopicContextFunctionalityType]) = { - val followingTopicId = topicWithScores.collectFirst { - case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.Following)) => topicId - } - - if (followingTopicId.nonEmpty) { - return (followingTopicId, Some(BasicTopicContextFunctionalityType)) - } - - val implicitFollowingId = topicWithScores.collectFirst { - case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.ImplicitFollow)) => - topicId - } - - if (implicitFollowingId.nonEmpty) { - return (implicitFollowingId, Some(RecommendationTopicContextFunctionalityType)) - } - - (None, None) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetMetaDataFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetMetaDataFeatureHydrator.scala deleted file mode 100644 index 1c237bf18..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetMetaDataFeatureHydrator.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.RichDataRecord -import com.twitter.ml.api.constant.SharedFeatures -import com.twitter.ml.api.util.DataRecordConverters._ -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch -import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures -import java.lang.{Long => JLong} - -object TweetMetaDataDataRecord - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -object TweetMetaDataFeatureHydrator - extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetMetaData") - - override def features: Set[Feature[_, _]] = Set(TweetMetaDataDataRecord) - - override def apply( - query: PipelineQuery, - candidate: TweetCandidate, - existingFeatures: FeatureMap - ): Stitch[FeatureMap] = OffloadFuturePools.offload { - val richDataRecord = new RichDataRecord() - setFeatures(richDataRecord, candidate, existingFeatures) - FeatureMapBuilder().add(TweetMetaDataDataRecord, richDataRecord.getRecord).build() - } - - private def setFeatures( - richDataRecord: RichDataRecord, - candidate: TweetCandidate, - existingFeatures: FeatureMap - ): Unit = { - richDataRecord.setFeatureValue[JLong](SharedFeatures.TWEET_ID, candidate.id) - - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.ORIGINAL_AUTHOR_ID, - CandidatesUtil.getOriginalAuthorId(existingFeatures)) - - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.CANDIDATE_TWEET_SOURCE_ID, - existingFeatures.getOrElse(CandidateSourceIdFeature, None).map(_.value.toLong)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetTimeFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetTimeFeatureHydrator.scala deleted file mode 100644 index 947016826..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetTimeFeatureHydrator.scala +++ /dev/null @@ -1,112 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature -import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.util.FDsl._ -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.stitch.Stitch -import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures._ -import com.twitter.util.Duration -import scala.collection.Searching._ - -object TweetTimeDataRecordFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -object TweetTimeFeatureHydrator extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetTime") - - override val features: Set[Feature[_, _]] = Set(TweetTimeDataRecordFeature) - - override def apply( - query: PipelineQuery, - candidate: TweetCandidate, - existingFeatures: FeatureMap - ): Stitch[FeatureMap] = { - val tweetFeatures = existingFeatures.getOrElse(EarlybirdFeature, None) - val timeSinceTweetCreation = SnowflakeId.timeFromIdOpt(candidate.id).map(query.queryTime.since) - val timeSinceTweetCreationMs = timeSinceTweetCreation.map(_.inMillis) - - val timeSinceSourceTweetCreationOpt = existingFeatures - .getOrElse(SourceTweetIdFeature, None) - .flatMap { sourceTweetId => - SnowflakeId.timeFromIdOpt(sourceTweetId).map(query.queryTime.since) - }.orElse(timeSinceTweetCreation) - - val lastFavSinceCreationHrs = - tweetFeatures.flatMap(_.lastFavSinceCreationHrs).map(_.toDouble) - val lastRetweetSinceCreationHrs = - tweetFeatures.flatMap(_.lastRetweetSinceCreationHrs).map(_.toDouble) - val lastReplySinceCreationHrs = - tweetFeatures.flatMap(_.lastReplySinceCreationHrs).map(_.toDouble) - val lastQuoteSinceCreationHrs = - tweetFeatures.flatMap(_.lastQuoteSinceCreationHrs).map(_.toDouble) - val timeSinceLastFavoriteHrs = - getTimeSinceLastEngagementHrs(lastFavSinceCreationHrs, timeSinceSourceTweetCreationOpt) - val timeSinceLastRetweetHrs = - getTimeSinceLastEngagementHrs(lastRetweetSinceCreationHrs, timeSinceSourceTweetCreationOpt) - val timeSinceLastReplyHrs = - getTimeSinceLastEngagementHrs(lastReplySinceCreationHrs, timeSinceSourceTweetCreationOpt) - val timeSinceLastQuoteHrs = - getTimeSinceLastEngagementHrs(lastQuoteSinceCreationHrs, timeSinceSourceTweetCreationOpt) - - val nonPollingTimestampsMs = query.features.get.getOrElse(NonPollingTimesFeature, Seq.empty) - val timeSinceLastNonPollingRequest = - nonPollingTimestampsMs.headOption.map(query.queryTime.inMillis - _) - - val nonPollingRequestsSinceTweetCreation = - if (nonPollingTimestampsMs.nonEmpty && timeSinceTweetCreationMs.isDefined) { - nonPollingTimestampsMs - .search(timeSinceTweetCreationMs.get)(Ordering[Long].reverse) - .insertionPoint - } else 0.0 - - val tweetAgeRatio = - if (timeSinceTweetCreationMs.exists(_ > 0.0) && timeSinceLastNonPollingRequest.isDefined) { - timeSinceLastNonPollingRequest.get / timeSinceTweetCreationMs.get.toDouble - } else 0.0 - - val dataRecord = new DataRecord() - .setFeatureValue(IS_TWEET_RECYCLED, false) - .setFeatureValue(TWEET_AGE_RATIO, tweetAgeRatio) - .setFeatureValueFromOption( - TIME_SINCE_TWEET_CREATION, - timeSinceTweetCreationMs.map(_.toDouble) - ) - .setFeatureValue( - NON_POLLING_REQUESTS_SINCE_TWEET_CREATION, - nonPollingRequestsSinceTweetCreation - ) - .setFeatureValueFromOption(LAST_FAVORITE_SINCE_CREATION_HRS, lastFavSinceCreationHrs) - .setFeatureValueFromOption(LAST_RETWEET_SINCE_CREATION_HRS, lastRetweetSinceCreationHrs) - .setFeatureValueFromOption(LAST_REPLY_SINCE_CREATION_HRS, lastReplySinceCreationHrs) - .setFeatureValueFromOption(LAST_QUOTE_SINCE_CREATION_HRS, lastQuoteSinceCreationHrs) - .setFeatureValueFromOption(TIME_SINCE_LAST_FAVORITE_HRS, timeSinceLastFavoriteHrs) - .setFeatureValueFromOption(TIME_SINCE_LAST_RETWEET_HRS, timeSinceLastRetweetHrs) - .setFeatureValueFromOption(TIME_SINCE_LAST_REPLY_HRS, timeSinceLastReplyHrs) - .setFeatureValueFromOption(TIME_SINCE_LAST_QUOTE_HRS, timeSinceLastQuoteHrs) - - Stitch.value(FeatureMapBuilder().add(TweetTimeDataRecordFeature, dataRecord).build()) - } - - private def getTimeSinceLastEngagementHrs( - lastEngagementTimeSinceCreationHrsOpt: Option[Double], - timeSinceTweetCreation: Option[Duration] - ): Option[Double] = lastEngagementTimeSinceCreationHrsOpt.flatMap { lastEngagementTimeHrs => - timeSinceTweetCreation.map(_.inHours - lastEngagementTimeHrs) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieContentFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieContentFeatureHydrator.scala deleted file mode 100644 index 93039e8bc..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieContentFeatureHydrator.scala +++ /dev/null @@ -1,144 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.escherbird.{thriftscala => esb} -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.model.HomeFeatures.MediaUnderstandingAnnotationIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetypieContentRepository -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.content.ContentFeatureAdapter -import com.twitter.home_mixer.util.ObservedKeyValueResultHandler -import com.twitter.home_mixer.util.tweetypie.content.FeatureExtractionHelper -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.servo.keyvalue.KeyValueResult -import com.twitter.servo.repository.KeyValueRepository -import com.twitter.stitch.Stitch -import com.twitter.timelines.prediction.common.util.MediaUnderstandingAnnotations -import com.twitter.tweetypie.{thriftscala => tp} -import com.twitter.util.Future -import com.twitter.util.Return -import com.twitter.util.Throw -import com.twitter.util.Try -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton -import scala.collection.JavaConverters._ - -object TweetypieContentDataRecordFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class TweetypieContentFeatureHydrator @Inject() ( - @Named(TweetypieContentRepository) client: KeyValueRepository[Seq[Long], Long, tp.Tweet], - override val statsReceiver: StatsReceiver) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] - with ObservedKeyValueResultHandler { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetypieContent") - - override val features: Set[Feature[_, _]] = Set( - MediaUnderstandingAnnotationIdsFeature, - TweetypieContentDataRecordFeature - ) - - override val statScope: String = identifier.toString - - private val bulkRequestLatencyStat = - statsReceiver.scope(statScope).scope("bulkRequest").stat("latency_ms") - private val postTransformerLatencyStat = - statsReceiver.scope(statScope).scope("postTransformer").stat("latency_ms") - private val bulkPostTransformerLatencyStat = - statsReceiver.scope(statScope).scope("bulkPostTransformer").stat("latency_ms") - - private val DefaultDataRecord: DataRecord = new DataRecord() - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { - val tweetIdsToHydrate = candidates.map(getCandidateOriginalTweetId).distinct - - val response: Future[KeyValueResult[Long, tp.Tweet]] = Stat.timeFuture(bulkRequestLatencyStat) { - if (tweetIdsToHydrate.isEmpty) Future.value(KeyValueResult.empty) - else client(tweetIdsToHydrate) - } - - response.flatMap { result => - Stat.timeFuture(bulkPostTransformerLatencyStat) { - OffloadFuturePools - .parallelize[CandidateWithFeatures[TweetCandidate], Try[(Seq[Long], DataRecord)]]( - candidates, - parTransformer(result, _), - parallelism = 32, - default = Return((Seq.empty, DefaultDataRecord)) - ).map { - _.map { - case Return(result) => - FeatureMapBuilder() - .add(MediaUnderstandingAnnotationIdsFeature, result._1) - .add(TweetypieContentDataRecordFeature, result._2) - .build() - case Throw(e) => - FeatureMapBuilder() - .add(MediaUnderstandingAnnotationIdsFeature, Throw(e)) - .add(TweetypieContentDataRecordFeature, Throw(e)) - .build() - } - } - } - } - } - - private def parTransformer( - result: KeyValueResult[Long, tp.Tweet], - candidate: CandidateWithFeatures[TweetCandidate] - ): Try[(Seq[Long], DataRecord)] = { - val originalTweetId = Some(getCandidateOriginalTweetId(candidate)) - val value = observedGet(key = originalTweetId, keyValueResult = result) - Stat.time(postTransformerLatencyStat)(postTransformer(value)) - } - - private def postTransformer( - result: Try[Option[tp.Tweet]] - ): Try[(Seq[Long], DataRecord)] = { - result.map { tweet => - val transformedValue = tweet.map(FeatureExtractionHelper.extractFeatures) - val semanticAnnotations = transformedValue - .flatMap { contentFeatures => - contentFeatures.semanticCoreAnnotations.map { - getNonSensitiveHighRecallMediaUnderstandingAnnotationEntityIds - } - }.getOrElse(Seq.empty) - val dataRecord = ContentFeatureAdapter.adaptToDataRecords(transformedValue).asScala.head - (semanticAnnotations, dataRecord) - } - } - - private def getCandidateOriginalTweetId( - candidate: CandidateWithFeatures[TweetCandidate] - ): Long = { - candidate.features - .getOrElse(SourceTweetIdFeature, None).getOrElse(candidate.candidate.id) - } - - private def getNonSensitiveHighRecallMediaUnderstandingAnnotationEntityIds( - semanticCoreAnnotations: Seq[esb.TweetEntityAnnotation] - ): Seq[Long] = - semanticCoreAnnotations - .filter(MediaUnderstandingAnnotations.isEligibleNonSensitiveHighRecallMUAnnotation) - .map(_.entityId) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieStaticEntitiesFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieStaticEntitiesFeatureHydrator.scala deleted file mode 100644 index 0ea9e8591..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieStaticEntitiesFeatureHydrator.scala +++ /dev/null @@ -1,161 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.google.inject.name.Named -import com.twitter.conversions.DurationOps.RichDuration -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.HasImageFeature -import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.MentionScreenNameFeature -import com.twitter.home_mixer.model.HomeFeatures.MentionUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetypieStaticEntitiesCache -import com.twitter.home_mixer.util.tweetypie.RequestFields -import com.twitter.home_mixer.util.tweetypie.content.TweetMediaFeaturesExtractor -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.cache.TtlCache -import com.twitter.spam.rtf.{thriftscala => sp} -import com.twitter.stitch.Stitch -import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient} -import com.twitter.tweetypie.{thriftscala => tp} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class TweetypieStaticEntitiesFeatureHydrator @Inject() ( - tweetypieStitchClient: TweetypieStitchClient, - @Named(TweetypieStaticEntitiesCache) cacheClient: TtlCache[Long, tp.Tweet]) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("TweetypieStaticEntities") - - override val features: Set[Feature[_, _]] = Set( - AuthorIdFeature, - DirectedAtUserIdFeature, - ExclusiveConversationAuthorIdFeature, - HasImageFeature, - HasVideoFeature, - InReplyToTweetIdFeature, - InReplyToUserIdFeature, - IsRetweetFeature, - MentionScreenNameFeature, - MentionUserIdFeature, - QuotedTweetIdFeature, - QuotedUserIdFeature, - SourceTweetIdFeature, - SourceUserIdFeature - ) - - private val CacheTTL = 24.hours - - private val DefaultFeatureMap = FeatureMapBuilder() - .add(AuthorIdFeature, None) - .add(DirectedAtUserIdFeature, None) - .add(ExclusiveConversationAuthorIdFeature, None) - .add(HasImageFeature, false) - .add(HasVideoFeature, false) - .add(InReplyToTweetIdFeature, None) - .add(InReplyToUserIdFeature, None) - .add(IsRetweetFeature, false) - .add(MentionScreenNameFeature, Seq.empty) - .add(MentionUserIdFeature, Seq.empty) - .add(QuotedTweetIdFeature, None) - .add(QuotedUserIdFeature, None) - .add(SourceTweetIdFeature, None) - .add(SourceUserIdFeature, None) - .build() - - /** - * Steps: - * 1. query cache with all candidates - * 2. create a cached feature map - * 3. iterate candidates to hydrate features - * 3.a transform cached candidates - * 3.b hydrate non-cached candidates from Tweetypie and write to cache - */ - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - val tweetIds = candidates.map(_.candidate.id) - val cachedTweetsMapFu = cacheClient - .get(tweetIds) - .map(_.found) - - Stitch.callFuture(cachedTweetsMapFu).flatMap { cachedTweets => - Stitch.collect { - candidates.map { candidate => - if (cachedTweets.contains(candidate.candidate.id)) - Stitch.value(createFeatureMap(cachedTweets(candidate.candidate.id))) - else readFromTweetypie(query, candidate) - } - } - } - } - - private def createFeatureMap(tweet: tp.Tweet): FeatureMap = { - val coreData = tweet.coreData - val quotedTweet = tweet.quotedTweet - val mentions = tweet.mentions.getOrElse(Seq.empty) - val share = coreData.flatMap(_.share) - val reply = coreData.flatMap(_.reply) - - FeatureMapBuilder() - .add(AuthorIdFeature, coreData.map(_.userId)) - .add(DirectedAtUserIdFeature, coreData.flatMap(_.directedAtUser.map(_.userId))) - .add( - ExclusiveConversationAuthorIdFeature, - tweet.exclusiveTweetControl.map(_.conversationAuthorId)) - .add(HasImageFeature, TweetMediaFeaturesExtractor.hasImage(tweet)) - .add(HasVideoFeature, TweetMediaFeaturesExtractor.hasVideo(tweet)) - .add(InReplyToTweetIdFeature, reply.flatMap(_.inReplyToStatusId)) - .add(InReplyToUserIdFeature, reply.map(_.inReplyToUserId)) - .add(IsRetweetFeature, share.isDefined) - .add(MentionScreenNameFeature, mentions.map(_.screenName)) - .add(MentionUserIdFeature, mentions.flatMap(_.userId)) - .add(QuotedTweetIdFeature, quotedTweet.map(_.tweetId)) - .add(QuotedUserIdFeature, quotedTweet.map(_.userId)) - .add(SourceTweetIdFeature, share.map(_.sourceStatusId)) - .add(SourceUserIdFeature, share.map(_.sourceUserId)) - .build() - } - - private def readFromTweetypie( - query: PipelineQuery, - candidate: CandidateWithFeatures[TweetCandidate] - ): Stitch[FeatureMap] = { - tweetypieStitchClient - .getTweetFields( - tweetId = candidate.candidate.id, - options = tp.GetTweetFieldsOptions( - tweetIncludes = RequestFields.TweetStaticEntitiesFields, - includeRetweetedTweet = false, - includeQuotedTweet = false, - forUserId = query.getOptionalUserId, // Needed to get protected Tweets for certain users - visibilityPolicy = tp.TweetVisibilityPolicy.UserVisible, - safetyLevel = Some(sp.SafetyLevel.FilterNone) // VF is handled in the For You product - ) - ).map { - case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), _, _) => - cacheClient.set(candidate.candidate.id, found.tweet, CacheTTL) - createFeatureMap(found.tweet) - case _ => - DefaultFeatureMap + (AuthorIdFeature, candidate.features.getOrElse(AuthorIdFeature, None)) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinAuthorFollowFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinAuthorFollowFeatureHydrator.scala deleted file mode 100644 index 2e531bfb2..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinAuthorFollowFeatureHydrator.scala +++ /dev/null @@ -1,85 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinAuthorFollowFeatureRepository -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.twhin_embeddings.TwhinAuthorFollowEmbeddingsAdapter -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.home_mixer.util.ObservedKeyValueResultHandler -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.{thriftscala => ml} -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.servo.repository.KeyValueRepository -import com.twitter.servo.repository.KeyValueResult -import com.twitter.stitch.Stitch -import com.twitter.util.Future -import com.twitter.util.Try -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton -import scala.collection.JavaConverters._ - -object TwhinAuthorFollowFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class TwhinAuthorFollowFeatureHydrator @Inject() ( - @Named(TwhinAuthorFollowFeatureRepository) - client: KeyValueRepository[Seq[Long], Long, ml.FloatTensor], - override val statsReceiver: StatsReceiver) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] - with ObservedKeyValueResultHandler { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("TwhinAuthorFollow") - - override val features: Set[Feature[_, _]] = Set(TwhinAuthorFollowFeature) - - override val statScope: String = identifier.toString - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { - val possiblyAuthorIds = extractKeys(candidates) - val authorIds = possiblyAuthorIds.flatten - - val response: Future[KeyValueResult[Long, ml.FloatTensor]] = - if (authorIds.isEmpty) Future.value(KeyValueResult.empty) else client(authorIds) - - response.map { result => - possiblyAuthorIds.map { possiblyAuthorId => - val value = observedGet(key = possiblyAuthorId, keyValueResult = result) - val transformedValue = postTransformer(value) - - FeatureMapBuilder().add(TwhinAuthorFollowFeature, transformedValue).build() - } - } - } - - private def postTransformer(embedding: Try[Option[ml.FloatTensor]]): Try[DataRecord] = { - embedding.map { floatTensor => - TwhinAuthorFollowEmbeddingsAdapter.adaptToDataRecords(floatTensor).asScala.head - } - } - - private def extractKeys( - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Seq[Option[Long]] = { - candidates.map { candidate => - CandidatesUtil.getOriginalAuthorId(candidate.features) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinUserEngagementQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinUserEngagementQueryFeatureHydrator.scala deleted file mode 100644 index f43abc98f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinUserEngagementQueryFeatureHydrator.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinUserEngagementFeatureRepository -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.twhin_embeddings.TwhinUserEngagementEmbeddingsAdapter -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.{thriftscala => ml} -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.repository.KeyValueRepository -import com.twitter.stitch.Stitch -import com.twitter.util.Return -import com.twitter.util.Throw -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton -import scala.collection.JavaConverters._ - -object TwhinUserEngagementFeature - extends DataRecordInAFeature[PipelineQuery] - with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class TwhinUserEngagementQueryFeatureHydrator @Inject() ( - @Named(TwhinUserEngagementFeatureRepository) - client: KeyValueRepository[Seq[Long], Long, ml.FloatTensor], - statsReceiver: StatsReceiver) - extends QueryFeatureHydrator[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("TwhinUserEngagement") - - override val features: Set[Feature[_, _]] = Set(TwhinUserEngagementFeature) - - private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) - private val keyFoundCounter = scopedStatsReceiver.counter("key/found") - private val keyLossCounter = scopedStatsReceiver.counter("key/loss") - private val keyFailureCounter = scopedStatsReceiver.counter("key/failure") - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - val userId = query.getRequiredUserId - Stitch.callFuture(client(Seq(userId))).map { results => - val embedding: Option[ml.FloatTensor] = results(userId) match { - case Return(value) => - if (value.exists(_.floats.nonEmpty)) keyFoundCounter.incr() - else keyLossCounter.incr() - value - case Throw(_) => - keyFailureCounter.incr() - None - case _ => - None - } - - val dataRecord = - TwhinUserEngagementEmbeddingsAdapter.adaptToDataRecords(embedding).asScala.head - - FeatureMapBuilder() - .add(TwhinUserEngagementFeature, dataRecord) - .build() - } - } - -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinUserFollowQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinUserFollowQueryFeatureHydrator.scala deleted file mode 100644 index 925043e1e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinUserFollowQueryFeatureHydrator.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinUserFollowFeatureRepository -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.twhin_embeddings.TwhinUserFollowEmbeddingsAdapter -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.{thriftscala => ml} -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.repository.KeyValueRepository -import com.twitter.stitch.Stitch -import com.twitter.util.Return -import com.twitter.util.Throw -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton -import scala.collection.JavaConverters._ - -object TwhinUserFollowFeature - extends DataRecordInAFeature[PipelineQuery] - with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class TwhinUserFollowQueryFeatureHydrator @Inject() ( - @Named(TwhinUserFollowFeatureRepository) - client: KeyValueRepository[Seq[Long], Long, ml.FloatTensor], - statsReceiver: StatsReceiver) - extends QueryFeatureHydrator[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("TwhinUserFollow") - - override val features: Set[Feature[_, _]] = Set(TwhinUserFollowFeature) - - private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) - private val keyFoundCounter = scopedStatsReceiver.counter("key/found") - private val keyLossCounter = scopedStatsReceiver.counter("key/loss") - private val keyFailureCounter = scopedStatsReceiver.counter("key/failure") - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - val userId = query.getRequiredUserId - Stitch.callFuture(client(Seq(userId))).map { results => - val embedding: Option[ml.FloatTensor] = results(userId) match { - case Return(value) => - if (value.exists(_.floats.nonEmpty)) keyFoundCounter.incr() - else keyLossCounter.incr() - value - case Throw(_) => - keyFailureCounter.incr() - None - case _ => - None - } - - val dataRecord = TwhinUserFollowEmbeddingsAdapter.adaptToDataRecords(embedding).asScala.head - - FeatureMapBuilder() - .add(TwhinUserFollowFeature, dataRecord) - .build() - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserFollowedTopicIdsFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserFollowedTopicIdsFeatureHydrator.scala deleted file mode 100644 index 90f618fd0..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserFollowedTopicIdsFeatureHydrator.scala +++ /dev/null @@ -1,76 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserFollowedTopicIdsRepository -import com.twitter.home_mixer.util.ObservedKeyValueResultHandler -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.servo.keyvalue.KeyValueResult -import com.twitter.servo.repository.KeyValueRepository -import com.twitter.stitch.Stitch -import com.twitter.util.Future -import com.twitter.util.Try -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -object UserFollowedTopicIdsFeature extends Feature[TweetCandidate, Seq[Long]] - -@Singleton -class UserFollowedTopicIdsFeatureHydrator @Inject() ( - @Named(UserFollowedTopicIdsRepository) - client: KeyValueRepository[Seq[Long], Long, Seq[Long]], - override val statsReceiver: StatsReceiver) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] - with ObservedKeyValueResultHandler { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("UserFollowedTopicIds") - - override val features: Set[Feature[_, _]] = Set(UserFollowedTopicIdsFeature) - - override val statScope: String = identifier.toString - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { - val possiblyAuthorIds = extractKeys(candidates) - val authorIds = possiblyAuthorIds.flatten - - val response: Future[KeyValueResult[Long, Seq[Long]]] = - if (authorIds.isEmpty) Future.value(KeyValueResult.empty) else client(authorIds) - - response.map { result => - possiblyAuthorIds.map { possiblyAuthorId => - val value = observedGet(key = possiblyAuthorId, keyValueResult = result) - val transformedValue = postTransformer(value) - - FeatureMapBuilder().add(UserFollowedTopicIdsFeature, transformedValue).build() - } - } - } - - private def postTransformer(input: Try[Option[Seq[Long]]]): Try[Seq[Long]] = { - input.map(_.getOrElse(Seq.empty[Long])) - } - - private def extractKeys( - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Seq[Option[Long]] = { - candidates.map { candidate => - candidate.features - .getTry(AuthorIdFeature) - .toOption - .flatten - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserLanguagesFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserLanguagesFeatureHydrator.scala deleted file mode 100644 index e2431c43e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserLanguagesFeatureHydrator.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserLanguagesRepository -import com.twitter.home_mixer.util.ObservedKeyValueResultHandler -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.search.common.constants.{thriftscala => scc} -import com.twitter.servo.repository.KeyValueRepository -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -object UserLanguagesFeature extends Feature[PipelineQuery, Seq[scc.ThriftLanguage]] - -@Singleton -case class UserLanguagesFeatureHydrator @Inject() ( - @Named(UserLanguagesRepository) client: KeyValueRepository[Seq[Long], Long, Seq[ - scc.ThriftLanguage - ]], - statsReceiver: StatsReceiver) - extends QueryFeatureHydrator[PipelineQuery] - with ObservedKeyValueResultHandler { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserLanguages") - - override val features: Set[Feature[_, _]] = Set(UserLanguagesFeature) - - override val statScope: String = identifier.toString - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - val key = query.getRequiredUserId - Stitch.callFuture(client(Seq(key))).map { result => - val feature = - observedGet(key = Some(key), keyValueResult = result).map(_.getOrElse(Seq.empty)) - FeatureMapBuilder() - .add(UserLanguagesFeature, feature) - .build() - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserStateQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserStateQueryFeatureHydrator.scala deleted file mode 100644 index 5a61f31fb..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserStateQueryFeatureHydrator.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelines.user_health.v1.{thriftscala => uhv1} -import com.twitter.timelines.user_health.{thriftscala => uh} -import com.twitter.user_session_store.ReadOnlyUserSessionStore -import com.twitter.user_session_store.ReadRequest -import com.twitter.user_session_store.UserSessionDataset -import com.twitter.user_session_store.UserSessionDataset.UserSessionDataset -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class UserStateQueryFeatureHydrator @Inject() ( - userSessionStore: ReadOnlyUserSessionStore) - extends QueryFeatureHydrator[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("UserState") - - override val features: Set[Feature[_, _]] = Set(UserStateFeature) - - private val datasets: Set[UserSessionDataset] = Set(UserSessionDataset.UserHealth) - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - userSessionStore - .read(ReadRequest(query.getRequiredUserId, datasets)) - .map { userSession => - val userState = userSession.flatMap { - _.userHealth match { - case Some(uh.UserHealth.V1(uhv1.UserHealth(userState))) => userState - case _ => None - } - } - - FeatureMapBuilder() - .add(UserStateFeature, userState) - .build() - } - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.9) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UtegFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UtegFeatureHydrator.scala deleted file mode 100644 index 96b9657e7..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UtegFeatureHydrator.scala +++ /dev/null @@ -1,102 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.FavoritedByCountFeature -import com.twitter.home_mixer.model.HomeFeatures.FavoritedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature -import com.twitter.home_mixer.model.HomeFeatures.RepliedByCountFeature -import com.twitter.home_mixer.model.HomeFeatures.RepliedByEngagerIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.RetweetedByCountFeature -import com.twitter.home_mixer.model.HomeFeatures.RetweetedByEngagerIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.home_mixer.param.HomeMixerInjectionNames.UtegSocialProofRepository -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.Conditionally -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.recos.recos_common.{thriftscala => rc} -import com.twitter.recos.user_tweet_entity_graph.{thriftscala => uteg} -import com.twitter.servo.keyvalue.KeyValueResult -import com.twitter.servo.repository.KeyValueRepository -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -@Singleton -class UtegFeatureHydrator @Inject() ( - @Named(UtegSocialProofRepository) client: KeyValueRepository[ - (Seq[Long], (Long, Map[Long, Double])), - Long, - uteg.TweetRecommendation - ]) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] - with Conditionally[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Uteg") - - override val features: Set[Feature[_, _]] = Set( - FavoritedByUserIdsFeature, - RetweetedByEngagerIdsFeature, - RepliedByEngagerIdsFeature, - FavoritedByCountFeature, - RetweetedByCountFeature, - RepliedByCountFeature - ) - - override def onlyIf(query: PipelineQuery): Boolean = query.features - .exists(_.getOrElse(RealGraphInNetworkScoresFeature, Map.empty[Long, Double]).nonEmpty) - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { - val seedUserWeights = query.features.map(_.get(RealGraphInNetworkScoresFeature)).get - - val sourceTweetIds = candidates.flatMap(_.features.getOrElse(SourceTweetIdFeature, None)) - val inReplyToTweetIds = candidates.flatMap(_.features.getOrElse(InReplyToTweetIdFeature, None)) - val tweetIds = candidates.map(_.candidate.id) - val tweetIdsToSend = (tweetIds ++ sourceTweetIds ++ inReplyToTweetIds).distinct - - val utegQuery = (tweetIdsToSend, (query.getRequiredUserId, seedUserWeights)) - - client(utegQuery).map(handleResponse(candidates, _)) - } - - private def handleResponse( - candidates: Seq[CandidateWithFeatures[TweetCandidate]], - results: KeyValueResult[Long, uteg.TweetRecommendation], - ): Seq[FeatureMap] = { - candidates.map { candidate => - val inNetwork = candidate.features.getOrElse(FromInNetworkSourceFeature, false) - val candidateProof = results(candidate.candidate.id).toOption.flatten - val sourceProof = candidate.features - .getOrElse(SourceTweetIdFeature, None).flatMap(results(_).toOption.flatten) - val proofs = Seq(candidateProof, sourceProof).flatten.map(_.socialProofByType) - - val favoritedBy = proofs.flatMap(_.get(rc.SocialProofType.Favorite)).flatten - val retweetedBy = proofs.flatMap(_.get(rc.SocialProofType.Retweet)).flatten - val repliedBy = proofs.flatMap(_.get(rc.SocialProofType.Reply)).flatten - - val (favoritedByCount, retweetedByCount, repliedByCount) = - if (!inNetwork) { - (favoritedBy.size.toDouble, retweetedBy.size.toDouble, repliedBy.size.toDouble) - } else { (0.0, 0.0, 0.0) } - - FeatureMapBuilder() - .add(FavoritedByUserIdsFeature, favoritedBy) - .add(RetweetedByEngagerIdsFeature, retweetedBy) - .add(RepliedByEngagerIdsFeature, repliedBy) - .add(FavoritedByCountFeature, favoritedByCount) - .add(RetweetedByCountFeature, retweetedByCount) - .add(RepliedByCountFeature, repliedByCount) - .build() - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features/AuthorFeaturesAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features/AuthorFeaturesAdapter.scala deleted file mode 100644 index cbaa3f18c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features/AuthorFeaturesAdapter.scala +++ /dev/null @@ -1,92 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.author_features - -import com.twitter.home_mixer.util.DataRecordUtil -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.Feature -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.util.CompactDataRecordConverter -import com.twitter.ml.api.util.FDsl._ -import com.twitter.timelines.author_features.v1.{thriftjava => af} -import com.twitter.timelines.prediction.common.adapters.TimelinesAdapterBase -import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig -import com.twitter.timelines.prediction.features.user_health.UserHealthFeatures -import scala.collection.JavaConverters._ - -object AuthorFeaturesAdapter extends TimelinesAdapterBase[af.AuthorFeatures] { - - private val Prefix = "original_author.timelines.original_author_aggregates." - - private val typedAggregateGroups = - TimelinesAggregationConfig.originalAuthorAggregatesV1.buildTypedAggregateGroups() - - private val aggregateFeaturesRenameMap: Map[Feature[_], Feature[_]] = - typedAggregateGroups.map(_.outputFeaturesToRenamedOutputFeatures(Prefix)).reduce(_ ++ _) - - private val prefixedOriginalAuthorAggregateFeatures = - typedAggregateGroups.flatMap(_.allOutputFeatures).map { feature => - aggregateFeaturesRenameMap.getOrElse(feature, feature) - } - - private val authorFeatures = prefixedOriginalAuthorAggregateFeatures ++ Seq( - UserHealthFeatures.AuthorState, - UserHealthFeatures.NumAuthorFollowers, - UserHealthFeatures.NumAuthorConnectDays, - UserHealthFeatures.NumAuthorConnect - ) - - private val aggregateFeatureContext: FeatureContext = - new FeatureContext(typedAggregateGroups.flatMap(_.allOutputFeatures).asJava) - - private lazy val prefixedAggregateFeatureContext: FeatureContext = - new FeatureContext(prefixedOriginalAuthorAggregateFeatures.asJava) - - override val getFeatureContext: FeatureContext = new FeatureContext(authorFeatures: _*) - - override val commonFeatures: Set[Feature[_]] = Set.empty - - private val compactDataRecordConverter = new CompactDataRecordConverter() - - override def adaptToDataRecords( - authorFeatures: af.AuthorFeatures - ): java.util.List[DataRecord] = { - val dataRecord = - if (authorFeatures.aggregates != null) { - val originalAuthorAggregatesDataRecord = - compactDataRecordConverter.compactDataRecordToDataRecord(authorFeatures.aggregates) - - DataRecordUtil.applyRename( - originalAuthorAggregatesDataRecord, - aggregateFeatureContext, - prefixedAggregateFeatureContext, - aggregateFeaturesRenameMap) - } else new DataRecord - - if (authorFeatures.user_health != null) { - val userHealth = authorFeatures.user_health - - if (userHealth.user_state != null) { - dataRecord.setFeatureValue( - UserHealthFeatures.AuthorState, - userHealth.user_state.getValue.toLong - ) - } - - dataRecord.setFeatureValue( - UserHealthFeatures.NumAuthorFollowers, - userHealth.num_followers.toDouble - ) - - dataRecord.setFeatureValue( - UserHealthFeatures.NumAuthorConnectDays, - userHealth.num_connect_days.toDouble - ) - - dataRecord.setFeatureValue( - UserHealthFeatures.NumAuthorConnect, - userHealth.num_connect.toDouble - ) - } - - List(dataRecord).asJava - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features/BUILD.bazel deleted file mode 100644 index 7a00a9b26..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features/BUILD.bazel +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "src/java/com/twitter/ml/api:api-base", - "src/java/com/twitter/ml/api/util", - "src/scala/com/twitter/ml/api/util", - "src/scala/com/twitter/timelines/prediction/common/adapters:base", - "src/scala/com/twitter/timelines/prediction/common/aggregates", - "src/scala/com/twitter/timelines/prediction/features/user_health", - "src/thrift/com/twitter/ml/api:data-java", - "src/thrift/com/twitter/timelines/author_features:thrift-java", - "timelines/data_processing/ml_util/aggregation_framework", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/BUILD.bazel deleted file mode 100644 index 6a9393121..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/BUILD.bazel +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "src/java/com/twitter/ml/api:api-base", - "src/scala/com/twitter/ml/api:api-base", - "src/scala/com/twitter/ml/api/util", - "src/scala/com/twitter/timelines/prediction/common/adapters", - "src/scala/com/twitter/timelines/prediction/common/adapters:base", - "src/scala/com/twitter/timelines/prediction/features/common", - "src/scala/com/twitter/timelines/prediction/features/conversation_features", - "src/thrift/com/twitter/ml/api:data-java", - "src/thrift/com/twitter/ml/api:data-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/ContentFeatureAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/ContentFeatureAdapter.scala deleted file mode 100644 index 2d73ece45..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/ContentFeatureAdapter.scala +++ /dev/null @@ -1,273 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.content - -import com.twitter.home_mixer.model.ContentFeatures -import com.twitter.ml.api.Feature -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.RichDataRecord -import com.twitter.ml.api.util.DataRecordConverters._ -import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase -import com.twitter.timelines.prediction.common.adapters.TweetLengthType -import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures -import com.twitter.timelines.prediction.features.conversation_features.ConversationFeatures -import scala.collection.JavaConverters._ - -object ContentFeatureAdapter extends TimelinesMutatingAdapterBase[Option[ContentFeatures]] { - - override val getFeatureContext: FeatureContext = new FeatureContext( - ConversationFeatures.IS_SELF_THREAD_TWEET, - ConversationFeatures.IS_LEAF_IN_SELF_THREAD, - TimelinesSharedFeatures.ASPECT_RATIO_DEN, - TimelinesSharedFeatures.ASPECT_RATIO_NUM, - TimelinesSharedFeatures.BIT_RATE, - TimelinesSharedFeatures.CLASSIFICATION_LABELS, - TimelinesSharedFeatures.COLOR_1_BLUE, - TimelinesSharedFeatures.COLOR_1_GREEN, - TimelinesSharedFeatures.COLOR_1_PERCENTAGE, - TimelinesSharedFeatures.COLOR_1_RED, - TimelinesSharedFeatures.FACE_AREAS, - TimelinesSharedFeatures.HAS_APP_INSTALL_CALL_TO_ACTION, - TimelinesSharedFeatures.HAS_DESCRIPTION, - TimelinesSharedFeatures.HAS_QUESTION, - TimelinesSharedFeatures.HAS_SELECTED_PREVIEW_IMAGE, - TimelinesSharedFeatures.HAS_TITLE, - TimelinesSharedFeatures.HAS_VISIT_SITE_CALL_TO_ACTION, - TimelinesSharedFeatures.HAS_WATCH_NOW_CALL_TO_ACTION, - TimelinesSharedFeatures.HEIGHT_1, - TimelinesSharedFeatures.HEIGHT_2, - TimelinesSharedFeatures.HEIGHT_3, - TimelinesSharedFeatures.HEIGHT_4, - TimelinesSharedFeatures.IS_360, - TimelinesSharedFeatures.IS_EMBEDDABLE, - TimelinesSharedFeatures.IS_MANAGED, - TimelinesSharedFeatures.IS_MONETIZABLE, - TimelinesSharedFeatures.MEDIA_PROVIDERS, - TimelinesSharedFeatures.NUM_CAPS, - TimelinesSharedFeatures.NUM_COLOR_PALLETTE_ITEMS, - TimelinesSharedFeatures.NUM_FACES, - TimelinesSharedFeatures.NUM_MEDIA_TAGS, - TimelinesSharedFeatures.NUM_NEWLINES, - TimelinesSharedFeatures.NUM_STICKERS, - TimelinesSharedFeatures.NUM_WHITESPACES, - TimelinesSharedFeatures.RESIZE_METHOD_1, - TimelinesSharedFeatures.RESIZE_METHOD_2, - TimelinesSharedFeatures.RESIZE_METHOD_3, - TimelinesSharedFeatures.RESIZE_METHOD_4, - TimelinesSharedFeatures.TWEET_LENGTH, - TimelinesSharedFeatures.TWEET_LENGTH_TYPE, - TimelinesSharedFeatures.VIDEO_DURATION, - TimelinesSharedFeatures.VIEW_COUNT, - TimelinesSharedFeatures.WIDTH_1, - TimelinesSharedFeatures.WIDTH_2, - TimelinesSharedFeatures.WIDTH_3, - TimelinesSharedFeatures.WIDTH_4, - ) - - override val commonFeatures: Set[Feature[_]] = Set.empty - - private def getTweetLengthType(tweetLength: Int): Long = { - tweetLength match { - case x if 0 > x || 280 < x => TweetLengthType.INVALID - case x if 0 <= x && x <= 30 => TweetLengthType.VERY_SHORT - case x if 30 < x && x <= 60 => TweetLengthType.SHORT - case x if 60 < x && x <= 90 => TweetLengthType.MEDIUM - case x if 90 < x && x <= 140 => TweetLengthType.LENGTHY - case x if 140 < x && x <= 210 => TweetLengthType.VERY_LENGTHY - case x if x > 210 => TweetLengthType.MAXIMUM_LENGTH - } - } - - override def setFeatures( - contentFeatures: Option[ContentFeatures], - richDataRecord: RichDataRecord - ): Unit = { - if (contentFeatures.nonEmpty) { - val features = contentFeatures.get - // Conversation Features - richDataRecord.setFeatureValueFromOption( - ConversationFeatures.IS_SELF_THREAD_TWEET, - Some(features.selfThreadMetadata.nonEmpty) - ) - richDataRecord.setFeatureValueFromOption( - ConversationFeatures.IS_LEAF_IN_SELF_THREAD, - features.selfThreadMetadata.map(_.isLeaf) - ) - - // Media Features - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.ASPECT_RATIO_DEN, - features.aspectRatioDen.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.ASPECT_RATIO_NUM, - features.aspectRatioNum.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.BIT_RATE, - features.bitRate.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.HEIGHT_1, - features.heights.flatMap(_.lift(0)).map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.HEIGHT_2, - features.heights.flatMap(_.lift(1)).map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.HEIGHT_3, - features.heights.flatMap(_.lift(2)).map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.HEIGHT_4, - features.heights.flatMap(_.lift(3)).map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.NUM_MEDIA_TAGS, - features.numMediaTags.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.RESIZE_METHOD_1, - features.resizeMethods.flatMap(_.lift(0)).map(_.toLong) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.RESIZE_METHOD_2, - features.resizeMethods.flatMap(_.lift(1)).map(_.toLong) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.RESIZE_METHOD_3, - features.resizeMethods.flatMap(_.lift(2)).map(_.toLong) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.RESIZE_METHOD_4, - features.resizeMethods.flatMap(_.lift(3)).map(_.toLong) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.VIDEO_DURATION, - features.videoDurationMs.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.WIDTH_1, - features.widths.flatMap(_.lift(0)).map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.WIDTH_2, - features.widths.flatMap(_.lift(1)).map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.WIDTH_3, - features.widths.flatMap(_.lift(2)).map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.WIDTH_4, - features.widths.flatMap(_.lift(3)).map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.NUM_COLOR_PALLETTE_ITEMS, - features.numColors.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.COLOR_1_RED, - features.dominantColorRed.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.COLOR_1_BLUE, - features.dominantColorBlue.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.COLOR_1_GREEN, - features.dominantColorGreen.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.COLOR_1_PERCENTAGE, - features.dominantColorPercentage - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.MEDIA_PROVIDERS, - features.mediaOriginProviders.map(_.toSet.asJava) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.IS_360, - features.is360 - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.VIEW_COUNT, - features.viewCount.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.IS_MANAGED, - features.isManaged - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.IS_MONETIZABLE, - features.isMonetizable - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.IS_EMBEDDABLE, - features.isEmbeddable - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.NUM_STICKERS, - features.stickerIds.map(_.length.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.NUM_FACES, - features.faceAreas.map(_.length.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.FACE_AREAS, - // guard for exception from max on empty seq - features.faceAreas.map(faceAreas => - faceAreas.map(_.toDouble).reduceOption(_ max _).getOrElse(0.0)) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.HAS_SELECTED_PREVIEW_IMAGE, - features.hasSelectedPreviewImage - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.HAS_TITLE, - features.hasTitle - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.HAS_DESCRIPTION, - features.hasDescription - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.HAS_VISIT_SITE_CALL_TO_ACTION, - features.hasVisitSiteCallToAction - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.HAS_APP_INSTALL_CALL_TO_ACTION, - features.hasAppInstallCallToAction - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.HAS_WATCH_NOW_CALL_TO_ACTION, - features.hasWatchNowCallToAction - ) - // text features - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.NUM_CAPS, - Some(features.numCaps.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.TWEET_LENGTH, - Some(features.length.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.TWEET_LENGTH_TYPE, - Some(getTweetLengthType(features.length.toInt)) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.NUM_WHITESPACES, - Some(features.numWhiteSpaces.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.HAS_QUESTION, - Some(features.hasQuestion) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.NUM_NEWLINES, - features.numNewlines.map(_.toDouble) - ) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/InReplyToContentFeatureAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/InReplyToContentFeatureAdapter.scala deleted file mode 100644 index fa27ddac0..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/InReplyToContentFeatureAdapter.scala +++ /dev/null @@ -1,75 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.content - -import com.twitter.home_mixer.model.ContentFeatures -import com.twitter.ml.api.Feature -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.RichDataRecord -import com.twitter.ml.api.util.DataRecordConverters.RichDataRecordWrapper -import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase -import com.twitter.timelines.prediction.features.common.InReplyToTweetTimelinesSharedFeatures - -object InReplyToContentFeatureAdapter - extends TimelinesMutatingAdapterBase[Option[ContentFeatures]] { - - override val getFeatureContext: FeatureContext = new FeatureContext( - // Media Features - InReplyToTweetTimelinesSharedFeatures.ASPECT_RATIO_DEN, - InReplyToTweetTimelinesSharedFeatures.ASPECT_RATIO_NUM, - InReplyToTweetTimelinesSharedFeatures.HEIGHT_1, - InReplyToTweetTimelinesSharedFeatures.HEIGHT_2, - InReplyToTweetTimelinesSharedFeatures.VIDEO_DURATION, - // TextFeatures - InReplyToTweetTimelinesSharedFeatures.NUM_CAPS, - InReplyToTweetTimelinesSharedFeatures.TWEET_LENGTH, - InReplyToTweetTimelinesSharedFeatures.HAS_QUESTION, - ) - - override val commonFeatures: Set[Feature[_]] = Set.empty - - override def setFeatures( - contentFeatures: Option[ContentFeatures], - richDataRecord: RichDataRecord - ): Unit = { - if (contentFeatures.nonEmpty) { - val features = contentFeatures.get - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.ASPECT_RATIO_DEN, - features.aspectRatioNum.map(_.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.ASPECT_RATIO_NUM, - features.aspectRatioNum.map(_.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.HEIGHT_1, - features.heights.flatMap(_.lift(0)).map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.HEIGHT_2, - features.heights.flatMap(_.lift(1)).map(_.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.VIDEO_DURATION, - features.videoDurationMs.map(_.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.NUM_CAPS, - Some(features.numCaps.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.TWEET_LENGTH, - Some(features.length.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.HAS_QUESTION, - Some(features.hasQuestion) - ) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/BUILD.bazel deleted file mode 100644 index 9428c0d39..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/BUILD.bazel +++ /dev/null @@ -1,19 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "src/java/com/twitter/ml/api:api-base", - "src/scala/com/twitter/ml/api:api-base", - "src/scala/com/twitter/ml/api/util", - "src/scala/com/twitter/timelines/prediction/common/adapters:base", - "src/scala/com/twitter/timelines/prediction/features/common", - "src/scala/com/twitter/timelines/prediction/features/recap", - "src/scala/com/twitter/timelines/util", - "src/thrift/com/twitter/ml/api:data-java", - "src/thrift/com/twitter/ml/api:data-scala", - "src/thrift/com/twitter/search/common:features-scala", - "timelines/src/main/scala/com/twitter/timelines/util", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/EarlybirdAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/EarlybirdAdapter.scala deleted file mode 100644 index 5a0207e8a..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/EarlybirdAdapter.scala +++ /dev/null @@ -1,457 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.earlybird - -import com.twitter.ml.api.Feature -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.RichDataRecord -import com.twitter.ml.api.util.DataRecordConverters._ -import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase -import com.twitter.search.common.features.{thriftscala => sc} -import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures -import com.twitter.timelines.prediction.features.recap.RecapFeatures -import com.twitter.timelines.util.UrlExtractorUtil -import java.lang.{Boolean => JBoolean} -import java.lang.{Double => JDouble} -import java.util.{Map => JMap} -import scala.collection.JavaConverters._ - -object EarlybirdAdapter extends TimelinesMutatingAdapterBase[Option[sc.ThriftTweetFeatures]] { - - override val getFeatureContext: FeatureContext = new FeatureContext( - RecapFeatures.BIDIRECTIONAL_FAV_COUNT, - RecapFeatures.BIDIRECTIONAL_REPLY_COUNT, - RecapFeatures.BIDIRECTIONAL_RETWEET_COUNT, - RecapFeatures.BLENDER_SCORE, - RecapFeatures.CONTAINS_MEDIA, - RecapFeatures.CONVERSATIONAL_COUNT, - RecapFeatures.EMBEDS_IMPRESSION_COUNT, - RecapFeatures.EMBEDS_URL_COUNT, - RecapFeatures.FAV_COUNT, - RecapFeatures.FAV_COUNT_V2, - RecapFeatures.FROM_INACTIVE_USER, - RecapFeatures.FROM_MUTUAL_FOLLOW, - RecapFeatures.FROM_VERIFIED_ACCOUNT, - RecapFeatures.HAS_CARD, - RecapFeatures.HAS_CONSUMER_VIDEO, - RecapFeatures.HAS_HASHTAG, - RecapFeatures.HAS_IMAGE, - RecapFeatures.HAS_LINK, - RecapFeatures.HAS_MENTION, - RecapFeatures.HAS_MULTIPLE_HASHTAGS_OR_TRENDS, - RecapFeatures.HAS_MULTIPLE_MEDIA, - RecapFeatures.HAS_NATIVE_IMAGE, - RecapFeatures.HAS_NATIVE_VIDEO, - RecapFeatures.HAS_NEWS, - RecapFeatures.HAS_PERISCOPE, - RecapFeatures.HAS_PRO_VIDEO, - RecapFeatures.HAS_TREND, - RecapFeatures.HAS_VIDEO, - RecapFeatures.HAS_VINE, - RecapFeatures.HAS_VISIBLE_LINK, - RecapFeatures.IS_AUTHOR_BOT, - RecapFeatures.IS_AUTHOR_NEW, - RecapFeatures.IS_AUTHOR_NSFW, - RecapFeatures.IS_AUTHOR_PROFILE_EGG, - RecapFeatures.IS_AUTHOR_SPAM, - RecapFeatures.IS_BUSINESS_SCORE, - RecapFeatures.IS_OFFENSIVE, - RecapFeatures.IS_REPLY, - RecapFeatures.IS_RETWEET, - RecapFeatures.IS_RETWEETER_BOT, - RecapFeatures.IS_RETWEETER_NEW, - RecapFeatures.IS_RETWEETER_NSFW, - RecapFeatures.IS_RETWEETER_PROFILE_EGG, - RecapFeatures.IS_RETWEETER_SPAM, - RecapFeatures.IS_RETWEET_OF_REPLY, - RecapFeatures.IS_SENSITIVE, - RecapFeatures.LANGUAGE, - RecapFeatures.LINK_COUNT, - RecapFeatures.LINK_LANGUAGE, - RecapFeatures.MATCH_SEARCHER_LANGS, - RecapFeatures.MATCH_SEARCHER_MAIN_LANG, - RecapFeatures.MATCH_UI_LANG, - RecapFeatures.MENTIONED_SCREEN_NAMES, - RecapFeatures.MENTION_SEARCHER, - RecapFeatures.NUM_HASHTAGS, - RecapFeatures.NUM_MENTIONS, - RecapFeatures.PREV_USER_TWEET_ENGAGEMENT, - RecapFeatures.PROBABLY_FROM_FOLLOWED_AUTHOR, - RecapFeatures.REPLY_COUNT, - RecapFeatures.REPLY_COUNT_V2, - RecapFeatures.REPLY_OTHER, - RecapFeatures.REPLY_SEARCHER, - RecapFeatures.RETWEET_COUNT, - RecapFeatures.RETWEET_COUNT_V2, - RecapFeatures.RETWEET_DIRECTED_AT_USER_IN_FIRST_DEGREE, - RecapFeatures.RETWEET_OF_MUTUAL_FOLLOW, - RecapFeatures.RETWEET_OTHER, - RecapFeatures.RETWEET_SEARCHER, - RecapFeatures.SIGNATURE, - RecapFeatures.SOURCE_AUTHOR_REP, - RecapFeatures.TEXT_SCORE, - RecapFeatures.TWEET_COUNT_FROM_USER_IN_SNAPSHOT, - RecapFeatures.UNIDIRECTIONAL_FAV_COUNT, - RecapFeatures.UNIDIRECTIONAL_REPLY_COUNT, - RecapFeatures.UNIDIRECTIONAL_RETWEET_COUNT, - RecapFeatures.URL_DOMAINS, - RecapFeatures.USER_REP, - RecapFeatures.VIDEO_VIEW_COUNT, - // shared features - TimelinesSharedFeatures.WEIGHTED_FAV_COUNT, - TimelinesSharedFeatures.WEIGHTED_RETWEET_COUNT, - TimelinesSharedFeatures.WEIGHTED_REPLY_COUNT, - TimelinesSharedFeatures.WEIGHTED_QUOTE_COUNT, - TimelinesSharedFeatures.EMBEDS_IMPRESSION_COUNT_V2, - TimelinesSharedFeatures.EMBEDS_URL_COUNT_V2, - TimelinesSharedFeatures.DECAYED_FAVORITE_COUNT, - TimelinesSharedFeatures.DECAYED_RETWEET_COUNT, - TimelinesSharedFeatures.DECAYED_REPLY_COUNT, - TimelinesSharedFeatures.DECAYED_QUOTE_COUNT, - TimelinesSharedFeatures.FAKE_FAVORITE_COUNT, - TimelinesSharedFeatures.FAKE_RETWEET_COUNT, - TimelinesSharedFeatures.FAKE_REPLY_COUNT, - TimelinesSharedFeatures.FAKE_QUOTE_COUNT, - TimelinesSharedFeatures.QUOTE_COUNT, - TimelinesSharedFeatures.EARLYBIRD_SCORE, - // Safety features - TimelinesSharedFeatures.LABEL_ABUSIVE_FLAG, - TimelinesSharedFeatures.LABEL_ABUSIVE_HI_RCL_FLAG, - TimelinesSharedFeatures.LABEL_DUP_CONTENT_FLAG, - TimelinesSharedFeatures.LABEL_NSFW_HI_PRC_FLAG, - TimelinesSharedFeatures.LABEL_NSFW_HI_RCL_FLAG, - TimelinesSharedFeatures.LABEL_SPAM_FLAG, - TimelinesSharedFeatures.LABEL_SPAM_HI_RCL_FLAG, - // periscope features - TimelinesSharedFeatures.PERISCOPE_EXISTS, - TimelinesSharedFeatures.PERISCOPE_IS_LIVE, - TimelinesSharedFeatures.PERISCOPE_HAS_BEEN_FEATURED, - TimelinesSharedFeatures.PERISCOPE_IS_CURRENTLY_FEATURED, - TimelinesSharedFeatures.PERISCOPE_IS_FROM_QUALITY_SOURCE, - // VISIBLE_TOKEN_RATIO - TimelinesSharedFeatures.VISIBLE_TOKEN_RATIO, - TimelinesSharedFeatures.HAS_QUOTE, - TimelinesSharedFeatures.IS_COMPOSER_SOURCE_CAMERA, - // health features - TimelinesSharedFeatures.PREPORTED_TWEET_SCORE, - // media - TimelinesSharedFeatures.CLASSIFICATION_LABELS - ) - - override val commonFeatures: Set[Feature[_]] = Set.empty - - override def setFeatures( - ebFeatures: Option[sc.ThriftTweetFeatures], - richDataRecord: RichDataRecord - ): Unit = { - if (ebFeatures.nonEmpty) { - val features = ebFeatures.get - richDataRecord.setFeatureValue[JDouble]( - RecapFeatures.PREV_USER_TWEET_ENGAGEMENT, - features.prevUserTweetEngagement.toDouble - ) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.IS_SENSITIVE, features.isSensitiveContent) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.HAS_MULTIPLE_MEDIA, features.hasMultipleMedia) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.IS_AUTHOR_PROFILE_EGG, features.isAuthorProfileEgg) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.IS_AUTHOR_NEW, features.isAuthorNew) - richDataRecord - .setFeatureValue[JDouble](RecapFeatures.NUM_MENTIONS, features.numMentions.toDouble) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_MENTION, features.numMentions > 0) - richDataRecord - .setFeatureValue[JDouble](RecapFeatures.NUM_HASHTAGS, features.numHashtags.toDouble) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_HASHTAG, features.numHashtags > 0) - richDataRecord - .setFeatureValue[JDouble](RecapFeatures.LINK_LANGUAGE, features.linkLanguage.toDouble) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.IS_AUTHOR_NSFW, features.isAuthorNSFW) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.IS_AUTHOR_SPAM, features.isAuthorSpam) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.IS_AUTHOR_BOT, features.isAuthorBot) - richDataRecord.setFeatureValueFromOption( - RecapFeatures.LANGUAGE, - features.language.map(_.getValue.toLong)) - richDataRecord.setFeatureValueFromOption( - RecapFeatures.SIGNATURE, - features.signature.map(_.toLong)) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.FROM_INACTIVE_USER, features.fromInActiveUser) - richDataRecord - .setFeatureValue[JBoolean]( - RecapFeatures.PROBABLY_FROM_FOLLOWED_AUTHOR, - features.probablyFromFollowedAuthor) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.FROM_MUTUAL_FOLLOW, features.fromMutualFollow) - richDataRecord.setFeatureValue[JBoolean]( - RecapFeatures.FROM_VERIFIED_ACCOUNT, - features.fromVerifiedAccount) - richDataRecord.setFeatureValue[JDouble](RecapFeatures.USER_REP, features.userRep) - richDataRecord - .setFeatureValue[JDouble](RecapFeatures.IS_BUSINESS_SCORE, features.isBusinessScore) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.HAS_CONSUMER_VIDEO, features.hasConsumerVideo) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_PRO_VIDEO, features.hasProVideo) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_VINE, features.hasVine) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_PERISCOPE, features.hasPeriscope) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.HAS_NATIVE_VIDEO, features.hasNativeVideo) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.HAS_NATIVE_IMAGE, features.hasNativeImage) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_CARD, features.hasCard) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_IMAGE, features.hasImage) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_NEWS, features.hasNews) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_VIDEO, features.hasVideo) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.CONTAINS_MEDIA, features.containsMedia) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.RETWEET_SEARCHER, features.retweetSearcher) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.REPLY_SEARCHER, features.replySearcher) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.MENTION_SEARCHER, features.mentionSearcher) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.REPLY_OTHER, features.replyOther) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.RETWEET_OTHER, features.retweetOther) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.IS_REPLY, features.isReply) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.IS_RETWEET, features.isRetweet) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.IS_OFFENSIVE, features.isOffensive) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.MATCH_UI_LANG, features.matchesUILang) - richDataRecord - .setFeatureValue[JBoolean]( - RecapFeatures.MATCH_SEARCHER_MAIN_LANG, - features.matchesSearcherMainLang) - richDataRecord.setFeatureValue[JBoolean]( - RecapFeatures.MATCH_SEARCHER_LANGS, - features.matchesSearcherLangs) - richDataRecord - .setFeatureValue[JDouble]( - RecapFeatures.BIDIRECTIONAL_FAV_COUNT, - features.bidirectionalFavCount) - richDataRecord - .setFeatureValue[JDouble]( - RecapFeatures.UNIDIRECTIONAL_FAV_COUNT, - features.unidirectionalFavCount) - richDataRecord - .setFeatureValue[JDouble]( - RecapFeatures.BIDIRECTIONAL_REPLY_COUNT, - features.bidirectionalReplyCount) - richDataRecord - .setFeatureValue[JDouble]( - RecapFeatures.UNIDIRECTIONAL_REPLY_COUNT, - features.unidirectionalReplyCount) - richDataRecord - .setFeatureValue[JDouble]( - RecapFeatures.BIDIRECTIONAL_RETWEET_COUNT, - features.bidirectionalRetweetCount) - richDataRecord - .setFeatureValue[JDouble]( - RecapFeatures.UNIDIRECTIONAL_RETWEET_COUNT, - features.unidirectionalRetweetCount) - richDataRecord - .setFeatureValue[JDouble](RecapFeatures.CONVERSATIONAL_COUNT, features.conversationCount) - richDataRecord.setFeatureValue[JDouble]( - RecapFeatures.TWEET_COUNT_FROM_USER_IN_SNAPSHOT, - features.tweetCountFromUserInSnapshot - ) - richDataRecord - .setFeatureValue[JBoolean]( - RecapFeatures.IS_RETWEETER_PROFILE_EGG, - features.isRetweeterProfileEgg) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.IS_RETWEETER_NEW, features.isRetweeterNew) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.IS_RETWEETER_BOT, features.isRetweeterBot) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.IS_RETWEETER_NSFW, features.isRetweeterNSFW) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.IS_RETWEETER_SPAM, features.isRetweeterSpam) - richDataRecord - .setFeatureValue[JBoolean]( - RecapFeatures.RETWEET_OF_MUTUAL_FOLLOW, - features.retweetOfMutualFollow) - richDataRecord - .setFeatureValue[JDouble](RecapFeatures.SOURCE_AUTHOR_REP, features.sourceAuthorRep) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.IS_RETWEET_OF_REPLY, features.isRetweetOfReply) - richDataRecord.setFeatureValueFromOption( - RecapFeatures.RETWEET_DIRECTED_AT_USER_IN_FIRST_DEGREE, - features.retweetDirectedAtUserInFirstDegree - ) - richDataRecord - .setFeatureValue[JDouble]( - RecapFeatures.EMBEDS_IMPRESSION_COUNT, - features.embedsImpressionCount.toDouble) - richDataRecord - .setFeatureValue[JDouble](RecapFeatures.EMBEDS_URL_COUNT, features.embedsUrlCount.toDouble) - richDataRecord - .setFeatureValue[JDouble](RecapFeatures.VIDEO_VIEW_COUNT, features.videoViewCount.toDouble) - richDataRecord - .setFeatureValue[JDouble](RecapFeatures.REPLY_COUNT, features.replyCount.toDouble) - richDataRecord - .setFeatureValue[JDouble](RecapFeatures.RETWEET_COUNT, features.retweetCount.toDouble) - richDataRecord.setFeatureValue[JDouble](RecapFeatures.FAV_COUNT, features.favCount.toDouble) - richDataRecord.setFeatureValue[JDouble](RecapFeatures.BLENDER_SCORE, features.blenderScore) - richDataRecord.setFeatureValue[JDouble](RecapFeatures.TEXT_SCORE, features.textScore) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.HAS_VISIBLE_LINK, features.hasVisibleLink) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_LINK, features.hasLink) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_TREND, features.hasTrend) - richDataRecord.setFeatureValue[JBoolean]( - RecapFeatures.HAS_MULTIPLE_HASHTAGS_OR_TRENDS, - features.hasMultipleHashtagsOrTrends - ) - richDataRecord.setFeatureValueFromOption( - RecapFeatures.FAV_COUNT_V2, - features.favCountV2.map(_.toDouble)) - richDataRecord.setFeatureValueFromOption( - RecapFeatures.RETWEET_COUNT_V2, - features.retweetCountV2.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - RecapFeatures.REPLY_COUNT_V2, - features.replyCountV2.map(_.toDouble)) - richDataRecord.setFeatureValueFromOption( - RecapFeatures.MENTIONED_SCREEN_NAMES, - features.mentionsList.map(_.toSet.asJava) - ) - val urls = features.urlsList.getOrElse(Seq.empty) - richDataRecord.setFeatureValue( - RecapFeatures.URL_DOMAINS, - urls.toSet.flatMap(UrlExtractorUtil.extractDomain).asJava) - richDataRecord.setFeatureValue[JDouble](RecapFeatures.LINK_COUNT, urls.size.toDouble) - // shared features - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.WEIGHTED_FAV_COUNT, - features.weightedFavoriteCount.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.WEIGHTED_RETWEET_COUNT, - features.weightedRetweetCount.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.WEIGHTED_REPLY_COUNT, - features.weightedReplyCount.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.WEIGHTED_QUOTE_COUNT, - features.weightedQuoteCount.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.EMBEDS_IMPRESSION_COUNT_V2, - features.embedsImpressionCountV2.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.EMBEDS_URL_COUNT_V2, - features.embedsUrlCountV2.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.DECAYED_FAVORITE_COUNT, - features.decayedFavoriteCount.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.DECAYED_RETWEET_COUNT, - features.decayedRetweetCount.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.DECAYED_REPLY_COUNT, - features.decayedReplyCount.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.DECAYED_QUOTE_COUNT, - features.decayedQuoteCount.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.FAKE_FAVORITE_COUNT, - features.fakeFavoriteCount.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.FAKE_RETWEET_COUNT, - features.fakeRetweetCount.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.FAKE_REPLY_COUNT, - features.fakeReplyCount.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.FAKE_QUOTE_COUNT, - features.fakeQuoteCount.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.QUOTE_COUNT, - features.quoteCount.map(_.toDouble) - ) - richDataRecord.setFeatureValue[JDouble]( - TimelinesSharedFeatures.EARLYBIRD_SCORE, - features.earlybirdScore - ) - // safety features - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.LABEL_ABUSIVE_FLAG, - features.labelAbusiveFlag - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.LABEL_ABUSIVE_HI_RCL_FLAG, - features.labelAbusiveHiRclFlag - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.LABEL_DUP_CONTENT_FLAG, - features.labelDupContentFlag - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.LABEL_NSFW_HI_PRC_FLAG, - features.labelNsfwHiPrcFlag - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.LABEL_NSFW_HI_RCL_FLAG, - features.labelNsfwHiRclFlag - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.LABEL_SPAM_FLAG, - features.labelSpamFlag - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.LABEL_SPAM_HI_RCL_FLAG, - features.labelSpamHiRclFlag - ) - // periscope features - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.PERISCOPE_EXISTS, - features.periscopeExists - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.PERISCOPE_IS_LIVE, - features.periscopeIsLive - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.PERISCOPE_HAS_BEEN_FEATURED, - features.periscopeHasBeenFeatured - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.PERISCOPE_IS_CURRENTLY_FEATURED, - features.periscopeIsCurrentlyFeatured - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.PERISCOPE_IS_FROM_QUALITY_SOURCE, - features.periscopeIsFromQualitySource - ) - // misc features - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.VISIBLE_TOKEN_RATIO, - features.visibleTokenRatio.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.HAS_QUOTE, - features.hasQuote - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.IS_COMPOSER_SOURCE_CAMERA, - features.isComposerSourceCamera - ) - // health scores - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.PREPORTED_TWEET_SCORE, - features.pReportedTweetScore - ) - // media - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.CLASSIFICATION_LABELS, - features.mediaClassificationInfo.map(_.toMap.asJava.asInstanceOf[JMap[String, JDouble]]) - ) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/InReplyToEarlybirdAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/InReplyToEarlybirdAdapter.scala deleted file mode 100644 index b6a7b0ad3..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/InReplyToEarlybirdAdapter.scala +++ /dev/null @@ -1,206 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.earlybird - -import com.twitter.ml.api.Feature -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.RichDataRecord -import com.twitter.ml.api.util.DataRecordConverters.RichDataRecordWrapper -import com.twitter.search.common.features.{thriftscala => sc} -import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase -import com.twitter.timelines.prediction.features.common.InReplyToTweetTimelinesSharedFeatures -import com.twitter.timelines.prediction.features.recap.InReplyToRecapFeatures -import java.lang.{Boolean => JBoolean} -import java.lang.{Double => JDouble} - -object InReplyToEarlybirdAdapter - extends TimelinesMutatingAdapterBase[Option[sc.ThriftTweetFeatures]] { - - override val getFeatureContext: FeatureContext = new FeatureContext( - // TextFeatures - InReplyToTweetTimelinesSharedFeatures.WEIGHTED_FAV_COUNT, - InReplyToTweetTimelinesSharedFeatures.WEIGHTED_RETWEET_COUNT, - InReplyToTweetTimelinesSharedFeatures.WEIGHTED_REPLY_COUNT, - InReplyToTweetTimelinesSharedFeatures.WEIGHTED_QUOTE_COUNT, - InReplyToTweetTimelinesSharedFeatures.DECAYED_FAVORITE_COUNT, - InReplyToTweetTimelinesSharedFeatures.DECAYED_RETWEET_COUNT, - InReplyToTweetTimelinesSharedFeatures.DECAYED_REPLY_COUNT, - InReplyToTweetTimelinesSharedFeatures.DECAYED_QUOTE_COUNT, - InReplyToTweetTimelinesSharedFeatures.QUOTE_COUNT, - InReplyToTweetTimelinesSharedFeatures.HAS_QUOTE, - InReplyToTweetTimelinesSharedFeatures.EARLYBIRD_SCORE, - InReplyToRecapFeatures.PREV_USER_TWEET_ENGAGEMENT, - InReplyToRecapFeatures.IS_SENSITIVE, - InReplyToRecapFeatures.IS_AUTHOR_NEW, - InReplyToRecapFeatures.NUM_MENTIONS, - InReplyToRecapFeatures.HAS_MENTION, - InReplyToRecapFeatures.HAS_HASHTAG, - InReplyToRecapFeatures.IS_AUTHOR_NSFW, - InReplyToRecapFeatures.IS_AUTHOR_SPAM, - InReplyToRecapFeatures.IS_AUTHOR_BOT, - InReplyToRecapFeatures.FROM_MUTUAL_FOLLOW, - InReplyToRecapFeatures.USER_REP, - InReplyToRecapFeatures.FROM_VERIFIED_ACCOUNT, - InReplyToRecapFeatures.HAS_IMAGE, - InReplyToRecapFeatures.HAS_NEWS, - InReplyToRecapFeatures.HAS_VIDEO, - InReplyToRecapFeatures.HAS_VISIBLE_LINK, - InReplyToRecapFeatures.IS_OFFENSIVE, - InReplyToRecapFeatures.IS_REPLY, - InReplyToRecapFeatures.BIDIRECTIONAL_REPLY_COUNT, - InReplyToRecapFeatures.UNIDIRECTIONAL_REPLY_COUNT, - InReplyToRecapFeatures.BIDIRECTIONAL_RETWEET_COUNT, - InReplyToRecapFeatures.UNIDIRECTIONAL_RETWEET_COUNT, - InReplyToRecapFeatures.BIDIRECTIONAL_FAV_COUNT, - InReplyToRecapFeatures.UNIDIRECTIONAL_FAV_COUNT, - InReplyToRecapFeatures.CONVERSATIONAL_COUNT, - InReplyToRecapFeatures.REPLY_COUNT, - InReplyToRecapFeatures.RETWEET_COUNT, - InReplyToRecapFeatures.FAV_COUNT, - InReplyToRecapFeatures.TEXT_SCORE, - InReplyToRecapFeatures.FAV_COUNT_V2, - InReplyToRecapFeatures.RETWEET_COUNT_V2, - InReplyToRecapFeatures.REPLY_COUNT_V2) - - override val commonFeatures: Set[Feature[_]] = Set.empty - - override def setFeatures( - ebFeatures: Option[sc.ThriftTweetFeatures], - richDataRecord: RichDataRecord - ): Unit = { - if (ebFeatures.nonEmpty) { - val features = ebFeatures.get - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.WEIGHTED_FAV_COUNT, - features.weightedFavoriteCount.map(_.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.WEIGHTED_RETWEET_COUNT, - features.weightedRetweetCount.map(_.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.WEIGHTED_REPLY_COUNT, - features.weightedReplyCount.map(_.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.WEIGHTED_QUOTE_COUNT, - features.weightedQuoteCount.map(_.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.DECAYED_FAVORITE_COUNT, - features.decayedFavoriteCount.map(_.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.DECAYED_RETWEET_COUNT, - features.decayedRetweetCount.map(_.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.DECAYED_REPLY_COUNT, - features.decayedReplyCount.map(_.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.DECAYED_QUOTE_COUNT, - features.decayedQuoteCount.map(_.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.QUOTE_COUNT, - features.quoteCount.map(_.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.HAS_QUOTE, - features.hasQuote - ) - - if (features.earlybirdScore > 0) - richDataRecord.setFeatureValue[JDouble]( - InReplyToTweetTimelinesSharedFeatures.EARLYBIRD_SCORE, - features.earlybirdScore - ) - - richDataRecord.setFeatureValue[JDouble]( - InReplyToRecapFeatures.PREV_USER_TWEET_ENGAGEMENT, - features.prevUserTweetEngagement.toDouble - ) - - richDataRecord - .setFeatureValue[JBoolean](InReplyToRecapFeatures.IS_SENSITIVE, features.isSensitiveContent) - richDataRecord - .setFeatureValue[JBoolean](InReplyToRecapFeatures.IS_AUTHOR_NEW, features.isAuthorNew) - richDataRecord.setFeatureValue[JDouble]( - InReplyToRecapFeatures.NUM_MENTIONS, - features.numMentions.toDouble) - richDataRecord - .setFeatureValue[JBoolean](InReplyToRecapFeatures.HAS_MENTION, (features.numMentions > 0)) - richDataRecord - .setFeatureValue[JBoolean](InReplyToRecapFeatures.HAS_HASHTAG, (features.numHashtags > 0)) - richDataRecord - .setFeatureValue[JBoolean](InReplyToRecapFeatures.IS_AUTHOR_NSFW, features.isAuthorNSFW) - richDataRecord - .setFeatureValue[JBoolean](InReplyToRecapFeatures.IS_AUTHOR_SPAM, features.isAuthorSpam) - richDataRecord - .setFeatureValue[JBoolean](InReplyToRecapFeatures.IS_AUTHOR_BOT, features.isAuthorBot) - richDataRecord.setFeatureValue[JBoolean]( - InReplyToRecapFeatures.FROM_MUTUAL_FOLLOW, - features.fromMutualFollow) - richDataRecord.setFeatureValue[JDouble](InReplyToRecapFeatures.USER_REP, features.userRep) - richDataRecord.setFeatureValue[JBoolean]( - InReplyToRecapFeatures.FROM_VERIFIED_ACCOUNT, - features.fromVerifiedAccount) - richDataRecord.setFeatureValue[JBoolean](InReplyToRecapFeatures.HAS_IMAGE, features.hasImage) - richDataRecord.setFeatureValue[JBoolean](InReplyToRecapFeatures.HAS_NEWS, features.hasNews) - richDataRecord.setFeatureValue[JBoolean](InReplyToRecapFeatures.HAS_VIDEO, features.hasVideo) - richDataRecord - .setFeatureValue[JBoolean](InReplyToRecapFeatures.HAS_VISIBLE_LINK, features.hasVisibleLink) - richDataRecord - .setFeatureValue[JBoolean](InReplyToRecapFeatures.IS_OFFENSIVE, features.isOffensive) - richDataRecord.setFeatureValue[JBoolean](InReplyToRecapFeatures.IS_REPLY, features.isReply) - richDataRecord.setFeatureValue[JDouble]( - InReplyToRecapFeatures.BIDIRECTIONAL_REPLY_COUNT, - features.bidirectionalReplyCount) - richDataRecord.setFeatureValue[JDouble]( - InReplyToRecapFeatures.UNIDIRECTIONAL_REPLY_COUNT, - features.unidirectionalReplyCount) - richDataRecord.setFeatureValue[JDouble]( - InReplyToRecapFeatures.BIDIRECTIONAL_RETWEET_COUNT, - features.bidirectionalRetweetCount) - richDataRecord.setFeatureValue[JDouble]( - InReplyToRecapFeatures.UNIDIRECTIONAL_RETWEET_COUNT, - features.unidirectionalRetweetCount) - richDataRecord.setFeatureValue[JDouble]( - InReplyToRecapFeatures.BIDIRECTIONAL_FAV_COUNT, - features.bidirectionalFavCount) - richDataRecord.setFeatureValue[JDouble]( - InReplyToRecapFeatures.UNIDIRECTIONAL_FAV_COUNT, - features.unidirectionalFavCount) - richDataRecord.setFeatureValue[JDouble]( - InReplyToRecapFeatures.CONVERSATIONAL_COUNT, - features.conversationCount) - richDataRecord - .setFeatureValue[JDouble](InReplyToRecapFeatures.REPLY_COUNT, features.replyCount.toDouble) - richDataRecord.setFeatureValue[JDouble]( - InReplyToRecapFeatures.RETWEET_COUNT, - features.retweetCount.toDouble) - richDataRecord - .setFeatureValue[JDouble](InReplyToRecapFeatures.FAV_COUNT, features.favCount.toDouble) - richDataRecord.setFeatureValue[JDouble](InReplyToRecapFeatures.TEXT_SCORE, features.textScore) - richDataRecord.setFeatureValueFromOption( - InReplyToRecapFeatures.FAV_COUNT_V2, - features.favCountV2.map(_.toDouble)) - richDataRecord.setFeatureValueFromOption( - InReplyToRecapFeatures.RETWEET_COUNT_V2, - features.retweetCountV2.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - InReplyToRecapFeatures.REPLY_COUNT_V2, - features.replyCountV2.map(_.toDouble)) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic/BUILD.bazel deleted file mode 100644 index acf19eabb..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "src/java/com/twitter/ml/api:api-base", - "src/scala/com/twitter/ml/api/util", - "src/scala/com/twitter/timelines/prediction/common/adapters:base", - "src/scala/com/twitter/timelines/prediction/features/common", - "src/thrift/com/twitter/ml/api:data-java", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic/InferredTopicAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic/InferredTopicAdapter.scala deleted file mode 100644 index 1439cc2ad..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic/InferredTopicAdapter.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.inferred_topic - -import com.twitter.ml.api.Feature -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.RichDataRecord -import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase -import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures -import scala.collection.JavaConverters._ - -object InferredTopicAdapter extends TimelinesMutatingAdapterBase[Map[Long, Double]] { - - override val getFeatureContext: FeatureContext = new FeatureContext( - TimelinesSharedFeatures.INFERRED_TOPIC_IDS) - - override val commonFeatures: Set[Feature[_]] = Set.empty - - override def setFeatures( - inferredTopicFeatures: Map[Long, Double], - richDataRecord: RichDataRecord - ): Unit = { - richDataRecord.setFeatureValue( - TimelinesSharedFeatures.INFERRED_TOPIC_IDS, - inferredTopicFeatures.keys.map(_.toString).toSet.asJava) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/BUILD.bazel deleted file mode 100644 index d1a7281c5..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/BUILD.bazel +++ /dev/null @@ -1,15 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "src/java/com/twitter/ml/api:api-base", - "src/java/com/twitter/ml/api/constant", - "src/scala/com/twitter/ml/api/util", - "src/scala/com/twitter/timelines/prediction/common/adapters:base", - "src/scala/com/twitter/timelines/prediction/features/common", - "src/thrift/com/twitter/ml/api:data-java", - "src/thrift/com/twitter/timelines/author_features:thrift-java", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/NonMLCandidateFeaturesAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/NonMLCandidateFeaturesAdapter.scala deleted file mode 100644 index 6c4b79eee..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/NonMLCandidateFeaturesAdapter.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.non_ml_features - -import com.twitter.ml.api.constant.SharedFeatures -import com.twitter.ml.api.Feature -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.RichDataRecord -import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase -import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures -import java.lang.{Long => JLong} - -case class NonMLCandidateFeatures( - tweetId: Long, - sourceTweetId: Option[Long], - originalAuthorId: Option[Long], -) - -/** - * define non ml features adapter to create a data record which includes many non ml features - * e.g. predictionRequestId, userId, tweetId to be used as joined key in batch pipeline. - */ -object NonMLCandidateFeaturesAdapter extends TimelinesMutatingAdapterBase[NonMLCandidateFeatures] { - - private val featureContext = new FeatureContext( - SharedFeatures.TWEET_ID, - // For Secondary Engagement data generation - TimelinesSharedFeatures.SOURCE_TWEET_ID, - TimelinesSharedFeatures.ORIGINAL_AUTHOR_ID, - ) - - override def getFeatureContext: FeatureContext = featureContext - - override val commonFeatures: Set[Feature[_]] = Set.empty - - override def setFeatures( - nonMLCandidateFeatures: NonMLCandidateFeatures, - richDataRecord: RichDataRecord - ): Unit = { - richDataRecord.setFeatureValue[JLong](SharedFeatures.TWEET_ID, nonMLCandidateFeatures.tweetId) - nonMLCandidateFeatures.sourceTweetId.foreach( - richDataRecord.setFeatureValue[JLong](TimelinesSharedFeatures.SOURCE_TWEET_ID, _)) - nonMLCandidateFeatures.originalAuthorId.foreach( - richDataRecord.setFeatureValue[JLong](TimelinesSharedFeatures.ORIGINAL_AUTHOR_ID, _)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/NonMLCommonFeaturesAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/NonMLCommonFeaturesAdapter.scala deleted file mode 100644 index a2777538c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/NonMLCommonFeaturesAdapter.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.non_ml_features - -import com.twitter.ml.api.constant.SharedFeatures -import com.twitter.ml.api.Feature -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.RichDataRecord -import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase -import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures -import java.lang.{Long => JLong} - -case class NonMLCommonFeatures( - userId: Long, - predictionRequestId: Option[Long], - servedTimestamp: Long, -) - -/** - * define non ml features adapter to create a data record which includes many non ml features - * e.g. predictionRequestId, userId, tweetId to be used as joined key in batch pipeline. - */ -object NonMLCommonFeaturesAdapter extends TimelinesMutatingAdapterBase[NonMLCommonFeatures] { - - private val featureContext = new FeatureContext( - SharedFeatures.USER_ID, - TimelinesSharedFeatures.PREDICTION_REQUEST_ID, - TimelinesSharedFeatures.SERVED_TIMESTAMP, - ) - - override def getFeatureContext: FeatureContext = featureContext - - override val commonFeatures: Set[Feature[_]] = Set( - SharedFeatures.USER_ID, - TimelinesSharedFeatures.PREDICTION_REQUEST_ID, - TimelinesSharedFeatures.SERVED_TIMESTAMP, - ) - - override def setFeatures( - nonMLCommonFeatures: NonMLCommonFeatures, - richDataRecord: RichDataRecord - ): Unit = { - richDataRecord.setFeatureValue[JLong](SharedFeatures.USER_ID, nonMLCommonFeatures.userId) - nonMLCommonFeatures.predictionRequestId.foreach( - richDataRecord.setFeatureValue[JLong](TimelinesSharedFeatures.PREDICTION_REQUEST_ID, _)) - richDataRecord.setFeatureValue[JLong]( - TimelinesSharedFeatures.SERVED_TIMESTAMP, - nonMLCommonFeatures.servedTimestamp) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/BUILD.bazel deleted file mode 100644 index bfff86d3f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "src/java/com/twitter/ml/api:api-base", - "src/scala/com/twitter/timelines/prediction/common/adapters:base", - "src/thrift/com/twitter/ml/api:data-java", - "timelines/data_processing/ml_util/aggregation_framework/conversion:for-timelines", - "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/PassThroughAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/PassThroughAdapter.scala deleted file mode 100644 index dce0a81db..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/PassThroughAdapter.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.offline_aggregates - -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.IRecordOneToOneAdapter - -object PassThroughAdapter extends IRecordOneToOneAdapter[Seq[DataRecord]] { - override def adaptToDataRecord(record: Seq[DataRecord]): DataRecord = - record.headOption.getOrElse(new DataRecord) - - // This is not necessary and should not be used. - override def getFeatureContext = ??? -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/SparseAggregatesToDenseAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/SparseAggregatesToDenseAdapter.scala deleted file mode 100644 index 4c54f78d8..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/SparseAggregatesToDenseAdapter.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.offline_aggregates - -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.RichDataRecord -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.conversion.CombineCountsPolicy -import com.twitter.timelines.prediction.common.adapters.TimelinesIRecordAdapter - -class SparseAggregatesToDenseAdapter(policy: CombineCountsPolicy) - extends TimelinesIRecordAdapter[Seq[DataRecord]] { - - override def setFeatures(input: Seq[DataRecord], mutableDataRecord: RichDataRecord): Unit = - policy.defaultMergeRecord(mutableDataRecord.getRecord, input.toList) - - override val getFeatureContext: FeatureContext = - new FeatureContext(policy.outputFeaturesPostMerge.toSeq: _*) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings/BUILD.bazel deleted file mode 100644 index c32c29ce5..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings/BUILD.bazel +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "src/java/com/twitter/ml/api:api-base", - "src/scala/com/twitter/ml/api:api-base", - "src/scala/com/twitter/ml/api/util", - "src/scala/com/twitter/timelines/prediction/common/adapters:base", - "src/thrift/com/twitter/ml/api:data-java", - "src/thrift/com/twitter/ml/api:data-scala", - "src/thrift/com/twitter/ml/api:embedding-java", - "src/thrift/com/twitter/ml/api:embedding-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings/TwhinEmbeddingsAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings/TwhinEmbeddingsAdapter.scala deleted file mode 100644 index c2830e462..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings/TwhinEmbeddingsAdapter.scala +++ /dev/null @@ -1,68 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.twhin_embeddings - -import com.twitter.ml.api.DataType -import com.twitter.ml.api.Feature -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.RichDataRecord -import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions -import com.twitter.ml.api.{thriftscala => ml} -import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase - -sealed trait TwhinEmbeddingsAdapter extends TimelinesMutatingAdapterBase[Option[ml.FloatTensor]] { - def twhinEmbeddingsFeature: Feature.Tensor - - override def getFeatureContext: FeatureContext = new FeatureContext( - twhinEmbeddingsFeature - ) - - override def setFeatures( - embedding: Option[ml.FloatTensor], - richDataRecord: RichDataRecord - ): Unit = { - embedding.foreach { floatTensor => - richDataRecord.setFeatureValue( - twhinEmbeddingsFeature, - ScalaToJavaDataRecordConversions.scalaTensor2Java( - ml.GeneralTensor - .FloatTensor(floatTensor))) - } - } -} - -object TwhinEmbeddingsFeatures { - val TwhinAuthorFollowEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( - "original_author.twhin.tw_hi_n.author_follow_as_float_tensor", - DataType.FLOAT - ) - - val TwhinUserEngagementEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( - "user.twhin.tw_hi_n.user_engagement_as_float_tensor", - DataType.FLOAT - ) - - val TwhinUserFollowEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( - "user.twhin.tw_hi_n.user_follow_as_float_tensor", - DataType.FLOAT - ) -} - -object TwhinAuthorFollowEmbeddingsAdapter extends TwhinEmbeddingsAdapter { - override val twhinEmbeddingsFeature: Feature.Tensor = - TwhinEmbeddingsFeatures.TwhinAuthorFollowEmbeddingsFeature - - override val commonFeatures: Set[Feature[_]] = Set.empty -} - -object TwhinUserEngagementEmbeddingsAdapter extends TwhinEmbeddingsAdapter { - override val twhinEmbeddingsFeature: Feature.Tensor = - TwhinEmbeddingsFeatures.TwhinUserEngagementEmbeddingsFeature - - override val commonFeatures: Set[Feature[_]] = Set(twhinEmbeddingsFeature) -} - -object TwhinUserFollowEmbeddingsAdapter extends TwhinEmbeddingsAdapter { - override val twhinEmbeddingsFeature: Feature.Tensor = - TwhinEmbeddingsFeatures.TwhinUserFollowEmbeddingsFeature - - override val commonFeatures: Set[Feature[_]] = Set(twhinEmbeddingsFeature) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/AggregateFeatureInfo.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/AggregateFeatureInfo.scala deleted file mode 100644 index 00dee1209..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/AggregateFeatureInfo.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates - -import com.twitter.ml.api.FeatureContext -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateType.AggregateType -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.TypedAggregateGroup -import scala.jdk.CollectionConverters.asJavaIterableConverter - -// A helper class deriving aggregate feature info from the given configuration parameters. -class AggregateFeatureInfo( - val aggregateGroups: Set[AggregateGroup], - val aggregateType: AggregateType) { - - private val typedAggregateGroups = aggregateGroups.flatMap(_.buildTypedAggregateGroups()).toList - - val featureContext: FeatureContext = - new FeatureContext( - (typedAggregateGroups.flatMap(_.allOutputFeatures) ++ - typedAggregateGroups.flatMap(_.allOutputKeys) ++ - Seq(TypedAggregateGroup.timestampFeature)).asJava) - - val feature: BaseAggregateRootFeature = - AggregateFeatureInfo.pickFeature(aggregateType) -} - -object AggregateFeatureInfo { - val features: Set[BaseAggregateRootFeature] = - Set(PartAAggregateRootFeature, PartBAggregateRootFeature) - - def pickFeature(aggregateType: AggregateType): BaseAggregateRootFeature = { - val filtered = features.filter(_.aggregateTypes.contains(aggregateType)) - require( - filtered.size == 1, - "requested AggregateType must be backed by exactly one physical store.") - filtered.head - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/AggregateFeaturesToDecodeWithMetadata.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/AggregateFeaturesToDecodeWithMetadata.scala deleted file mode 100644 index 5fb599240..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/AggregateFeaturesToDecodeWithMetadata.scala +++ /dev/null @@ -1,68 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.timelinemixer.injection.repository.uss.VersionedAggregateFeaturesDecoder -import com.twitter.ml.api.DataRecord -import com.twitter.timelines.aggregate_interactions.thriftjava.UserAggregateInteractions -import com.twitter.timelines.aggregate_interactions.v17.thriftjava.{ - UserAggregateInteractions => V17UserAggregateInteractions -} -import com.twitter.timelines.aggregate_interactions.v1.thriftjava.{ - UserAggregateInteractions => V1UserAggregateInteractions -} -import com.twitter.timelines.suggests.common.dense_data_record.thriftjava.DenseCompactDataRecord -import com.twitter.timelines.suggests.common.dense_data_record.thriftscala.DenseFeatureMetadata -import java.lang.{Long => JLong} -import java.util.Collections -import java.util.{Map => JMap} - -private[offline_aggregates] case class AggregateFeaturesToDecodeWithMetadata( - metadataOpt: Option[DenseFeatureMetadata], - aggregates: UserAggregateInteractions) { - def toDataRecord(dr: DenseCompactDataRecord): DataRecord = - VersionedAggregateFeaturesDecoder.fromJDenseCompact( - metadataOpt, - dr.versionId, - NullStatsReceiver, - s"V${dr.versionId}" - )(dr) - - def userAggregatesOpt: Option[DenseCompactDataRecord] = { - aggregates.getSetField match { - case UserAggregateInteractions._Fields.V17 => - Option(aggregates.getV17.user_aggregates) - case _ => - None - } - } - - def userAuthorAggregates = extract(_.user_author_aggregates) - def userEngagerAggregates = extract(_.user_engager_aggregates) - def userMentionAggregates = extract(_.user_mention_aggregates) - def userOriginalAuthorAggregates = extract(_.user_original_author_aggregates) - def userRequestDowAggregates = extract(_.user_request_dow_aggregates) - def userRequestHourAggregates = extract(_.user_request_hour_aggregates) - def rectweetUserSimclustersTweetAggregates = extract(_.rectweet_user_simclusters_tweet_aggregates) - def userTwitterListAggregates = extract(_.user_list_aggregates) - def userTopicAggregates = extract(_.user_topic_aggregates) - def userInferredTopicAggregates = extract(_.user_inferred_topic_aggregates) - def userMediaUnderstandingAnnotationAggregates = extract( - _.user_media_understanding_annotation_aggregates) - - private def extract[T]( - v17Fn: V17UserAggregateInteractions => JMap[JLong, DenseCompactDataRecord] - ): JMap[JLong, DenseCompactDataRecord] = { - aggregates.getSetField match { - case UserAggregateInteractions._Fields.V17 => - v17Fn(aggregates.getV17) - case _ => - Collections.emptyMap() - } - } -} - -object AggregateFeaturesToDecodeWithMetadata { - val empty = new AggregateFeaturesToDecodeWithMetadata( - None, - UserAggregateInteractions.v1(new V1UserAggregateInteractions())) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BUILD.bazel deleted file mode 100644 index 10f696409..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BUILD.bazel +++ /dev/null @@ -1,40 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-core/src/main/scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "servo/repo/src/main/scala", - "src/java/com/twitter/ml/api:api-base", - "src/scala/com/twitter/timelines/prediction/adapters/request_context", - "src/scala/com/twitter/timelines/prediction/common/adapters:base", - "src/scala/com/twitter/timelines/prediction/common/aggregates", - "src/scala/com/twitter/timelines/util", - "src/thrift/com/twitter/ml/api:data-java", - "src/thrift/com/twitter/timelines/suggests/common:data_record_metadata-scala", - "src/thrift/com/twitter/timelines/suggests/common:dense_data_record-scala", - "src/thrift/com/twitter/tweetypie:service-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - "src/thrift/com/twitter/user_session_store:thrift-java", - "stitch/stitch-core", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/repository/uss:versioned-aggregate-features-decoder", - "timelines/data_processing/jobs/timeline_ranking_user_features:mini", - "timelines/data_processing/ml_util/aggregation_framework:common_types", - "timelines/data_processing/ml_util/aggregation_framework/conversion:for-timelines", - "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", - "user_session_store/src/main/scala/com/twitter/user_session_store", - "util/util-core", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BaseAggregateQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BaseAggregateQueryFeatureHydrator.scala deleted file mode 100644 index c81ed5054..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BaseAggregateQueryFeatureHydrator.scala +++ /dev/null @@ -1,76 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates - -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.repository.Repository -import com.twitter.stitch.Stitch -import com.twitter.timelines.aggregate_interactions.thriftjava.UserAggregateInteractions -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateType.AggregateType -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.StoreConfig -import com.twitter.timelines.suggests.common.dense_data_record.thriftscala.DenseFeatureMetadata -import com.twitter.user_session_store.thriftjava.UserSession -import com.twitter.util.Future - -abstract class BaseAggregateQueryFeatureHydrator( - featureRepository: Repository[Long, Option[UserSession]], - metadataRepository: Repository[Int, Option[DenseFeatureMetadata]], - feature: Feature[PipelineQuery, Option[AggregateFeaturesToDecodeWithMetadata]]) - extends QueryFeatureHydrator[PipelineQuery] { - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - val viewerId = query.getRequiredUserId - - Stitch.callFuture( - featureRepository(viewerId) - .flatMap { userSession: Option[UserSession] => - val featuresWithMetadata: Option[Future[AggregateFeaturesToDecodeWithMetadata]] = - userSession - .flatMap(decodeUserSession(_)) - - featuresWithMetadata - .map { fu: Future[AggregateFeaturesToDecodeWithMetadata] => fu.map(Some(_)) } - .getOrElse(Future.None) - .map { value => - FeatureMapBuilder() - .add(feature, value) - .build() - } - } - ) - } - - private def decodeUserSession( - session: UserSession - ): Option[Future[AggregateFeaturesToDecodeWithMetadata]] = { - Option(session.user_aggregate_interactions).flatMap { aggregates => - aggregates.getSetField match { - case UserAggregateInteractions._Fields.V17 => - Some( - getAggregateFeaturesWithMetadata( - aggregates.getV17.user_aggregates.versionId, - UserAggregateInteractions.v17(aggregates.getV17)) - ) - case _ => - None - } - } - } - - private def getAggregateFeaturesWithMetadata( - versionId: Int, - userAggregateInteractions: UserAggregateInteractions, - ): Future[AggregateFeaturesToDecodeWithMetadata] = { - metadataRepository(versionId) - .map(AggregateFeaturesToDecodeWithMetadata(_, userAggregateInteractions)) - } -} - -trait BaseAggregateRootFeature - extends Feature[PipelineQuery, Option[AggregateFeaturesToDecodeWithMetadata]] { - def aggregateStores: Set[StoreConfig[_]] - - lazy val aggregateTypes: Set[AggregateType] = aggregateStores.map(_.aggregateType) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BaseEdgeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BaseEdgeAggregateFeatureHydrator.scala deleted file mode 100644 index 43cfa967b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BaseEdgeAggregateFeatureHydrator.scala +++ /dev/null @@ -1,93 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates - -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.IRecordOneToOneAdapter -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateType.AggregateType -import com.twitter.timelines.suggests.common.dense_data_record.thriftjava.DenseCompactDataRecord -import java.lang.{Long => JLong} -import java.util.{Map => JMap} - -abstract case class BaseEdgeAggregateFeature( - aggregateGroups: Set[AggregateGroup], - aggregateType: AggregateType, - extractMapFn: AggregateFeaturesToDecodeWithMetadata => JMap[JLong, DenseCompactDataRecord], - adapter: IRecordOneToOneAdapter[Seq[DataRecord]], - getSecondaryKeysFn: CandidateWithFeatures[TweetCandidate] => Seq[Long]) - extends DataRecordInAFeature[PipelineQuery] - with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { - override def defaultValue: DataRecord = new DataRecord - - private val rootFeatureInfo = new AggregateFeatureInfo(aggregateGroups, aggregateType) - val featureContext: FeatureContext = rootFeatureInfo.featureContext - val rootFeature: BaseAggregateRootFeature = rootFeatureInfo.feature -} - -trait BaseEdgeAggregateFeatureHydrator - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - def aggregateFeatures: Set[BaseEdgeAggregateFeature] - - override def features = aggregateFeatures.asInstanceOf[Set[Feature[_, _]]] - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { - val featureMapBuilders: Seq[FeatureMapBuilder] = - for (_ <- candidates) yield FeatureMapBuilder() - - aggregateFeatures.foreach { feature => - val featureValues = hydrateAggregateFeature(query, candidates, feature) - (featureMapBuilders zip featureValues).foreach { - case (featureMapBuilder, featureValue) => featureMapBuilder.add(feature, featureValue) - } - } - - featureMapBuilders.map(_.build()) - } - - private def hydrateAggregateFeature( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]], - feature: BaseEdgeAggregateFeature - ): Seq[DataRecord] = { - val rootFeature = feature.rootFeature - val extractMapFn = feature.extractMapFn - val featureContext = feature.featureContext - val secondaryIds: Seq[Seq[Long]] = candidates.map(feature.getSecondaryKeysFn) - - val featuresToDecodeWithMetadata = query.features - .flatMap(_.getOrElse(rootFeature, None)) - .getOrElse(AggregateFeaturesToDecodeWithMetadata.empty) - - // Decode the DenseCompactDataRecords into DataRecords for each required secondary id. - val decoded: Map[Long, DataRecord] = Utils.selectAndTransform( - secondaryIds.flatten.distinct, - featuresToDecodeWithMetadata.toDataRecord, - extractMapFn(featuresToDecodeWithMetadata) - ) - - // Remove unnecessary features in-place. This is safe because the underlying DataRecords - // are unique and have just been generated in the previous step. - decoded.values.foreach(Utils.filterDataRecord(_, featureContext)) - - // Put features into the FeatureMapBuilders - secondaryIds.map { ids => - val dataRecords = ids.flatMap(decoded.get) - feature.adapter.adaptToDataRecord(dataRecords) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/EdgeAggregateFeatures.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/EdgeAggregateFeatures.scala deleted file mode 100644 index f769bb980..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/EdgeAggregateFeatures.scala +++ /dev/null @@ -1,118 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates - -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.offline_aggregates.PassThroughAdapter -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.offline_aggregates.SparseAggregatesToDenseAdapter -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.MentionScreenNameFeature -import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TSPInferredTopicFeature -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateType -import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig -import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig.CombineCountPolicies - -object EdgeAggregateFeatures { - - object UserAuthorAggregateFeature - extends BaseEdgeAggregateFeature( - aggregateGroups = TimelinesAggregationConfig.userAuthorAggregatesV2 ++ Set( - TimelinesAggregationConfig.userAuthorAggregatesV5, - TimelinesAggregationConfig.tweetSourceUserAuthorAggregatesV1, - TimelinesAggregationConfig.twitterWideUserAuthorAggregates - ), - aggregateType = AggregateType.UserAuthor, - extractMapFn = _.userAuthorAggregates, - adapter = PassThroughAdapter, - getSecondaryKeysFn = _.features.getOrElse(AuthorIdFeature, None).toSeq - ) - - object UserOriginalAuthorAggregateFeature - extends BaseEdgeAggregateFeature( - aggregateGroups = Set(TimelinesAggregationConfig.userOriginalAuthorAggregatesV1), - aggregateType = AggregateType.UserOriginalAuthor, - extractMapFn = _.userOriginalAuthorAggregates, - adapter = PassThroughAdapter, - getSecondaryKeysFn = candidate => - CandidatesUtil.getOriginalAuthorId(candidate.features).toSeq - ) - - object UserTopicAggregateFeature - extends BaseEdgeAggregateFeature( - aggregateGroups = Set( - TimelinesAggregationConfig.userTopicAggregates, - TimelinesAggregationConfig.userTopicAggregatesV2, - ), - aggregateType = AggregateType.UserTopic, - extractMapFn = _.userTopicAggregates, - adapter = PassThroughAdapter, - getSecondaryKeysFn = candidate => - candidate.features.getOrElse(TopicIdSocialContextFeature, None).toSeq - ) - - object UserMentionAggregateFeature - extends BaseEdgeAggregateFeature( - aggregateGroups = Set(TimelinesAggregationConfig.userMentionAggregates), - aggregateType = AggregateType.UserMention, - extractMapFn = _.userMentionAggregates, - adapter = new SparseAggregatesToDenseAdapter(CombineCountPolicies.MentionCountsPolicy), - getSecondaryKeysFn = candidate => - candidate.features.getOrElse(MentionScreenNameFeature, Seq.empty).map(_.hashCode.toLong) - ) - - object UserInferredTopicAggregateFeature - extends BaseEdgeAggregateFeature( - aggregateGroups = Set( - TimelinesAggregationConfig.userInferredTopicAggregates, - ), - aggregateType = AggregateType.UserInferredTopic, - extractMapFn = _.userInferredTopicAggregates, - adapter = new SparseAggregatesToDenseAdapter( - CombineCountPolicies.UserInferredTopicCountsPolicy), - getSecondaryKeysFn = candidate => - candidate.features.getOrElse(TSPInferredTopicFeature, Map.empty[Long, Double]).keys.toSeq - ) - - object UserInferredTopicAggregateV2Feature - extends BaseEdgeAggregateFeature( - aggregateGroups = Set( - TimelinesAggregationConfig.userInferredTopicAggregatesV2 - ), - aggregateType = AggregateType.UserInferredTopic, - extractMapFn = _.userInferredTopicAggregates, - adapter = new SparseAggregatesToDenseAdapter( - CombineCountPolicies.UserInferredTopicV2CountsPolicy), - getSecondaryKeysFn = candidate => - candidate.features.getOrElse(TSPInferredTopicFeature, Map.empty[Long, Double]).keys.toSeq - ) - - object UserMediaUnderstandingAnnotationAggregateFeature - extends BaseEdgeAggregateFeature( - aggregateGroups = Set( - TimelinesAggregationConfig.userMediaUnderstandingAnnotationAggregates), - aggregateType = AggregateType.UserMediaUnderstandingAnnotation, - extractMapFn = _.userMediaUnderstandingAnnotationAggregates, - adapter = new SparseAggregatesToDenseAdapter( - CombineCountPolicies.UserMediaUnderstandingAnnotationCountsPolicy), - getSecondaryKeysFn = candidate => - CandidatesUtil.getMediaUnderstandingAnnotationIds(candidate.features) - ) - - object UserEngagerAggregateFeature - extends BaseEdgeAggregateFeature( - aggregateGroups = Set(TimelinesAggregationConfig.userEngagerAggregates), - aggregateType = AggregateType.UserEngager, - extractMapFn = _.userEngagerAggregates, - adapter = new SparseAggregatesToDenseAdapter(CombineCountPolicies.EngagerCountsPolicy), - getSecondaryKeysFn = candidate => CandidatesUtil.getEngagerUserIds(candidate.features) - ) - - object UserEngagerGoodClickAggregateFeature - extends BaseEdgeAggregateFeature( - aggregateGroups = Set(TimelinesAggregationConfig.userEngagerGoodClickAggregates), - aggregateType = AggregateType.UserEngager, - extractMapFn = _.userEngagerAggregates, - adapter = new SparseAggregatesToDenseAdapter( - CombineCountPolicies.EngagerGoodClickCountsPolicy), - getSecondaryKeysFn = candidate => CandidatesUtil.getEngagerUserIds(candidate.features) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/PartAAggregateQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/PartAAggregateQueryFeatureHydrator.scala deleted file mode 100644 index f18cceaca..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/PartAAggregateQueryFeatureHydrator.scala +++ /dev/null @@ -1,35 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates - -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelineAggregateMetadataRepository -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelineAggregatePartARepository -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.servo.repository.Repository -import com.twitter.timelines.data_processing.jobs.timeline_ranking_user_features.TimelinesPartAStoreRegister -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.StoreConfig -import com.twitter.timelines.suggests.common.dense_data_record.thriftscala.DenseFeatureMetadata -import com.twitter.user_session_store.thriftjava.UserSession -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -object PartAAggregateRootFeature extends BaseAggregateRootFeature { - override val aggregateStores: Set[StoreConfig[_]] = TimelinesPartAStoreRegister.allStores -} - -@Singleton -class PartAAggregateQueryFeatureHydrator @Inject() ( - @Named(TimelineAggregatePartARepository) - repository: Repository[Long, Option[UserSession]], - @Named(TimelineAggregateMetadataRepository) - metadataRepository: Repository[Int, Option[DenseFeatureMetadata]]) - extends BaseAggregateQueryFeatureHydrator( - repository, - metadataRepository, - PartAAggregateRootFeature - ) { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("PartAAggregateQuery") - - override val features = Set(PartAAggregateRootFeature) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/PartBAggregateQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/PartBAggregateQueryFeatureHydrator.scala deleted file mode 100644 index fa0336165..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/PartBAggregateQueryFeatureHydrator.scala +++ /dev/null @@ -1,144 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates - -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelineAggregateMetadataRepository -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelineAggregatePartBRepository -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.DataRecordMerger -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.RichDataRecord -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.repository.Repository -import com.twitter.stitch.Stitch -import com.twitter.timelines.data_processing.jobs.timeline_ranking_user_features.TimelinesPartBStoreRegister -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateType -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.StoreConfig -import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter -import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig -import com.twitter.timelines.suggests.common.dense_data_record.thriftscala.DenseFeatureMetadata -import com.twitter.user_session_store.thriftjava.UserSession -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -object PartBAggregateRootFeature extends BaseAggregateRootFeature { - override val aggregateStores: Set[StoreConfig[_]] = TimelinesPartBStoreRegister.allStores -} - -object UserAggregateFeature - extends DataRecordInAFeature[PipelineQuery] - with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class PartBAggregateQueryFeatureHydrator @Inject() ( - @Named(TimelineAggregatePartBRepository) - repository: Repository[Long, Option[UserSession]], - @Named(TimelineAggregateMetadataRepository) - metadataRepository: Repository[Int, Option[DenseFeatureMetadata]]) - extends BaseAggregateQueryFeatureHydrator( - repository, - metadataRepository, - PartBAggregateRootFeature - ) { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("PartBAggregateQuery") - - override val features: Set[Feature[_, _]] = - Set(PartBAggregateRootFeature, UserAggregateFeature) - - private val userAggregateFeatureInfo = new AggregateFeatureInfo( - aggregateGroups = Set( - TimelinesAggregationConfig.userAggregatesV2, - TimelinesAggregationConfig.userAggregatesV5Continuous, - TimelinesAggregationConfig.userAggregatesV6, - TimelinesAggregationConfig.twitterWideUserAggregates, - ), - aggregateType = AggregateType.User - ) - - private val userHourAggregateFeatureInfo = new AggregateFeatureInfo( - aggregateGroups = Set( - TimelinesAggregationConfig.userRequestHourAggregates, - ), - aggregateType = AggregateType.UserRequestHour - ) - - private val userDowAggregateFeatureInfo = new AggregateFeatureInfo( - aggregateGroups = Set( - TimelinesAggregationConfig.userRequestDowAggregates - ), - aggregateType = AggregateType.UserRequestDow - ) - - require( - userAggregateFeatureInfo.feature == PartBAggregateRootFeature, - "UserAggregates feature must be provided by the PartB data source.") - require( - userHourAggregateFeatureInfo.feature == PartBAggregateRootFeature, - "UserRequstHourAggregates feature must be provided by the PartB data source.") - require( - userDowAggregateFeatureInfo.feature == PartBAggregateRootFeature, - "UserRequestDowAggregates feature must be provided by the PartB data source.") - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - // Hydrate TimelineAggregatePartBFeature and UserAggregateFeature sequentially. - super.hydrate(query).map { featureMap => - val time: Time = Time.now - val hourOfDay = RequestContextAdapter.hourFromTimestamp(time.inMilliseconds) - val dayOfWeek = RequestContextAdapter.dowFromTimestamp(time.inMilliseconds) - - val dr = featureMap - .get(PartBAggregateRootFeature).map { featuresWithMetadata => - val userAggregatesDr = - featuresWithMetadata.userAggregatesOpt - .map(featuresWithMetadata.toDataRecord) - val userRequestHourAggregatesDr = - Option(featuresWithMetadata.userRequestHourAggregates.get(hourOfDay)) - .map(featuresWithMetadata.toDataRecord) - val userRequestDowAggregatesDr = - Option(featuresWithMetadata.userRequestDowAggregates.get(dayOfWeek)) - .map(featuresWithMetadata.toDataRecord) - - dropUnknownFeatures(userAggregatesDr, userAggregateFeatureInfo.featureContext) - - dropUnknownFeatures( - userRequestHourAggregatesDr, - userHourAggregateFeatureInfo.featureContext) - - dropUnknownFeatures( - userRequestDowAggregatesDr, - userDowAggregateFeatureInfo.featureContext) - - mergeDataRecordOpts( - userAggregatesDr, - userRequestHourAggregatesDr, - userRequestDowAggregatesDr) - - }.getOrElse(new DataRecord()) - - featureMap + (UserAggregateFeature, dr) - } - } - - private val drMerger = new DataRecordMerger - private def mergeDataRecordOpts(dataRecordOpts: Option[DataRecord]*): DataRecord = - dataRecordOpts.flatten.foldLeft(new DataRecord) { (l, r) => - drMerger.merge(l, r) - l - } - - private def dropUnknownFeatures( - dataRecordOpt: Option[DataRecord], - featureContext: FeatureContext - ): Unit = - dataRecordOpt.foreach(new RichDataRecord(_, featureContext).dropUnknownFeatures()) - -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Phase1EdgeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Phase1EdgeAggregateFeatureHydrator.scala deleted file mode 100644 index d9df51d5c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Phase1EdgeAggregateFeatureHydrator.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates - -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.EdgeAggregateFeatures._ -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class Phase1EdgeAggregateFeatureHydrator @Inject() extends BaseEdgeAggregateFeatureHydrator { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("Phase1EdgeAggregate") - - override val aggregateFeatures: Set[BaseEdgeAggregateFeature] = Set( - UserAuthorAggregateFeature, - UserOriginalAuthorAggregateFeature, - UserMentionAggregateFeature - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Phase2EdgeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Phase2EdgeAggregateFeatureHydrator.scala deleted file mode 100644 index cfbb0b2c9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Phase2EdgeAggregateFeatureHydrator.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates - -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserEngagerAggregateFeature -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserEngagerGoodClickAggregateFeature -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserInferredTopicAggregateFeature -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserInferredTopicAggregateV2Feature -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserMediaUnderstandingAnnotationAggregateFeature -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserTopicAggregateFeature -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class Phase2EdgeAggregateFeatureHydrator @Inject() extends BaseEdgeAggregateFeatureHydrator { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("Phase2EdgeAggregate") - - override val aggregateFeatures: Set[BaseEdgeAggregateFeature] = - Set( - UserEngagerAggregateFeature, - UserEngagerGoodClickAggregateFeature, - UserInferredTopicAggregateFeature, - UserInferredTopicAggregateV2Feature, - UserTopicAggregateFeature, - UserMediaUnderstandingAnnotationAggregateFeature - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Utils.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Utils.scala deleted file mode 100644 index 6ae1a0f4b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Utils.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates - -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.RichDataRecord -import com.twitter.timelines.suggests.common.dense_data_record.thriftjava.DenseCompactDataRecord - -private[offline_aggregates] object Utils { - - /** - * Selects only those values in map that correspond to the keys in ids and apply the provided - * transform to the selected values. This is a convenience method for use by Timelines Aggregation - * Framework based features. - * - * @param idsToSelect The set of ids to extract values for. - * @param transform A transform to apply to the selected values. - * @param map Map[Long, DenseCompactDataRecord] - */ - def selectAndTransform( - idsToSelect: Seq[Long], - transform: DenseCompactDataRecord => DataRecord, - map: java.util.Map[java.lang.Long, DenseCompactDataRecord], - ): Map[Long, DataRecord] = { - val filtered: Seq[(Long, DataRecord)] = - for { - id <- idsToSelect if map.containsKey(id) - } yield { - id -> transform(map.get(id)) - } - filtered.toMap - } - - def filterDataRecord(dr: DataRecord, featureContext: FeatureContext): Unit = { - new RichDataRecord(dr, featureContext).dropUnknownFeatures() - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BUILD.bazel deleted file mode 100644 index 1954dd77d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BUILD.bazel +++ /dev/null @@ -1,26 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-core/src/main/scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "servo/repo/src/main/scala", - "src/java/com/twitter/ml/api:api-base", - "src/java/com/twitter/ml/api/constant", - "src/scala/com/twitter/ml/api/util", - "src/scala/com/twitter/timelines/prediction/common/aggregates/real_time:base-config", - "src/thrift/com/twitter/ml/api:data-java", - "src/thrift/com/twitter/wtf/real_time_interaction_graph:wtf-real_time_interaction_graph-thrift-java", - "stitch/stitch-core", - "timelines/data_processing/ml_util/aggregation_framework:common_types", - "timelines/data_processing/ml_util/aggregation_framework/heron", - "util/util-core", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateBulkCandidateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateBulkCandidateFeatureHydrator.scala deleted file mode 100644 index 41b565ded..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateBulkCandidateFeatureHydrator.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates - -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch - -trait BaseRealTimeAggregateBulkCandidateFeatureHydrator[K] - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] - with BaseRealtimeAggregateHydrator[K] { - - val outputFeature: DataRecordInAFeature[TweetCandidate] - - override def features: Set[Feature[_, _]] = Set(outputFeature) - - override lazy val statScope: String = identifier.toString - - def keysFromQueryAndCandidates( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Seq[Option[K]] - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { - val possiblyKeys = keysFromQueryAndCandidates(query, candidates) - fetchAndConstructDataRecords(possiblyKeys).map { dataRecords => - dataRecords.map { dataRecord => - FeatureMapBuilder().add(outputFeature, dataRecord).build() - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateQueryFeatureHydrator.scala deleted file mode 100644 index 8f5b17d64..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateQueryFeatureHydrator.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates - -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch - -trait BaseRealTimeAggregateQueryFeatureHydrator[K] - extends QueryFeatureHydrator[PipelineQuery] - with BaseRealtimeAggregateHydrator[K] { - - val outputFeature: DataRecordInAFeature[PipelineQuery] - - override def features: Set[Feature[_, _]] = Set(outputFeature) - - override lazy val statScope: String = identifier.toString - - def keysFromQueryAndCandidates( - query: PipelineQuery - ): Option[K] - - override def hydrate( - query: PipelineQuery - ): Stitch[FeatureMap] = OffloadFuturePools.offloadFuture { - val possiblyKeys = keysFromQueryAndCandidates(query) - fetchAndConstructDataRecords(Seq(possiblyKeys)).map { dataRecords => - FeatureMapBuilder() - .add(outputFeature, dataRecords.head) - .build() - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealtimeAggregateHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealtimeAggregateHydrator.scala deleted file mode 100644 index f97820be0..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealtimeAggregateHydrator.scala +++ /dev/null @@ -1,138 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates - -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates.BaseRealtimeAggregateHydrator._ -import com.twitter.home_mixer.util.DataRecordUtil -import com.twitter.home_mixer.util.ObservedKeyValueResultHandler -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.DataRecordMerger -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.constant.SharedFeatures -import com.twitter.ml.api.util.SRichDataRecord -import com.twitter.ml.api.{Feature => MLApiFeature} -import com.twitter.servo.cache.ReadCache -import com.twitter.servo.keyvalue.KeyValueResult -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup -import com.twitter.util.Future -import com.twitter.util.Time -import com.twitter.util.Try -import java.lang.{Double => JDouble} -import scala.collection.JavaConverters._ - -trait BaseRealtimeAggregateHydrator[K] extends ObservedKeyValueResultHandler { - - val client: ReadCache[K, DataRecord] - - val aggregateGroups: Seq[AggregateGroup] - - val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map.empty - - private lazy val typedAggregateGroupsList = aggregateGroups.map(_.buildTypedAggregateGroups()) - - private lazy val featureContexts: Seq[FeatureContext] = typedAggregateGroupsList.map { - typedAggregateGroups => - new FeatureContext( - (SharedFeatures.TIMESTAMP +: typedAggregateGroups.flatMap(_.allOutputFeatures)).asJava - ) - } - - private lazy val aggregateFeaturesRenameMap: Map[MLApiFeature[_], MLApiFeature[_]] = { - val prefixes: Seq[Option[String]] = aggregateGroups.map(aggregateGroupToPrefix.get) - - typedAggregateGroupsList - .zip(prefixes).map { - case (typedAggregateGroups, prefix) => - if (prefix.nonEmpty) - typedAggregateGroups - .map { - _.outputFeaturesToRenamedOutputFeatures(prefix.get) - }.reduce(_ ++ _) - else - Map.empty[MLApiFeature[_], MLApiFeature[_]] - }.reduce(_ ++ _) - } - - private lazy val renamedFeatureContexts: Seq[FeatureContext] = - typedAggregateGroupsList.map { typedAggregateGroups => - val renamedAllOutputFeatures = typedAggregateGroups.flatMap(_.allOutputFeatures).map { - feature => aggregateFeaturesRenameMap.getOrElse(feature, feature) - } - - new FeatureContext(renamedAllOutputFeatures.asJava) - } - - private lazy val decays: Seq[TimeDecay] = typedAggregateGroupsList.map { typedAggregateGroups => - RealTimeAggregateTimeDecay( - typedAggregateGroups.flatMap(_.continuousFeatureIdsToHalfLives).toMap) - .apply(_, _) - } - - private val drMerger = new DataRecordMerger - - private def postTransformer(dataRecord: Try[Option[DataRecord]]): Try[DataRecord] = { - dataRecord.map { - case Some(dr) => - val newDr = new DataRecord() - featureContexts.zip(renamedFeatureContexts).zip(decays).foreach { - case ((featureContext, renamedFeatureContext), decay) => - val decayedDr = applyDecay(dr, featureContext, decay) - val renamedDr = DataRecordUtil.applyRename( - dataRecord = decayedDr, - featureContext, - renamedFeatureContext, - aggregateFeaturesRenameMap) - drMerger.merge(newDr, renamedDr) - } - newDr - case _ => new DataRecord - } - } - - def fetchAndConstructDataRecords(possiblyKeys: Seq[Option[K]]): Future[Seq[Try[DataRecord]]] = { - val keys = possiblyKeys.flatten - - val response: Future[KeyValueResult[K, DataRecord]] = - if (keys.isEmpty) Future.value(KeyValueResult.empty) - else { - val batchResponses = keys - .grouped(RequestBatchSize) - .map(keyGroup => client.get(keyGroup)) - .toSeq - - Future.collect(batchResponses).map(_.reduce(_ ++ _)) - } - - response.map { result => - possiblyKeys.map { possiblyKey => - val value = observedGet(key = possiblyKey, keyValueResult = result) - postTransformer(value) - } - } - } -} - -object BaseRealtimeAggregateHydrator { - private val RequestBatchSize = 5 - - type TimeDecay = scala.Function2[com.twitter.ml.api.DataRecord, scala.Long, scala.Unit] - - private def applyDecay( - dataRecord: DataRecord, - featureContext: FeatureContext, - decay: TimeDecay - ): DataRecord = { - def time: Long = Time.now.inMillis - - val richFullDr = new SRichDataRecord(dataRecord, featureContext) - val richNewDr = new SRichDataRecord(new DataRecord, featureContext) - val featureIterator = featureContext.iterator() - featureIterator.forEachRemaining { feature => - if (richFullDr.hasFeature(feature)) { - val typedFeature = feature.asInstanceOf[MLApiFeature[JDouble]] - richNewDr.setFeatureValue(typedFeature, richFullDr.getFeatureValue(typedFeature)) - } - } - val resultDr = richNewDr.getRecord - decay(resultDr, time) - resultDr - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator.scala deleted file mode 100644 index 990fe1ca4..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates - -import com.google.inject.name.Named -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.EngagementsReceivedByAuthorCache -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.cache.ReadCache -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup -import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ -import javax.inject.Inject -import javax.inject.Singleton - -object EngagementsReceivedByAuthorRealTimeAggregateFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator @Inject() ( - @Named(EngagementsReceivedByAuthorCache) override val client: ReadCache[Long, DataRecord], - override val statsReceiver: StatsReceiver) - extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[Long] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("EngagementsReceivedByAuthorRealTimeAggregate") - - override val outputFeature: DataRecordInAFeature[TweetCandidate] = - EngagementsReceivedByAuthorRealTimeAggregateFeature - - override val aggregateGroups: Seq[AggregateGroup] = Seq( - authorEngagementRealTimeAggregatesProd, - authorShareEngagementsRealTimeAggregates - ) - - override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( - authorShareEngagementsRealTimeAggregates -> "original_author.timelines.author_share_engagements_real_time_aggregates." - ) - - override def keysFromQueryAndCandidates( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Seq[Option[Long]] = - candidates.map(candidate => CandidatesUtil.getOriginalAuthorId(candidate.features)) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/RealTimeAggregateTimeDecay.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/RealTimeAggregateTimeDecay.scala deleted file mode 100644 index dbecd12c6..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/RealTimeAggregateTimeDecay.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates - -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.constant.SharedFeatures.TIMESTAMP -import com.twitter.util.Duration - -/** - * The default TimeDecay implementation for real time aggregates. - * - * @param featureIdToHalfLife A precomputed map from aggregate feature ids to their half lives. - * @param timestampFeatureId A discrete timestamp feature id. - */ -case class RealTimeAggregateTimeDecay( - featureIdToHalfLife: Map[Long, Duration], - timestampFeatureId: Long = TIMESTAMP.getFeatureId) { - - /** - * Mutates the data record which is just a reference to the input. - * - * @param record Data record to apply decay to (is mutated). - * @param timeNow The current read time (in milliseconds) to decay counts forward to. - */ - def apply(record: DataRecord, timeNow: Long): Unit = { - if (record.isSetDiscreteFeatures) { - val discreteFeatures = record.getDiscreteFeatures - if (discreteFeatures.containsKey(timestampFeatureId)) { - if (record.isSetContinuousFeatures) { - val ctsFeatures = record.getContinuousFeatures - - val storedTimestamp: Long = discreteFeatures.get(timestampFeatureId) - val scaledDt = if (timeNow > storedTimestamp) { - (timeNow - storedTimestamp).toDouble * math.log(2) - } else 0.0 - featureIdToHalfLife.foreach { - case (featureId, halfLife) => - if (ctsFeatures.containsKey(featureId)) { - val storedValue = ctsFeatures.get(featureId) - val alpha = - if (halfLife.inMilliseconds != 0) math.exp(-scaledDt / halfLife.inMilliseconds) - else 0 - val decayedValue: Double = alpha * storedValue - record.putToContinuousFeatures(featureId, decayedValue) - } - } - } - discreteFeatures.remove(timestampFeatureId) - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TopicCountryEngagementRealTimeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TopicCountryEngagementRealTimeAggregateFeatureHydrator.scala deleted file mode 100644 index 13b2a0d2d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TopicCountryEngagementRealTimeAggregateFeatureHydrator.scala +++ /dev/null @@ -1,64 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates - -import com.google.inject.name.Named -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TopicCountryEngagementCache -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.cache.ReadCache -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup -import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ -import javax.inject.Inject -import javax.inject.Singleton - -object TopicCountryEngagementRealTimeAggregateFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class TopicCountryEngagementRealTimeAggregateFeatureHydrator @Inject() ( - @Named(TopicCountryEngagementCache) override val client: ReadCache[(Long, String), DataRecord], - override val statsReceiver: StatsReceiver) - extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[(Long, String)] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("TopicCountryEngagementRealTimeAggregate") - - override val outputFeature: DataRecordInAFeature[TweetCandidate] = - TopicCountryEngagementRealTimeAggregateFeature - - override val aggregateGroups: Seq[AggregateGroup] = Seq( - topicCountryRealTimeAggregates - ) - - override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( - topicCountryRealTimeAggregates -> "topic-country_code.timelines.topic_country_engagement_real_time_aggregates." - ) - - override def keysFromQueryAndCandidates( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Seq[Option[(Long, String)]] = { - candidates.map { candidate => - val maybeTopicId = candidate.features - .getTry(TopicIdSocialContextFeature) - .toOption - .flatten - - val maybeCountryCode = query.clientContext.countryCode - - for { - topicId <- maybeTopicId - countryCode <- maybeCountryCode - } yield (topicId, countryCode) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TopicEngagementRealTimeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TopicEngagementRealTimeAggregateFeatureHydrator.scala deleted file mode 100644 index 12a2e1c47..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TopicEngagementRealTimeAggregateFeatureHydrator.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TopicEngagementCache -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.cache.ReadCache -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup -import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -object TopicEngagementRealTimeAggregateFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class TopicEngagementRealTimeAggregateFeatureHydrator @Inject() ( - @Named(TopicEngagementCache) override val client: ReadCache[Long, DataRecord], - override val statsReceiver: StatsReceiver) - extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[Long] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("TopicEngagementRealTimeAggregate") - - override val outputFeature: DataRecordInAFeature[TweetCandidate] = - TopicEngagementRealTimeAggregateFeature - - override val aggregateGroups: Seq[AggregateGroup] = Seq( - topicEngagementRealTimeAggregatesProd, - topicEngagement24HourRealTimeAggregatesProd, - topicShareEngagementsRealTimeAggregates - ) - - override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( - topicEngagement24HourRealTimeAggregatesProd -> "topic.timelines.topic_engagement_24_hour_real_time_aggregates.", - topicShareEngagementsRealTimeAggregates -> "topic.timelines.topic_share_engagements_real_time_aggregates." - ) - - override def keysFromQueryAndCandidates( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Seq[Option[Long]] = { - candidates.map { candidate => - candidate.features - .getTry(TopicIdSocialContextFeature) - .toOption - .flatten - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TweetCountryEngagementRealTimeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TweetCountryEngagementRealTimeAggregateFeatureHydrator.scala deleted file mode 100644 index cd9e34a3e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TweetCountryEngagementRealTimeAggregateFeatureHydrator.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates - -import com.google.inject.name.Named -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetCountryEngagementCache -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.cache.ReadCache -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup -import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ -import javax.inject.Inject -import javax.inject.Singleton - -object TweetCountryEngagementRealTimeAggregateFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class TweetCountryEngagementRealTimeAggregateFeatureHydrator @Inject() ( - @Named(TweetCountryEngagementCache) override val client: ReadCache[(Long, String), DataRecord], - override val statsReceiver: StatsReceiver) - extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[(Long, String)] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("TweetCountryEngagementRealTimeAggregate") - - override val outputFeature: DataRecordInAFeature[TweetCandidate] = - TweetCountryEngagementRealTimeAggregateFeature - - override val aggregateGroups: Seq[AggregateGroup] = Seq( - tweetCountryRealTimeAggregates, - tweetCountryPrivateEngagementsRealTimeAggregates - ) - - override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( - tweetCountryRealTimeAggregates -> "tweet-country_code.timelines.tweet_country_engagement_real_time_aggregates.", - tweetCountryPrivateEngagementsRealTimeAggregates -> "tweet-country_code.timelines.tweet_country_private_engagement_real_time_aggregates." - ) - - override def keysFromQueryAndCandidates( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Seq[Option[(Long, String)]] = { - val countryCode = query.clientContext.countryCode - candidates.map { candidate => - val originalTweetId = CandidatesUtil.getOriginalTweetId(candidate) - countryCode.map((originalTweetId, _)) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TweetEngagementRealTimeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TweetEngagementRealTimeAggregateFeatureHydrator.scala deleted file mode 100644 index 99bae79d9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TweetEngagementRealTimeAggregateFeatureHydrator.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates - -import com.google.inject.name.Named -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetEngagementCache -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.cache.ReadCache -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup -import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ -import javax.inject.Inject -import javax.inject.Singleton - -object TweetEngagementRealTimeAggregateFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class TweetEngagementRealTimeAggregateFeatureHydrator @Inject() ( - @Named(TweetEngagementCache) override val client: ReadCache[Long, DataRecord], - override val statsReceiver: StatsReceiver) - extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[Long] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("TweetEngagementRealTimeAggregate") - - override val outputFeature: DataRecordInAFeature[TweetCandidate] = - TweetEngagementRealTimeAggregateFeature - - override val aggregateGroups: Seq[AggregateGroup] = Seq( - tweetEngagement30MinuteCountsProd, - tweetEngagementTotalCountsProd, - tweetEngagementUserStateRealTimeAggregatesProd, - tweetNegativeEngagementUserStateRealTimeAggregates, - tweetNegativeEngagement6HourCounts, - tweetNegativeEngagementTotalCounts, - tweetShareEngagementsRealTimeAggregates, - tweetBCEDwellEngagementsRealTimeAggregates - ) - - override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( - tweetShareEngagementsRealTimeAggregates -> "original_tweet.timelines.tweet_share_engagements_real_time_aggregates.", - tweetBCEDwellEngagementsRealTimeAggregates -> "original_tweet.timelines.tweet_bce_dwell_engagements_real_time_aggregates." - ) - - override def keysFromQueryAndCandidates( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Seq[Option[Long]] = { - candidates - .map(candidate => Some(CandidatesUtil.getOriginalTweetId(candidate))) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TwitterListEngagementRealTimeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TwitterListEngagementRealTimeAggregateFeatureHydrator.scala deleted file mode 100644 index b5f3af2a9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TwitterListEngagementRealTimeAggregateFeatureHydrator.scala +++ /dev/null @@ -1,57 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates - -import com.google.inject.name.Named -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.model.HomeFeatures.TwitterListIdFeature -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwitterListEngagementCache -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.cache.ReadCache -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup -import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ -import javax.inject.Inject -import javax.inject.Singleton - -object TwitterListEngagementRealTimeAggregateFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class TwitterListEngagementRealTimeAggregateFeatureHydrator @Inject() ( - @Named(TwitterListEngagementCache) override val client: ReadCache[Long, DataRecord], - override val statsReceiver: StatsReceiver) - extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[Long] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("TwitterListEngagementRealTimeAggregate") - - override val outputFeature: DataRecordInAFeature[TweetCandidate] = - TwitterListEngagementRealTimeAggregateFeature - - override val aggregateGroups: Seq[AggregateGroup] = Seq( - listEngagementRealTimeAggregatesProd - ) - - override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( - listEngagementRealTimeAggregatesProd -> "twitter_list.timelines.twitter_list_engagement_real_time_aggregates." - ) - - override def keysFromQueryAndCandidates( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Seq[Option[Long]] = { - candidates.map { candidate => - candidate.features - .getTry(TwitterListIdFeature) - .toOption - .flatten - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/UserAuthorEngagementRealTimeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/UserAuthorEngagementRealTimeAggregateFeatureHydrator.scala deleted file mode 100644 index 2c13fc0f6..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/UserAuthorEngagementRealTimeAggregateFeatureHydrator.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates - -import com.google.inject.name.Named -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserAuthorEngagementCache -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.cache.ReadCache -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup -import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ -import javax.inject.Inject -import javax.inject.Singleton - -object UserAuthorEngagementRealTimeAggregateFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class UserAuthorEngagementRealTimeAggregateFeatureHydrator @Inject() ( - @Named(UserAuthorEngagementCache) override val client: ReadCache[(Long, Long), DataRecord], - override val statsReceiver: StatsReceiver) - extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[(Long, Long)] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("UserAuthorEngagementRealTimeAggregate") - - override val outputFeature: DataRecordInAFeature[TweetCandidate] = - UserAuthorEngagementRealTimeAggregateFeature - - override val aggregateGroups: Seq[AggregateGroup] = Seq( - userAuthorEngagementRealTimeAggregatesProd, - userAuthorShareEngagementsRealTimeAggregates - ) - - override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( - userAuthorEngagementRealTimeAggregatesProd -> "user-author.timelines.user_author_engagement_real_time_aggregates.", - userAuthorShareEngagementsRealTimeAggregates -> "user-author.timelines.user_author_share_engagements_real_time_aggregates." - ) - - override def keysFromQueryAndCandidates( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Seq[Option[(Long, Long)]] = { - val userId = query.getRequiredUserId - candidates.map { candidate => - CandidatesUtil - .getOriginalAuthorId(candidate.features) - .map((userId, _)) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/UserEngagementRealTimeAggregatesFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/UserEngagementRealTimeAggregatesFeatureHydrator.scala deleted file mode 100644 index cc05b52aa..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/UserEngagementRealTimeAggregatesFeatureHydrator.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates - -import com.google.inject.name.Named -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserEngagementCache -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.cache.ReadCache -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup -import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ -import javax.inject.Inject -import javax.inject.Singleton - -object UserEngagementRealTimeAggregateFeature - extends DataRecordInAFeature[PipelineQuery] - with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class UserEngagementRealTimeAggregatesFeatureHydrator @Inject() ( - @Named(UserEngagementCache) override val client: ReadCache[Long, DataRecord], - override val statsReceiver: StatsReceiver) - extends BaseRealTimeAggregateQueryFeatureHydrator[Long] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("UserEngagementRealTimeAggregates") - - override val outputFeature: DataRecordInAFeature[PipelineQuery] = - UserEngagementRealTimeAggregateFeature - - val aggregateGroups: Seq[AggregateGroup] = Seq( - userEngagementRealTimeAggregatesProd, - userShareEngagementsRealTimeAggregates, - userBCEDwellEngagementsRealTimeAggregates, - userEngagement48HourRealTimeAggregatesProd, - userNegativeEngagementAuthorUserState72HourRealTimeAggregates, - userNegativeEngagementAuthorUserStateRealTimeAggregates, - userProfileEngagementRealTimeAggregates, - ) - - override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( - userShareEngagementsRealTimeAggregates -> "user.timelines.user_share_engagements_real_time_aggregates.", - userBCEDwellEngagementsRealTimeAggregates -> "user.timelines.user_bce_dwell_engagements_real_time_aggregates.", - userEngagement48HourRealTimeAggregatesProd -> "user.timelines.user_engagement_48_hour_real_time_aggregates.", - userNegativeEngagementAuthorUserState72HourRealTimeAggregates -> "user.timelines.user_negative_engagement_author_user_state_72_hour_real_time_aggregates.", - userProfileEngagementRealTimeAggregates -> "user.timelines.user_profile_engagement_real_time_aggregates." - ) - - override def keysFromQueryAndCandidates(query: PipelineQuery): Option[Long] = { - Some(query.getRequiredUserId) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/BUILD.bazel deleted file mode 100644 index 07a2e7cb3..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/BUILD.bazel +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter", - "stitch/stitch-core", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/DuplicateConversationTweetsFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/DuplicateConversationTweetsFilter.scala deleted file mode 100644 index adc11d255..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/DuplicateConversationTweetsFilter.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.filter - -import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -/** - * Remove any candidate that is in the ancestor list of any reply, including retweets of ancestors. - * - * E.g. if B replied to A and D was a retweet of A, we would prefer to drop D since otherwise - * we may end up serving the same tweet twice in the timeline (e.g. serving both A->B and D). - */ -object DuplicateConversationTweetsFilter extends Filter[PipelineQuery, TweetCandidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("DuplicateConversationTweets") - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - val allAncestors = candidates - .flatMap(_.features.getOrElse(AncestorsFeature, Seq.empty)) - .map(_.tweetId).toSet - - val (kept, removed) = candidates.partition { candidate => - !allAncestors.contains(CandidatesUtil.getOriginalTweetId(candidate)) - } - - Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/OutOfNetworkCompetitorFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/OutOfNetworkCompetitorFilter.scala deleted file mode 100644 index 3c6024268..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/OutOfNetworkCompetitorFilter.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.filter - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CompetitorSetParam -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -object OutOfNetworkCompetitorFilter extends Filter[PipelineQuery, TweetCandidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("OutOfNetworkCompetitor") - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - val competitorAuthors = query.params(CompetitorSetParam) - val (removed, kept) = - candidates.partition(isOutOfNetworkTweetFromCompetitor(_, competitorAuthors)) - - Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) - } - - def isOutOfNetworkTweetFromCompetitor( - candidate: CandidateWithFeatures[TweetCandidate], - competitorAuthors: Set[Long] - ): Boolean = { - !candidate.features.getOrElse(InNetworkFeature, true) && - !candidate.features.getOrElse(IsRetweetFeature, false) && - candidate.features.getOrElse(AuthorIdFeature, None).exists(competitorAuthors.contains) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/OutOfNetworkCompetitorURLFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/OutOfNetworkCompetitorURLFilter.scala deleted file mode 100644 index 00e2bb200..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/OutOfNetworkCompetitorURLFilter.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.filter - -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CompetitorURLSeqParam -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -object OutOfNetworkCompetitorURLFilter extends Filter[PipelineQuery, TweetCandidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("OutOfNetworkCompetitorURL") - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - val competitorUrls = query.params(CompetitorURLSeqParam).toSet - val (removed, kept) = candidates.partition(hasOutOfNetworkUrlFromCompetitor(_, competitorUrls)) - - Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) - } - - def hasOutOfNetworkUrlFromCompetitor( - candidate: CandidateWithFeatures[TweetCandidate], - competitorUrls: Set[String] - ): Boolean = { - !candidate.features.getOrElse(InNetworkFeature, true) && - !candidate.features.getOrElse(IsRetweetFeature, false) && - candidate.features - .getOrElse(TweetUrlsFeature, Seq.empty).toSet.intersect(competitorUrls).nonEmpty - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/RetweetSourceTweetRemovingFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/RetweetSourceTweetRemovingFilter.scala deleted file mode 100644 index 5900756d3..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/RetweetSourceTweetRemovingFilter.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.filter - -import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.util.ReplyRetweetUtil -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -/** - * This filter removes source tweets of retweets, added via second EB call in TLR - */ -object RetweetSourceTweetRemovingFilter extends Filter[PipelineQuery, TweetCandidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("RetweetSourceTweetRemoving") - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - val (kept, removed) = - candidates.partition( - _.features.getOrElse(EarlybirdFeature, None).exists(_.isSourceTweet)) match { - case (sourceTweets, nonSourceTweets) => - val inReplyToTweetIds: Set[Long] = - nonSourceTweets - .filter(ReplyRetweetUtil.isEligibleReply(_)).flatMap( - _.features.getOrElse(InReplyToTweetIdFeature, None)).toSet - val (keptSourceTweets, removedSourceTweets) = sourceTweets - .map(_.candidate) - .partition(candidate => inReplyToTweetIds.contains(candidate.id)) - (nonSourceTweets.map(_.candidate) ++ keptSourceTweets, removedSourceTweets) - } - Stitch.value(FilterResult(kept = kept, removed = removed)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/ScoredTweetsSocialContextFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/ScoredTweetsSocialContextFilter.scala deleted file mode 100644 index fef427a6d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/ScoredTweetsSocialContextFilter.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.filter - -import com.twitter.home_mixer.model.HomeFeatures._ -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelineservice.suggests.{thriftscala => st} - -object ScoredTweetsSocialContextFilter extends Filter[PipelineQuery, TweetCandidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("ScoredTweetsSocialContext") - - // Tweets from candidate sources which don't need generic like/follow/topic proof - private val AllowedSources: Set[st.SuggestType] = Set( - st.SuggestType.RankedListTweet, - st.SuggestType.RecommendedTrendTweet, - st.SuggestType.MediaTweet - ) - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - val validTweetIds = candidates - .filter { candidate => - candidate.features.getOrElse(InNetworkFeature, true) || - candidate.features.getOrElse(SuggestTypeFeature, None).exists(AllowedSources.contains) || - candidate.features.getOrElse(InReplyToTweetIdFeature, None).isDefined || - hasLikedBySocialContext(candidate.features) || - hasFollowedBySocialContext(candidate.features) || - hasTopicSocialContext(candidate.features) - }.map(_.candidate.id).toSet - - val (kept, removed) = - candidates.map(_.candidate).partition(candidate => validTweetIds.contains(candidate.id)) - - Stitch.value(FilterResult(kept = kept, removed = removed)) - } - - private def hasLikedBySocialContext(candidateFeatures: FeatureMap): Boolean = - candidateFeatures - .getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty) - .exists( - candidateFeatures - .getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Seq.empty) - .toSet.contains - ) - - private def hasFollowedBySocialContext(candidateFeatures: FeatureMap): Boolean = - candidateFeatures.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty).nonEmpty - - private def hasTopicSocialContext(candidateFeatures: FeatureMap): Boolean = { - candidateFeatures.getOrElse(TopicIdSocialContextFeature, None).isDefined && - candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None).isDefined - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/BUILD.bazel deleted file mode 100644 index 6d3fd9b1c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/BUILD.bazel +++ /dev/null @@ -1,12 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/MinCachedTweetsGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/MinCachedTweetsGate.scala deleted file mode 100644 index bdeb33a92..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/MinCachedTweetsGate.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.gate - -import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate.identifierSuffix -import com.twitter.home_mixer.util.CachedScoredTweetsHelper -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.Param - -case class MinCachedTweetsGate( - candidatePipelineIdentifier: CandidatePipelineIdentifier, - minCachedTweetsParam: Param[Int]) - extends Gate[PipelineQuery] { - - override val identifier: GateIdentifier = - GateIdentifier(candidatePipelineIdentifier + identifierSuffix) - - override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { - val minCachedTweets = query.params(minCachedTweetsParam) - val cachedScoredTweets = - query.features.map(CachedScoredTweetsHelper.unseenCachedScoredTweets).getOrElse(Seq.empty) - val numCachedTweets = cachedScoredTweets.count { tweet => - tweet.candidatePipelineIdentifier.exists( - CandidatePipelineIdentifier(_).equals(candidatePipelineIdentifier)) - } - Stitch.value(numCachedTweets < minCachedTweets) - } -} - -object MinCachedTweetsGate { - val identifierSuffix = "MinCachedTweets" -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/MinTimeSinceLastRequestGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/MinTimeSinceLastRequestGate.scala deleted file mode 100644 index 050f2ab67..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/MinTimeSinceLastRequestGate.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.gate - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.model.HomeFeatures.LastNonPollingTimeFeature -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -/** - * Gate continues if the amount of time passed since the previous request is greater - * than the configured amount or if the previous request time in not available - */ -object MinTimeSinceLastRequestGate extends Gate[PipelineQuery] { - - override val identifier: GateIdentifier = GateIdentifier("TimeSinceLastRequest") - - private val MinTimeSinceLastRequest = 24.hours - - override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = Stitch.value { - query.features.exists { features => - features - .getOrElse(LastNonPollingTimeFeature, None) - .forall(lnpt => (query.queryTime - lnpt) > MinTimeSinceLastRequest) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/BUILD.bazel deleted file mode 100644 index 4347c1093..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/BUILD.bazel +++ /dev/null @@ -1,15 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/premarshaller", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/ScoredTweetsResponseDomainMarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/ScoredTweetsResponseDomainMarshaller.scala deleted file mode 100644 index 796970bec..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/ScoredTweetsResponseDomainMarshaller.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.marshaller - -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse -import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller -import com.twitter.product_mixer.core.model.common.identifier.DomainMarshallerIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails - -/** - * Creates a domain model of the Scored Tweets product response from the set of candidates selected - */ -object ScoredTweetsResponseDomainMarshaller - extends DomainMarshaller[ScoredTweetsQuery, ScoredTweetsResponse] { - - override val identifier: DomainMarshallerIdentifier = - DomainMarshallerIdentifier("ScoredTweetsResponse") - - override def apply( - query: ScoredTweetsQuery, - selections: Seq[CandidateWithDetails] - ): ScoredTweetsResponse = ScoredTweetsResponse(scoredTweets = selections) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/ScoredTweetsResponseTransportMarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/ScoredTweetsResponseTransportMarshaller.scala deleted file mode 100644 index 27486768b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/ScoredTweetsResponseTransportMarshaller.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.marshaller - -import com.twitter.home_mixer.model.HomeFeatures._ -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse -import com.twitter.home_mixer.{thriftscala => t} -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller -import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.metadata.TopicContextFunctionalityTypeMarshaller -import com.twitter.product_mixer.core.model.common.identifier.TransportMarshallerIdentifier - -/** - * Marshall the domain model into our transport (Thrift) model. - */ -object ScoredTweetsResponseTransportMarshaller - extends TransportMarshaller[ScoredTweetsResponse, t.ScoredTweetsResponse] { - - override val identifier: TransportMarshallerIdentifier = - TransportMarshallerIdentifier("ScoredTweetsResponse") - - override def apply(input: ScoredTweetsResponse): t.ScoredTweetsResponse = { - val scoredTweets = input.scoredTweets.map { tweet => - mkScoredTweet(tweet.candidateIdLong, tweet.features) - } - t.ScoredTweetsResponse(scoredTweets) - } - - private def mkScoredTweet(tweetId: Long, features: FeatureMap): t.ScoredTweet = { - val topicFunctionalityType = features - .getOrElse(TopicContextFunctionalityTypeFeature, None) - .map(TopicContextFunctionalityTypeMarshaller(_)) - - t.ScoredTweet( - tweetId = tweetId, - authorId = features.get(AuthorIdFeature).get, - score = features.get(ScoreFeature), - suggestType = features.get(SuggestTypeFeature), - sourceTweetId = features.getOrElse(SourceTweetIdFeature, None), - sourceUserId = features.getOrElse(SourceUserIdFeature, None), - quotedTweetId = features.getOrElse(QuotedTweetIdFeature, None), - quotedUserId = features.getOrElse(QuotedUserIdFeature, None), - inReplyToTweetId = features.getOrElse(InReplyToTweetIdFeature, None), - inReplyToUserId = features.getOrElse(InReplyToUserIdFeature, None), - directedAtUserId = features.getOrElse(DirectedAtUserIdFeature, None), - inNetwork = Some(features.getOrElse(InNetworkFeature, true)), - sgsValidLikedByUserIds = Some(features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty)), - sgsValidFollowedByUserIds = - Some(features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty)), - topicId = features.getOrElse(TopicIdSocialContextFeature, None), - topicFunctionalityType = topicFunctionalityType, - ancestors = Some(features.getOrElse(AncestorsFeature, Seq.empty)), - isReadFromCache = Some(features.getOrElse(IsReadFromCacheFeature, false)), - streamToKafka = Some(features.getOrElse(StreamToKafkaFeature, false)), - exclusiveConversationAuthorId = - features.getOrElse(ExclusiveConversationAuthorIdFeature, None), - authorMetadata = Some( - t.AuthorMetadata( - blueVerified = features.getOrElse(AuthorIsBlueVerifiedFeature, false), - goldVerified = features.getOrElse(AuthorIsGoldVerifiedFeature, false), - grayVerified = features.getOrElse(AuthorIsGrayVerifiedFeature, false), - legacyVerified = features.getOrElse(AuthorIsLegacyVerifiedFeature, false), - creator = features.getOrElse(AuthorIsCreatorFeature, false) - )), - lastScoredTimestampMs = None, - candidatePipelineIdentifier = None, - tweetUrls = None, - perspectiveFilteredLikedByUserIds = - Some(features.getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Seq.empty)), - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/BUILD.bazel deleted file mode 100644 index 6414453db..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/BUILD.bazel +++ /dev/null @@ -1,17 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/ScoredTweetsQuery.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/ScoredTweetsQuery.scala deleted file mode 100644 index a2eb3a466..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/ScoredTweetsQuery.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.model - -import com.twitter.home_mixer.model.request.DeviceContext -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.home_mixer.model.request.HasSeenTweetIds -import com.twitter.home_mixer.model.request.ScoredTweetsProduct -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.request._ -import com.twitter.product_mixer.core.pipeline.HasPipelineCursor -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus -import com.twitter.product_mixer.core.quality_factor.QualityFactorStatus -import com.twitter.timelines.configapi.Params - -case class ScoredTweetsQuery( - override val params: Params, - override val clientContext: ClientContext, - override val pipelineCursor: Option[UrtOrderedCursor], - override val requestedMaxResults: Option[Int], - override val debugOptions: Option[DebugOptions], - override val features: Option[FeatureMap], - override val deviceContext: Option[DeviceContext], - override val seenTweetIds: Option[Seq[Long]], - override val qualityFactorStatus: Option[QualityFactorStatus]) - extends PipelineQuery - with HasPipelineCursor[UrtOrderedCursor] - with HasDeviceContext - with HasSeenTweetIds - with HasQualityFactorStatus { - override val product: Product = ScoredTweetsProduct - - override def withFeatureMap(features: FeatureMap): ScoredTweetsQuery = - copy(features = Some(features)) - - override def withQualityFactorStatus( - qualityFactorStatus: QualityFactorStatus - ): ScoredTweetsQuery = copy(qualityFactorStatus = Some(qualityFactorStatus)) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/ScoredTweetsResponse.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/ScoredTweetsResponse.scala deleted file mode 100644 index e9bd7cd61..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/ScoredTweetsResponse.scala +++ /dev/null @@ -1,6 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.model - -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling - -case class ScoredTweetsResponse(scoredTweets: Seq[CandidateWithDetails]) extends HasMarshalling diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/BUILD.bazel deleted file mode 100644 index 065841642..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/BUILD.bazel +++ /dev/null @@ -1,11 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/ScoredTweetsParam.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/ScoredTweetsParam.scala deleted file mode 100644 index 9720b9344..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/ScoredTweetsParam.scala +++ /dev/null @@ -1,361 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.param - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.param.decider.DeciderKey -import com.twitter.timelines.configapi.DurationConversion -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.timelines.configapi.decider.BooleanDeciderParam -import com.twitter.util.Duration - -object ScoredTweetsParam { - val SupportedClientFSName = "scored_tweets_supported_client" - - object CandidatePipeline { - object EnableInNetworkParam - extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsInNetworkCandidatePipeline) - - object EnableTweetMixerParam - extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsTweetMixerCandidatePipeline) - - object EnableUtegParam - extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsUtegCandidatePipeline) - - object EnableFrsParam - extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsFrsCandidatePipeline) - - object EnableListsParam - extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsListsCandidatePipeline) - - object EnablePopularVideosParam - extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsPopularVideosCandidatePipeline) - - object EnableBackfillParam - extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsBackfillCandidatePipeline) - } - - object EnableBackfillCandidatePipelineParam - extends FSParam[Boolean]( - name = "scored_tweets_enable_backfill_candidate_pipeline", - default = true - ) - - object QualityFactor { - object InNetworkMaxTweetsToScoreParam - extends FSBoundedParam[Int]( - name = "scored_tweets_quality_factor_earlybird_max_tweets_to_score", - default = 500, - min = 0, - max = 10000 - ) - - object UtegMaxTweetsToScoreParam - extends FSBoundedParam[Int]( - name = "scored_tweets_quality_factor_uteg_max_tweets_to_score", - default = 500, - min = 0, - max = 10000 - ) - - object FrsMaxTweetsToScoreParam - extends FSBoundedParam[Int]( - name = "scored_tweets_quality_factor_frs_max_tweets_to_score", - default = 500, - min = 0, - max = 10000 - ) - - object TweetMixerMaxTweetsToScoreParam - extends FSBoundedParam[Int]( - name = "scored_tweets_quality_factor_tweet_mixer_max_tweets_to_score", - default = 500, - min = 0, - max = 10000 - ) - - object ListsMaxTweetsToScoreParam - extends FSBoundedParam[Int]( - name = "scored_tweets_quality_factor_lists_max_tweets_to_score", - default = 500, - min = 0, - max = 100 - ) - - object PopularVideosMaxTweetsToScoreParam - extends FSBoundedParam[Int]( - name = "scored_tweets_quality_factor_popular_videos_max_tweets_to_score", - default = 40, - min = 0, - max = 10000 - ) - - object BackfillMaxTweetsToScoreParam - extends FSBoundedParam[Int]( - name = "scored_tweets_quality_factor_backfill_max_tweets_to_score", - default = 500, - min = 0, - max = 10000 - ) - } - - object ServerMaxResultsParam - extends FSBoundedParam[Int]( - name = "scored_tweets_server_max_results", - default = 120, - min = 1, - max = 500 - ) - - object MaxInNetworkResultsParam - extends FSBoundedParam[Int]( - name = "scored_tweets_max_in_network_results", - default = 60, - min = 1, - max = 500 - ) - - object MaxOutOfNetworkResultsParam - extends FSBoundedParam[Int]( - name = "scored_tweets_max_out_of_network_results", - default = 60, - min = 1, - max = 500 - ) - - object CachedScoredTweets { - object TTLParam - extends FSBoundedParam[Duration]( - name = "scored_tweets_cached_scored_tweets_ttl_minutes", - default = 3.minutes, - min = 0.minute, - max = 60.minutes - ) - with HasDurationConversion { - override val durationConversion: DurationConversion = DurationConversion.FromMinutes - } - - object MinCachedTweetsParam - extends FSBoundedParam[Int]( - name = "scored_tweets_cached_scored_tweets_min_cached_tweets", - default = 30, - min = 0, - max = 1000 - ) - } - - object Scoring { - object HomeModelParam - extends FSParam[String](name = "scored_tweets_home_model", default = "Home") - - object ModelWeights { - - object FavParam - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_fav", - default = 1.0, - min = 0.0, - max = 100.0 - ) - - object RetweetParam - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_retweet", - default = 1.0, - min = 0.0, - max = 100.0 - ) - - object ReplyParam - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_reply", - default = 1.0, - min = 0.0, - max = 100.0 - ) - - object GoodProfileClickParam - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_good_profile_click", - default = 1.0, - min = 0.0, - max = 1000000.0 - ) - - object VideoPlayback50Param - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_video_playback50", - default = 1.0, - min = 0.0, - max = 100.0 - ) - - object ReplyEngagedByAuthorParam - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_reply_engaged_by_author", - default = 1.0, - min = 0.0, - max = 200.0 - ) - - object GoodClickParam - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_good_click", - default = 1.0, - min = 0.0, - max = 1000000.0 - ) - - object GoodClickV2Param - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_good_click_v2", - default = 1.0, - min = 0.0, - max = 1000000.0 - ) - - object TweetDetailDwellParam - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_tweet_detail_dwell", - default = 0.0, - min = 0.0, - max = 100.0 - ) - - object ProfileDwelledParam - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_profile_dwelled", - default = 0.0, - min = 0.0, - max = 100.0 - ) - - object BookmarkParam - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_bookmark", - default = 0.0, - min = 0.0, - max = 100.0 - ) - - object ShareParam - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_share", - default = 0.0, - min = 0.0, - max = 100.0 - ) - - object ShareMenuClickParam - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_share_menu_click", - default = 0.0, - min = 0.0, - max = 100.0 - ) - - object NegativeFeedbackV2Param - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_negative_feedback_v2", - default = 1.0, - min = -1000.0, - max = 0.0 - ) - - object ReportParam - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_report", - default = 1.0, - min = -20000.0, - max = 0.0 - ) - - object WeakNegativeFeedbackParam - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_weak_negative_feedback", - default = 0.0, - min = -1000.0, - max = 0.0 - ) - - object StrongNegativeFeedbackParam - extends FSBoundedParam[Double]( - name = "scored_tweets_model_weight_strong_negative_feedback", - default = 0.0, - min = -1000.0, - max = 0.0 - ) - } - } - - object EnableSimClustersSimilarityFeatureHydrationDeciderParam - extends BooleanDeciderParam(decider = DeciderKey.EnableSimClustersSimilarityFeatureHydration) - - object CompetitorSetParam - extends FSParam[Set[Long]](name = "scored_tweets_competitor_list", default = Set.empty) - - object CompetitorURLSeqParam - extends FSParam[Seq[String]](name = "scored_tweets_competitor_url_list", default = Seq.empty) - - object BlueVerifiedAuthorInNetworkMultiplierParam - extends FSBoundedParam[Double]( - name = "scored_tweets_blue_verified_author_in_network_multiplier", - default = 4.0, - min = 0.0, - max = 100.0 - ) - - object BlueVerifiedAuthorOutOfNetworkMultiplierParam - extends FSBoundedParam[Double]( - name = "scored_tweets_blue_verified_author_out_of_network_multiplier", - default = 2.0, - min = 0.0, - max = 100.0 - ) - - object CreatorInNetworkMultiplierParam - extends FSBoundedParam[Double]( - name = "scored_tweets_creator_in_network_multiplier", - default = 1.1, - min = 0.0, - max = 100.0 - ) - - object CreatorOutOfNetworkMultiplierParam - extends FSBoundedParam[Double]( - name = "scored_tweets_creator_out_of_network_multiplier", - default = 1.3, - min = 0.0, - max = 100.0 - ) - - object OutOfNetworkScaleFactorParam - extends FSBoundedParam[Double]( - name = "scored_tweets_out_of_network_scale_factor", - default = 1.0, - min = 0.0, - max = 100.0 - ) - - object EnableScribeScoredCandidatesParam - extends FSParam[Boolean](name = "scored_tweets_enable_scribing", default = false) - - object EarlybirdTensorflowModel { - - object InNetworkParam - extends FSParam[String]( - name = "scored_tweets_in_network_earlybird_tensorflow_model", - default = "timelines_recap_replica") - - object FrsParam - extends FSParam[String]( - name = "scored_tweets_frs_earlybird_tensorflow_model", - default = "timelines_rectweet_replica") - - object UtegParam - extends FSParam[String]( - name = "scored_tweets_uteg_earlybird_tensorflow_model", - default = "timelines_rectweet_replica") - } - -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/ScoredTweetsParamConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/ScoredTweetsParamConfig.scala deleted file mode 100644 index 10b9de49d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/ScoredTweetsParamConfig.scala +++ /dev/null @@ -1,89 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.param - -import com.twitter.home_mixer.param.decider.DeciderKey -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam._ -import com.twitter.product_mixer.core.product.ProductParamConfig -import com.twitter.servo.decider.DeciderKeyName -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ScoredTweetsParamConfig @Inject() () extends ProductParamConfig { - override val enabledDeciderKey: DeciderKeyName = DeciderKey.EnableScoredTweetsProduct - override val supportedClientFSName: String = SupportedClientFSName - - override val booleanDeciderOverrides = Seq( - CandidatePipeline.EnableBackfillParam, - CandidatePipeline.EnableTweetMixerParam, - CandidatePipeline.EnableFrsParam, - CandidatePipeline.EnableInNetworkParam, - CandidatePipeline.EnableListsParam, - CandidatePipeline.EnablePopularVideosParam, - CandidatePipeline.EnableUtegParam, - ScoredTweetsParam.EnableSimClustersSimilarityFeatureHydrationDeciderParam - ) - - override val booleanFSOverrides = Seq( - EnableBackfillCandidatePipelineParam, - EnableScribeScoredCandidatesParam - ) - - override val boundedIntFSOverrides = Seq( - CachedScoredTweets.MinCachedTweetsParam, - MaxInNetworkResultsParam, - MaxOutOfNetworkResultsParam, - QualityFactor.BackfillMaxTweetsToScoreParam, - QualityFactor.TweetMixerMaxTweetsToScoreParam, - QualityFactor.FrsMaxTweetsToScoreParam, - QualityFactor.InNetworkMaxTweetsToScoreParam, - QualityFactor.ListsMaxTweetsToScoreParam, - QualityFactor.PopularVideosMaxTweetsToScoreParam, - QualityFactor.UtegMaxTweetsToScoreParam, - ServerMaxResultsParam - ) - - override val boundedDurationFSOverrides = Seq( - CachedScoredTweets.TTLParam - ) - - override val stringFSOverrides = Seq( - Scoring.HomeModelParam, - EarlybirdTensorflowModel.InNetworkParam, - EarlybirdTensorflowModel.FrsParam, - EarlybirdTensorflowModel.UtegParam - ) - - override val boundedDoubleFSOverrides = Seq( - BlueVerifiedAuthorInNetworkMultiplierParam, - BlueVerifiedAuthorOutOfNetworkMultiplierParam, - CreatorInNetworkMultiplierParam, - CreatorOutOfNetworkMultiplierParam, - OutOfNetworkScaleFactorParam, - // Model Weights - Scoring.ModelWeights.FavParam, - Scoring.ModelWeights.ReplyParam, - Scoring.ModelWeights.RetweetParam, - Scoring.ModelWeights.GoodClickParam, - Scoring.ModelWeights.GoodClickV2Param, - Scoring.ModelWeights.GoodProfileClickParam, - Scoring.ModelWeights.ReplyEngagedByAuthorParam, - Scoring.ModelWeights.VideoPlayback50Param, - Scoring.ModelWeights.ReportParam, - Scoring.ModelWeights.NegativeFeedbackV2Param, - Scoring.ModelWeights.TweetDetailDwellParam, - Scoring.ModelWeights.ProfileDwelledParam, - Scoring.ModelWeights.BookmarkParam, - Scoring.ModelWeights.ShareParam, - Scoring.ModelWeights.ShareMenuClickParam, - Scoring.ModelWeights.StrongNegativeFeedbackParam, - Scoring.ModelWeights.WeakNegativeFeedbackParam - ) - - override val longSetFSOverrides = Seq( - CompetitorSetParam - ) - - override val stringSeqFSOverrides = Seq( - CompetitorURLSeqParam - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/BUILD.bazel deleted file mode 100644 index 15fc94f47..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/BUILD.bazel +++ /dev/null @@ -1,23 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/thrift/com/twitter/timelineranker:thrift-scala", - "timelineranker/common/src/main/scala/com/twitter/timelineranker/model", - "timelines:util", - "timelines/src/main/scala/com/twitter/timelines/common/model", - "timelines/src/main/scala/com/twitter/timelines/earlybird/common/options", - "timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils", - "timelines/src/main/scala/com/twitter/timelines/model/candidate", - "timelineservice/common:model", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerFrsQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerFrsQueryTransformer.scala deleted file mode 100644 index 2592ec82e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerFrsQueryTransformer.scala +++ /dev/null @@ -1,64 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.query_transformer - -import com.twitter.conversions.DurationOps._ -import com.twitter.core_workflows.user_model.{thriftscala => um} -import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.FrsSeedUserIdsFeature -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam -import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerFrsQueryTransformer._ -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus -import com.twitter.timelineranker.{thriftscala => t} -import com.twitter.timelines.common.model.TweetKindOption -import com.twitter.timelines.model.candidate.CandidateTweetSourceId - -object TimelineRankerFrsQueryTransformer { - private val DefaultSinceDuration = 24.hours - private val ExpandedSinceDuration = 48.hours - private val MaxTweetsToFetch = 100 - - private val tweetKindOptions: TweetKindOption.ValueSet = - TweetKindOption(includeOriginalTweetsAndQuotes = true) - - private val UserStatesForExtendedSinceDuration: Set[um.UserState] = Set( - um.UserState.Light, - um.UserState.MediumNonTweeter, - um.UserState.MediumTweeter, - um.UserState.NearZero, - um.UserState.New, - um.UserState.VeryLight - ) -} - -case class TimelineRankerFrsQueryTransformer[ - Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext -]( - override val candidatePipelineIdentifier: CandidatePipelineIdentifier, - override val maxTweetsToFetch: Int = MaxTweetsToFetch) - extends CandidatePipelineQueryTransformer[Query, t.RecapQuery] - with TimelineRankerQueryTransformer[Query] { - - override val candidateTweetSourceId = CandidateTweetSourceId.FrsTweet - override val options = tweetKindOptions - - override def getTensorflowModel(query: Query): Option[String] = { - Some(query.params(ScoredTweetsParam.EarlybirdTensorflowModel.FrsParam)) - } - - override def seedAuthorIds(query: Query): Option[Seq[Long]] = { - query.features.flatMap(_.getOrElse(FrsSeedUserIdsFeature, None)) - } - - override def transform(input: Query): t.RecapQuery = { - val userState = input.features.get.getOrElse(UserStateFeature, None) - - val sinceDuration = - if (userState.exists(UserStatesForExtendedSinceDuration.contains)) ExpandedSinceDuration - else DefaultSinceDuration - - buildTimelineRankerQuery(input, sinceDuration).toThriftRecapQuery - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerInNetworkQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerInNetworkQueryTransformer.scala deleted file mode 100644 index 4514dc2c4..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerInNetworkQueryTransformer.scala +++ /dev/null @@ -1,63 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.query_transformer - -import com.twitter.conversions.DurationOps._ -import com.twitter.core_workflows.user_model.{thriftscala => um} -import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam -import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerInNetworkQueryTransformer._ -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus -import com.twitter.timelineranker.{thriftscala => t} -import com.twitter.timelines.common.model.TweetKindOption -import com.twitter.timelines.model.candidate.CandidateTweetSourceId - -object TimelineRankerInNetworkQueryTransformer { - private val DefaultSinceDuration = 24.hours - private val ExpandedSinceDuration = 48.hours - private val MaxTweetsToFetch = 600 - - private val tweetKindOptions: TweetKindOption.ValueSet = TweetKindOption( - includeReplies = true, - includeRetweets = true, - includeOriginalTweetsAndQuotes = true, - includeExtendedReplies = true - ) - - private val UserStatesForExtendedSinceDuration: Set[um.UserState] = Set( - um.UserState.Light, - um.UserState.MediumNonTweeter, - um.UserState.MediumTweeter, - um.UserState.NearZero, - um.UserState.New, - um.UserState.VeryLight - ) -} - -case class TimelineRankerInNetworkQueryTransformer[ - Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext -]( - override val candidatePipelineIdentifier: CandidatePipelineIdentifier, - override val maxTweetsToFetch: Int = MaxTweetsToFetch) - extends CandidatePipelineQueryTransformer[Query, t.RecapQuery] - with TimelineRankerQueryTransformer[Query] { - - override val candidateTweetSourceId = CandidateTweetSourceId.RecycledTweet - override val options = tweetKindOptions - - override def getTensorflowModel(query: Query): Option[String] = { - Some(query.params(ScoredTweetsParam.EarlybirdTensorflowModel.InNetworkParam)) - } - - override def transform(input: Query): t.RecapQuery = { - val userState = input.features.get.getOrElse(UserStateFeature, None) - - val sinceDuration = - if (userState.exists(UserStatesForExtendedSinceDuration.contains)) ExpandedSinceDuration - else DefaultSinceDuration - - buildTimelineRankerQuery(input, sinceDuration).toThriftRecapQuery - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerQueryTransformer.scala deleted file mode 100644 index e187a0aa0..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerQueryTransformer.scala +++ /dev/null @@ -1,108 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.query_transformer - -import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerQueryTransformer._ -import com.twitter.home_mixer.util.CachedScoredTweetsHelper -import com.twitter.home_mixer.util.earlybird.EarlybirdRequestUtil -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus -import com.twitter.timelineranker.{model => tlr} -import com.twitter.timelines.common.model.TweetKindOption -import com.twitter.timelines.earlybird.common.options.EarlybirdOptions -import com.twitter.timelines.earlybird.common.options.EarlybirdScoringModelConfig -import com.twitter.timelines.earlybird.common.utils.SearchOperator -import com.twitter.timelines.model.UserId -import com.twitter.timelines.model.candidate.CandidateTweetSourceId -import com.twitter.timelines.util.SnowflakeSortIndexHelper -import com.twitter.util.Duration -import com.twitter.util.Time - -object TimelineRankerQueryTransformer { - - /** - * Specifies the maximum number of excluded tweet ids to include in the search index query. - * Earlybird's named multi term disjunction map feature supports up to 1500 tweet ids. - */ - private val EarlybirdMaxExcludedTweets = 1500 - - /** - * Maximum number of query hits each earlybird shard is allowed to accumulate before - * early-terminating the query and reducing the hits to MaxNumEarlybirdResults. - */ - private val EarlybirdMaxHits = 1000 - - /** - * Maximum number of results TLR should retrieve from each earlybird shard. - */ - private val EarlybirdMaxResults = 300 -} - -trait TimelineRankerQueryTransformer[ - Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext] { - def maxTweetsToFetch: Int - def options: TweetKindOption.ValueSet = TweetKindOption.Default - def candidateTweetSourceId: CandidateTweetSourceId.Value - def utegLikedByTweetsOptions(query: Query): Option[tlr.UtegLikedByTweetsOptions] = None - def seedAuthorIds(query: Query): Option[Seq[Long]] = None - def candidatePipelineIdentifier: CandidatePipelineIdentifier - def earlybirdModels: Seq[EarlybirdScoringModelConfig] = - EarlybirdRequestUtil.EarlybirdScoringModels.UnifiedEngagementProd - def getTensorflowModel(query: Query): Option[String] = None - - def buildTimelineRankerQuery(query: Query, sinceDuration: Duration): tlr.RecapQuery = { - val sinceTime: Time = sinceDuration.ago - val untilTime: Time = Time.now - - val fromTweetIdExclusive = SnowflakeSortIndexHelper.timestampToFakeId(sinceTime) - val toTweetIdExclusive = SnowflakeSortIndexHelper.timestampToFakeId(untilTime) - val range = tlr.TweetIdRange(Some(fromTweetIdExclusive), Some(toTweetIdExclusive)) - - val excludedTweetIds = query.features.map { featureMap => - CachedScoredTweetsHelper.tweetImpressionsAndCachedScoredTweetsInRange( - featureMap, - candidatePipelineIdentifier, - EarlybirdMaxExcludedTweets, - sinceTime, - untilTime) - } - - val maxCount = - (query.getQualityFactorCurrentValue(candidatePipelineIdentifier) * maxTweetsToFetch).toInt - - val authorScoreMap = query.features - .map(_.getOrElse(RealGraphInNetworkScoresFeature, Map.empty[UserId, Double])) - .getOrElse(Map.empty) - - val deviceContext = - query.deviceContext.map(_.toTimelineServiceDeviceContext(query.clientContext)) - - val tensorflowModel = getTensorflowModel(query) - - val earlyBirdOptions = EarlybirdOptions( - maxNumHitsPerShard = EarlybirdMaxHits, - maxNumResultsPerShard = EarlybirdMaxResults, - models = earlybirdModels, - authorScoreMap = authorScoreMap, - skipVeryRecentTweets = true, - tensorflowModel = tensorflowModel - ) - - tlr.RecapQuery( - userId = query.getRequiredUserId, - maxCount = Some(maxCount), - range = Some(range), - options = options, - searchOperator = SearchOperator.Exclude, - earlybirdOptions = Some(earlyBirdOptions), - deviceContext = deviceContext, - authorIds = seedAuthorIds(query), - excludedTweetIds = excludedTweetIds, - utegLikedByTweetsOptions = utegLikedByTweetsOptions(query), - searchClientSubId = None, - candidateTweetSourceId = Some(candidateTweetSourceId), - hydratesContentFeatures = Some(false) - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerUtegQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerUtegQueryTransformer.scala deleted file mode 100644 index ea051f331..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerUtegQueryTransformer.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.query_transformer - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam -import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerUtegQueryTransformer._ -import com.twitter.home_mixer.util.earlybird.EarlybirdRequestUtil -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus -import com.twitter.timelineranker.{model => tlr} -import com.twitter.timelineranker.{thriftscala => t} -import com.twitter.timelines.common.model.TweetKindOption -import com.twitter.timelines.earlybird.common.options.EarlybirdScoringModelConfig -import com.twitter.timelines.model.UserId -import com.twitter.timelines.model.candidate.CandidateTweetSourceId - -object TimelineRankerUtegQueryTransformer { - private val SinceDuration = 24.hours - private val MaxTweetsToFetch = 300 - private val MaxUtegCandidates = 800 - - private val tweetKindOptions = - TweetKindOption(includeOriginalTweetsAndQuotes = true, includeReplies = true) - - def utegEarlybirdModels: Seq[EarlybirdScoringModelConfig] = - EarlybirdRequestUtil.EarlybirdScoringModels.UnifiedEngagementRectweet -} - -case class TimelineRankerUtegQueryTransformer[ - Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext -]( - override val candidatePipelineIdentifier: CandidatePipelineIdentifier, - override val maxTweetsToFetch: Int = MaxTweetsToFetch) - extends CandidatePipelineQueryTransformer[Query, t.UtegLikedByTweetsQuery] - with TimelineRankerQueryTransformer[Query] { - - override val candidateTweetSourceId = CandidateTweetSourceId.RecommendedTweet - override val options = tweetKindOptions - override val earlybirdModels = utegEarlybirdModels - override def getTensorflowModel(query: Query): Option[String] = { - Some(query.params(ScoredTweetsParam.EarlybirdTensorflowModel.UtegParam)) - } - - override def utegLikedByTweetsOptions(input: Query): Option[tlr.UtegLikedByTweetsOptions] = Some( - tlr.UtegLikedByTweetsOptions( - utegCount = MaxUtegCandidates, - isInNetwork = false, - weightedFollowings = input.features - .map(_.getOrElse(RealGraphInNetworkScoresFeature, Map.empty[UserId, Double])) - .getOrElse(Map.empty) - ) - ) - - override def transform(input: Query): t.UtegLikedByTweetsQuery = - buildTimelineRankerQuery(input, SinceDuration).toThriftUtegLikedByTweetsQuery -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/BUILD.bazel deleted file mode 100644 index a337a328b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/BUILD.bazel +++ /dev/null @@ -1,23 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/thrift/com/twitter/search:earlybird-scala", - "timelines:util", - "timelines/src/main/scala/com/twitter/timelines/clients/relevance_search", - "timelines/src/main/scala/com/twitter/timelines/common/model", - "timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils", - "timelines/src/main/scala/com/twitter/timelines/model/candidate", - "timelines/src/main/scala/com/twitter/timelines/model/types", - "timelines/src/main/scala/com/twitter/timelines/util/stats", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdFrsQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdFrsQueryTransformer.scala deleted file mode 100644 index 9b2ac341e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdFrsQueryTransformer.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.FrsSeedUserIdsFeature -import com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird.EarlybirdFrsQueryTransformer._ -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus -import com.twitter.search.earlybird.{thriftscala => eb} -import com.twitter.timelines.common.model.TweetKindOption - -object EarlybirdFrsQueryTransformer { - private val SinceDuration = 24.hours - private val MaxTweetsToFetch = 100 - private val TensorflowModel = Some("timelines_rectweet_replica") - - private val TweetKindOptions: TweetKindOption.ValueSet = - TweetKindOption(includeOriginalTweetsAndQuotes = true) -} - -case class EarlybirdFrsQueryTransformer[ - Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext -]( - candidatePipelineIdentifier: CandidatePipelineIdentifier, - override val clientId: Option[String]) - extends CandidatePipelineQueryTransformer[Query, eb.EarlybirdRequest] - with EarlybirdQueryTransformer[Query] { - - override val tweetKindOptions: TweetKindOption.ValueSet = TweetKindOptions - override val maxTweetsToFetch: Int = MaxTweetsToFetch - override val tensorflowModel: Option[String] = TensorflowModel - - override def transform(query: Query): eb.EarlybirdRequest = { - val seedUserIds = query.features - .flatMap(_.getOrElse(FrsSeedUserIdsFeature, None)) - .getOrElse(Seq.empty).toSet - buildEarlybirdQuery(query, SinceDuration, seedUserIds) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdInNetworkQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdInNetworkQueryTransformer.scala deleted file mode 100644 index 5c5464b8b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdInNetworkQueryTransformer.scala +++ /dev/null @@ -1,68 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird - -import com.twitter.conversions.DurationOps._ -import com.twitter.core_workflows.user_model.{thriftscala => um} -import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird.EarlybirdInNetworkQueryTransformer._ -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus -import com.twitter.search.earlybird.{thriftscala => eb} -import com.twitter.timelines.common.model.TweetKindOption - -object EarlybirdInNetworkQueryTransformer { - private val DefaultSinceDuration = 24.hours - private val ExpandedSinceDuration = 48.hours - private val MaxTweetsToFetch = 600 - private val TensorflowModel = Some("timelines_recap_replica") - - private val TweetKindOptions: TweetKindOption.ValueSet = TweetKindOption( - includeReplies = true, - includeRetweets = true, - includeOriginalTweetsAndQuotes = true, - includeExtendedReplies = true - ) - - private val UserStatesForExtendedSinceDuration: Set[um.UserState] = Set( - um.UserState.Light, - um.UserState.MediumNonTweeter, - um.UserState.MediumTweeter, - um.UserState.NearZero, - um.UserState.New, - um.UserState.VeryLight - ) -} - -case class EarlybirdInNetworkQueryTransformer[ - Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext -]( - candidatePipelineIdentifier: CandidatePipelineIdentifier, - override val clientId: Option[String]) - extends CandidatePipelineQueryTransformer[Query, eb.EarlybirdRequest] - with EarlybirdQueryTransformer[Query] { - - override val tweetKindOptions: TweetKindOption.ValueSet = TweetKindOptions - override val maxTweetsToFetch: Int = MaxTweetsToFetch - override val tensorflowModel: Option[String] = TensorflowModel - - override def transform(query: Query): eb.EarlybirdRequest = { - - val userState = query.features.get.getOrElse(UserStateFeature, None) - - val sinceDuration = - if (userState.exists(UserStatesForExtendedSinceDuration.contains)) ExpandedSinceDuration - else DefaultSinceDuration - - val followedUserIds = - query.features - .map( - _.getOrElse( - SGSFollowedUsersFeature, - Seq.empty)).toSeq.flatten.toSet + query.getRequiredUserId - - buildEarlybirdQuery(query, sinceDuration, followedUserIds) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdQueryTransformer.scala deleted file mode 100644 index 0e51c54ea..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdQueryTransformer.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird - -import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.home_mixer.util.CachedScoredTweetsHelper -import com.twitter.home_mixer.util.earlybird.EarlybirdRequestUtil -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus -import com.twitter.search.earlybird.{thriftscala => eb} -import com.twitter.timelines.clients.relevance_search.SearchClient.TweetTypes -import com.twitter.timelines.common.model.TweetKindOption -import com.twitter.timelines.util.SnowflakeSortIndexHelper -import com.twitter.util.Duration -import com.twitter.util.Time - -trait EarlybirdQueryTransformer[ - Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext] { - - def candidatePipelineIdentifier: CandidatePipelineIdentifier - def clientId: Option[String] = None - def maxTweetsToFetch: Int = 100 - def tweetKindOptions: TweetKindOption.ValueSet - def tensorflowModel: Option[String] = None - - private val EarlybirdMaxExcludedTweets = 1500 - - def buildEarlybirdQuery( - query: Query, - sinceDuration: Duration, - followedUserIds: Set[Long] = Set.empty - ): eb.EarlybirdRequest = { - val sinceTime: Time = sinceDuration.ago - val untilTime: Time = Time.now - - val fromTweetIdExclusive = SnowflakeSortIndexHelper.timestampToFakeId(sinceTime) - val toTweetIdExclusive = SnowflakeSortIndexHelper.timestampToFakeId(untilTime) - - val excludedTweetIds = query.features.map { featureMap => - CachedScoredTweetsHelper.tweetImpressionsAndCachedScoredTweetsInRange( - featureMap, - candidatePipelineIdentifier, - EarlybirdMaxExcludedTweets, - sinceTime, - untilTime) - } - - val maxCount = - (query.getQualityFactorCurrentValue(candidatePipelineIdentifier) * maxTweetsToFetch).toInt - - val authorScoreMap = query.features - .map(_.getOrElse(RealGraphInNetworkScoresFeature, Map.empty[Long, Double])) - .getOrElse(Map.empty) - - EarlybirdRequestUtil.getTweetsRequest( - userId = Some(query.getRequiredUserId), - clientId = clientId, - skipVeryRecentTweets = true, - followedUserIds = followedUserIds, - retweetsMutedUserIds = Set.empty, - beforeTweetIdExclusive = Some(toTweetIdExclusive), - afterTweetIdExclusive = Some(fromTweetIdExclusive), - excludedTweetIds = excludedTweetIds.map(_.toSet), - maxCount = maxCount, - tweetTypes = TweetTypes.fromTweetKindOption(tweetKindOptions), - authorScoreMap = Some(authorScoreMap), - tensorflowModel = tensorflowModel - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/BUILD.bazel deleted file mode 100644 index 30774b91f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/BUILD.bazel +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - dependencies = [ - "explore/explore-ranker/thrift/src/main/thrift:thrift-scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", - "src/thrift/com/twitter/timelineranker:thrift-scala", - "topic-social-proof/server/src/main/thrift:thrift-scala", - "tweet-mixer/thrift/src/main/thrift:thrift-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/CachedScoredTweetsResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/CachedScoredTweetsResponseFeatureTransformer.scala deleted file mode 100644 index 0fbe7a438..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/CachedScoredTweetsResponseFeatureTransformer.scala +++ /dev/null @@ -1,119 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.response_transformer - -import com.twitter.home_mixer.marshaller.timelines.TopicContextFunctionalityTypeUnmarshaller -import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorIsCreatorFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorIsGoldVerifiedFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorIsGrayVerifiedFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorIsLegacyVerifiedFeature -import com.twitter.home_mixer.model.HomeFeatures.CachedCandidatePipelineIdentifierFeature -import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsReadFromCacheFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.LastScoredTimestampMsFeature -import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.StreamToKafkaFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature -import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature -import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature -import com.twitter.home_mixer.model.HomeFeatures.WeightedModelScoreFeature -import com.twitter.home_mixer.{thriftscala => hmt} -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier - -object CachedScoredTweetsResponseFeatureTransformer - extends CandidateFeatureTransformer[hmt.ScoredTweet] { - - override val identifier: TransformerIdentifier = - TransformerIdentifier("CachedScoredTweetsResponse") - - override val features: Set[Feature[_, _]] = Set( - AncestorsFeature, - AuthorIdFeature, - AuthorIsBlueVerifiedFeature, - AuthorIsCreatorFeature, - AuthorIsGoldVerifiedFeature, - AuthorIsGrayVerifiedFeature, - AuthorIsLegacyVerifiedFeature, - CachedCandidatePipelineIdentifierFeature, - DirectedAtUserIdFeature, - ExclusiveConversationAuthorIdFeature, - InNetworkFeature, - InReplyToTweetIdFeature, - InReplyToUserIdFeature, - IsReadFromCacheFeature, - IsRetweetFeature, - LastScoredTimestampMsFeature, - PerspectiveFilteredLikedByUserIdsFeature, - QuotedTweetIdFeature, - QuotedUserIdFeature, - SGSValidFollowedByUserIdsFeature, - SGSValidLikedByUserIdsFeature, - ScoreFeature, - SourceTweetIdFeature, - SourceUserIdFeature, - StreamToKafkaFeature, - SuggestTypeFeature, - TopicContextFunctionalityTypeFeature, - TopicIdSocialContextFeature, - TweetUrlsFeature, - WeightedModelScoreFeature - ) - - override def transform(candidate: hmt.ScoredTweet): FeatureMap = - FeatureMapBuilder() - .add(AncestorsFeature, candidate.ancestors.getOrElse(Seq.empty)) - .add(AuthorIdFeature, Some(candidate.authorId)) - .add(AuthorIsBlueVerifiedFeature, candidate.authorMetadata.exists(_.blueVerified)) - .add(AuthorIsGoldVerifiedFeature, candidate.authorMetadata.exists(_.goldVerified)) - .add(AuthorIsGrayVerifiedFeature, candidate.authorMetadata.exists(_.grayVerified)) - .add(AuthorIsLegacyVerifiedFeature, candidate.authorMetadata.exists(_.legacyVerified)) - .add(AuthorIsCreatorFeature, candidate.authorMetadata.exists(_.creator)) - .add(CachedCandidatePipelineIdentifierFeature, candidate.candidatePipelineIdentifier) - .add(DirectedAtUserIdFeature, candidate.directedAtUserId) - .add(ExclusiveConversationAuthorIdFeature, candidate.exclusiveConversationAuthorId) - .add(InNetworkFeature, candidate.inNetwork.getOrElse(true)) - .add(InReplyToTweetIdFeature, candidate.inReplyToTweetId) - .add(InReplyToUserIdFeature, candidate.inReplyToUserId) - .add(IsReadFromCacheFeature, true) - .add(IsRetweetFeature, candidate.sourceTweetId.isDefined) - .add(LastScoredTimestampMsFeature, candidate.lastScoredTimestampMs) - .add( - PerspectiveFilteredLikedByUserIdsFeature, - candidate.perspectiveFilteredLikedByUserIds.getOrElse(Seq.empty)) - .add(QuotedTweetIdFeature, candidate.quotedTweetId) - .add(QuotedUserIdFeature, candidate.quotedUserId) - .add(ScoreFeature, candidate.score) - .add(SGSValidLikedByUserIdsFeature, candidate.sgsValidLikedByUserIds.getOrElse(Seq.empty)) - .add( - SGSValidFollowedByUserIdsFeature, - candidate.sgsValidFollowedByUserIds.getOrElse(Seq.empty)) - .add(SourceTweetIdFeature, candidate.sourceTweetId) - .add(SourceUserIdFeature, candidate.sourceUserId) - .add(StreamToKafkaFeature, false) - .add(SuggestTypeFeature, candidate.suggestType) - .add( - TopicContextFunctionalityTypeFeature, - candidate.topicFunctionalityType.map(TopicContextFunctionalityTypeUnmarshaller(_))) - .add(TopicIdSocialContextFeature, candidate.topicId) - .add(TweetUrlsFeature, candidate.tweetUrls.getOrElse(Seq.empty)) - .add(WeightedModelScoreFeature, candidate.score) - .build() -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsBackfillResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsBackfillResponseFeatureTransformer.scala deleted file mode 100644 index 806c0d11d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsBackfillResponseFeatureTransformer.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.response_transformer - -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature -import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts} -import com.twitter.timelineservice.suggests.{thriftscala => st} - -object ScoredTweetsBackfillResponseFeatureTransformer extends CandidateFeatureTransformer[Long] { - - override val identifier: TransformerIdentifier = - TransformerIdentifier("ScoredTweetsBackfillResponse") - - override val features: Set[Feature[_, _]] = Set( - CandidateSourceIdFeature, - FromInNetworkSourceFeature, - SuggestTypeFeature - ) - - override def transform(candidate: Long): FeatureMap = FeatureMapBuilder() - .add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.BackfillOrganicTweet)) - .add(FromInNetworkSourceFeature, true) - .add(SuggestTypeFeature, Some(st.SuggestType.RankedOrganicTweet)) - .build() -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsFrsResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsFrsResponseFeatureTransformer.scala deleted file mode 100644 index 077dccf24..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsFrsResponseFeatureTransformer.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.response_transformer - -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.timelineranker.{thriftscala => tlr} -import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts} -import com.twitter.timelineservice.suggests.{thriftscala => st} - -object ScoredTweetsFrsResponseFeatureTransformer - extends CandidateFeatureTransformer[tlr.CandidateTweet] { - - override val identifier: TransformerIdentifier = TransformerIdentifier("ScoredTweetsFrsResponse") - - override val features: Set[Feature[_, _]] = TimelineRankerResponseTransformer.features - - override def transform(candidate: tlr.CandidateTweet): FeatureMap = { - val baseFeatures = TimelineRankerResponseTransformer.transform(candidate) - - val features = FeatureMapBuilder() - .add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.FrsTweet)) - .add(SuggestTypeFeature, Some(st.SuggestType.FrsTweet)) - .build() - - baseFeatures ++ features - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsInNetworkResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsInNetworkResponseFeatureTransformer.scala deleted file mode 100644 index d7d23bc98..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsInNetworkResponseFeatureTransformer.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.response_transformer - -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature -import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.timelineranker.{thriftscala => tlr} -import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts} -import com.twitter.timelineservice.suggests.{thriftscala => st} - -object ScoredTweetsInNetworkResponseFeatureTransformer - extends CandidateFeatureTransformer[tlr.CandidateTweet] { - - override val identifier: TransformerIdentifier = - TransformerIdentifier("ScoredTweetsInNetworkResponse") - - override val features: Set[Feature[_, _]] = TimelineRankerResponseTransformer.features - - override def transform(candidate: tlr.CandidateTweet): FeatureMap = { - val baseFeatures = TimelineRankerResponseTransformer.transform(candidate) - - val features = FeatureMapBuilder() - .add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.RecycledTweet)) - .add(FromInNetworkSourceFeature, true) - .add(SuggestTypeFeature, Some(st.SuggestType.RankedTimelineTweet)) - .build() - - baseFeatures ++ features - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsListsResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsListsResponseFeatureTransformer.scala deleted file mode 100644 index 6a48e84c8..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsListsResponseFeatureTransformer.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.response_transformer - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature -import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.timelineservice.{thriftscala => t} -import com.twitter.timelineservice.suggests.{thriftscala => st} -import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts} - -object ScoredTweetsListsResponseFeatureTransformer extends CandidateFeatureTransformer[t.Tweet] { - - override val identifier: TransformerIdentifier = - TransformerIdentifier("ScoredTweetsListsResponse") - - override val features: Set[Feature[_, _]] = Set( - AuthorIdFeature, - CandidateSourceIdFeature, - FromInNetworkSourceFeature, - IsRetweetFeature, - SuggestTypeFeature, - SourceTweetIdFeature, - SourceUserIdFeature, - ) - - override def transform(candidate: t.Tweet): FeatureMap = { - FeatureMapBuilder() - .add(AuthorIdFeature, candidate.userId) - .add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.ListTweet)) - .add(FromInNetworkSourceFeature, false) - .add(IsRetweetFeature, candidate.sourceStatusId.isDefined) - .add(SuggestTypeFeature, Some(st.SuggestType.RankedListTweet)) - .add(SourceTweetIdFeature, candidate.sourceStatusId) - .add(SourceUserIdFeature, candidate.sourceUserId) - .build() - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsPopularVideosResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsPopularVideosResponseFeatureTransformer.scala deleted file mode 100644 index 6657e6f5c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsPopularVideosResponseFeatureTransformer.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.response_transformer - -import com.twitter.explore_ranker.{thriftscala => ert} -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature -import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature -import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRandomTweetFeature -import com.twitter.home_mixer.model.HomeFeatures.StreamToKafkaFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts} -import com.twitter.timelineservice.suggests.{thriftscala => st} - -object ScoredTweetsPopularVideosResponseFeatureTransformer - extends CandidateFeatureTransformer[ert.ExploreTweetRecommendation] { - - override val identifier: TransformerIdentifier = - TransformerIdentifier("ScoredTweetsPopularVideosResponse") - - override val features: Set[Feature[_, _]] = Set( - AuthorIdFeature, - CandidateSourceIdFeature, - FromInNetworkSourceFeature, - HasVideoFeature, - IsRandomTweetFeature, - StreamToKafkaFeature, - SuggestTypeFeature - ) - - override def transform(candidate: ert.ExploreTweetRecommendation): FeatureMap = { - FeatureMapBuilder() - .add(AuthorIdFeature, candidate.authorId) - .add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.MediaTweet)) - .add(FromInNetworkSourceFeature, false) - .add(HasVideoFeature, candidate.mediaType.contains(ert.MediaType.Video)) - .add(IsRandomTweetFeature, false) - .add(StreamToKafkaFeature, true) - .add(SuggestTypeFeature, Some(st.SuggestType.MediaTweet)) - .build() - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsTweetMixerResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsTweetMixerResponseFeatureTransformer.scala deleted file mode 100644 index 4312b5104..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsTweetMixerResponseFeatureTransformer.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.response_transformer - -import com.twitter.tweet_mixer.{thriftscala => tmt} -import com.twitter.home_mixer.model.HomeFeatures._ -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts} -import com.twitter.timelineservice.suggests.{thriftscala => st} -import com.twitter.tsp.{thriftscala => tsp} - -object ScoredTweetsTweetMixerResponseFeatureTransformer - extends CandidateFeatureTransformer[tmt.TweetResult] { - - override val identifier: TransformerIdentifier = - TransformerIdentifier("ScoredTweetsTweetMixerResponse") - - override val features: Set[Feature[_, _]] = Set( - CandidateSourceIdFeature, - FromInNetworkSourceFeature, - IsRandomTweetFeature, - StreamToKafkaFeature, - SuggestTypeFeature, - TSPMetricTagFeature - ) - - override def transform(candidate: tmt.TweetResult): FeatureMap = { - val tweetMixerMetricTags = candidate.metricTags.getOrElse(Seq.empty) - val tspMetricTag = tweetMixerMetricTags - .map(TweetMixerMetricTagToTspMetricTag) - .filter(_.nonEmpty).map(_.get).toSet - - FeatureMapBuilder() - .add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.Simcluster)) - .add(FromInNetworkSourceFeature, false) - .add(IsRandomTweetFeature, false) - .add(StreamToKafkaFeature, true) - .add(SuggestTypeFeature, Some(st.SuggestType.ScTweet)) - .add(TSPMetricTagFeature, tspMetricTag) - .build() - } - - private def TweetMixerMetricTagToTspMetricTag( - tweetMixerMetricTag: tmt.MetricTag - ): Option[tsp.MetricTag] = tweetMixerMetricTag match { - case tmt.MetricTag.TweetFavorite => Some(tsp.MetricTag.TweetFavorite) - case tmt.MetricTag.Retweet => Some(tsp.MetricTag.Retweet) - case _ => None - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsUtegResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsUtegResponseFeatureTransformer.scala deleted file mode 100644 index e7bd61b2b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsUtegResponseFeatureTransformer.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.response_transformer - -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.timelineranker.{thriftscala => tlr} -import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts} -import com.twitter.timelineservice.suggests.{thriftscala => st} - -object ScoredTweetsUtegResponseFeatureTransformer - extends CandidateFeatureTransformer[tlr.CandidateTweet] { - - override val identifier: TransformerIdentifier = TransformerIdentifier("ScoredTweetsUtegResponse") - - override val features: Set[Feature[_, _]] = TimelineRankerResponseTransformer.features - - override def transform(candidate: tlr.CandidateTweet): FeatureMap = { - val baseFeatures = TimelineRankerResponseTransformer.transform(candidate) - - val features = FeatureMapBuilder() - .add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.RecommendedTweet)) - .add(SuggestTypeFeature, Some(st.SuggestType.ActivityTweet)) - .build() - - baseFeatures ++ features - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/TimelineRankerResponseTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/TimelineRankerResponseTransformer.scala deleted file mode 100644 index a261b2fc2..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/TimelineRankerResponseTransformer.scala +++ /dev/null @@ -1,91 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.response_transformer - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature -import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature -import com.twitter.home_mixer.model.HomeFeatures.EarlybirdScoreFeature -import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature -import com.twitter.home_mixer.model.HomeFeatures.HasImageFeature -import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRandomTweetFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.MentionScreenNameFeature -import com.twitter.home_mixer.model.HomeFeatures.MentionUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.StreamToKafkaFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature -import com.twitter.home_mixer.util.tweetypie.content.TweetMediaFeaturesExtractor -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.timelineranker.{thriftscala => tlr} - -object TimelineRankerResponseTransformer { - - val features: Set[Feature[_, _]] = Set( - AuthorIdFeature, - CandidateSourceIdFeature, - DirectedAtUserIdFeature, - EarlybirdFeature, - EarlybirdScoreFeature, - ExclusiveConversationAuthorIdFeature, - FromInNetworkSourceFeature, - HasImageFeature, - HasVideoFeature, - InReplyToTweetIdFeature, - InReplyToUserIdFeature, - IsRandomTweetFeature, - IsRetweetFeature, - MentionScreenNameFeature, - MentionUserIdFeature, - StreamToKafkaFeature, - QuotedTweetIdFeature, - QuotedUserIdFeature, - SourceTweetIdFeature, - SourceUserIdFeature, - SuggestTypeFeature, - TweetUrlsFeature - ) - - def transform(candidate: tlr.CandidateTweet): FeatureMap = { - val tweet = candidate.tweet - val quotedTweet = tweet.filter(_.quotedTweet.exists(_.tweetId != 0)).flatMap(_.quotedTweet) - val mentions = tweet.flatMap(_.mentions).getOrElse(Seq.empty) - val coreData = tweet.flatMap(_.coreData) - val share = coreData.flatMap(_.share) - val reply = coreData.flatMap(_.reply) - - FeatureMapBuilder() - .add(AuthorIdFeature, coreData.map(_.userId)) - .add(DirectedAtUserIdFeature, coreData.flatMap(_.directedAtUser.map(_.userId))) - .add(EarlybirdFeature, candidate.features) - .add(EarlybirdScoreFeature, candidate.features.map(_.earlybirdScore)) - .add( - ExclusiveConversationAuthorIdFeature, - tweet.flatMap(_.exclusiveTweetControl.map(_.conversationAuthorId))) - .add(FromInNetworkSourceFeature, false) - .add(HasImageFeature, tweet.exists(TweetMediaFeaturesExtractor.hasImage)) - .add(HasVideoFeature, tweet.exists(TweetMediaFeaturesExtractor.hasVideo)) - .add(InReplyToTweetIdFeature, reply.flatMap(_.inReplyToStatusId)) - .add(InReplyToUserIdFeature, reply.map(_.inReplyToUserId)) - .add(IsRandomTweetFeature, candidate.features.exists(_.isRandomTweet.getOrElse(false))) - .add(IsRetweetFeature, share.isDefined) - .add(MentionScreenNameFeature, mentions.map(_.screenName)) - .add(MentionUserIdFeature, mentions.flatMap(_.userId)) - .add(StreamToKafkaFeature, true) - .add(QuotedTweetIdFeature, quotedTweet.map(_.tweetId)) - .add(QuotedUserIdFeature, quotedTweet.map(_.userId)) - .add(SourceTweetIdFeature, share.map(_.sourceStatusId)) - .add(SourceUserIdFeature, share.map(_.sourceUserId)) - .add(TweetUrlsFeature, candidate.features.flatMap(_.urlsList).getOrElse(Seq.empty)) - .build() - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/BUILD.bazel deleted file mode 100644 index 81cf19e9b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", - "src/thrift/com/twitter/search:earlybird-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/EarlybirdResponseTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/EarlybirdResponseTransformer.scala deleted file mode 100644 index f0b1b59b1..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/EarlybirdResponseTransformer.scala +++ /dev/null @@ -1,92 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature -import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.EarlybirdScoreFeature -import com.twitter.home_mixer.model.HomeFeatures.EarlybirdSearchResultFeature -import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature -import com.twitter.home_mixer.model.HomeFeatures.HasImageFeature -import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRandomTweetFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.MentionScreenNameFeature -import com.twitter.home_mixer.model.HomeFeatures.MentionUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.StreamToKafkaFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature -import com.twitter.home_mixer.util.tweetypie.content.TweetMediaFeaturesExtractor -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.search.earlybird.{thriftscala => eb} - -object EarlybirdResponseTransformer { - - val features: Set[Feature[_, _]] = Set( - AuthorIdFeature, - CandidateSourceIdFeature, - DirectedAtUserIdFeature, - EarlybirdScoreFeature, - EarlybirdSearchResultFeature, - ExclusiveConversationAuthorIdFeature, - FromInNetworkSourceFeature, - HasImageFeature, - HasVideoFeature, - InReplyToTweetIdFeature, - InReplyToUserIdFeature, - IsRandomTweetFeature, - IsRetweetFeature, - MentionScreenNameFeature, - MentionUserIdFeature, - StreamToKafkaFeature, - QuotedTweetIdFeature, - QuotedUserIdFeature, - SourceTweetIdFeature, - SourceUserIdFeature, - SuggestTypeFeature, - TweetUrlsFeature - ) - - def transform(candidate: eb.ThriftSearchResult): FeatureMap = { - val tweet = candidate.tweetypieTweet - val quotedTweet = tweet.flatMap(_.quotedTweet) - val mentions = tweet.flatMap(_.mentions).getOrElse(Seq.empty) - val coreData = tweet.flatMap(_.coreData) - val share = coreData.flatMap(_.share) - val reply = coreData.flatMap(_.reply) - FeatureMapBuilder() - .add(AuthorIdFeature, coreData.map(_.userId)) - .add(DirectedAtUserIdFeature, coreData.flatMap(_.directedAtUser.map(_.userId))) - .add(EarlybirdSearchResultFeature, Some(candidate)) - .add(EarlybirdScoreFeature, candidate.metadata.flatMap(_.score)) - .add( - ExclusiveConversationAuthorIdFeature, - tweet.flatMap(_.exclusiveTweetControl.map(_.conversationAuthorId))) - .add(FromInNetworkSourceFeature, false) - .add(HasImageFeature, tweet.exists(TweetMediaFeaturesExtractor.hasImage)) - .add(HasVideoFeature, tweet.exists(TweetMediaFeaturesExtractor.hasVideo)) - .add(InReplyToTweetIdFeature, reply.flatMap(_.inReplyToStatusId)) - .add(InReplyToUserIdFeature, reply.map(_.inReplyToUserId)) - .add(IsRandomTweetFeature, candidate.tweetFeatures.exists(_.isRandomTweet.getOrElse(false))) - .add(IsRetweetFeature, share.isDefined) - .add(MentionScreenNameFeature, mentions.map(_.screenName)) - .add(MentionUserIdFeature, mentions.flatMap(_.userId)) - .add(StreamToKafkaFeature, true) - .add(QuotedTweetIdFeature, quotedTweet.map(_.tweetId)) - .add(QuotedUserIdFeature, quotedTweet.map(_.userId)) - .add(SourceTweetIdFeature, share.map(_.sourceStatusId)) - .add(SourceUserIdFeature, share.map(_.sourceUserId)) - .add( - TweetUrlsFeature, - candidate.metadata.flatMap(_.tweetUrls.map(_.map(_.originalUrl))).getOrElse(Seq.empty)) - .build() - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsEarlybirdFrsResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsEarlybirdFrsResponseFeatureTransformer.scala deleted file mode 100644 index bb9ea8bee..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsEarlybirdFrsResponseFeatureTransformer.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird - -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.search.earlybird.{thriftscala => eb} -import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts} -import com.twitter.timelineservice.suggests.{thriftscala => st} - -object ScoredTweetsEarlybirdFrsResponseFeatureTransformer - extends CandidateFeatureTransformer[eb.ThriftSearchResult] { - - override val identifier: TransformerIdentifier = - TransformerIdentifier("ScoredTweetsEarlybirdFrsResponse") - - override val features: Set[Feature[_, _]] = EarlybirdResponseTransformer.features - - override def transform(candidate: eb.ThriftSearchResult): FeatureMap = { - - val baseFeatures = EarlybirdResponseTransformer.transform(candidate) - - val features = FeatureMapBuilder() - .add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.FrsTweet)) - .add(SuggestTypeFeature, Some(st.SuggestType.FrsTweet)) - .build() - - baseFeatures ++ features - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsEarlybirdInNetworkResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsEarlybirdInNetworkResponseFeatureTransformer.scala deleted file mode 100644 index 6b6a9d003..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsEarlybirdInNetworkResponseFeatureTransformer.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird - -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.search.earlybird.{thriftscala => eb} -import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts} -import com.twitter.timelineservice.suggests.{thriftscala => st} - -object ScoredTweetsEarlybirdInNetworkResponseFeatureTransformer - extends CandidateFeatureTransformer[eb.ThriftSearchResult] { - override val identifier: TransformerIdentifier = - TransformerIdentifier("ScoredTweetsEarlybirdInNetworkResponse") - - override val features: Set[Feature[_, _]] = EarlybirdResponseTransformer.features - - override def transform(candidate: eb.ThriftSearchResult): FeatureMap = { - - val baseFeatures = EarlybirdResponseTransformer.transform(candidate) - - val features = FeatureMapBuilder() - .add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.RecycledTweet)) - .add(SuggestTypeFeature, Some(st.SuggestType.RecycledTweet)) - .build() - - baseFeatures ++ features - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/BUILD.bazel deleted file mode 100644 index 91ff12ee1..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/BUILD.bazel +++ /dev/null @@ -1,20 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/module", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "src/scala/com/twitter/timelines/prediction/features/recap", - "timelineservice/common:model", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/DiversityDiscountProvider.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/DiversityDiscountProvider.scala deleted file mode 100644 index 62dea8748..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/DiversityDiscountProvider.scala +++ /dev/null @@ -1,63 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.scorer - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures - -trait DiversityDiscountProvider { - - /** - * Fetch the ID of the entity to diversify - */ - def entityId(candidate: CandidateWithFeatures[TweetCandidate]): Option[Long] - - /** - * Compute discount factor for each candidate based on position (zero-based) - * relative to other candidates associated with the same entity - */ - def discount(position: Int): Double - - /** - * Return candidate IDs sorted by score in descending order - */ - def sort(candidates: Seq[CandidateWithFeatures[TweetCandidate]]): Seq[Long] = candidates - .map { candidate => - (candidate.candidate.id, candidate.features.getOrElse(ScoreFeature, None).getOrElse(0.0)) - } - .sortBy(_._2)(Ordering.Double.reverse) - .map(_._1) - - /** - * Group by the specified entity ID (e.g. authors, likers, followers) - * Sort each group by score in descending order - * Determine the discount factor based on the position of each candidate - */ - def apply( - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Map[Long, Double] = candidates - .groupBy(entityId) - .flatMap { - case (entityIdOpt, entityCandidates) => - val sortedCandidateIds = sort(entityCandidates) - - if (entityIdOpt.isDefined) { - sortedCandidateIds.zipWithIndex.map { - case (candidateId, index) => - candidateId -> discount(index) - } - } else sortedCandidateIds.map(_ -> 1.0) - } -} - -object AuthorDiversityDiscountProvider extends DiversityDiscountProvider { - private val Decay = 0.5 - private val Floor = 0.25 - - override def entityId(candidate: CandidateWithFeatures[TweetCandidate]): Option[Long] = - candidate.features.getOrElse(AuthorIdFeature, None) - - // Provides an exponential decay based discount by position (with a floor) - override def discount(position: Int): Double = - (1 - Floor) * Math.pow(Decay, position) + Floor -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/HeuristicScorer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/HeuristicScorer.scala deleted file mode 100644 index 765523b53..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/HeuristicScorer.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.scorer - -import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier -import com.twitter.stitch.Stitch - -/** - * Apply various heuristics to the model score - */ -object HeuristicScorer extends Scorer[ScoredTweetsQuery, TweetCandidate] { - - override val identifier: ScorerIdentifier = ScorerIdentifier("Heuristic") - - override val features: Set[Feature[_, _]] = Set(ScoreFeature) - - override def apply( - query: ScoredTweetsQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - val rescorers = Seq( - RescoreOutOfNetwork, - RescoreReplies, - RescoreBlueVerified, - RescoreCreators, - RescoreMTLNormalization, - RescoreAuthorDiversity(AuthorDiversityDiscountProvider(candidates)), - RescoreFeedbackFatigue(query) - ) - - val updatedScores = candidates.map { candidate => - val score = candidate.features.getOrElse(ScoreFeature, None) - val scaleFactor = rescorers.map(_(query, candidate)).product - val updatedScore = score.map(_ * scaleFactor) - FeatureMapBuilder().add(ScoreFeature, updatedScore).build() - } - - Stitch.value(updatedScores) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/NaviModelScorer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/NaviModelScorer.scala deleted file mode 100644 index cbde87f73..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/NaviModelScorer.scala +++ /dev/null @@ -1,179 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.scorer - -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature -import com.twitter.home_mixer.model.HomeFeatures.WeightedModelScoreFeature -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.scorer.PredictedScoreFeature.PredictedScoreFeatures -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.datarecord.AllFeatures -import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordConverter -import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordExtractor -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.pipeline_failure.IllegalStateFailure -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch -import com.twitter.timelines.clients.predictionservice.PredictionGRPCService -import com.twitter.timelines.clients.predictionservice.PredictionServiceGRPCClient -import com.twitter.util.Future -import com.twitter.util.Return -import javax.inject.Inject -import javax.inject.Singleton - -object CommonFeaturesDataRecordFeature - extends DataRecordInAFeature[PipelineQuery] - with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -object CandidateFeaturesDataRecordFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -case class NaviModelScorer @Inject() ( - predictionGRPCService: PredictionGRPCService, - statsReceiver: StatsReceiver) - extends Scorer[ScoredTweetsQuery, TweetCandidate] { - - override val identifier: ScorerIdentifier = ScorerIdentifier("NaviModel") - - override val features: Set[Feature[_, _]] = Set( - CommonFeaturesDataRecordFeature, - CandidateFeaturesDataRecordFeature, - WeightedModelScoreFeature, - ScoreFeature - ) ++ PredictedScoreFeatures.asInstanceOf[Set[Feature[_, _]]] - - private val queryDataRecordAdapter = new DataRecordConverter(AllFeatures()) - private val candidatesDataRecordAdapter = new DataRecordConverter(AllFeatures()) - private val resultDataRecordExtractor = new DataRecordExtractor(PredictedScoreFeatures) - - private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) - private val failuresStat = scopedStatsReceiver.stat("failures") - private val responsesStat = scopedStatsReceiver.stat("responses") - private val invalidResponsesCounter = scopedStatsReceiver.counter("invalidResponses") - private val candidatesDataRecordAdapterLatencyStat = - scopedStatsReceiver.scope("candidatesDataRecordAdapter").stat("latency_ms") - - private val StatsReadabilityMultiplier = 1000 - private val Epsilon = 0.001 - private val PredictedScoreStatName = f"predictedScore${StatsReadabilityMultiplier}x" - private val MissingScoreStatName = "missingScore" - private val scoreStat = scopedStatsReceiver.stat(f"score${StatsReadabilityMultiplier}x") - - private val RequestBatchSize = 64 - private val DataRecordConstructionParallelism = 32 - private val ModelId = "Home" - - private val modelClient = new PredictionServiceGRPCClient( - service = predictionGRPCService, - statsReceiver = statsReceiver, - requestBatchSize = RequestBatchSize, - useCompact = false - ) - - override def apply( - query: ScoredTweetsQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - val commonRecord = query.features.map(queryDataRecordAdapter.toDataRecord) - val candidateRecords: Future[Seq[DataRecord]] = - Stat.time(candidatesDataRecordAdapterLatencyStat) { - OffloadFuturePools.parallelize[FeatureMap, DataRecord]( - inputSeq = candidates.map(_.features), - transformer = candidatesDataRecordAdapter.toDataRecord(_), - parallelism = DataRecordConstructionParallelism, - default = new DataRecord - ) - } - - val scoreFeatureMaps = candidateRecords.flatMap { records => - val predictionResponses = - modelClient.getPredictions(records, commonRecord, modelId = Some(ModelId)) - - predictionResponses.map { responses => - failuresStat.add(responses.count(_.isThrow)) - responsesStat.add(responses.size) - - if (responses.size == candidates.size) { - val predictedScoreFeatureMaps = responses.map { - case Return(dataRecord) => resultDataRecordExtractor.fromDataRecord(dataRecord) - case _ => resultDataRecordExtractor.fromDataRecord(new DataRecord()) - } - - // Add Data Record to candidate Feature Map for logging in later stages - predictedScoreFeatureMaps.zip(records).map { - case (predictedScoreFeatureMap, candidateRecord) => - val weightedModelScore = computeWeightedModelScore(query, predictedScoreFeatureMap) - scoreStat.add((weightedModelScore * StatsReadabilityMultiplier).toFloat) - - predictedScoreFeatureMap + - (CandidateFeaturesDataRecordFeature, candidateRecord) + - (CommonFeaturesDataRecordFeature, commonRecord.getOrElse(new DataRecord())) + - (ScoreFeature, Some(weightedModelScore)) + - (WeightedModelScoreFeature, Some(weightedModelScore)) - } - } else { - invalidResponsesCounter.incr() - throw PipelineFailure(IllegalStateFailure, "Result size mismatched candidates size") - } - } - } - - Stitch.callFuture(scoreFeatureMaps) - } - - /** - * Compute the weighted sum of predicted scores of all engagements - * Convert negative score to positive, if needed - */ - private def computeWeightedModelScore( - query: PipelineQuery, - features: FeatureMap - ): Double = { - val weightedScoreAndModelWeightSeq = PredictedScoreFeatures.toSeq.map { predictedScoreFeature => - val predictedScoreOpt = predictedScoreFeature.extractScore(features) - - predictedScoreOpt match { - case Some(predictedScore) => - scopedStatsReceiver - .stat(predictedScoreFeature.statName, PredictedScoreStatName) - .add((predictedScore * StatsReadabilityMultiplier).toFloat) - case None => - scopedStatsReceiver.counter(predictedScoreFeature.statName, MissingScoreStatName).incr() - } - - val weight = query.params(predictedScoreFeature.modelWeightParam) - val weightedScore = predictedScoreOpt.getOrElse(0.0) * weight - (weightedScore, weight) - } - - val (weightedScores, modelWeights) = weightedScoreAndModelWeightSeq.unzip - val combinedScoreSum = weightedScores.sum - - val positiveModelWeightsSum = modelWeights.filter(_ > 0.0).sum - val negativeModelWeightsSum = modelWeights.filter(_ < 0).sum.abs - val modelWeightsSum = positiveModelWeightsSum + negativeModelWeightsSum - - val weightedScoresSum = - if (modelWeightsSum == 0) combinedScoreSum.max(0.0) - else if (combinedScoreSum < 0) - (combinedScoreSum + negativeModelWeightsSum) / modelWeightsSum * Epsilon - else combinedScoreSum + Epsilon - - weightedScoresSum - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/PredictedScoreFeature.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/PredictedScoreFeature.scala deleted file mode 100644 index f24223a99..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/PredictedScoreFeature.scala +++ /dev/null @@ -1,166 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.scorer - -import com.twitter.dal.personal_data.{thriftjava => pd} -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.Scoring.ModelWeights -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.datarecord.DataRecordOptionalFeature -import com.twitter.product_mixer.core.feature.datarecord.DoubleDataRecordCompatible -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.prediction.features.recap.RecapFeatures - -sealed trait PredictedScoreFeature - extends DataRecordOptionalFeature[TweetCandidate, Double] - with DoubleDataRecordCompatible { - - override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty - def statName: String - def modelWeightParam: FSBoundedParam[Double] - def extractScore: FeatureMap => Option[Double] = _.getOrElse(this, None) -} - -object PredictedFavoriteScoreFeature extends PredictedScoreFeature { - override val featureName: String = RecapFeatures.PREDICTED_IS_FAVORITED.getFeatureName - override val statName = "fav" - override val modelWeightParam = ModelWeights.FavParam -} - -object PredictedReplyScoreFeature extends PredictedScoreFeature { - override val featureName: String = RecapFeatures.PREDICTED_IS_REPLIED.getFeatureName - override val statName = "reply" - override val modelWeightParam = ModelWeights.ReplyParam -} - -object PredictedRetweetScoreFeature extends PredictedScoreFeature { - override val featureName: String = RecapFeatures.PREDICTED_IS_RETWEETED.getFeatureName - override val statName = "retweet" - override val modelWeightParam = ModelWeights.RetweetParam -} - -object PredictedReplyEngagedByAuthorScoreFeature extends PredictedScoreFeature { - override val featureName: String = - RecapFeatures.PREDICTED_IS_REPLIED_REPLY_ENGAGED_BY_AUTHOR.getFeatureName - override val statName = "reply_engaged_by_author" - override val modelWeightParam = ModelWeights.ReplyEngagedByAuthorParam -} - -object PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature extends PredictedScoreFeature { - override val featureName: String = RecapFeatures.PREDICTED_IS_GOOD_CLICKED_V1.getFeatureName - override val statName = "good_click_convo_desc_favorited_or_replied" - override val modelWeightParam = ModelWeights.GoodClickParam - - override def extractScore: FeatureMap => Option[Double] = { featureMap => - val goodClickV1Opt = featureMap.getOrElse(this, None) - val goodClickV2Opt = featureMap.getOrElse(PredictedGoodClickConvoDescUamGt2ScoreFeature, None) - - (goodClickV1Opt, goodClickV2Opt) match { - case (Some(v1Score), Some(v2Score)) => Some(Math.max(v1Score, v2Score)) - case _ => goodClickV1Opt.orElse(goodClickV2Opt) - } - } -} - -object PredictedGoodClickConvoDescUamGt2ScoreFeature extends PredictedScoreFeature { - override val featureName: String = RecapFeatures.PREDICTED_IS_GOOD_CLICKED_V2.getFeatureName - override val statName = "good_click_convo_desc_uam_gt_2" - override val modelWeightParam = ModelWeights.GoodClickV2Param -} - -object PredictedGoodProfileClickScoreFeature extends PredictedScoreFeature { - override val featureName: String = - RecapFeatures.PREDICTED_IS_PROFILE_CLICKED_AND_PROFILE_ENGAGED.getFeatureName - override val statName = "good_profile_click" - override val modelWeightParam = ModelWeights.GoodProfileClickParam -} - -object PredictedVideoPlayback50ScoreFeature extends PredictedScoreFeature { - override val featureName: String = RecapFeatures.PREDICTED_IS_VIDEO_PLAYBACK_50.getFeatureName - override val statName = "video_playback_50" - override val modelWeightParam = ModelWeights.VideoPlayback50Param -} - -object PredictedTweetDetailDwellScoreFeature extends PredictedScoreFeature { - override val featureName: String = - RecapFeatures.PREDICTED_IS_TWEET_DETAIL_DWELLED_15_SEC.getFeatureName - override val statName = "tweet_detail_dwell" - override val modelWeightParam = ModelWeights.TweetDetailDwellParam -} - -object PredictedProfileDwelledScoreFeature extends PredictedScoreFeature { - override val featureName: String = - RecapFeatures.PREDICTED_IS_PROFILE_DWELLED_20_SEC.getFeatureName - override val statName = "profile_dwell" - override val modelWeightParam = ModelWeights.ProfileDwelledParam -} - -object PredictedBookmarkScoreFeature extends PredictedScoreFeature { - override val featureName: String = RecapFeatures.PREDICTED_IS_BOOKMARKED.getFeatureName - override val statName = "bookmark" - override val modelWeightParam = ModelWeights.BookmarkParam -} - -object PredictedShareScoreFeature extends PredictedScoreFeature { - override val featureName: String = - RecapFeatures.PREDICTED_IS_SHARED.getFeatureName - override val statName = "share" - override val modelWeightParam = ModelWeights.ShareParam -} - -object PredictedShareMenuClickScoreFeature extends PredictedScoreFeature { - override val featureName: String = - RecapFeatures.PREDICTED_IS_SHARE_MENU_CLICKED.getFeatureName - override val statName = "share_menu_click" - override val modelWeightParam = ModelWeights.ShareMenuClickParam -} - -// Negative Engagements -object PredictedNegativeFeedbackV2ScoreFeature extends PredictedScoreFeature { - override val featureName: String = - RecapFeatures.PREDICTED_IS_NEGATIVE_FEEDBACK_V2.getFeatureName - override val statName = "negative_feedback_v2" - override val modelWeightParam = ModelWeights.NegativeFeedbackV2Param -} - -object PredictedReportedScoreFeature extends PredictedScoreFeature { - override val featureName: String = - RecapFeatures.PREDICTED_IS_REPORT_TWEET_CLICKED.getFeatureName - override val statName = "reported" - override val modelWeightParam = ModelWeights.ReportParam -} - -object PredictedStrongNegativeFeedbackScoreFeature extends PredictedScoreFeature { - override val featureName: String = - RecapFeatures.PREDICTED_IS_STRONG_NEGATIVE_FEEDBACK.getFeatureName - override val statName = "strong_negative_feedback" - override val modelWeightParam = ModelWeights.StrongNegativeFeedbackParam -} - -object PredictedWeakNegativeFeedbackScoreFeature extends PredictedScoreFeature { - override val featureName: String = - RecapFeatures.PREDICTED_IS_WEAK_NEGATIVE_FEEDBACK.getFeatureName - override val statName = "weak_negative_feedback" - override val modelWeightParam = ModelWeights.WeakNegativeFeedbackParam -} - -object PredictedScoreFeature { - val PredictedScoreFeatures: Set[PredictedScoreFeature] = Set( - PredictedFavoriteScoreFeature, - PredictedReplyScoreFeature, - PredictedRetweetScoreFeature, - PredictedReplyEngagedByAuthorScoreFeature, - PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature, - PredictedGoodClickConvoDescUamGt2ScoreFeature, - PredictedGoodProfileClickScoreFeature, - PredictedVideoPlayback50ScoreFeature, - PredictedTweetDetailDwellScoreFeature, - PredictedProfileDwelledScoreFeature, - PredictedBookmarkScoreFeature, - PredictedShareScoreFeature, - PredictedShareMenuClickScoreFeature, - // Negative Engagements - PredictedNegativeFeedbackV2ScoreFeature, - PredictedReportedScoreFeature, - PredictedStrongNegativeFeedbackScoreFeature, - PredictedWeakNegativeFeedbackScoreFeature, - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/RescoringFactorProvider.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/RescoringFactorProvider.scala deleted file mode 100644 index d9538b66d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/RescoringFactorProvider.scala +++ /dev/null @@ -1,180 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.scorer - -import com.twitter.home_mixer.functional_component.scorer.FeedbackFatigueScorer -import com.twitter.home_mixer.model.HomeFeatures -import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorIsCreatorFeature -import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.BlueVerifiedAuthorInNetworkMultiplierParam -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.BlueVerifiedAuthorOutOfNetworkMultiplierParam -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CreatorInNetworkMultiplierParam -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CreatorOutOfNetworkMultiplierParam -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.OutOfNetworkScaleFactorParam -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelineservice.{thriftscala => tls} - -trait RescoringFactorProvider { - - def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean - - def factor( - query: PipelineQuery, - candidate: CandidateWithFeatures[TweetCandidate] - ): Double - - def apply( - query: PipelineQuery, - candidate: CandidateWithFeatures[TweetCandidate], - ): Double = if (selector(candidate)) factor(query, candidate) else 1.0 -} - -/** - * Re-scoring multiplier to apply to authors who are eligible subscription content creators - */ -object RescoreCreators extends RescoringFactorProvider { - - def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = - candidate.features.getOrElse(AuthorIsCreatorFeature, false) && - CandidatesUtil.isOriginalTweet(candidate) - - def factor( - query: PipelineQuery, - candidate: CandidateWithFeatures[TweetCandidate] - ): Double = - if (candidate.features.getOrElse(InNetworkFeature, false)) - query.params(CreatorInNetworkMultiplierParam) - else query.params(CreatorOutOfNetworkMultiplierParam) -} - -/** - * Re-scoring multiplier to apply to authors who are verified by Twitter Blue - */ -object RescoreBlueVerified extends RescoringFactorProvider { - - def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = - candidate.features.getOrElse(AuthorIsBlueVerifiedFeature, false) && - CandidatesUtil.isOriginalTweet(candidate) - - def factor( - query: PipelineQuery, - candidate: CandidateWithFeatures[TweetCandidate] - ): Double = - if (candidate.features.getOrElse(InNetworkFeature, false)) - query.params(BlueVerifiedAuthorInNetworkMultiplierParam) - else query.params(BlueVerifiedAuthorOutOfNetworkMultiplierParam) -} - -/** - * Re-scoring multiplier to apply to out-of-network tweets - */ -object RescoreOutOfNetwork extends RescoringFactorProvider { - - def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = - !candidate.features.getOrElse(InNetworkFeature, false) - - def factor( - query: PipelineQuery, - candidate: CandidateWithFeatures[TweetCandidate] - ): Double = query.params(OutOfNetworkScaleFactorParam) -} - -/** - * Re-scoring multiplier to apply to reply candidates - */ -object RescoreReplies extends RescoringFactorProvider { - - private val ScaleFactor = 0.75 - - def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = - candidate.features.getOrElse(InReplyToTweetIdFeature, None).isDefined - - def factor( - query: PipelineQuery, - candidate: CandidateWithFeatures[TweetCandidate] - ): Double = ScaleFactor -} - -/** - * Re-scoring multiplier to calibrate multi-tasks learning model prediction - */ -object RescoreMTLNormalization extends RescoringFactorProvider { - - private val ScaleFactor = 1.0 - - def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = { - candidate.features.contains(HomeFeatures.FocalTweetAuthorIdFeature) - } - - def factor( - query: PipelineQuery, - candidate: CandidateWithFeatures[TweetCandidate] - ): Double = ScaleFactor -} - -/** - * Re-scoring multiplier to apply to multiple tweets from the same author - */ -case class RescoreAuthorDiversity(diversityDiscounts: Map[Long, Double]) - extends RescoringFactorProvider { - - def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = - diversityDiscounts.contains(candidate.candidate.id) - - def factor( - query: PipelineQuery, - candidate: CandidateWithFeatures[TweetCandidate] - ): Double = diversityDiscounts(candidate.candidate.id) -} - -case class RescoreFeedbackFatigue(query: PipelineQuery) extends RescoringFactorProvider { - - def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = true - - private val feedbackEntriesByEngagementType = - query.features - .getOrElse(FeatureMap.empty).getOrElse(FeedbackHistoryFeature, Seq.empty) - .filter { entry => - val timeSinceFeedback = query.queryTime.minus(entry.timestamp) - timeSinceFeedback < FeedbackFatigueScorer.DurationForFiltering + FeedbackFatigueScorer.DurationForDiscounting && - entry.feedbackType == tls.FeedbackType.SeeFewer - }.groupBy(_.engagementType) - - private val authorsToDiscount = - FeedbackFatigueScorer.getUserDiscounts( - query.queryTime, - feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Tweet, Seq.empty)) - - private val likersToDiscount = - FeedbackFatigueScorer.getUserDiscounts( - query.queryTime, - feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Like, Seq.empty)) - - private val followersToDiscount = - FeedbackFatigueScorer.getUserDiscounts( - query.queryTime, - feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Follow, Seq.empty)) - - private val retweetersToDiscount = - FeedbackFatigueScorer.getUserDiscounts( - query.queryTime, - feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Retweet, Seq.empty)) - - def factor( - query: PipelineQuery, - candidate: CandidateWithFeatures[TweetCandidate] - ): Double = { - FeedbackFatigueScorer.getScoreMultiplier( - candidate, - authorsToDiscount, - likersToDiscount, - followersToDiscount, - retweetersToDiscount - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/BUILD.bazel deleted file mode 100644 index a7fccd8ff..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/BUILD.bazel +++ /dev/null @@ -1,25 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/module", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsHeuristicScoringPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsHeuristicScoringPipelineConfig.scala deleted file mode 100644 index aedfc15b5..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsHeuristicScoringPipelineConfig.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.scoring_pipeline - -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.scorer.HeuristicScorer -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.component_library.selector.InsertAppendResults -import com.twitter.product_mixer.core.functional_component.common.AllPipelines -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier -import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipelineConfig - -object ScoredTweetsHeuristicScoringPipelineConfig - extends ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate] { - - override val identifier: ScoringPipelineIdentifier = - ScoringPipelineIdentifier("ScoredTweetsHeuristic") - - override val selectors: Seq[Selector[ScoredTweetsQuery]] = Seq(InsertAppendResults(AllPipelines)) - - override val scorers: Seq[Scorer[ScoredTweetsQuery, TweetCandidate]] = - Seq(HeuristicScorer) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsModelScoringPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsModelScoringPipelineConfig.scala deleted file mode 100644 index ab1b49a83..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsModelScoringPipelineConfig.scala +++ /dev/null @@ -1,223 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.scoring_pipeline - -import com.twitter.home_mixer.functional_component.feature_hydrator._ -import com.twitter.home_mixer.model.HomeFeatures.EarlybirdScoreFeature -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.CachedScoredTweetsCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsBackfillCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsFrsCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsInNetworkCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsListsCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsPopularVideosCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsTweetMixerCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsUtegCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.AncestorFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.AuthorFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.AuthorIsCreatorFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.EarlybirdFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.GizmoduckAuthorFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.GraphTwoHopFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.MetricCenterUserCountingFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RealGraphViewerAuthorFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RealGraphViewerRelatedUsersFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RealTimeInteractionGraphEdgeFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.SimClustersEngagementSimilarityFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.SimClustersUserTweetScoresHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TSPInferredTopicFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TweetMetaDataFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TweetTimeFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TweetypieContentFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TwhinAuthorFollowFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.UtegFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.Phase1EdgeAggregateFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.Phase2EdgeAggregateFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates._ -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.QualityFactor -import com.twitter.home_mixer.product.scored_tweets.scorer.NaviModelScorer -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.component_library.selector.DropMaxCandidates -import com.twitter.product_mixer.component_library.selector.InsertAppendResults -import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates -import com.twitter.product_mixer.core.functional_component.common.AllExceptPipelines -import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.gate.BaseGate -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.pipeline_failure.UnexpectedCandidateResult -import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipelineConfig -import com.twitter.timelines.configapi.Param - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ScoredTweetsModelScoringPipelineConfig @Inject() ( - // candidate sources - scoredTweetsInNetworkCandidatePipelineConfig: ScoredTweetsInNetworkCandidatePipelineConfig, - scoredTweetsUtegCandidatePipelineConfig: ScoredTweetsUtegCandidatePipelineConfig, - scoredTweetsTweetMixerCandidatePipelineConfig: ScoredTweetsTweetMixerCandidatePipelineConfig, - scoredTweetsFrsCandidatePipelineConfig: ScoredTweetsFrsCandidatePipelineConfig, - scoredTweetsListsCandidatePipelineConfig: ScoredTweetsListsCandidatePipelineConfig, - scoredTweetsPopularVideosCandidatePipelineConfig: ScoredTweetsPopularVideosCandidatePipelineConfig, - scoredTweetsBackfillCandidatePipelineConfig: ScoredTweetsBackfillCandidatePipelineConfig, - // feature hydrators - ancestorFeatureHydrator: AncestorFeatureHydrator, - authorFeatureHydrator: AuthorFeatureHydrator, - authorIsCreatorFeatureHydrator: AuthorIsCreatorFeatureHydrator, - earlybirdFeatureHydrator: EarlybirdFeatureHydrator, - gizmoduckAuthorSafetyFeatureHydrator: GizmoduckAuthorFeatureHydrator, - graphTwoHopFeatureHydrator: GraphTwoHopFeatureHydrator, - metricCenterUserCountingFeatureHydrator: MetricCenterUserCountingFeatureHydrator, - perspectiveFilteredSocialContextFeatureHydrator: PerspectiveFilteredSocialContextFeatureHydrator, - realGraphViewerAuthorFeatureHydrator: RealGraphViewerAuthorFeatureHydrator, - realGraphViewerRelatedUsersFeatureHydrator: RealGraphViewerRelatedUsersFeatureHydrator, - realTimeInteractionGraphEdgeFeatureHydrator: RealTimeInteractionGraphEdgeFeatureHydrator, - sgsValidSocialContextFeatureHydrator: SGSValidSocialContextFeatureHydrator, - simClustersEngagementSimilarityFeatureHydrator: SimClustersEngagementSimilarityFeatureHydrator, - simClustersUserTweetScoresHydrator: SimClustersUserTweetScoresHydrator, - tspInferredTopicFeatureHydrator: TSPInferredTopicFeatureHydrator, - tweetypieContentFeatureHydrator: TweetypieContentFeatureHydrator, - twhinAuthorFollowFeatureHydrator: TwhinAuthorFollowFeatureHydrator, - utegFeatureHydrator: UtegFeatureHydrator, - // real time aggregate feature hydrators - engagementsReceivedByAuthorRealTimeAggregateFeatureHydrator: EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator, - topicCountryEngagementRealTimeAggregateFeatureHydrator: TopicCountryEngagementRealTimeAggregateFeatureHydrator, - topicEngagementRealTimeAggregateFeatureHydrator: TopicEngagementRealTimeAggregateFeatureHydrator, - tweetCountryEngagementRealTimeAggregateFeatureHydrator: TweetCountryEngagementRealTimeAggregateFeatureHydrator, - tweetEngagementRealTimeAggregateFeatureHydrator: TweetEngagementRealTimeAggregateFeatureHydrator, - twitterListEngagementRealTimeAggregateFeatureHydrator: TwitterListEngagementRealTimeAggregateFeatureHydrator, - userAuthorEngagementRealTimeAggregateFeatureHydrator: UserAuthorEngagementRealTimeAggregateFeatureHydrator, - // offline aggregate feature hydrators - phase1EdgeAggregateFeatureHydrator: Phase1EdgeAggregateFeatureHydrator, - phase2EdgeAggregateFeatureHydrator: Phase2EdgeAggregateFeatureHydrator, - // model - naviModelScorer: NaviModelScorer) - extends ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate] { - - override val identifier: ScoringPipelineIdentifier = - ScoringPipelineIdentifier("ScoredTweetsModel") - - private val nonCachedScoringPipelineScope = AllExceptPipelines( - pipelinesToExclude = Set(CachedScoredTweetsCandidatePipelineConfig.Identifier) - ) - - override val gates: Seq[BaseGate[ScoredTweetsQuery]] = Seq( - NonEmptyCandidatesGate(nonCachedScoringPipelineScope) - ) - - private val earlybirdScorePipelineScope = Set( - scoredTweetsInNetworkCandidatePipelineConfig.identifier, - scoredTweetsUtegCandidatePipelineConfig.identifier, - scoredTweetsFrsCandidatePipelineConfig.identifier - ) - - private val earlybirdScoreOrdering: Ordering[CandidateWithDetails] = - Ordering.by[CandidateWithDetails, Double] { - case ItemCandidateWithDetails(_, _, features) => - -features.getOrElse(EarlybirdScoreFeature, None).getOrElse(0.0) - case _ => throw PipelineFailure(UnexpectedCandidateResult, "Invalid candidate type") - } - - private def qualityFactorDropMaxCandidates( - pipelineIdentifier: CandidatePipelineIdentifier, - qualityFactorParam: Param[Int] - ): DropMaxCandidates[ScoredTweetsQuery] = { - new DropMaxCandidates( - pipelineScope = SpecificPipelines(pipelineIdentifier), - maxSelector = (query, _, _) => - (query.getQualityFactorCurrentValue(identifier) * - query.params(qualityFactorParam)).toInt - ) - } - - override val selectors: Seq[Selector[ScoredTweetsQuery]] = Seq( - UpdateSortCandidates(SpecificPipelines(earlybirdScorePipelineScope), earlybirdScoreOrdering), - UpdateSortCandidates( - SpecificPipeline(scoredTweetsBackfillCandidatePipelineConfig.identifier), - CandidatesUtil.reverseChronTweetsOrdering - ), - qualityFactorDropMaxCandidates( - scoredTweetsInNetworkCandidatePipelineConfig.identifier, - QualityFactor.InNetworkMaxTweetsToScoreParam - ), - qualityFactorDropMaxCandidates( - scoredTweetsUtegCandidatePipelineConfig.identifier, - QualityFactor.UtegMaxTweetsToScoreParam - ), - qualityFactorDropMaxCandidates( - scoredTweetsFrsCandidatePipelineConfig.identifier, - QualityFactor.FrsMaxTweetsToScoreParam - ), - qualityFactorDropMaxCandidates( - scoredTweetsTweetMixerCandidatePipelineConfig.identifier, - QualityFactor.TweetMixerMaxTweetsToScoreParam - ), - qualityFactorDropMaxCandidates( - scoredTweetsListsCandidatePipelineConfig.identifier, - QualityFactor.ListsMaxTweetsToScoreParam - ), - qualityFactorDropMaxCandidates( - scoredTweetsPopularVideosCandidatePipelineConfig.identifier, - QualityFactor.PopularVideosMaxTweetsToScoreParam - ), - qualityFactorDropMaxCandidates( - scoredTweetsBackfillCandidatePipelineConfig.identifier, - QualityFactor.BackfillMaxTweetsToScoreParam - ), - // Select candidates for Heavy Ranker Feature Hydration and Scoring - InsertAppendResults(nonCachedScoringPipelineScope) - ) - - override val preScoringFeatureHydrationPhase1: Seq[ - BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] - ] = Seq( - TweetMetaDataFeatureHydrator, - ancestorFeatureHydrator, - authorFeatureHydrator, - authorIsCreatorFeatureHydrator, - earlybirdFeatureHydrator, - gizmoduckAuthorSafetyFeatureHydrator, - graphTwoHopFeatureHydrator, - metricCenterUserCountingFeatureHydrator, - realTimeInteractionGraphEdgeFeatureHydrator, - realGraphViewerAuthorFeatureHydrator, - simClustersEngagementSimilarityFeatureHydrator, - simClustersUserTweetScoresHydrator, - InNetworkFeatureHydrator, - tspInferredTopicFeatureHydrator, - tweetypieContentFeatureHydrator, - twhinAuthorFollowFeatureHydrator, - utegFeatureHydrator, - // real time aggregates - engagementsReceivedByAuthorRealTimeAggregateFeatureHydrator, - tweetCountryEngagementRealTimeAggregateFeatureHydrator, - tweetEngagementRealTimeAggregateFeatureHydrator, - twitterListEngagementRealTimeAggregateFeatureHydrator, - userAuthorEngagementRealTimeAggregateFeatureHydrator, - // offline aggregates - phase1EdgeAggregateFeatureHydrator - ) - - override val preScoringFeatureHydrationPhase2: Seq[ - BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] - ] = Seq( - perspectiveFilteredSocialContextFeatureHydrator, - phase2EdgeAggregateFeatureHydrator, - realGraphViewerRelatedUsersFeatureHydrator, - sgsValidSocialContextFeatureHydrator, - TweetTimeFeatureHydrator, - topicCountryEngagementRealTimeAggregateFeatureHydrator, - topicEngagementRealTimeAggregateFeatureHydrator - ) - - override val scorers: Seq[Scorer[ScoredTweetsQuery, TweetCandidate]] = Seq(naviModelScorer) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/selector/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/selector/BUILD.bazel deleted file mode 100644 index c5dbb187f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/selector/BUILD.bazel +++ /dev/null @@ -1,12 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/selector/KeepBestOutOfNetworkCandidatePerAuthorPerSuggestType.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/selector/KeepBestOutOfNetworkCandidatePerAuthorPerSuggestType.scala deleted file mode 100644 index c485c4c2a..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/selector/KeepBestOutOfNetworkCandidatePerAuthorPerSuggestType.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.selector - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class KeepBestOutOfNetworkCandidatePerAuthorPerSuggestType( - override val pipelineScope: CandidateScope) - extends Selector[PipelineQuery] { - - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val (selectedCandidates, otherCandidates) = - remainingCandidates.partition(candidate => - pipelineScope.contains(candidate) && !candidate.features.getOrElse(InNetworkFeature, true)) - - val filteredCandidates = selectedCandidates - .groupBy { candidate => - ( - candidate.features.getOrElse(AuthorIdFeature, None), - candidate.features.getOrElse(SuggestTypeFeature, None) - ) - } - .values.map(_.maxBy(_.features.getOrElse(ScoreFeature, None))) - .toSeq - - val updatedCandidates = otherCandidates ++ filteredCandidates - SelectorResult(remainingCandidates = updatedCandidates, result = result) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/BUILD.bazel deleted file mode 100644 index 2147ee217..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/BUILD.bazel +++ /dev/null @@ -1,33 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "finagle/finagle-mysql/src/main/scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", - "servo/repo/src/main/scala", - "servo/util/src/main/scala", - "src/scala/com/twitter/timelines/prediction/common/adapters", - "src/thrift/com/twitter/timelines/suggests/common:data_record_metadata-scala", - "src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java", - "src/thrift/com/twitter/timelines/timeline_logging:thrift-scala", - "timelines/ml:pldr-client", - "timelines/ml:pldr-conversion", - "timelines/ml/cont_train/common/domain/src/main/scala/com/twitter/timelines/ml/cont_train/common/domain/non_scalding", - "timelines/src/main/scala/com/twitter/timelines/util/stats", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CachedScoredTweetsSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CachedScoredTweetsSideEffect.scala deleted file mode 100644 index 3d66ff54a..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CachedScoredTweetsSideEffect.scala +++ /dev/null @@ -1,134 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.side_effect - -import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorIsCreatorFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorIsGoldVerifiedFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorIsGrayVerifiedFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorIsLegacyVerifiedFeature -import com.twitter.home_mixer.model.HomeFeatures.CachedCandidatePipelineIdentifierFeature -import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.LastScoredTimestampMsFeature -import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature -import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature -import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature -import com.twitter.home_mixer.model.HomeFeatures.WeightedModelScoreFeature -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.home_mixer.{thriftscala => hmt} -import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.metadata.TopicContextFunctionalityTypeMarshaller -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.cache.TtlCache -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class CachedScoredTweetsSideEffect @Inject() ( - scoredTweetsCache: TtlCache[Long, hmt.ScoredTweetsResponse]) - extends PipelineResultSideEffect[PipelineQuery, ScoredTweetsResponse] { - - override val identifier: SideEffectIdentifier = SideEffectIdentifier("CachedScoredTweets") - - private val MaxTweetsToCache = 1000 - - def buildCachedScoredTweets( - query: PipelineQuery, - candidates: Seq[CandidateWithDetails] - ): hmt.ScoredTweetsResponse = { - val tweets = candidates.map { candidate => - val sgsValidLikedByUserIds = - candidate.features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty) - val sgsValidFollowedByUserIds = - candidate.features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty) - val perspectiveFilteredLikedByUserIds = - candidate.features.getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Seq.empty) - val ancestors = candidate.features.getOrElse(AncestorsFeature, Seq.empty) - - hmt.ScoredTweet( - tweetId = candidate.candidateIdLong, - authorId = candidate.features.get(AuthorIdFeature).get, - // Cache the model score instead of the final score because rescoring is per-request - score = candidate.features.getOrElse(WeightedModelScoreFeature, None), - suggestType = candidate.features.getOrElse(SuggestTypeFeature, None), - sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None), - sourceUserId = candidate.features.getOrElse(SourceUserIdFeature, None), - quotedTweetId = candidate.features.getOrElse(QuotedTweetIdFeature, None), - quotedUserId = candidate.features.getOrElse(QuotedUserIdFeature, None), - inReplyToTweetId = candidate.features.getOrElse(InReplyToTweetIdFeature, None), - inReplyToUserId = candidate.features.getOrElse(InReplyToUserIdFeature, None), - directedAtUserId = candidate.features.getOrElse(DirectedAtUserIdFeature, None), - inNetwork = Some(candidate.features.getOrElse(InNetworkFeature, true)), - sgsValidLikedByUserIds = Some(sgsValidLikedByUserIds), - sgsValidFollowedByUserIds = Some(sgsValidFollowedByUserIds), - topicId = candidate.features.getOrElse(TopicIdSocialContextFeature, None), - topicFunctionalityType = candidate.features - .getOrElse(TopicContextFunctionalityTypeFeature, None).map( - TopicContextFunctionalityTypeMarshaller(_)), - ancestors = if (ancestors.nonEmpty) Some(ancestors) else None, - isReadFromCache = Some(true), - streamToKafka = Some(false), - exclusiveConversationAuthorId = candidate.features - .getOrElse(ExclusiveConversationAuthorIdFeature, None), - authorMetadata = Some( - hmt.AuthorMetadata( - blueVerified = candidate.features.getOrElse(AuthorIsBlueVerifiedFeature, false), - goldVerified = candidate.features.getOrElse(AuthorIsGoldVerifiedFeature, false), - grayVerified = candidate.features.getOrElse(AuthorIsGrayVerifiedFeature, false), - legacyVerified = candidate.features.getOrElse(AuthorIsLegacyVerifiedFeature, false), - creator = candidate.features.getOrElse(AuthorIsCreatorFeature, false) - )), - lastScoredTimestampMs = candidate.features - .getOrElse(LastScoredTimestampMsFeature, Some(query.queryTime.inMilliseconds)), - candidatePipelineIdentifier = candidate.features - .getOrElse(CachedCandidatePipelineIdentifierFeature, Some(candidate.source.name)), - tweetUrls = Some(candidate.features.getOrElse(TweetUrlsFeature, Seq.empty)), - perspectiveFilteredLikedByUserIds = Some(perspectiveFilteredLikedByUserIds) - ) - } - - hmt.ScoredTweetsResponse(tweets) - } - - final override def apply( - inputs: PipelineResultSideEffect.Inputs[PipelineQuery, ScoredTweetsResponse] - ): Stitch[Unit] = { - val candidates = - (inputs.selectedCandidates ++ inputs.remainingCandidates ++ inputs.droppedCandidates) - .filter(_.features.getOrElse(ScoreFeature, None).exists(_ > 0.0)) - - val truncatedCandidates = - if (candidates.size > MaxTweetsToCache) - candidates - .sortBy(-_.features.getOrElse(ScoreFeature, None).getOrElse(0.0)).take(MaxTweetsToCache) - else candidates - - if (truncatedCandidates.nonEmpty) { - val ttl = inputs.query.params(CachedScoredTweets.TTLParam) - val scoredTweets = buildCachedScoredTweets(inputs.query, truncatedCandidates) - Stitch.callFuture(scoredTweetsCache.set(inputs.query.getRequiredUserId, scoredTweets, ttl)) - } else Stitch.Unit - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.4) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScribeScoredCandidatesSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScribeScoredCandidatesSideEffect.scala deleted file mode 100644 index ef7e3b41a..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScribeScoredCandidatesSideEffect.scala +++ /dev/null @@ -1,126 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.side_effect - -import com.twitter.finagle.tracing.Trace -import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.EarlybirdScoreFeature -import com.twitter.home_mixer.model.HomeFeatures.FavoritedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.FollowedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.RequestJoinIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeScoredCandidatesFlag -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableScribeScoredCandidatesParam -import com.twitter.inject.annotations.Flag -import com.twitter.logpipeline.client.common.EventPublisher -import com.twitter.product_mixer.component_library.side_effect.ScribeLogEventSideEffect -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidatePipelines -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.timelines.timeline_logging.{thriftscala => t} -import javax.inject.Inject -import javax.inject.Singleton -import com.twitter.util.logging.Logging - -/** - * Side effect that logs scored candidates from scoring pipelines - */ -@Singleton -class ScribeScoredCandidatesSideEffect @Inject() ( - @Flag(ScribeScoredCandidatesFlag) enableScribeScoredCandidates: Boolean, - eventBusPublisher: EventPublisher[t.ScoredCandidate]) - extends ScribeLogEventSideEffect[ - t.ScoredCandidate, - ScoredTweetsQuery, - ScoredTweetsResponse - ] - with PipelineResultSideEffect.Conditionally[ - ScoredTweetsQuery, - ScoredTweetsResponse - ] - with Logging { - - override val identifier: SideEffectIdentifier = - SideEffectIdentifier("ScribeScoredCandidates") - - override def onlyIf( - query: ScoredTweetsQuery, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: ScoredTweetsResponse - ): Boolean = enableScribeScoredCandidates && query.params(EnableScribeScoredCandidatesParam) - - /** - * Build the log events from query, selections and response - * - * @param query PipelineQuery - * @param selectedCandidates Result after Selectors are executed - * @param remainingCandidates Candidates which were not selected - * @param droppedCandidates Candidates dropped during selection - * @param response Result after Unmarshalling - * - * @return LogEvent in thrift - */ - override def buildLogEvents( - query: ScoredTweetsQuery, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: ScoredTweetsResponse - ): Seq[t.ScoredCandidate] = { - val returned = (selectedCandidates ++ remainingCandidates).map(toThrift(_, query, false)) - val dropped = droppedCandidates.map(toThrift(_, query, true)) - returned ++ dropped - } - - private def toThrift( - candidate: CandidateWithDetails, - query: ScoredTweetsQuery, - isDropped: Boolean - ): t.ScoredCandidate = { - t.ScoredCandidate( - tweetId = candidate.candidateIdLong, - viewerId = query.getOptionalUserId, - authorId = candidate.features.getOrElse(AuthorIdFeature, None), - traceId = Some(Trace.id.traceId.toLong), - requestJoinId = query.features.flatMap(_.getOrElse(RequestJoinIdFeature, None)), - score = candidate.features.getOrElse(ScoreFeature, None), - suggestType = candidate.features.getOrElse(SuggestTypeFeature, None).map(_.name), - isInNetwork = candidate.features.getTry(FromInNetworkSourceFeature).toOption, - inReplyToTweetId = candidate.features.getOrElse(InReplyToTweetIdFeature, None), - inReplyToUserId = candidate.features.getOrElse(InReplyToUserIdFeature, None), - quotedTweetId = candidate.features.getOrElse(QuotedTweetIdFeature, None), - quotedUserId = candidate.features.getOrElse(QuotedUserIdFeature, None), - directedAtUserId = candidate.features.getOrElse(DirectedAtUserIdFeature, None), - favoritedByUserIds = convertSeqFeature(candidate, FavoritedByUserIdsFeature), - followedByUserIds = convertSeqFeature(candidate, FollowedByUserIdsFeature), - ancestors = convertSeqFeature(candidate, AncestorsFeature), - requestTimeMs = Some(query.queryTime.inMilliseconds), - candidatePipelineIdentifier = - candidate.features.getTry(CandidatePipelines).toOption.map(_.head.name), - earlybirdScore = candidate.features.getOrElse(EarlybirdScoreFeature, None), - isDropped = Some(isDropped) - ) - } - - private def convertSeqFeature[T]( - candidateWithDetails: CandidateWithDetails, - feature: Feature[_, Seq[T]] - ): Option[Seq[T]] = - Option( - candidateWithDetails.features - .getOrElse(feature, Seq.empty)).filter(_.nonEmpty) - - override val logPipelinePublisher: EventPublisher[t.ScoredCandidate] = eventBusPublisher -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScribeServedCommonFeaturesAndCandidateFeaturesSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScribeServedCommonFeaturesAndCandidateFeaturesSideEffect.scala deleted file mode 100644 index c93adee27..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScribeServedCommonFeaturesAndCandidateFeaturesSideEffect.scala +++ /dev/null @@ -1,225 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.side_effect - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.mysql.Client -import com.twitter.finagle.mysql.Transactions -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.util.DefaultTimer -import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.home_mixer.param.HomeMixerFlagName.DataRecordMetadataStoreConfigsYmlFlag -import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeServedCommonFeaturesAndCandidateFeaturesFlag -import com.twitter.home_mixer.param.HomeMixerInjectionNames.CandidateFeaturesScribeEventPublisher -import com.twitter.home_mixer.param.HomeMixerInjectionNames.CommonFeaturesScribeEventPublisher -import com.twitter.home_mixer.param.HomeMixerInjectionNames.MinimumFeaturesScribeEventPublisher -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.non_ml_features.NonMLCandidateFeatures -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.non_ml_features.NonMLCandidateFeaturesAdapter -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.non_ml_features.NonMLCommonFeatures -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.non_ml_features.NonMLCommonFeaturesAdapter -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse -import com.twitter.home_mixer.product.scored_tweets.scorer.CandidateFeaturesDataRecordFeature -import com.twitter.home_mixer.product.scored_tweets.scorer.CommonFeaturesDataRecordFeature -import com.twitter.home_mixer.product.scored_tweets.scorer.PredictedScoreFeature.PredictedScoreFeatures -import com.twitter.home_mixer.util.CandidatesUtil.getOriginalAuthorId -import com.twitter.inject.annotations.Flag -import com.twitter.logpipeline.client.common.EventPublisher -import com.twitter.ml.api.DataRecordMerger -import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordConverter -import com.twitter.product_mixer.core.feature.featuremap.datarecord.SpecificFeatures -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.stitch.Stitch -import com.twitter.timelines.ml.cont_train.common.domain.non_scalding.CandidateAndCommonFeaturesStreamingUtils -import com.twitter.timelines.ml.pldr.client.MysqlClientUtils -import com.twitter.timelines.ml.pldr.client.VersionedMetadataCacheClient -import com.twitter.timelines.ml.pldr.conversion.VersionIdAndFeatures -import com.twitter.timelines.suggests.common.data_record_metadata.{thriftscala => drmd} -import com.twitter.timelines.suggests.common.poly_data_record.{thriftjava => pldr} -import com.twitter.timelines.util.stats.OptionObserver -import com.twitter.util.Time -import com.twitter.util.Try -import com.twitter.util.logging.Logging -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton -import scala.collection.JavaConverters._ - -/** - * (1) Scribe common features sent to prediction service + some other features as PLDR format into logs - * (2) Scribe candidate features sent to prediction service + some other features as PLDR format into another logs - */ -@Singleton -class ScribeServedCommonFeaturesAndCandidateFeaturesSideEffect @Inject() ( - @Flag(DataRecordMetadataStoreConfigsYmlFlag) dataRecordMetadataStoreConfigsYml: String, - @Flag(ScribeServedCommonFeaturesAndCandidateFeaturesFlag) enableScribeServedCommonFeaturesAndCandidateFeatures: Boolean, - @Named(CommonFeaturesScribeEventPublisher) commonFeaturesScribeEventPublisher: EventPublisher[ - pldr.PolyDataRecord - ], - @Named(CandidateFeaturesScribeEventPublisher) candidateFeaturesScribeEventPublisher: EventPublisher[ - pldr.PolyDataRecord - ], - @Named(MinimumFeaturesScribeEventPublisher) minimumFeaturesScribeEventPublisher: EventPublisher[ - pldr.PolyDataRecord - ], - statsReceiver: StatsReceiver, -) extends PipelineResultSideEffect[ScoredTweetsQuery, ScoredTweetsResponse] - with PipelineResultSideEffect.Conditionally[ScoredTweetsQuery, ScoredTweetsResponse] - with Logging { - - override val identifier: SideEffectIdentifier = - SideEffectIdentifier("ScribeServedCommonFeaturesAndCandidateFeatures") - - private val drMerger = new DataRecordMerger - private val postScoringCandidateFeatures = SpecificFeatures(PredictedScoreFeatures) - private val postScoringCandidateFeaturesDataRecordAdapter = - new DataRecordConverter(postScoringCandidateFeatures) - - private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) - private val metadataFetchFailedCounter = scopedStatsReceiver.counter("metadataFetchFailed") - private val commonFeaturesScribeCounter = scopedStatsReceiver.counter("commonFeaturesScribe") - private val commonFeaturesPLDROptionObserver = - OptionObserver(scopedStatsReceiver.scope("commonFeaturesPLDR")) - private val candidateFeaturesScribeCounter = - scopedStatsReceiver.counter("candidateFeaturesScribe") - private val candidateFeaturesPLDROptionObserver = - OptionObserver(scopedStatsReceiver.scope("candidateFeaturesPLDR")) - private val minimumFeaturesPLDROptionObserver = - OptionObserver(scopedStatsReceiver.scope("minimumFeaturesPLDR")) - private val minimumFeaturesScribeCounter = - scopedStatsReceiver.counter("minimumFeaturesScribe") - - lazy private val dataRecordMetadataStoreClient: Option[Client with Transactions] = - Try { - MysqlClientUtils.mysqlClientProvider( - MysqlClientUtils.parseConfigFromYaml(dataRecordMetadataStoreConfigsYml)) - }.onFailure { e => info(s"Error building MySQL client: $e") }.toOption - - lazy private val versionedMetadataCacheClientOpt: Option[ - VersionedMetadataCacheClient[Map[drmd.FeaturesCategory, Option[VersionIdAndFeatures]]] - ] = - dataRecordMetadataStoreClient.map { mysqlClient => - new VersionedMetadataCacheClient[Map[drmd.FeaturesCategory, Option[VersionIdAndFeatures]]]( - maximumSize = 1, - expireDurationOpt = None, - mysqlClient = mysqlClient, - transform = CandidateAndCommonFeaturesStreamingUtils.metadataTransformer, - statsReceiver = statsReceiver - ) - } - - versionedMetadataCacheClientOpt.foreach { versionedMetadataCacheClient => - versionedMetadataCacheClient - .metadataFetchTimerTask( - CandidateAndCommonFeaturesStreamingUtils.metadataFetchKey, - metadataFetchTimer = DefaultTimer, - metadataFetchInterval = 90.seconds, - metadataFetchFailedCounter = metadataFetchFailedCounter - ) - } - - override def onlyIf( - query: ScoredTweetsQuery, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: ScoredTweetsResponse - ): Boolean = enableScribeServedCommonFeaturesAndCandidateFeatures - - override def apply( - inputs: PipelineResultSideEffect.Inputs[ScoredTweetsQuery, ScoredTweetsResponse] - ): Stitch[Unit] = { - Stitch.value { - val servedTimestamp: Long = Time.now.inMilliseconds - val nonMLCommonFeatures = NonMLCommonFeatures( - userId = inputs.query.getRequiredUserId, - predictionRequestId = - inputs.query.features.flatMap(_.getOrElse(ServedRequestIdFeature, None)), - servedTimestamp = servedTimestamp - ) - val nonMLCommonFeaturesDataRecord = - NonMLCommonFeaturesAdapter.adaptToDataRecords(nonMLCommonFeatures).asScala.head - - /** - * Steps of scribing common features - * (1) fetch common features as data record - * (2) extract additional feature as data record, e.g. predictionRequestId which is used as join key in downstream jobs - * (3) merge two data records above and convert the merged data record to pldr - * (4) publish pldr - */ - val commonFeaturesDataRecordOpt = - inputs.selectedCandidates.headOption.map(_.features.get(CommonFeaturesDataRecordFeature)) - val commonFeaturesPLDROpt = commonFeaturesDataRecordOpt.flatMap { commonFeaturesDataRecord => - drMerger.merge(commonFeaturesDataRecord, nonMLCommonFeaturesDataRecord) - - CandidateAndCommonFeaturesStreamingUtils.commonFeaturesToPolyDataRecord( - versionedMetadataCacheClientOpt = versionedMetadataCacheClientOpt, - commonFeatures = commonFeaturesDataRecord, - valueFormat = pldr.PolyDataRecord._Fields.LITE_COMPACT_DATA_RECORD - ) - } - - commonFeaturesPLDROptionObserver(commonFeaturesPLDROpt).foreach { pldr => - commonFeaturesScribeEventPublisher.publish(pldr) - commonFeaturesScribeCounter.incr() - } - - /** - * steps of scribing candidate features - * (1) fetch candidate features as data record - * (2) extract additional features (mostly non ML features including predicted scores, predictionRequestId, userId, tweetId) - * (3) merge data records and convert the merged data record into pldr - * (4) publish pldr - */ - inputs.selectedCandidates.foreach { candidate => - val candidateFeaturesDataRecord = candidate.features.get(CandidateFeaturesDataRecordFeature) - - /** - * extract predicted scores as data record and merge it into original data record - */ - val postScoringCandidateFeaturesDataRecord = - postScoringCandidateFeaturesDataRecordAdapter.toDataRecord(candidate.features) - drMerger.merge(candidateFeaturesDataRecord, postScoringCandidateFeaturesDataRecord) - - /** - * extract non ML common features as data record and merge it into original data record - */ - drMerger.merge(candidateFeaturesDataRecord, nonMLCommonFeaturesDataRecord) - - /** - * extract non ML candidate features as data record and merge it into original data record - */ - val nonMLCandidateFeatures = NonMLCandidateFeatures( - tweetId = candidate.candidateIdLong, - sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None), - originalAuthorId = getOriginalAuthorId(candidate.features) - ) - val nonMLCandidateFeaturesDataRecord = - NonMLCandidateFeaturesAdapter.adaptToDataRecords(nonMLCandidateFeatures).asScala.head - drMerger.merge(candidateFeaturesDataRecord, nonMLCandidateFeaturesDataRecord) - - val candidateFeaturesPLDROpt = - CandidateAndCommonFeaturesStreamingUtils.candidateFeaturesToPolyDataRecord( - versionedMetadataCacheClientOpt = versionedMetadataCacheClientOpt, - candidateFeatures = candidateFeaturesDataRecord, - valueFormat = pldr.PolyDataRecord._Fields.LITE_COMPACT_DATA_RECORD - ) - - candidateFeaturesPLDROptionObserver(candidateFeaturesPLDROpt).foreach { pldr => - candidateFeaturesScribeEventPublisher.publish(pldr) - candidateFeaturesScribeCounter.incr() - } - - // scribe minimum features which are used to join labels from client events. - val minimumFeaturesPLDROpt = candidateFeaturesPLDROpt - .map(CandidateAndCommonFeaturesStreamingUtils.extractMinimumFeaturesFromPldr) - .map(pldr.PolyDataRecord.dataRecord) - minimumFeaturesPLDROptionObserver(minimumFeaturesPLDROpt).foreach { pldr => - minimumFeaturesScribeEventPublisher.publish(pldr) - minimumFeaturesScribeCounter.incr() - } - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/BUILD.bazel deleted file mode 100644 index c1b5032db..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/BUILD.bazel +++ /dev/null @@ -1,82 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "ads-injection/lib/src/main/scala/com/twitter/goldfinch/api", - "finagle/finagle-memcached/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/async", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/param_gated", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", - "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/item", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", - "src/java/com/twitter/search/common/schema/base", - "src/java/com/twitter/search/common/schema/earlybird", - "src/java/com/twitter/search/common/util/lang", - "src/java/com/twitter/search/queryparser/query:core-query-nodes", - "src/java/com/twitter/search/queryparser/query/search:search-query-nodes", - "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/search/common:constants-java", - "src/thrift/com/twitter/tweetypie:service-scala", - "stitch/stitch-gizmoduck", - "stitch/stitch-tweetypie", - "stringcenter/client", - "stringcenter/client/src/main/java", - "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", - "timelines/src/main/scala/com/twitter/timelines/clients/relevance_search", - "timelines/src/main/scala/com/twitter/timelines/injection/scribe", - ], - exports = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "src/thrift/com/twitter/timelines/render:thrift-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedEarlybirdCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedEarlybirdCandidatePipelineConfig.scala deleted file mode 100644 index c331f955b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedEarlybirdCandidatePipelineConfig.scala +++ /dev/null @@ -1,64 +0,0 @@ -package com.twitter.home_mixer.product.subscribed - -import com.google.inject.Inject -import com.twitter.home_mixer.functional_component.candidate_source.EarlybirdCandidateSource -import com.twitter.home_mixer.product.subscribed.model.SubscribedQuery -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSSubscribedUsersFeature -import com.twitter.product_mixer.component_library.filter.TweetVisibilityFilter -import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.search.earlybird.{thriftscala => t} -import com.twitter.spam.rtf.thriftscala.SafetyLevel.TimelineHomeSubscribed -import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient} -import com.twitter.tweetypie.thriftscala.TweetVisibilityPolicy - -class SubscribedEarlybirdCandidatePipelineConfig @Inject() ( - earlybirdCandidateSource: EarlybirdCandidateSource, - tweetyPieStitchClient: TweetypieStitchClient, - subscribedEarlybirdQueryTransformer: SubscribedEarlybirdQueryTransformer) - extends CandidatePipelineConfig[ - SubscribedQuery, - t.EarlybirdRequest, - t.ThriftSearchResult, - TweetCandidate - ] { - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("SubscribedEarlybird") - - override val candidateSource: BaseCandidateSource[t.EarlybirdRequest, t.ThriftSearchResult] = - earlybirdCandidateSource - - override val gates: Seq[Gate[SubscribedQuery]] = Seq( - NonEmptySeqFeatureGate(SGSSubscribedUsersFeature) - ) - - override def filters: Seq[Filter[SubscribedQuery, TweetCandidate]] = Seq( - new TweetVisibilityFilter( - tweetyPieStitchClient, - TweetVisibilityPolicy.UserVisible, - TimelineHomeSubscribed - ) - ) - - override val queryTransformer: CandidatePipelineQueryTransformer[ - SubscribedQuery, - t.EarlybirdRequest - ] = subscribedEarlybirdQueryTransformer - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[t.ThriftSearchResult] - ] = Seq(SubscribedEarlybirdResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - t.ThriftSearchResult, - TweetCandidate - ] = { sourceResult => TweetCandidate(id = sourceResult.id) } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedEarlybirdQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedEarlybirdQueryTransformer.scala deleted file mode 100644 index 6e0d57c13..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedEarlybirdQueryTransformer.scala +++ /dev/null @@ -1,67 +0,0 @@ -package com.twitter.home_mixer.product.subscribed - -import com.twitter.finagle.thrift.ClientId -import com.twitter.finagle.tracing.Trace -import com.twitter.home_mixer.product.subscribed.model.SubscribedQuery -import com.twitter.home_mixer.product.subscribed.param.SubscribedParam.ServerMaxResultsParam -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSSubscribedUsersFeature -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.BottomCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor -import com.twitter.product_mixer.core.pipeline.pipeline_failure.MalformedCursor -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.search.common.schema.earlybird.EarlybirdFieldConstants.EarlybirdFieldConstant -import com.twitter.search.earlybird.{thriftscala => t} -import com.twitter.search.queryparser.query.Conjunction -import com.twitter.search.queryparser.query.search.SearchOperator -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class SubscribedEarlybirdQueryTransformer @Inject() (clientId: ClientId) - extends CandidatePipelineQueryTransformer[SubscribedQuery, t.EarlybirdRequest] { - - override def transform(query: SubscribedQuery): t.EarlybirdRequest = { - val subscribedUserIds = - query.features.map(_.get(SGSSubscribedUsersFeature)).getOrElse(Seq.empty) - - val subscribedUsersQuery = new SearchOperator.Builder() - .setType(SearchOperator.Type.FILTER) - .addOperand(EarlybirdFieldConstant.EXCLUSIVE_FILTER_TERM) - .build() - - val searchQuery = query.pipelineCursor - .map { cursor => - val sinceIdQuery = - (id: Long) => new SearchOperator(SearchOperator.Type.SINCE_ID, id.toString) - val maxIdQuery = // max ID is inclusive, so subtract 1 - (id: Long) => new SearchOperator(SearchOperator.Type.MAX_ID, (id - 1).toString) - - (cursor.cursorType, cursor.id, cursor.gapBoundaryId) match { - case (Some(TopCursor), Some(sinceId), _) => - new Conjunction(sinceIdQuery(sinceId), subscribedUsersQuery) - case (Some(BottomCursor), Some(maxId), _) => - new Conjunction(maxIdQuery(maxId), subscribedUsersQuery) - case (Some(GapCursor), Some(maxId), Some(sinceId)) => - new Conjunction(sinceIdQuery(sinceId), maxIdQuery(maxId), subscribedUsersQuery) - case (Some(GapCursor), _, _) => - throw PipelineFailure(MalformedCursor, "Invalid cursor " + cursor.toString) - case _ => subscribedUsersQuery - } - }.getOrElse(subscribedUsersQuery) - - t.EarlybirdRequest( - searchQuery = t.ThriftSearchQuery( - serializedQuery = Some(searchQuery.serialize), - fromUserIDFilter64 = Some(subscribedUserIds), - numResults = query.requestedMaxResults.getOrElse(query.params(ServerMaxResultsParam)), - rankingMode = t.ThriftSearchRankingMode.Recency, - ), - getOlderResults = Some(true), // needed for archive access to older tweets - clientRequestID = Some(s"${Trace.id.traceId}"), - numResultsToReturnAtRoot = Some(query.params(ServerMaxResultsParam)), - clientId = Some(clientId.name), - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedEarlybirdResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedEarlybirdResponseFeatureTransformer.scala deleted file mode 100644 index a8136102c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedEarlybirdResponseFeatureTransformer.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.home_mixer.product.subscribed - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.search.earlybird.{thriftscala => t} - -object SubscribedEarlybirdResponseFeatureTransformer - extends CandidateFeatureTransformer[t.ThriftSearchResult] { - - override val identifier: TransformerIdentifier = - TransformerIdentifier("SubscribedEarlybirdResponse") - - override val features: Set[Feature[_, _]] = Set( - AuthorIdFeature, - InReplyToTweetIdFeature, - IsRetweetFeature, - SourceTweetIdFeature, - SourceUserIdFeature, - ) - - override def transform(candidate: t.ThriftSearchResult): FeatureMap = FeatureMapBuilder() - .add(AuthorIdFeature, candidate.tweetypieTweet.flatMap(_.coreData.map(_.userId))) - .add( - InReplyToTweetIdFeature, - candidate.tweetypieTweet.flatMap(_.coreData.flatMap(_.reply.flatMap(_.inReplyToStatusId)))) - .add(IsRetweetFeature, candidate.metadata.exists(_.isRetweet.contains(true))) - .add(SourceTweetIdFeature, candidate.sourceTweetypieTweet.map(_.id)) - .add(SourceUserIdFeature, candidate.sourceTweetypieTweet.flatMap(_.coreData.map(_.userId))) - .build() -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedMixerPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedMixerPipelineConfig.scala deleted file mode 100644 index 5b1f74b37..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedMixerPipelineConfig.scala +++ /dev/null @@ -1,221 +0,0 @@ -package com.twitter.home_mixer.product.subscribed - -import com.twitter.clientapp.{thriftscala => ca} -import com.twitter.home_mixer.candidate_pipeline.ConversationServiceCandidatePipelineConfigBuilder -import com.twitter.home_mixer.candidate_pipeline.EditedTweetsCandidatePipelineConfig -import com.twitter.home_mixer.candidate_pipeline.NewTweetsPillCandidatePipelineConfig -import com.twitter.home_mixer.functional_component.decorator.HomeConversationServiceCandidateDecorator -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder -import com.twitter.home_mixer.functional_component.feature_hydrator._ -import com.twitter.home_mixer.functional_component.selector.UpdateHomeClientEventDetails -import com.twitter.home_mixer.functional_component.selector.UpdateNewTweetsPillDecoration -import com.twitter.home_mixer.functional_component.side_effect._ -import com.twitter.home_mixer.model.GapIncludeInstruction -import com.twitter.home_mixer.param.HomeGlobalParams.MaxNumberReplaceInstructionsParam -import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeClientEventsFlag -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.home_mixer.product.subscribed.model.SubscribedQuery -import com.twitter.home_mixer.product.subscribed.param.SubscribedParam.ServerMaxResultsParam -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.inject.annotations.Flag -import com.twitter.logpipeline.client.common.EventPublisher -import com.twitter.product_mixer.component_library.feature_hydrator.query.async.AsyncQueryFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweetsQueryFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSSubscribedUsersQueryFeatureHydrator -import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.AddEntriesWithReplaceAndShowAlertInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedBottomCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedGapCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedTopCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceAllEntries -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceEntryInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowAlertInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.StaticTimelineScribeConfigBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtMetadataBuilder -import com.twitter.product_mixer.component_library.selector.DropMaxCandidates -import com.twitter.product_mixer.component_library.selector.InsertAppendResults -import com.twitter.product_mixer.component_library.selector.SelectConditionally -import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates -import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller -import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.UrtTransportMarshaller -import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.product_mixer.core.pipeline.FailOpenPolicy -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import com.twitter.timelines.render.{thriftscala => urt} -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton - -@Singleton -class SubscribedMixerPipelineConfig @Inject() ( - subscribedEarlybirdCandidatePipelineConfig: SubscribedEarlybirdCandidatePipelineConfig, - conversationServiceCandidatePipelineConfigBuilder: ConversationServiceCandidatePipelineConfigBuilder[ - SubscribedQuery - ], - homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder, - editedTweetsCandidatePipelineConfig: EditedTweetsCandidatePipelineConfig, - newTweetsPillCandidatePipelineConfig: NewTweetsPillCandidatePipelineConfig[SubscribedQuery], - dismissInfoQueryFeatureHydrator: DismissInfoQueryFeatureHydrator, - gizmoduckUserQueryFeatureHydrator: GizmoduckUserQueryFeatureHydrator, - requestQueryFeatureHydrator: RequestQueryFeatureHydrator[SubscribedQuery], - sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, - sgsSubscribedUsersQueryFeatureHydrator: SGSSubscribedUsersQueryFeatureHydrator, - manhattanTweetImpressionsQueryFeatureHydrator: TweetImpressionsQueryFeatureHydrator[ - SubscribedQuery - ], - memcacheTweetImpressionsQueryFeatureHydrator: ImpressedTweetsQueryFeatureHydrator, - publishClientSentImpressionsEventBusSideEffect: PublishClientSentImpressionsEventBusSideEffect, - publishClientSentImpressionsManhattanSideEffect: PublishClientSentImpressionsManhattanSideEffect, - homeTimelineServedCandidatesSideEffect: HomeScribeServedCandidatesSideEffect, - clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent], - externalStrings: HomeMixerExternalStrings, - @ProductScoped stringCenterProvider: Provider[StringCenter], - urtTransportMarshaller: UrtTransportMarshaller, - @Flag(ScribeClientEventsFlag) enableScribeClientEvents: Boolean) - extends MixerPipelineConfig[SubscribedQuery, Timeline, urt.TimelineResponse] { - - override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("Subscribed") - - private val dependentCandidatesStep = MixerPipelineConfig.dependentCandidatePipelinesStep - private val resultSelectorsStep = MixerPipelineConfig.resultSelectorsStep - - override val fetchQueryFeatures: Seq[QueryFeatureHydrator[SubscribedQuery]] = Seq( - requestQueryFeatureHydrator, - sgsFollowedUsersQueryFeatureHydrator, - sgsSubscribedUsersQueryFeatureHydrator, - AsyncQueryFeatureHydrator(dependentCandidatesStep, dismissInfoQueryFeatureHydrator), - AsyncQueryFeatureHydrator(dependentCandidatesStep, gizmoduckUserQueryFeatureHydrator), - AsyncQueryFeatureHydrator(resultSelectorsStep, manhattanTweetImpressionsQueryFeatureHydrator), - AsyncQueryFeatureHydrator(resultSelectorsStep, memcacheTweetImpressionsQueryFeatureHydrator) - ) - - private val earlybirdCandidatePipelineScope = - SpecificPipeline(subscribedEarlybirdCandidatePipelineConfig.identifier) - - private val conversationServiceCandidatePipelineConfig = - conversationServiceCandidatePipelineConfigBuilder.build( - Seq(NonEmptyCandidatesGate(earlybirdCandidatePipelineScope)), - HomeConversationServiceCandidateDecorator(homeFeedbackActionInfoBuilder) - ) - - override val candidatePipelines: Seq[CandidatePipelineConfig[SubscribedQuery, _, _, _]] = - Seq(subscribedEarlybirdCandidatePipelineConfig) - - override val dependentCandidatePipelines: Seq[ - DependentCandidatePipelineConfig[SubscribedQuery, _, _, _] - ] = Seq( - conversationServiceCandidatePipelineConfig, - editedTweetsCandidatePipelineConfig, - newTweetsPillCandidatePipelineConfig - ) - - override val failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = Map( - editedTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - newTweetsPillCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - ) - - override val resultSelectors: Seq[Selector[SubscribedQuery]] = Seq( - UpdateSortCandidates( - ordering = CandidatesUtil.reverseChronTweetsOrdering, - candidatePipeline = conversationServiceCandidatePipelineConfig.identifier - ), - DropMaxCandidates( - candidatePipeline = editedTweetsCandidatePipelineConfig.identifier, - maxSelectionsParam = MaxNumberReplaceInstructionsParam - ), - DropMaxCandidates( - candidatePipeline = conversationServiceCandidatePipelineConfig.identifier, - maxSelectionsParam = ServerMaxResultsParam - ), - InsertAppendResults(candidatePipeline = conversationServiceCandidatePipelineConfig.identifier), - // This selector must come after the tweets are inserted into the results - UpdateNewTweetsPillDecoration( - pipelineScope = SpecificPipelines( - conversationServiceCandidatePipelineConfig.identifier, - newTweetsPillCandidatePipelineConfig.identifier - ), - stringCenter = stringCenterProvider.get(), - seeNewTweetsString = externalStrings.seeNewTweetsString, - tweetedString = externalStrings.tweetedString - ), - InsertAppendResults(candidatePipeline = editedTweetsCandidatePipelineConfig.identifier), - SelectConditionally( - selector = - InsertAppendResults(candidatePipeline = newTweetsPillCandidatePipelineConfig.identifier), - includeSelector = (_, _, results) => CandidatesUtil.containsType[TweetCandidate](results) - ), - UpdateHomeClientEventDetails( - candidatePipelines = Set(conversationServiceCandidatePipelineConfig.identifier) - ), - ) - - private val homeScribeClientEventSideEffect = HomeScribeClientEventSideEffect( - enableScribeClientEvents = enableScribeClientEvents, - logPipelinePublisher = clientEventsScribeEventPublisher, - injectedTweetsCandidatePipelineIdentifiers = - Seq(conversationServiceCandidatePipelineConfig.identifier), - ) - - override val resultSideEffects: Seq[PipelineResultSideEffect[SubscribedQuery, Timeline]] = Seq( - homeScribeClientEventSideEffect, - homeTimelineServedCandidatesSideEffect, - publishClientSentImpressionsEventBusSideEffect, - publishClientSentImpressionsManhattanSideEffect - ) - - override val domainMarshaller: DomainMarshaller[SubscribedQuery, Timeline] = { - val instructionBuilders = Seq( - ReplaceEntryInstructionBuilder(ReplaceAllEntries), - AddEntriesWithReplaceAndShowAlertInstructionBuilder(), - ShowAlertInstructionBuilder(), - ) - - val idSelector: PartialFunction[UniversalNoun[_], Long] = { - // exclude ads while determining tweet cursor values - case item: TweetItem if item.promotedMetadata.isEmpty => item.id - case module: TimelineModule - if module.items.headOption.exists(_.item.isInstanceOf[TweetItem]) => - module.items.last.item match { case item: TweetItem => item.id } - } - - val topCursorBuilder = OrderedTopCursorBuilder(idSelector) - val bottomCursorBuilder = - OrderedBottomCursorBuilder(idSelector, GapIncludeInstruction.inverse()) - val gapCursorBuilder = OrderedGapCursorBuilder(idSelector, GapIncludeInstruction) - - val metadataBuilder = UrtMetadataBuilder( - title = None, - scribeConfigBuilder = Some( - StaticTimelineScribeConfigBuilder( - TimelineScribeConfig(page = Some("subscribed"), section = None, entityToken = None))) - ) - - UrtDomainMarshaller( - instructionBuilders = instructionBuilders, - metadataBuilder = Some(metadataBuilder), - cursorBuilders = Seq(topCursorBuilder, bottomCursorBuilder, gapCursorBuilder) - ) - } - - override val transportMarshaller: TransportMarshaller[Timeline, urt.TimelineResponse] = - urtTransportMarshaller -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedProductPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedProductPipelineConfig.scala deleted file mode 100644 index 0d524391a..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedProductPipelineConfig.scala +++ /dev/null @@ -1,123 +0,0 @@ -package com.twitter.home_mixer.product.subscribed - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.marshaller.timelines.ChronologicalCursorUnmarshaller -import com.twitter.home_mixer.model.request.HomeMixerRequest -import com.twitter.home_mixer.model.request.SubscribedProduct -import com.twitter.home_mixer.model.request.SubscribedProductContext -import com.twitter.home_mixer.product.subscribed.model.SubscribedQuery -import com.twitter.home_mixer.product.subscribed.param.SubscribedParam.ServerMaxResultsParam -import com.twitter.home_mixer.service.HomeMixerAccessPolicy.DefaultHomeMixerAccessPolicy -import com.twitter.home_mixer.service.HomeMixerAlertConfig.DefaultNotificationGroup -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer -import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.common.alert.LatencyAlert -import com.twitter.product_mixer.core.functional_component.common.alert.P99 -import com.twitter.product_mixer.core.functional_component.common.alert.SuccessRateAlert -import com.twitter.product_mixer.core.functional_component.common.alert.ThroughputAlert -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfAbove -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfBelow -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfLatencyAbove -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.Product -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor -import com.twitter.product_mixer.core.pipeline.PipelineConfig -import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest -import com.twitter.product_mixer.core.pipeline.pipeline_failure.MalformedCursor -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.product.ProductPipelineConfig -import com.twitter.product_mixer.core.product.ProductParamConfig -import com.twitter.product_mixer.core.util.SortIndexBuilder -import com.twitter.timelines.configapi.Params -import com.twitter.timelines.render.{thriftscala => urt} -import com.twitter.timelines.util.RequestCursorSerializer -import com.twitter.util.Time -import com.twitter.util.Try -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class SubscribedProductPipelineConfig @Inject() ( - subscribedMixerPipelineConfig: SubscribedMixerPipelineConfig, - subscribedParamConfig: param.SubscribedParamConfig) - extends ProductPipelineConfig[HomeMixerRequest, SubscribedQuery, urt.TimelineResponse] { - - override val identifier: ProductPipelineIdentifier = ProductPipelineIdentifier("Subscribed") - - override val product: Product = SubscribedProduct - override val paramConfig: ProductParamConfig = subscribedParamConfig - - override def pipelineQueryTransformer( - request: HomeMixerRequest, - params: Params - ): SubscribedQuery = { - val context = request.productContext match { - case Some(context: SubscribedProductContext) => context - case _ => throw PipelineFailure(BadRequest, "SubscribedProductContext not found") - } - - val debugOptions = request.debugParams.flatMap(_.debugOptions) - - /** - * Unlike other clients, newly created tweets on Android have the sort index set to the current - * time instead of the top sort index + 1, so these tweets get stuck at the top of the timeline - * if subsequent timeline responses use the sort index from the previous response instead of - * the current time. - */ - val pipelineCursor = request.serializedRequestCursor.flatMap { cursor => - Try(UrtCursorSerializer.deserializeOrderedCursor(cursor)) - .getOrElse(ChronologicalCursorUnmarshaller(RequestCursorSerializer.deserialize(cursor))) - .map { - case UrtOrderedCursor(_, id, Some(GapCursor), gapBoundaryId) - if id.isEmpty || gapBoundaryId.isEmpty => - throw PipelineFailure(MalformedCursor, "Gap Cursor bounds not defined") - case topCursor @ UrtOrderedCursor(_, _, Some(TopCursor), _) => - val queryTime = debugOptions.flatMap(_.requestTimeOverride).getOrElse(Time.now) - topCursor.copy(initialSortIndex = SortIndexBuilder.timeToId(queryTime)) - case cursor => cursor - } - } - - SubscribedQuery( - params = params, - clientContext = request.clientContext, - features = None, - pipelineCursor = pipelineCursor, - requestedMaxResults = Some(params(ServerMaxResultsParam)), - debugOptions = debugOptions, - deviceContext = context.deviceContext, - seenTweetIds = context.seenTweetIds - ) - } - - override val pipelines: Seq[PipelineConfig] = Seq(subscribedMixerPipelineConfig) - - override def pipelineSelector( - query: SubscribedQuery - ): ComponentIdentifier = subscribedMixerPipelineConfig.identifier - - override val alerts: Seq[Alert] = Seq( - SuccessRateAlert( - notificationGroup = DefaultNotificationGroup, - warnPredicate = TriggerIfBelow(99.9, 20, 30), - criticalPredicate = TriggerIfBelow(99.9, 30, 30), - ), - LatencyAlert( - notificationGroup = DefaultNotificationGroup, - percentile = P99, - warnPredicate = TriggerIfLatencyAbove(1100.millis, 15, 30), - criticalPredicate = TriggerIfLatencyAbove(1200.millis, 15, 30) - ), - ThroughputAlert( - notificationGroup = DefaultNotificationGroup, - warnPredicate = TriggerIfAbove(18000), - criticalPredicate = TriggerIfAbove(20000) - ) - ) - - override val debugAccessPolicies: Set[AccessPolicy] = DefaultHomeMixerAccessPolicy -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/model/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/model/BUILD.bazel deleted file mode 100644 index b821a8f8d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/model/BUILD.bazel +++ /dev/null @@ -1,20 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "stringcenter/client", - "stringcenter/client/src/main/java", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/model/SubscribedQuery.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/model/SubscribedQuery.scala deleted file mode 100644 index 2085ef54f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/model/SubscribedQuery.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.home_mixer.product.subscribed.model - -import com.twitter.home_mixer.model.request.DeviceContext -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.home_mixer.model.request.HasSeenTweetIds -import com.twitter.home_mixer.model.request.SubscribedProduct -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.request._ -import com.twitter.product_mixer.core.pipeline.HasPipelineCursor -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.Params - -case class SubscribedQuery( - override val params: Params, - override val clientContext: ClientContext, - override val pipelineCursor: Option[UrtOrderedCursor], - override val requestedMaxResults: Option[Int], - override val debugOptions: Option[DebugOptions], - override val features: Option[FeatureMap], - override val deviceContext: Option[DeviceContext], - override val seenTweetIds: Option[Seq[Long]]) - extends PipelineQuery - with HasPipelineCursor[UrtOrderedCursor] - with HasDeviceContext - with HasSeenTweetIds { - override val product: Product = SubscribedProduct - - override def withFeatureMap(features: FeatureMap): SubscribedQuery = - copy(features = Some(features)) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/BUILD.bazel deleted file mode 100644 index a56e3a1fd..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/BUILD.bazel +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "util/util-core/src/main/scala/com/twitter/conversions", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/SubscribedParam.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/SubscribedParam.scala deleted file mode 100644 index 9e4ade43a..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/SubscribedParam.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.home_mixer.product.subscribed.param - -import com.twitter.timelines.configapi.FSBoundedParam - -object SubscribedParam { - val SupportedClientFSName = "subscribed_supported_client" - - object ServerMaxResultsParam - extends FSBoundedParam[Int]( - name = "subscribed_server_max_results", - default = 100, - min = 1, - max = 500 - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/SubscribedParamConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/SubscribedParamConfig.scala deleted file mode 100644 index 58ce7ec35..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/SubscribedParamConfig.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.home_mixer.product.subscribed.param - -import com.twitter.home_mixer.param.decider.DeciderKey -import com.twitter.home_mixer.product.subscribed.param.SubscribedParam._ -import com.twitter.product_mixer.core.product.ProductParamConfig -import com.twitter.servo.decider.DeciderKeyName -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class SubscribedParamConfig @Inject() () extends ProductParamConfig { - override val enabledDeciderKey: DeciderKeyName = DeciderKey.EnableSubscribedProduct - override val supportedClientFSName: String = SupportedClientFSName - - override val boundedIntFSOverrides = Seq( - ServerMaxResultsParam - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/BUILD.bazel deleted file mode 100644 index c0211ff72..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/BUILD.bazel +++ /dev/null @@ -1,15 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "configapi/configapi-core", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry", - "stitch/stitch-core", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/HomeMixerAccessPolicy.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/HomeMixerAccessPolicy.scala deleted file mode 100644 index 853f4b56a..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/HomeMixerAccessPolicy.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.home_mixer.service - -import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy -import com.twitter.product_mixer.core.functional_component.common.access_policy.AllowedLdapGroups - -object HomeMixerAccessPolicy { - - /** - * Access policies can be configured on a product-by-product basis but you may also want products - * to have a common policy. - */ - val DefaultHomeMixerAccessPolicy: Set[AccessPolicy] = Set(AllowedLdapGroups(Set.empty[String])) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/HomeMixerAlertConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/HomeMixerAlertConfig.scala deleted file mode 100644 index 597fd4d36..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/HomeMixerAlertConfig.scala +++ /dev/null @@ -1,66 +0,0 @@ -package com.twitter.home_mixer.service - -import com.twitter.conversions.DurationOps._ -import com.twitter.product_mixer.core.functional_component.common.alert.Destination -import com.twitter.product_mixer.core.functional_component.common.alert.EmptyResponseRateAlert -import com.twitter.product_mixer.core.functional_component.common.alert.LatencyAlert -import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup -import com.twitter.product_mixer.core.functional_component.common.alert.P99 -import com.twitter.product_mixer.core.functional_component.common.alert.Percentile -import com.twitter.product_mixer.core.functional_component.common.alert.SuccessRateAlert -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfAbove -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfBelow -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfLatencyAbove -import com.twitter.util.Duration - -/** - * Notifications (email, pagerduty, etc) can be specific per-alert but it is common for multiple - * products to share notification configuration. - */ -object HomeMixerAlertConfig { - val DefaultNotificationGroup: NotificationGroup = NotificationGroup( - warn = Destination(emails = Seq("")), - critical = Destination(emails = Seq("")) - ) - - object BusinessHours { - val DefaultNotificationGroup: NotificationGroup = NotificationGroup( - warn = Destination(emails = Seq("")), - critical = Destination(emails = - Seq("")) - ) - - def defaultEmptyResponseRateAlert(warnThreshold: Double = 50, criticalThreshold: Double = 80) = - EmptyResponseRateAlert( - notificationGroup = DefaultNotificationGroup, - warnPredicate = TriggerIfAbove(warnThreshold), - criticalPredicate = TriggerIfAbove(criticalThreshold) - ) - - def defaultSuccessRateAlert( - threshold: Double = 99.5, - warnDatapointsPastThreshold: Int = 20, - criticalDatapointsPastThreshold: Int = 30, - duration: Int = 30 - ) = SuccessRateAlert( - notificationGroup = DefaultNotificationGroup, - warnPredicate = TriggerIfBelow(threshold, warnDatapointsPastThreshold, duration), - criticalPredicate = TriggerIfBelow(threshold, criticalDatapointsPastThreshold, duration), - ) - - def defaultLatencyAlert( - latencyThreshold: Duration = 200.millis, - warningDatapointsPastThreshold: Int = 15, - criticalDatapointsPastThreshold: Int = 30, - duration: Int = 30, - percentile: Percentile = P99 - ): LatencyAlert = LatencyAlert( - notificationGroup = DefaultNotificationGroup, - percentile = percentile, - warnPredicate = - TriggerIfLatencyAbove(latencyThreshold, warningDatapointsPastThreshold, duration), - criticalPredicate = - TriggerIfLatencyAbove(latencyThreshold, criticalDatapointsPastThreshold, duration) - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/ScoredTweetsService.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/ScoredTweetsService.scala deleted file mode 100644 index 158e5ee45..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/service/ScoredTweetsService.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.home_mixer.service - -import com.twitter.home_mixer.{thriftscala => t} -import com.twitter.product_mixer.core.model.marshalling.request.Request -import com.twitter.product_mixer.core.pipeline.product.ProductPipelineRequest -import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.Params -import javax.inject.Inject -import javax.inject.Singleton -import scala.reflect.runtime.universe._ - -@Singleton -class ScoredTweetsService @Inject() (productPipelineRegistry: ProductPipelineRegistry) { - - def getScoredTweetsResponse[RequestType <: Request]( - request: RequestType, - params: Params - )( - implicit requestTypeTag: TypeTag[RequestType] - ): Stitch[t.ScoredTweetsResponse] = productPipelineRegistry - .getProductPipeline[RequestType, t.ScoredTweetsResponse](request.product) - .process(ProductPipelineRequest(request, params)) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/BUILD.bazel deleted file mode 100644 index c4855d9e7..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/twitter/bijection:scrooge", - "3rdparty/jvm/com/twitter/storehaus:core", - "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", - "stitch/stitch-core", - "storage/clients/manhattan/client/src/main/scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/RealGraphInNetworkScoresStore.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/RealGraphInNetworkScoresStore.scala deleted file mode 100644 index ce0b182be..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/store/RealGraphInNetworkScoresStore.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.home_mixer.store - -import com.twitter.bijection.Injection -import com.twitter.home_mixer.store.ManhattanRealGraphKVDescriptor._ -import com.twitter.stitch.Stitch -import com.twitter.storage.client.manhattan.bijections.Bijections -import com.twitter.storage.client.manhattan.bijections.Bijections.BinaryScalaInjection -import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpoint -import com.twitter.storage.client.manhattan.kv.impl.ReadOnlyKeyDescriptor -import com.twitter.storage.client.manhattan.kv.impl.ValueDescriptor -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future -import com.twitter.wtf.candidate.{thriftscala => wtf} - -object ManhattanRealGraphKVDescriptor { - implicit val byteArray2Buf = Bijections.BytesBijection - - val realGraphDatasetName = "real_graph_scores_in" - val keyInjection = Injection.connect[Long, Array[Byte]].andThen(Bijections.BytesInjection) - val keyDesc = ReadOnlyKeyDescriptor(keyInjection) - val valueDesc = ValueDescriptor(BinaryScalaInjection(wtf.CandidateSeq)) - val realGraphDatasetKey = keyDesc.withDataset(realGraphDatasetName) -} - -/** - * Hydrates real graph in network scores for a viewer - */ -class RealGraphInNetworkScoresStore(manhattanKVEndpoint: ManhattanKVEndpoint) - extends ReadableStore[Long, Seq[wtf.Candidate]] { - - override def get(viewerId: Long): Future[Option[Seq[wtf.Candidate]]] = Stitch - .run(manhattanKVEndpoint.get(realGraphDatasetKey.withPkey(viewerId), valueDesc)) - .map(_.map(mhResponse => mhResponse.contents.candidates)) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/BUILD.bazel deleted file mode 100644 index 8e04fa11f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/BUILD.bazel +++ /dev/null @@ -1,25 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", - "servo/repo/src/main/scala", - "snowflake/src/main/scala/com/twitter/snowflake/id", - "src/java/com/twitter/ml/api:api-base", - "src/java/com/twitter/ml/api/util", - "src/java/com/twitter/search/common/util/lang", - "src/scala/com/twitter/ml/api/util", - "src/thrift/com/twitter/ml/api:data-java", - "src/thrift/com/twitter/search/common:constants-java", - "src/thrift/com/twitter/search/common:constants-scala", - "src/thrift/com/twitter/service/metastore/gen:thrift-java", - "src/thrift/com/twitter/service/metastore/gen:thrift-scala", - "storage/clients/manhattan/client/src/main/scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/CachedScoredTweetsHelper.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/CachedScoredTweetsHelper.scala deleted file mode 100644 index e0fbdd76f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/CachedScoredTweetsHelper.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.home_mixer.util - -import com.twitter.home_mixer.model.HomeFeatures.CachedScoredTweetsFeature -import com.twitter.home_mixer.{thriftscala => hmt} -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.util.Time - -object CachedScoredTweetsHelper { - - def tweetImpressionsAndCachedScoredTweets( - features: FeatureMap, - candidatePipelineIdentifier: CandidatePipelineIdentifier - ): Seq[Long] = { - val tweetImpressions = TweetImpressionsHelper.tweetImpressions(features) - val cachedScoredTweets = features - .getOrElse(CachedScoredTweetsFeature, Seq.empty) - .filter { tweet => - tweet.candidatePipelineIdentifier.exists( - CandidatePipelineIdentifier(_).equals(candidatePipelineIdentifier)) - }.map(_.tweetId) - - (tweetImpressions ++ cachedScoredTweets).toSeq - } - - def tweetImpressionsAndCachedScoredTweetsInRange( - features: FeatureMap, - candidatePipelineIdentifier: CandidatePipelineIdentifier, - maxNumImpressions: Int, - sinceTime: Time, - untilTime: Time - ): Seq[Long] = - tweetImpressionsAndCachedScoredTweets(features, candidatePipelineIdentifier) - .filter { tweetId => SnowflakeId.isSnowflakeId(tweetId) } - .filter { tweetId => - val creationTime = SnowflakeId.timeFromId(tweetId) - sinceTime <= creationTime && untilTime >= creationTime - }.take(maxNumImpressions) - - def unseenCachedScoredTweets( - features: FeatureMap - ): Seq[hmt.ScoredTweet] = { - val seenTweetIds = TweetImpressionsHelper.tweetImpressions(features) - - features - .getOrElse(CachedScoredTweetsFeature, Seq.empty) - .filter(tweet => !seenTweetIds.contains(tweet.tweetId)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/CandidatesUtil.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/CandidatesUtil.scala deleted file mode 100644 index 06fe5646c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/CandidatesUtil.scala +++ /dev/null @@ -1,116 +0,0 @@ -package com.twitter.home_mixer.util - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.FavoritedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.HasImageFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.MediaUnderstandingAnnotationIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.RepliedByEngagerIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.RetweetedByEngagerIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.product_mixer.component_library.model.candidate.CursorCandidate -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.pipeline_failure.UnexpectedCandidateResult -import scala.reflect.ClassTag - -object CandidatesUtil { - def getItemCandidates(candidates: Seq[CandidateWithDetails]): Seq[ItemCandidateWithDetails] = { - candidates.collect { - case item: ItemCandidateWithDetails if !item.isCandidateType[CursorCandidate] => Seq(item) - case module: ModuleCandidateWithDetails => module.candidates - }.flatten - } - - def getItemCandidatesWithOnlyModuleLast( - candidates: Seq[CandidateWithDetails] - ): Seq[ItemCandidateWithDetails] = { - candidates.collect { - case item: ItemCandidateWithDetails if !item.isCandidateType[CursorCandidate] => item - case module: ModuleCandidateWithDetails => module.candidates.last - } - } - - def containsType[CandidateType <: UniversalNoun[_]]( - candidates: Seq[CandidateWithDetails] - )( - implicit tag: ClassTag[CandidateType] - ): Boolean = candidates.exists { - case ItemCandidateWithDetails(_: CandidateType, _, _) => true - case module: ModuleCandidateWithDetails => - module.candidates.head.isCandidateType[CandidateType]() - case _ => false - } - - def getOriginalTweetId(candidate: CandidateWithFeatures[TweetCandidate]): Long = { - if (candidate.features.getOrElse(IsRetweetFeature, false)) - candidate.features.getOrElse(SourceTweetIdFeature, None).getOrElse(candidate.candidate.id) - else - candidate.candidate.id - } - - def getOriginalAuthorId(candidateFeatures: FeatureMap): Option[Long] = - if (candidateFeatures.getOrElse(IsRetweetFeature, false)) - candidateFeatures.getOrElse(SourceUserIdFeature, None) - else candidateFeatures.getOrElse(AuthorIdFeature, None) - - def isOriginalTweet(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = - !candidate.features.getOrElse(IsRetweetFeature, false) && - candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty - - def getEngagerUserIds( - candidateFeatures: FeatureMap - ): Seq[Long] = { - candidateFeatures.getOrElse(FavoritedByUserIdsFeature, Seq.empty) ++ - candidateFeatures.getOrElse(RetweetedByEngagerIdsFeature, Seq.empty) ++ - candidateFeatures.getOrElse(RepliedByEngagerIdsFeature, Seq.empty) - } - - def getMediaUnderstandingAnnotationIds( - candidateFeatures: FeatureMap - ): Seq[Long] = { - if (candidateFeatures.get(HasImageFeature)) - candidateFeatures.getOrElse(MediaUnderstandingAnnotationIdsFeature, Seq.empty) - else Seq.empty - } - - def getTweetIdAndSourceId(candidate: CandidateWithFeatures[TweetCandidate]): Seq[Long] = - Seq(candidate.candidate.id) ++ candidate.features.getOrElse(SourceTweetIdFeature, None) - - def isAuthoredByViewer(query: PipelineQuery, candidateFeatures: FeatureMap): Boolean = - candidateFeatures.getOrElse(AuthorIdFeature, None).contains(query.getRequiredUserId) || - (candidateFeatures.getOrElse(IsRetweetFeature, false) && - candidateFeatures.getOrElse(SourceUserIdFeature, None).contains(query.getRequiredUserId)) - - val reverseChronTweetsOrdering: Ordering[CandidateWithDetails] = - Ordering.by[CandidateWithDetails, Long] { - case ItemCandidateWithDetails(candidate: TweetCandidate, _, _) => -candidate.id - case ModuleCandidateWithDetails(candidates, _, _) if candidates.nonEmpty => - -candidates.last.candidateIdLong - case _ => throw PipelineFailure(UnexpectedCandidateResult, "Invalid candidate type") - } - - val scoreOrdering: Ordering[CandidateWithDetails] = Ordering.by[CandidateWithDetails, Double] { - case ItemCandidateWithDetails(_, _, features) => - -features.getOrElse(ScoreFeature, None).getOrElse(0.0) - case ModuleCandidateWithDetails(candidates, _, _) => - -candidates.last.features.getOrElse(ScoreFeature, None).getOrElse(0.0) - case _ => throw PipelineFailure(UnexpectedCandidateResult, "Invalid candidate type") - } - - val conversationModuleTweetsOrdering: Ordering[CandidateWithDetails] = - Ordering.by[CandidateWithDetails, Long] { - case ItemCandidateWithDetails(candidate: TweetCandidate, _, _) => candidate.id - case _ => throw PipelineFailure(UnexpectedCandidateResult, "Only Item candidate expected") - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/DataRecordUtil.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/DataRecordUtil.scala deleted file mode 100644 index b972b8158..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/DataRecordUtil.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.home_mixer.util - -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.util.SRichDataRecord -import com.twitter.ml.api.Feature -import java.lang.{Double => JDouble} - -object DataRecordUtil { - def applyRename( - dataRecord: DataRecord, - featureContext: FeatureContext, - renamedFeatureContext: FeatureContext, - featureRenamingMap: Map[Feature[_], Feature[_]] - ): DataRecord = { - val richFullDr = new SRichDataRecord(dataRecord, featureContext) - val richNewDr = new SRichDataRecord(new DataRecord, renamedFeatureContext) - val featureIterator = featureContext.iterator() - featureIterator.forEachRemaining { feature => - if (richFullDr.hasFeature(feature)) { - val renamedFeature = featureRenamingMap.getOrElse(feature, feature) - - val typedFeature = feature.asInstanceOf[Feature[JDouble]] - val typedRenamedFeature = renamedFeature.asInstanceOf[Feature[JDouble]] - - richNewDr.setFeatureValue(typedRenamedFeature, richFullDr.getFeatureValue(typedFeature)) - } - } - richNewDr.getRecord - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/InjectionTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/InjectionTransformer.scala deleted file mode 100644 index f45102f37..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/InjectionTransformer.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.twitter.home_mixer.util - -import com.twitter.bijection.Injection -import com.twitter.io.Buf -import com.twitter.servo.util.Transformer -import com.twitter.storage.client.manhattan.bijections.Bijections -import com.twitter.util.Return -import com.twitter.util.Try -import java.nio.ByteBuffer - -object InjectionTransformerImplicits { - implicit class ByteArrayInjectionToByteBufferTransformer[A](baInj: Injection[A, Array[Byte]]) { - - private val bbInj: Injection[A, ByteBuffer] = baInj - .andThen(Bijections.byteArray2Buf) - .andThen(Bijections.byteBuffer2Buf.inverse) - - def toByteBufferTransformer(): Transformer[A, ByteBuffer] = new InjectionTransformer(bbInj) - def toByteArrayTransformer(): Transformer[A, Array[Byte]] = new InjectionTransformer(baInj) - } - - implicit class BufInjectionToByteBufferTransformer[A](bufInj: Injection[A, Buf]) { - - private val bbInj: Injection[A, ByteBuffer] = bufInj.andThen(Bijections.byteBuffer2Buf.inverse) - private val baInj: Injection[A, Array[Byte]] = bufInj.andThen(Bijections.byteArray2Buf.inverse) - - def toByteBufferTransformer(): Transformer[A, ByteBuffer] = new InjectionTransformer(bbInj) - def toByteArrayTransformer(): Transformer[A, Array[Byte]] = new InjectionTransformer(baInj) - } - - implicit class ByteBufferInjectionToByteBufferTransformer[A](bbInj: Injection[A, ByteBuffer]) { - - private val baInj: Injection[A, Array[Byte]] = bbInj.andThen(Bijections.bb2ba) - - def toByteBufferTransformer(): Transformer[A, ByteBuffer] = new InjectionTransformer(bbInj) - def toByteArrayTransformer(): Transformer[A, Array[Byte]] = new InjectionTransformer(baInj) - } -} - -class InjectionTransformer[A, B](inj: Injection[A, B]) extends Transformer[A, B] { - override def to(a: A): Try[B] = Return(inj(a)) - override def from(b: B): Try[A] = Try.fromScala(inj.invert(b)) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/LanguageUtil.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/LanguageUtil.scala deleted file mode 100644 index 23c77c27f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/LanguageUtil.scala +++ /dev/null @@ -1,93 +0,0 @@ -package com.twitter.home_mixer.util - -import com.twitter.search.common.constants.{thriftscala => scc} -import com.twitter.search.common.util.lang.ThriftLanguageUtil -import com.twitter.service.metastore.gen.{thriftscala => smg} - -object LanguageUtil { - - private val DefaultMinProducedLanguageRatio = 0.05 - private val DefaultMinConsumedLanguageConfidence = 0.8 - - /** - * Computes a list of languages based on UserLanguages information retrieved from Metastore. - * - * The list is sorted in descending order of confidence score associated with each language. - * That is, language with highest confidence value is in index 0. - */ - def computeLanguages( - userLanguages: smg.UserLanguages, - minProducedLanguageRatio: Double = DefaultMinProducedLanguageRatio, - minConsumedLanguageConfidence: Double = DefaultMinConsumedLanguageConfidence - ): Seq[scc.ThriftLanguage] = { - val languageConfidenceMap = computeLanguageConfidenceMap( - userLanguages, - minProducedLanguageRatio, - minConsumedLanguageConfidence - ) - languageConfidenceMap.toSeq.sortBy(-_._2).map(_._1) // _1 = language, _2 = score - } - - /** - * Computes confidence map based on UserLanguages information retrieved from Metastore. - * where, - * key = language code - * value = level of confidence that the language is applicable to a user. - */ - private def computeLanguageConfidenceMap( - userLanguages: smg.UserLanguages, - minProducedLanguageRatio: Double, - minConsumedLanguageConfidence: Double - ): Map[scc.ThriftLanguage, Double] = { - - val producedLanguages = getLanguageMap(userLanguages.produced) - val consumedLanguages = getLanguageMap(userLanguages.consumed) - val languages = (producedLanguages.keys ++ consumedLanguages.keys).toSet - var maxConfidence = 0.0 - - val confidenceMap = languages.map { language => - val produceRatio = producedLanguages - .get(language) - .map { score => if (score < minProducedLanguageRatio) 0.0 else score } - .getOrElse(0.0) - - val consumeConfidence = consumedLanguages - .get(language) - .map { score => if (score < minConsumedLanguageConfidence) 0.0 else score } - .getOrElse(0.0) - - val overallConfidence = (0.3 + 4 * produceRatio) * (0.1 + consumeConfidence) - maxConfidence = Math.max(maxConfidence, overallConfidence) - - (language -> overallConfidence) - }.toMap - - val normalizedConfidenceMap = if (maxConfidence > 0) { - confidenceMap.map { - case (language, confidenceScore) => - val normalizedScore = (confidenceScore / maxConfidence * 0.9) + 0.1 - (language -> normalizedScore) - } - } else { - confidenceMap - } - normalizedConfidenceMap - } - - private def getLanguageMap( - scoredLanguages: Seq[smg.ScoredString] - ): Map[scc.ThriftLanguage, Double] = { - scoredLanguages.flatMap { scoredLanguage => - getThriftLanguage(scoredLanguage.item).map { language => (language -> scoredLanguage.weight) } - }.toMap - } - - private def getThriftLanguage(languageName: String): Option[scc.ThriftLanguage] = { - val languageOrdinal = ThriftLanguageUtil.getThriftLanguageOf(languageName).ordinal - val language = scc.ThriftLanguage(languageOrdinal) - language match { - case scc.ThriftLanguage.Unknown => None - case _ => Some(language) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/MissingKeyException.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/MissingKeyException.scala deleted file mode 100644 index ae8fd4ded..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/MissingKeyException.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.home_mixer.util - -object MissingKeyException extends Exception("Missing key") { - override def toString: String = getMessage -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/ObservedKeyValueResultHandler.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/ObservedKeyValueResultHandler.scala deleted file mode 100644 index 0cd0cd60b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/ObservedKeyValueResultHandler.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.twitter.home_mixer.util - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.servo.keyvalue.KeyValueResult -import com.twitter.util.Return -import com.twitter.util.Throw -import com.twitter.util.Try - -trait ObservedKeyValueResultHandler { - val statsReceiver: StatsReceiver - val statScope: String - - private lazy val scopedStatsReceiver = statsReceiver.scope(statScope) - private lazy val keyTotalCounter = scopedStatsReceiver.counter("key/total") - private lazy val keyFoundCounter = scopedStatsReceiver.counter("key/found") - private lazy val keyLossCounter = scopedStatsReceiver.counter("key/loss") - private lazy val keyFailureCounter = scopedStatsReceiver.counter("key/failure") - - def observedGet[K, V]( - key: Option[K], - keyValueResult: KeyValueResult[K, V], - ): Try[Option[V]] = { - if (key.nonEmpty) { - keyTotalCounter.incr() - keyValueResult(key.get) match { - case Return(Some(value)) => - keyFoundCounter.incr() - Return(Some(value)) - case Return(None) => - keyLossCounter.incr() - Return(None) - case Throw(exception) => - keyFailureCounter.incr() - Throw(exception) - case _ => - // never reaches here - Return(None) - } - } else { - Throw(MissingKeyException) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/ReplyRetweetUtil.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/ReplyRetweetUtil.scala deleted file mode 100644 index 13758ec5d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/ReplyRetweetUtil.scala +++ /dev/null @@ -1,120 +0,0 @@ -package com.twitter.home_mixer.util - -import com.twitter.home_mixer.model.HomeFeatures._ -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures - -object ReplyRetweetUtil { - - def isEligibleReply(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = { - candidate.features.getOrElse(InReplyToTweetIdFeature, None).nonEmpty && - !candidate.features.getOrElse(IsRetweetFeature, false) - } - - /** - * Builds a map from reply tweet to all ancestors that are also hydrated candidates. If a reply - * does not have any ancestors which are also candidates, it will not add to the returned Map. - * Make sure ancestors are bottom-up ordered such that: - * (1) if parent tweet is a candidate, it should be the first item at the returned ancestors; - * (2) if root tweet is a candidate, it should be the last item at the returned ancestors. - * Retweets of replies or replies to retweets are not included. - */ - def replyToAncestorTweetCandidatesMap( - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Map[Long, Seq[CandidateWithFeatures[TweetCandidate]]] = { - val replyToAncestorTweetIdsMap: Map[Long, Seq[Long]] = - candidates.flatMap { candidate => - if (isEligibleReply(candidate)) { - val ancestorIds = - if (candidate.features.getOrElse(AncestorsFeature, Seq.empty).nonEmpty) { - candidate.features.getOrElse(AncestorsFeature, Seq.empty).map(_.tweetId) - } else { - Seq( - candidate.features.getOrElse(InReplyToTweetIdFeature, None), - candidate.features.getOrElse(ConversationModuleIdFeature, None) - ).flatten.distinct - } - Some(candidate.candidate.id -> ancestorIds) - } else { - None - } - }.toMap - - val ancestorTweetIds = replyToAncestorTweetIdsMap.values.flatten.toSet - val ancestorTweetsMapById: Map[Long, CandidateWithFeatures[TweetCandidate]] = candidates - .filter { maybeAncestor => - ancestorTweetIds.contains(maybeAncestor.candidate.id) - }.map { ancestor => - ancestor.candidate.id -> ancestor - }.toMap - - replyToAncestorTweetIdsMap - .mapValues { ancestorTweetIds => - ancestorTweetIds.flatMap { ancestorTweetId => - ancestorTweetsMapById.get(ancestorTweetId) - } - }.filter { - case (reply, ancestors) => - ancestors.nonEmpty - } - } - - /** - * This map is the opposite of [[replyToAncestorTweetCandidatesMap]]. - * Builds a map from ancestor tweet to all descendant replies that are also hydrated candidates. - * Currently, we only return two ancestors at most: one is inReplyToTweetId and the other - * is conversationId. - * Retweets of replies are not included. - */ - def ancestorTweetIdToDescendantRepliesMap( - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Map[Long, Seq[CandidateWithFeatures[TweetCandidate]]] = { - val tweetToCandidateMap = candidates.map(c => c.candidate.id -> c).toMap - replyToAncestorTweetCandidatesMap(candidates).toSeq - .flatMap { - case (reply, ancestorTweets) => - ancestorTweets.map { ancestor => - (ancestor.candidate.id, reply) - } - }.groupBy { case (ancestor, reply) => ancestor } - .mapValues { ancestorReplyPairs => - ancestorReplyPairs.map(_._2).distinct - }.mapValues(tweetIds => tweetIds.map(tid => tweetToCandidateMap(tid))) - } - - /** - * Builds a map from reply tweet to inReplyToTweet which is also a candidate. - * Retweets of replies or replies to retweets are not included - */ - def replyTweetIdToInReplyToTweetMap( - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Map[Long, CandidateWithFeatures[TweetCandidate]] = { - val eligibleReplyCandidates = candidates.filter { candidate => - isEligibleReply(candidate) && candidate.features - .getOrElse(InReplyToTweetIdFeature, None) - .nonEmpty - } - - val inReplyToTweetIds = eligibleReplyCandidates - .flatMap(_.features.getOrElse(InReplyToTweetIdFeature, None)) - .toSet - - val inReplyToTweetIdToTweetMap: Map[Long, CandidateWithFeatures[TweetCandidate]] = candidates - .filter { maybeInReplyToTweet => - inReplyToTweetIds.contains(maybeInReplyToTweet.candidate.id) - }.map { inReplyToTweet => - inReplyToTweet.candidate.id -> inReplyToTweet - }.toMap - - eligibleReplyCandidates.flatMap { reply => - val inReplyToTweetId = reply.features.getOrElse(InReplyToTweetIdFeature, None) - if (inReplyToTweetId.nonEmpty) { - inReplyToTweetIdToTweetMap.get(inReplyToTweetId.get).map { inReplyToTweet => - reply.candidate.id -> inReplyToTweet - } - } else { - None - } - }.toMap - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/TensorFlowUtil.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/TensorFlowUtil.scala deleted file mode 100644 index 05d7c2127..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/TensorFlowUtil.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.home_mixer.util - -import com.twitter.ml.api.thriftscala.FloatTensor -import com.twitter.ml.api.util.BufferToIterators.RichFloatBuffer -import java.nio.ByteBuffer -import java.nio.ByteOrder - -/** - * Contains functionality to transform data records and Tensors - */ - -object TensorFlowUtil { - - private def skipEmbeddingBBHeader(bb: ByteBuffer): ByteBuffer = { - val bb_copy = bb.duplicate() - bb_copy.getLong() - bb_copy - } - - private def byteBufferToFloatIterator( - bb: ByteBuffer - ): Iterator[Float] = { - bb.order(ByteOrder.LITTLE_ENDIAN).asFloatBuffer.iterator - } - - def embeddingByteBufferToFloatTensor( - bb: ByteBuffer - ): FloatTensor = { - val bb_content = skipEmbeddingBBHeader(bb) - FloatTensor(byteBufferToFloatIterator(bb_content).map(_.toDouble).toList) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/TweetImpressionsHelper.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/TweetImpressionsHelper.scala deleted file mode 100644 index eabaac9e3..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/TweetImpressionsHelper.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.home_mixer.util - -import com.twitter.home_mixer.model.HomeFeatures.TweetImpressionsFeature -import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweets -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap - -object TweetImpressionsHelper { - def tweetImpressions(features: FeatureMap): Set[Long] = { - val manhattanImpressions = - features.getOrElse(TweetImpressionsFeature, Seq.empty).flatMap(_.tweetIds) - val memcacheImpressions = features.getOrElse(ImpressedTweets, Seq.empty) - - (manhattanImpressions ++ memcacheImpressions).toSet - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/BUILD.bazel deleted file mode 100644 index 2b5722179..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/BUILD.bazel +++ /dev/null @@ -1,23 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/thrift/src/main/thrift:thrift-scala", - "src/java/com/twitter/search/common/schema/base", - "src/java/com/twitter/search/common/schema/earlybird", - "src/java/com/twitter/search/common/util/lang", - "src/java/com/twitter/search/queryparser/query:core-query-nodes", - "src/java/com/twitter/search/queryparser/query/search:search-query-nodes", - "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/search/common:constants-scala", - "src/thrift/com/twitter/search/common:query-scala", - "src/thrift/com/twitter/search/common:ranking-scala", - "timelines/src/main/scala/com/twitter/timelines/clients/relevance_search", - "timelines/src/main/scala/com/twitter/timelines/earlybird/common/options", - "timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils", - "timelines/src/main/scala/com/twitter/timelines/model/types", - "timelines/src/main/scala/com/twitter/timelines/util/stats", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/EarlybirdRequestUtil.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/EarlybirdRequestUtil.scala deleted file mode 100644 index 4c482b251..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/EarlybirdRequestUtil.scala +++ /dev/null @@ -1,182 +0,0 @@ -package com.twitter.home_mixer.util.earlybird - -import com.twitter.conversions.DurationOps._ -import com.twitter.search.common.query.thriftjava.{thriftscala => scq} -import com.twitter.search.common.ranking.{thriftscala => scr} -import com.twitter.search.earlybird.{thriftscala => eb} -import com.twitter.timelines.clients.relevance_search.SearchClient.TweetFeatures -import com.twitter.timelines.clients.relevance_search.SearchClient.TweetTypes -import com.twitter.timelines.clients.relevance_search.SearchQueryBuilder -import com.twitter.timelines.clients.relevance_search.SearchQueryBuilder.QueryWithNamedDisjunctions -import com.twitter.timelines.earlybird.common.options.EarlybirdScoringModelConfig -import com.twitter.timelines.earlybird.common.utils.SearchOperator -import com.twitter.util.Duration - -object EarlybirdRequestUtil { - - val DefaultMaxHitsToProcess = 1000 - val DefaultSearchProcessingTimeout: Duration = 200.milliseconds - val DefaultHydrationMaxNumResultsPerShard = 1000 - val DefaultQueryMaxNumResultsPerShard = 300 - val DefaultHydrationCollectorParams = mkCollectorParams(DefaultHydrationMaxNumResultsPerShard) - - private val queryBuilder = new SearchQueryBuilder - - object EarlybirdScoringModels { - val UnifiedEngagementProd: Seq[EarlybirdScoringModelConfig] = Seq( - EarlybirdScoringModelConfig("timelines_unified_engagement_prod.schema_based", 1.0) - ) - - val UnifiedEngagementRectweet: Seq[EarlybirdScoringModelConfig] = Seq( - EarlybirdScoringModelConfig("timelines_unified_engagement_rectweet.schema_based", 1.0) - ) - } - - private[earlybird] def mkCollectorParams(numResultsToReturn: Int): scq.CollectorParams = { - scq.CollectorParams( - // numResultsToReturn defines how many results each EB shard will return to search root - numResultsToReturn = numResultsToReturn, - // terminationParams.maxHitsToProcess is used for early terminating per shard results fetching. - terminationParams = Some( - scq.CollectorTerminationParams( - maxHitsToProcess = Some(DefaultMaxHitsToProcess), - timeoutMs = DefaultSearchProcessingTimeout.inMilliseconds.toInt - )) - ) - } - - private def getRankingParams( - authorScoreMap: Option[Map[Long, Double]], - tensorflowModel: Option[String], - ebModels: Seq[EarlybirdScoringModelConfig] - ): Option[scr.ThriftRankingParams] = { - if (tensorflowModel.nonEmpty) { - Some( - scr.ThriftRankingParams( - `type` = Some(scr.ThriftScoringFunctionType.TensorflowBased), - selectedTensorflowModel = tensorflowModel, - minScore = -1.0e100, - applyBoosts = false, - authorSpecificScoreAdjustments = authorScoreMap - ) - ) - } else if (ebModels.nonEmpty) { - Some( - scr.ThriftRankingParams( - `type` = Some(scr.ThriftScoringFunctionType.ModelBased), - selectedModels = Some(ebModels.map(m => m.name -> m.weight).toMap), - applyBoosts = false, - minScore = -1.0e100, - authorSpecificScoreAdjustments = authorScoreMap - ) - ) - } else None - } - - def getTweetsRequest( - userId: Option[Long], - clientId: Option[String], - skipVeryRecentTweets: Boolean, - followedUserIds: Set[Long], - retweetsMutedUserIds: Set[Long], - beforeTweetIdExclusive: Option[Long], - afterTweetIdExclusive: Option[Long], - excludedTweetIds: Option[Set[Long]] = None, - maxCount: Int, - tweetTypes: TweetTypes.ValueSet, - authorScoreMap: Option[Map[Long, Double]] = None, - tensorflowModel: Option[String] = None, - ebModels: Seq[EarlybirdScoringModelConfig] = Seq.empty, - queryMaxNumResultsPerShard: Int = DefaultQueryMaxNumResultsPerShard - ): eb.EarlybirdRequest = { - - val QueryWithNamedDisjunctions(query, namedDisjunctionMap) = queryBuilder.create( - followedUserIds, - retweetsMutedUserIds, - beforeTweetIdExclusive, - afterTweetIdExclusive, - semanticCoreIds = None, - languages = None, - tweetTypes = tweetTypes, - searchOperator = SearchOperator.Exclude, - tweetFeatures = TweetFeatures.All, - excludedTweetIds = excludedTweetIds.getOrElse(Set.empty), - enableExcludeSourceTweetIdsQuery = false - ) - val ebRankingParams = getRankingParams(authorScoreMap, tensorflowModel, ebModels) - val relOptions = RelevanceSearchUtil.RelevanceOptions.copy( - rankingParams = ebRankingParams - ) - - val followedUserIdsSeq = followedUserIds.toSeq - val namedDisjunctionMapOpt = - if (namedDisjunctionMap.isEmpty) None - else Some(namedDisjunctionMap.mapValues(_.toSeq)) - - val thriftQuery = eb.ThriftSearchQuery( - serializedQuery = Some(query.serialize), - fromUserIDFilter64 = Some(followedUserIdsSeq), - numResults = maxCount, - collectConversationId = true, - rankingMode = eb.ThriftSearchRankingMode.Relevance, - relevanceOptions = Some(relOptions), - collectorParams = Some(mkCollectorParams(queryMaxNumResultsPerShard)), - facetFieldNames = Some(RelevanceSearchUtil.FacetsToFetch), - resultMetadataOptions = Some(RelevanceSearchUtil.MetadataOptions), - searcherId = userId, - searchStatusIds = None, - namedDisjunctionMap = namedDisjunctionMapOpt - ) - - eb.EarlybirdRequest( - searchQuery = thriftQuery, - clientId = clientId, - getOlderResults = Some(false), - followedUserIds = Some(followedUserIdsSeq), - getProtectedTweetsOnly = Some(false), - timeoutMs = DefaultSearchProcessingTimeout.inMilliseconds.toInt, - skipVeryRecentTweets = skipVeryRecentTweets, - numResultsToReturnAtRoot = Some(maxCount) - ) - } - - def getTweetsFeaturesRequest( - userId: Option[Long], - tweetIds: Option[Seq[Long]], - clientId: Option[String], - getOnlyProtectedTweets: Boolean = false, - authorScoreMap: Option[Map[Long, Double]] = None, - tensorflowModel: Option[String] = None, - ebModels: Seq[EarlybirdScoringModelConfig] = Seq.empty - ): eb.EarlybirdRequest = { - - val candidateSize = tweetIds.getOrElse(Seq.empty).size - val ebRankingParams = getRankingParams(authorScoreMap, tensorflowModel, ebModels) - val relOptions = RelevanceSearchUtil.RelevanceOptions.copy( - rankingParams = ebRankingParams - ) - val thriftQuery = eb.ThriftSearchQuery( - numResults = candidateSize, - collectConversationId = true, - rankingMode = eb.ThriftSearchRankingMode.Relevance, - relevanceOptions = Some(relOptions), - collectorParams = Some(DefaultHydrationCollectorParams), - facetFieldNames = Some(RelevanceSearchUtil.FacetsToFetch), - resultMetadataOptions = Some(RelevanceSearchUtil.MetadataOptions), - searcherId = userId, - searchStatusIds = tweetIds.map(_.toSet), - ) - - eb.EarlybirdRequest( - searchQuery = thriftQuery, - clientId = clientId, - getOlderResults = Some(false), - getProtectedTweetsOnly = Some(getOnlyProtectedTweets), - timeoutMs = DefaultSearchProcessingTimeout.inMilliseconds.toInt, - skipVeryRecentTweets = true, - // This param decides # of tweets to return from search superRoot and realtime/protected/Archive roots. - // It takes higher precedence than ThriftSearchQuery.numResults - numResultsToReturnAtRoot = Some(candidateSize) - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/EarlybirdResponseUtil.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/EarlybirdResponseUtil.scala deleted file mode 100644 index 06e0dd708..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/EarlybirdResponseUtil.scala +++ /dev/null @@ -1,411 +0,0 @@ -package com.twitter.home_mixer.util.earlybird - -import com.twitter.search.common.constants.{thriftscala => scc} -import com.twitter.search.common.features.{thriftscala => sc} -import com.twitter.search.common.schema.earlybird.EarlybirdFieldConstants.EarlybirdFieldConstant -import com.twitter.search.common.schema.earlybird.EarlybirdFieldConstants.EarlybirdFieldConstant._ -import com.twitter.search.common.util.lang.ThriftLanguageUtil -import com.twitter.search.earlybird.{thriftscala => eb} -import com.twitter.timelines.earlybird.common.utils.InNetworkEngagement - -object EarlybirdResponseUtil { - - private[earlybird] val Mentions: String = "mentions" - private[earlybird] val Hashtags: String = "hashtags" - private val CharsToRemoveFromMentions: Set[Char] = "@".toSet - private val CharsToRemoveFromHashtags: Set[Char] = "#".toSet - - // Default value of settings of ThriftTweetFeatures. - private[earlybird] val DefaultEarlybirdFeatures: sc.ThriftTweetFeatures = sc.ThriftTweetFeatures() - private[earlybird] val DefaultCount = 0 - private[earlybird] val DefaultLanguage = 0 - private[earlybird] val DefaultScore = 0.0 - - private[earlybird] def getTweetCountByAuthorId( - searchResults: Seq[eb.ThriftSearchResult] - ): Map[Long, Int] = { - searchResults - .groupBy { result => - result.metadata.map(_.fromUserId).getOrElse(0L) - }.mapValues(_.size).withDefaultValue(0) - } - - private[earlybird] def getLanguage(uiLanguageCode: Option[String]): Option[scc.ThriftLanguage] = { - uiLanguageCode.flatMap { languageCode => - scc.ThriftLanguage.get(ThriftLanguageUtil.getThriftLanguageOf(languageCode).getValue) - } - } - - private def getMentions(result: eb.ThriftSearchResult): Seq[String] = { - val facetLabels = result.metadata.flatMap(_.facetLabels).getOrElse(Seq.empty) - getFacets(facetLabels, Mentions, CharsToRemoveFromMentions) - } - - private def getHashtags(result: eb.ThriftSearchResult): Seq[String] = { - val facetLabels = result.metadata.flatMap(_.facetLabels).getOrElse(Seq.empty) - getFacets(facetLabels, Hashtags, CharsToRemoveFromHashtags) - } - - private def getFacets( - facetLabels: Seq[eb.ThriftFacetLabel], - facetName: String, - charsToRemove: Set[Char] - ): Seq[String] = { - facetLabels.filter(_.fieldName == facetName).map(_.label.filterNot(charsToRemove)) - } - - private def isUserMentioned( - screenName: Option[String], - mentions: Seq[String], - mentionsInSourceTweet: Seq[String] - ): Boolean = - isUserMentioned(screenName, mentions) || isUserMentioned(screenName, mentionsInSourceTweet) - - private def isUserMentioned( - screenName: Option[String], - mentions: Seq[String] - ): Boolean = { - screenName - .exists { screenName => mentions.exists(_.equalsIgnoreCase(screenName)) } - } - - private[earlybird] def isUsersMainLanguage( - tweetLanguage: scc.ThriftLanguage, - userLanguages: Seq[scc.ThriftLanguage] - ): Boolean = { - (tweetLanguage != scc.ThriftLanguage.Unknown) && userLanguages.headOption.contains( - tweetLanguage) - } - - private[earlybird] def isUsersLanguage( - tweetLanguage: scc.ThriftLanguage, - userLanguages: Seq[scc.ThriftLanguage] - ): Boolean = { - (tweetLanguage != scc.ThriftLanguage.Unknown) && userLanguages.contains(tweetLanguage) - } - - private[earlybird] def isUILanguage( - tweetLanguage: scc.ThriftLanguage, - uiLanguage: Option[scc.ThriftLanguage] - ): Boolean = { - (tweetLanguage != scc.ThriftLanguage.Unknown) && uiLanguage.contains(tweetLanguage) - } - - private def getBooleanOptFeature( - featureName: EarlybirdFieldConstant, - resultMapOpt: Option[scala.collection.Map[Int, Boolean]], - defaultValue: Boolean = false, - ): Option[Boolean] = { - resultMapOpt.map { - _.getOrElse(featureName.getFieldId, defaultValue) - } - } - - private def getDoubleAsIntOptFeature( - featureName: EarlybirdFieldConstant, - resultMapOpt: Option[scala.collection.Map[Int, Double]] - ): Option[Int] = { - if (resultMapOpt.exists(_.contains(featureName.getFieldId))) - resultMapOpt - .map { - _.get(featureName.getFieldId) - } - .flatMap { doubleValue => - doubleValue.map(_.toInt) - } - else - None - } - - private def getIntOptFeature( - featureName: EarlybirdFieldConstant, - resultMapOpt: Option[scala.collection.Map[Int, Int]] - ): Option[Int] = { - if (resultMapOpt.exists(_.contains(featureName.getFieldId))) - resultMapOpt.flatMap { - _.get(featureName.getFieldId) - } - else - None - } - - def getTweetThriftFeaturesByTweetId( - searcherUserId: Long, - screenName: Option[String], - userLanguages: Seq[scc.ThriftLanguage], - uiLanguageCode: Option[String] = None, - followedUserIds: Set[Long], - mutuallyFollowingUserIds: Set[Long], - searchResults: Seq[eb.ThriftSearchResult], - sourceTweetSearchResults: Seq[eb.ThriftSearchResult], - ): Map[Long, sc.ThriftTweetFeatures] = { - - val allSearchResults = searchResults ++ sourceTweetSearchResults - val sourceTweetSearchResultById = - sourceTweetSearchResults.map(result => (result.id -> result)).toMap - val inNetworkEngagement = - InNetworkEngagement(followedUserIds.toSeq, mutuallyFollowingUserIds, allSearchResults) - searchResults.map { searchResult => - val features = getThriftTweetFeaturesFromSearchResult( - searcherUserId, - screenName, - userLanguages, - getLanguage(uiLanguageCode), - getTweetCountByAuthorId(searchResults), - followedUserIds, - mutuallyFollowingUserIds, - sourceTweetSearchResultById, - inNetworkEngagement, - searchResult - ) - (searchResult.id -> features) - }.toMap - } - - private[earlybird] def getThriftTweetFeaturesFromSearchResult( - searcherUserId: Long, - screenName: Option[String], - userLanguages: Seq[scc.ThriftLanguage], - uiLanguage: Option[scc.ThriftLanguage], - tweetCountByAuthorId: Map[Long, Int], - followedUserIds: Set[Long], - mutuallyFollowingUserIds: Set[Long], - sourceTweetSearchResultById: Map[Long, eb.ThriftSearchResult], - inNetworkEngagement: InNetworkEngagement, - searchResult: eb.ThriftSearchResult, - ): sc.ThriftTweetFeatures = { - val applyFeatures = (applyUserIndependentFeatures( - searchResult - )(_)).andThen( - applyUserDependentFeatures( - searcherUserId, - screenName, - userLanguages, - uiLanguage, - tweetCountByAuthorId, - followedUserIds, - mutuallyFollowingUserIds, - sourceTweetSearchResultById, - inNetworkEngagement, - searchResult - )(_) - ) - val tweetFeatures = searchResult.tweetFeatures.getOrElse(DefaultEarlybirdFeatures) - applyFeatures(tweetFeatures) - } - - private[earlybird] def applyUserIndependentFeatures( - result: eb.ThriftSearchResult - )( - thriftTweetFeatures: sc.ThriftTweetFeatures - ): sc.ThriftTweetFeatures = { - - val features = result.metadata - .map { metadata => - val isRetweet = metadata.isRetweet.getOrElse(false) - val isReply = metadata.isReply.getOrElse(false) - - // Facets. - val mentions = getMentions(result) - val hashtags = getHashtags(result) - - val searchResultSchemaFeatures = metadata.extraMetadata.flatMap(_.features) - val booleanSearchResultSchemaFeatures = searchResultSchemaFeatures.flatMap(_.boolValues) - val intSearchResultSchemaFeatures = searchResultSchemaFeatures.flatMap(_.intValues) - val doubleSearchResultSchemaFeatures = searchResultSchemaFeatures.flatMap(_.doubleValues) - - thriftTweetFeatures.copy( - // Info about the Tweet. - isRetweet = isRetweet, - isOffensive = metadata.isOffensive.getOrElse(false), - isReply = isReply, - fromVerifiedAccount = metadata.fromVerifiedAccount.getOrElse(false), - cardType = metadata.cardType, - signature = metadata.signature, - language = metadata.language, - isAuthorNSFW = metadata.isUserNSFW.getOrElse(false), - isAuthorBot = metadata.isUserBot.getOrElse(false), - isAuthorSpam = metadata.isUserSpam.getOrElse(false), - isSensitiveContent = - metadata.extraMetadata.flatMap(_.isSensitiveContent).getOrElse(false), - isAuthorProfileEgg = metadata.extraMetadata.flatMap(_.profileIsEggFlag).getOrElse(false), - isAuthorNew = metadata.extraMetadata.flatMap(_.isUserNewFlag).getOrElse(false), - linkLanguage = metadata.extraMetadata.flatMap(_.linkLanguage).getOrElse(DefaultLanguage), - // Info about Tweet content/media. - hasCard = metadata.hasCard.getOrElse(false), - hasImage = metadata.hasImage.getOrElse(false), - hasNews = metadata.hasNews.getOrElse(false), - hasVideo = metadata.hasVideo.getOrElse(false), - hasConsumerVideo = metadata.hasConsumerVideo.getOrElse(false), - hasProVideo = metadata.hasProVideo.getOrElse(false), - hasVine = metadata.hasVine.getOrElse(false), - hasPeriscope = metadata.hasPeriscope.getOrElse(false), - hasNativeVideo = metadata.hasNativeVideo.getOrElse(false), - hasNativeImage = metadata.hasNativeImage.getOrElse(false), - hasLink = metadata.hasLink.getOrElse(false), - hasVisibleLink = metadata.hasVisibleLink.getOrElse(false), - hasTrend = metadata.hasTrend.getOrElse(false), - hasMultipleHashtagsOrTrends = metadata.hasMultipleHashtagsOrTrends.getOrElse(false), - hasQuote = metadata.extraMetadata.flatMap(_.hasQuote), - urlsList = metadata.tweetUrls.map { - _.map(_.originalUrl) - }, - hasMultipleMedia = - metadata.extraMetadata.flatMap(_.hasMultipleMediaFlag).getOrElse(false), - visibleTokenRatio = getIntOptFeature(VISIBLE_TOKEN_RATIO, intSearchResultSchemaFeatures), - // Various counts. - favCount = metadata.favCount.getOrElse(DefaultCount), - replyCount = metadata.replyCount.getOrElse(DefaultCount), - retweetCount = metadata.retweetCount.getOrElse(DefaultCount), - quoteCount = metadata.extraMetadata.flatMap(_.quotedCount), - embedsImpressionCount = metadata.embedsImpressionCount.getOrElse(DefaultCount), - embedsUrlCount = metadata.embedsUrlCount.getOrElse(DefaultCount), - videoViewCount = metadata.videoViewCount.getOrElse(DefaultCount), - numMentions = metadata.extraMetadata.flatMap(_.numMentions).getOrElse(DefaultCount), - numHashtags = metadata.extraMetadata.flatMap(_.numHashtags).getOrElse(DefaultCount), - favCountV2 = metadata.extraMetadata.flatMap(_.favCountV2), - replyCountV2 = metadata.extraMetadata.flatMap(_.replyCountV2), - retweetCountV2 = metadata.extraMetadata.flatMap(_.retweetCountV2), - weightedFavoriteCount = metadata.extraMetadata.flatMap(_.weightedFavCount), - weightedReplyCount = metadata.extraMetadata.flatMap(_.weightedReplyCount), - weightedRetweetCount = metadata.extraMetadata.flatMap(_.weightedRetweetCount), - weightedQuoteCount = metadata.extraMetadata.flatMap(_.weightedQuoteCount), - embedsImpressionCountV2 = - getDoubleAsIntOptFeature(EMBEDS_IMPRESSION_COUNT_V2, doubleSearchResultSchemaFeatures), - embedsUrlCountV2 = - getDoubleAsIntOptFeature(EMBEDS_URL_COUNT_V2, doubleSearchResultSchemaFeatures), - decayedFavoriteCount = - getDoubleAsIntOptFeature(DECAYED_FAVORITE_COUNT, doubleSearchResultSchemaFeatures), - decayedRetweetCount = - getDoubleAsIntOptFeature(DECAYED_RETWEET_COUNT, doubleSearchResultSchemaFeatures), - decayedReplyCount = - getDoubleAsIntOptFeature(DECAYED_REPLY_COUNT, doubleSearchResultSchemaFeatures), - decayedQuoteCount = - getDoubleAsIntOptFeature(DECAYED_QUOTE_COUNT, doubleSearchResultSchemaFeatures), - fakeFavoriteCount = - getDoubleAsIntOptFeature(FAKE_FAVORITE_COUNT, doubleSearchResultSchemaFeatures), - fakeRetweetCount = - getDoubleAsIntOptFeature(FAKE_RETWEET_COUNT, doubleSearchResultSchemaFeatures), - fakeReplyCount = - getDoubleAsIntOptFeature(FAKE_REPLY_COUNT, doubleSearchResultSchemaFeatures), - fakeQuoteCount = - getDoubleAsIntOptFeature(FAKE_QUOTE_COUNT, doubleSearchResultSchemaFeatures), - // Scores. - textScore = metadata.textScore.getOrElse(DefaultScore), - earlybirdScore = metadata.score.getOrElse(DefaultScore), - parusScore = metadata.parusScore.getOrElse(DefaultScore), - userRep = metadata.userRep.getOrElse(DefaultScore), - pBlockScore = metadata.extraMetadata.flatMap(_.pBlockScore), - toxicityScore = metadata.extraMetadata.flatMap(_.toxicityScore), - pSpammyTweetScore = metadata.extraMetadata.flatMap(_.pSpammyTweetScore), - pReportedTweetScore = metadata.extraMetadata.flatMap(_.pReportedTweetScore), - pSpammyTweetContent = metadata.extraMetadata.flatMap(_.spammyTweetContentScore), - // Safety Signals - labelAbusiveFlag = - getBooleanOptFeature(LABEL_ABUSIVE_FLAG, booleanSearchResultSchemaFeatures), - labelAbusiveHiRclFlag = - getBooleanOptFeature(LABEL_ABUSIVE_HI_RCL_FLAG, booleanSearchResultSchemaFeatures), - labelDupContentFlag = - getBooleanOptFeature(LABEL_DUP_CONTENT_FLAG, booleanSearchResultSchemaFeatures), - labelNsfwHiPrcFlag = - getBooleanOptFeature(LABEL_NSFW_HI_PRC_FLAG, booleanSearchResultSchemaFeatures), - labelNsfwHiRclFlag = - getBooleanOptFeature(LABEL_NSFW_HI_RCL_FLAG, booleanSearchResultSchemaFeatures), - labelSpamFlag = getBooleanOptFeature(LABEL_SPAM_FLAG, booleanSearchResultSchemaFeatures), - labelSpamHiRclFlag = - getBooleanOptFeature(LABEL_SPAM_HI_RCL_FLAG, booleanSearchResultSchemaFeatures), - // Periscope Features - periscopeExists = - getBooleanOptFeature(PERISCOPE_EXISTS, booleanSearchResultSchemaFeatures), - periscopeHasBeenFeatured = - getBooleanOptFeature(PERISCOPE_HAS_BEEN_FEATURED, booleanSearchResultSchemaFeatures), - periscopeIsCurrentlyFeatured = getBooleanOptFeature( - PERISCOPE_IS_CURRENTLY_FEATURED, - booleanSearchResultSchemaFeatures), - periscopeIsFromQualitySource = getBooleanOptFeature( - PERISCOPE_IS_FROM_QUALITY_SOURCE, - booleanSearchResultSchemaFeatures), - periscopeIsLive = - getBooleanOptFeature(PERISCOPE_IS_LIVE, booleanSearchResultSchemaFeatures), - // Last Engagement Features - lastFavSinceCreationHrs = - getIntOptFeature(LAST_FAVORITE_SINCE_CREATION_HRS, intSearchResultSchemaFeatures), - lastRetweetSinceCreationHrs = - getIntOptFeature(LAST_RETWEET_SINCE_CREATION_HRS, intSearchResultSchemaFeatures), - lastReplySinceCreationHrs = - getIntOptFeature(LAST_REPLY_SINCE_CREATION_HRS, intSearchResultSchemaFeatures), - lastQuoteSinceCreationHrs = - getIntOptFeature(LAST_QUOTE_SINCE_CREATION_HRS, intSearchResultSchemaFeatures), - likedByUserIds = metadata.extraMetadata.flatMap(_.likedByUserIds), - mentionsList = if (mentions.nonEmpty) Some(mentions) else None, - hashtagsList = if (hashtags.nonEmpty) Some(hashtags) else None, - isComposerSourceCamera = - getBooleanOptFeature(COMPOSER_SOURCE_IS_CAMERA_FLAG, booleanSearchResultSchemaFeatures), - ) - } - .getOrElse(thriftTweetFeatures) - - features - } - - private def applyUserDependentFeatures( - searcherUserId: Long, - screenName: Option[String], - userLanguages: Seq[scc.ThriftLanguage], - uiLanguage: Option[scc.ThriftLanguage], - tweetCountByAuthorId: Map[Long, Int], - followedUserIds: Set[Long], - mutuallyFollowingUserIds: Set[Long], - sourceTweetSearchResultById: Map[Long, eb.ThriftSearchResult], - inNetworkEngagement: InNetworkEngagement, - result: eb.ThriftSearchResult - )( - thriftTweetFeatures: sc.ThriftTweetFeatures - ): sc.ThriftTweetFeatures = { - result.metadata - .map { metadata => - val isRetweet = metadata.isRetweet.getOrElse(false) - val sourceTweet = - if (isRetweet) sourceTweetSearchResultById.get(metadata.sharedStatusId) - else None - val mentionsInSourceTweet = sourceTweet.map(getMentions).getOrElse(Seq.empty) - - val isReply = metadata.isReply.getOrElse(false) - val replyToSearcher = isReply && (metadata.referencedTweetAuthorId == searcherUserId) - val replyOther = isReply && !replyToSearcher - val retweetOther = isRetweet && (metadata.referencedTweetAuthorId != searcherUserId) - val tweetLanguage = metadata.language.getOrElse(scc.ThriftLanguage.Unknown) - - val referencedTweetAuthorId = - if (metadata.referencedTweetAuthorId > 0) Some(metadata.referencedTweetAuthorId) else None - val inReplyToUserId = if (!isRetweet) referencedTweetAuthorId else None - - thriftTweetFeatures.copy( - // Info about the Tweet. - fromSearcher = metadata.fromUserId == searcherUserId, - probablyFromFollowedAuthor = followedUserIds.contains(metadata.fromUserId), - fromMutualFollow = mutuallyFollowingUserIds.contains(metadata.fromUserId), - replySearcher = replyToSearcher, - replyOther = replyOther, - retweetOther = retweetOther, - mentionSearcher = isUserMentioned(screenName, getMentions(result), mentionsInSourceTweet), - // Info about Tweet content/media. - matchesSearcherMainLang = isUsersMainLanguage(tweetLanguage, userLanguages), - matchesSearcherLangs = isUsersLanguage(tweetLanguage, userLanguages), - matchesUILang = isUILanguage(tweetLanguage, uiLanguage), - // Various counts. - prevUserTweetEngagement = - metadata.extraMetadata.flatMap(_.prevUserTweetEngagement).getOrElse(DefaultCount), - tweetCountFromUserInSnapshot = tweetCountByAuthorId(metadata.fromUserId), - bidirectionalReplyCount = inNetworkEngagement.biDirectionalReplyCounts(result.id), - unidirectionalReplyCount = inNetworkEngagement.uniDirectionalReplyCounts(result.id), - bidirectionalRetweetCount = inNetworkEngagement.biDirectionalRetweetCounts(result.id), - unidirectionalRetweetCount = inNetworkEngagement.uniDirectionalRetweetCounts(result.id), - conversationCount = inNetworkEngagement.descendantReplyCounts(result.id), - directedAtUserIdIsInFirstDegree = - if (isReply) inReplyToUserId.map(followedUserIds.contains) else None, - ) - } - .getOrElse(thriftTweetFeatures) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/RelevanceSearchUtil.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/RelevanceSearchUtil.scala deleted file mode 100644 index 40b727141..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/RelevanceSearchUtil.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.home_mixer.util.earlybird - -import com.twitter.search.common.schema.earlybird.EarlybirdFieldConstants.EarlybirdFieldConstant -import com.twitter.search.earlybird.{thriftscala => eb} - -object RelevanceSearchUtil { - - val Mentions: String = EarlybirdFieldConstant.MENTIONS_FACET - val Hashtags: String = EarlybirdFieldConstant.HASHTAGS_FACET - val FacetsToFetch: Seq[String] = Seq(Mentions, Hashtags) - - val MetadataOptions: eb.ThriftSearchResultMetadataOptions = { - eb.ThriftSearchResultMetadataOptions( - getTweetUrls = true, - getResultLocation = false, - getLuceneScore = false, - getInReplyToStatusId = true, - getReferencedTweetAuthorId = true, - getMediaBits = true, - getAllFeatures = true, - returnSearchResultFeatures = true, - // Set getExclusiveConversationAuthorId in order to retrieve Exclusive / SuperFollow tweets. - getExclusiveConversationAuthorId = true - ) - } - - val RelevanceOptions: eb.ThriftSearchRelevanceOptions = { - eb.ThriftSearchRelevanceOptions( - proximityScoring = true, - maxConsecutiveSameUser = Some(2), - rankingParams = None, - maxHitsToProcess = Some(500), - maxUserBlendCount = Some(3), - proximityPhraseWeight = 9.0, - returnAllResults = Some(true) - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/BUILD.bazel deleted file mode 100644 index dbfee21f4..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/BUILD.bazel +++ /dev/null @@ -1,10 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/thrift/src/main/thrift:thrift-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/RequestFields.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/RequestFields.scala deleted file mode 100644 index e20aead94..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/RequestFields.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.home_mixer.util.tweetypie - -import com.twitter.tweetypie.{thriftscala => tp} - -object RequestFields { - - val CoreTweetFields: Set[tp.TweetInclude] = Set[tp.TweetInclude]( - tp.TweetInclude.TweetFieldId(tp.Tweet.IdField.id), - tp.TweetInclude.TweetFieldId(tp.Tweet.CoreDataField.id) - ) - val MediaFields: Set[tp.TweetInclude] = Set[tp.TweetInclude]( - tp.TweetInclude.TweetFieldId(tp.Tweet.MediaField.id), - ) - val SelfThreadFields: Set[tp.TweetInclude] = Set[tp.TweetInclude]( - tp.TweetInclude.TweetFieldId(tp.Tweet.SelfThreadMetadataField.id) - ) - val MentionsTweetFields: Set[tp.TweetInclude] = Set[tp.TweetInclude]( - tp.TweetInclude.TweetFieldId(tp.Tweet.MentionsField.id) - ) - val SemanticAnnotationTweetFields: Set[tp.TweetInclude] = Set[tp.TweetInclude]( - tp.TweetInclude.TweetFieldId(tp.Tweet.EscherbirdEntityAnnotationsField.id) - ) - val NsfwLabelFields: Set[tp.TweetInclude] = Set[tp.TweetInclude]( - // Tweet fields containing NSFW related attributes. - tp.TweetInclude.TweetFieldId(tp.Tweet.NsfwHighRecallLabelField.id), - tp.TweetInclude.TweetFieldId(tp.Tweet.NsfwHighPrecisionLabelField.id), - tp.TweetInclude.TweetFieldId(tp.Tweet.NsfaHighRecallLabelField.id) - ) - val SafetyLabelFields: Set[tp.TweetInclude] = Set[tp.TweetInclude]( - // Tweet fields containing RTF labels for abuse and spam. - tp.TweetInclude.TweetFieldId(tp.Tweet.SpamLabelField.id), - tp.TweetInclude.TweetFieldId(tp.Tweet.AbusiveLabelField.id) - ) - val ConversationControlField: Set[tp.TweetInclude] = - Set[tp.TweetInclude](tp.TweetInclude.TweetFieldId(tp.Tweet.ConversationControlField.id)) - - val TweetTPHydrationFields: Set[tp.TweetInclude] = CoreTweetFields ++ - NsfwLabelFields ++ - SafetyLabelFields ++ - SemanticAnnotationTweetFields ++ - Set( - tp.TweetInclude.TweetFieldId(tp.Tweet.TakedownCountryCodesField.id), - // QTs imply a TweetyPie -> SGS request dependency - tp.TweetInclude.TweetFieldId(tp.Tweet.QuotedTweetField.id), - tp.TweetInclude.TweetFieldId(tp.Tweet.CommunitiesField.id), - // Field required for determining if a Tweet was created via News Camera. - tp.TweetInclude.TweetFieldId(tp.Tweet.ComposerSourceField.id), - tp.TweetInclude.TweetFieldId(tp.Tweet.LanguageField.id) - ) - - val TweetStaticEntitiesFields: Set[tp.TweetInclude] = - MentionsTweetFields ++ CoreTweetFields ++ SemanticAnnotationTweetFields ++ MediaFields - - val ContentFields: Set[tp.TweetInclude] = CoreTweetFields ++ MediaFields ++ SelfThreadFields ++ - ConversationControlField ++ SemanticAnnotationTweetFields ++ - Set[tp.TweetInclude]( - tp.TweetInclude.MediaEntityFieldId(tp.MediaEntity.AdditionalMetadataField.id)) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content/BUILD.bazel deleted file mode 100644 index 5acfd98e0..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content/BUILD.bazel +++ /dev/null @@ -1,19 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "src/java/com/twitter/common/text/tagger", - "src/java/com/twitter/common/text/token", - "src/java/com/twitter/common_internal/text", - "src/java/com/twitter/common_internal/text/version", - "src/java/com/twitter/search/common/util/text", - "src/thrift/com/twitter/search/common:features-scala", - "src/thrift/com/twitter/tweetypie:media-entity-scala", - "src/thrift/com/twitter/tweetypie:service-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content/FeatureExtractionHelper.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content/FeatureExtractionHelper.scala deleted file mode 100644 index 07cbdebe4..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content/FeatureExtractionHelper.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.home_mixer.util.tweetypie.content - -import com.twitter.home_mixer.model.ContentFeatures -import com.twitter.tweetypie.{thriftscala => tp} - -object FeatureExtractionHelper { - - def extractFeatures( - tweet: tp.Tweet - ): ContentFeatures = { - val contentFeaturesFromTweet = ContentFeatures.Empty.copy( - selfThreadMetadata = tweet.selfThreadMetadata - ) - - val contentFeaturesWithText = TweetTextFeaturesExtractor.addTextFeaturesFromTweet( - contentFeaturesFromTweet, - tweet - ) - val contentFeaturesWithMedia = TweetMediaFeaturesExtractor.addMediaFeaturesFromTweet( - contentFeaturesWithText, - tweet - ) - - contentFeaturesWithMedia.copy( - conversationControl = tweet.conversationControl, - semanticCoreAnnotations = tweet.escherbirdEntityAnnotations.map(_.entityAnnotations) - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content/TweetMediaFeaturesExtractor.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content/TweetMediaFeaturesExtractor.scala deleted file mode 100644 index 0a5a93a2e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content/TweetMediaFeaturesExtractor.scala +++ /dev/null @@ -1,285 +0,0 @@ -package com.twitter.home_mixer.util.tweetypie.content - -import com.twitter.home_mixer.model.ContentFeatures -import com.twitter.mediaservices.commons.mediainformation.{thriftscala => mi} -import com.twitter.mediaservices.commons.tweetmedia.{thriftscala => tm} -import com.twitter.mediaservices.commons.{thriftscala => ms} -import com.twitter.tweetypie.{thriftscala => tp} -import scala.collection.Map - -object TweetMediaFeaturesExtractor { - - private val ImageCategories = Set( - ms.MediaCategory.TweetImage.value, - ms.MediaCategory.TweetGif.value - ) - private val VideoCategories = Set( - ms.MediaCategory.TweetVideo.value, - ms.MediaCategory.AmplifyVideo.value - ) - - def hasImage(tweet: tp.Tweet): Boolean = hasMediaByCategory(tweet, ImageCategories) - - def hasVideo(tweet: tp.Tweet): Boolean = hasMediaByCategory(tweet, VideoCategories) - - private def hasMediaByCategory(tweet: tp.Tweet, categories: Set[Int]): Boolean = { - tweet.media.exists { mediaEntities => - mediaEntities.exists { mediaEntity => - mediaEntity.mediaKey.map(_.mediaCategory).exists { mediaCategory => - categories.contains(mediaCategory.value) - } - } - } - } - - def addMediaFeaturesFromTweet( - inputFeatures: ContentFeatures, - tweet: tp.Tweet, - ): ContentFeatures = { - val featuresWithMediaEntity = tweet.media - .map { mediaEntities => - val sizeFeatures = getSizeFeatures(mediaEntities) - val playbackFeatures = getPlaybackFeatures(mediaEntities) - val mediaWidths = sizeFeatures.map(_.width.toShort) - val mediaHeights = sizeFeatures.map(_.height.toShort) - val resizeMethods = sizeFeatures.map(_.resizeMethod.toShort) - val faceMapAreas = getFaceMapAreas(mediaEntities) - val sortedColorPalette = getSortedColorPalette(mediaEntities) - val stickerFeatures = getStickerFeatures(mediaEntities) - val mediaOriginProviders = getMediaOriginProviders(mediaEntities) - val isManaged = getIsManaged(mediaEntities) - val is360 = getIs360(mediaEntities) - val viewCount = getViewCount(mediaEntities) - val userDefinedProductMetadataFeatures = - getUserDefinedProductMetadataFeatures(mediaEntities) - val isMonetizable = - getOptBooleanFromSeqOpt(userDefinedProductMetadataFeatures.map(_.isMonetizable)) - val isEmbeddable = - getOptBooleanFromSeqOpt(userDefinedProductMetadataFeatures.map(_.isEmbeddable)) - val hasSelectedPreviewImage = - getOptBooleanFromSeqOpt(userDefinedProductMetadataFeatures.map(_.hasSelectedPreviewImage)) - val hasTitle = getOptBooleanFromSeqOpt(userDefinedProductMetadataFeatures.map(_.hasTitle)) - val hasDescription = - getOptBooleanFromSeqOpt(userDefinedProductMetadataFeatures.map(_.hasDescription)) - val hasVisitSiteCallToAction = getOptBooleanFromSeqOpt( - userDefinedProductMetadataFeatures.map(_.hasVisitSiteCallToAction)) - val hasAppInstallCallToAction = getOptBooleanFromSeqOpt( - userDefinedProductMetadataFeatures.map(_.hasAppInstallCallToAction)) - val hasWatchNowCallToAction = - getOptBooleanFromSeqOpt(userDefinedProductMetadataFeatures.map(_.hasWatchNowCallToAction)) - - inputFeatures.copy( - videoDurationMs = playbackFeatures.durationMs, - bitRate = playbackFeatures.bitRate, - aspectRatioNum = playbackFeatures.aspectRatioNum, - aspectRatioDen = playbackFeatures.aspectRatioDen, - widths = Some(mediaWidths), - heights = Some(mediaHeights), - resizeMethods = Some(resizeMethods), - faceAreas = Some(faceMapAreas), - dominantColorRed = sortedColorPalette.headOption.map(_.rgb.red), - dominantColorBlue = sortedColorPalette.headOption.map(_.rgb.blue), - dominantColorGreen = sortedColorPalette.headOption.map(_.rgb.green), - dominantColorPercentage = sortedColorPalette.headOption.map(_.percentage), - numColors = Some(sortedColorPalette.size.toShort), - stickerIds = Some(stickerFeatures), - mediaOriginProviders = Some(mediaOriginProviders), - isManaged = Some(isManaged), - is360 = Some(is360), - viewCount = viewCount, - isMonetizable = isMonetizable, - isEmbeddable = isEmbeddable, - hasSelectedPreviewImage = hasSelectedPreviewImage, - hasTitle = hasTitle, - hasDescription = hasDescription, - hasVisitSiteCallToAction = hasVisitSiteCallToAction, - hasAppInstallCallToAction = hasAppInstallCallToAction, - hasWatchNowCallToAction = hasWatchNowCallToAction - ) - } - .getOrElse(inputFeatures) - - val featuresWithMediaTags = tweet.mediaTags - .map { mediaTags => - val mediaTagScreenNames = getMediaTagScreenNames(mediaTags.tagMap) - val numMediaTags = mediaTagScreenNames.size - - featuresWithMediaEntity.copy( - mediaTagScreenNames = Some(mediaTagScreenNames), - numMediaTags = Some(numMediaTags.toShort) - ) - } - .getOrElse(featuresWithMediaEntity) - - featuresWithMediaTags - .copy(media = tweet.media) - } - - private def getSizeFeatures(mediaEntities: Seq[tp.MediaEntity]): Seq[MediaSizeFeatures] = { - mediaEntities.map { mediaEntity => - mediaEntity.sizes.foldLeft(MediaSizeFeatures(0, 0, 0))((accDimensions, dimensions) => - MediaSizeFeatures( - width = math.max(dimensions.width, accDimensions.width), - height = math.max(dimensions.height, accDimensions.height), - resizeMethod = math.max(dimensions.resizeMethod.getValue, accDimensions.resizeMethod) - )) - } - } - - private def getPlaybackFeatures(mediaEntities: Seq[tp.MediaEntity]): PlaybackFeatures = { - val allPlaybackFeatures = mediaEntities - .flatMap { mediaEntity => - mediaEntity.mediaInfo map { - case videoEntity: tm.MediaInfo.VideoInfo => - PlaybackFeatures( - durationMs = Some(videoEntity.videoInfo.durationMillis), - bitRate = videoEntity.videoInfo.variants.maxBy(_.bitRate).bitRate, - aspectRatioNum = Some(videoEntity.videoInfo.aspectRatio.numerator), - aspectRatioDen = Some(videoEntity.videoInfo.aspectRatio.denominator) - ) - case gifEntity: tm.MediaInfo.AnimatedGifInfo => - PlaybackFeatures( - durationMs = None, - bitRate = gifEntity.animatedGifInfo.variants.maxBy(_.bitRate).bitRate, - aspectRatioNum = Some(gifEntity.animatedGifInfo.aspectRatio.numerator), - aspectRatioDen = Some(gifEntity.animatedGifInfo.aspectRatio.denominator) - ) - case _ => PlaybackFeatures(None, None, None, None) - } - } - .collect { - case playbackFeatures: PlaybackFeatures => playbackFeatures - } - - if (allPlaybackFeatures.nonEmpty) allPlaybackFeatures.maxBy(_.durationMs) - else PlaybackFeatures(None, None, None, None) - } - - private def getMediaTagScreenNames(tagMap: Map[Long, Seq[tp.MediaTag]]): Seq[String] = - tagMap.values - .flatMap(seqMediaTag => seqMediaTag.flatMap(_.screenName)) - .toSeq - - // Areas of the faces identified in the media entities - private def getFaceMapAreas(mediaEntities: Seq[tp.MediaEntity]): Seq[Int] = { - for { - mediaEntity <- mediaEntities - metadata <- mediaEntity.additionalMetadata.toSeq - faceData <- metadata.faceData - faces <- faceData.faces - } yield { - faces - .getOrElse("orig", Seq.empty[mi.Face]) - .flatMap(f => f.boundingBox.map(bb => bb.width * bb.height)) - } - }.flatten - - // All ColorPalettes in the media sorted by the percentage in descending order - private def getSortedColorPalette( - mediaEntities: Seq[tp.MediaEntity] - ): Seq[mi.ColorPaletteItem] = { - for { - mediaEntity <- mediaEntities - metadata <- mediaEntity.additionalMetadata.toSeq - colorInfo <- metadata.colorInfo - } yield { - colorInfo.palette - } - }.flatten.sortBy(-_.percentage) - - // Id's of stickers applied by the user - private def getStickerFeatures(mediaEntities: Seq[tp.MediaEntity]): Seq[Long] = { - for { - mediaEntity <- mediaEntities - metadata <- mediaEntity.additionalMetadata.toSeq - stickerInfo <- metadata.stickerInfo - } yield { - stickerInfo.stickers.map(_.id) - } - }.flatten - - // 3rd party media providers. eg. giphy for gifs - private def getMediaOriginProviders(mediaEntities: Seq[tp.MediaEntity]): Seq[String] = - for { - mediaEntity <- mediaEntities - metadata <- mediaEntity.additionalMetadata.toSeq - mediaOrigin <- metadata.foundMediaOrigin - } yield { - mediaOrigin.provider - } - - private def getIsManaged(mediaEntities: Seq[tp.MediaEntity]): Boolean = { - for { - mediaEntity <- mediaEntities - metadata <- mediaEntity.additionalMetadata.toSeq - managementInfo <- metadata.managementInfo - } yield { - managementInfo.managed - } - }.contains(true) - - private def getIs360(mediaEntities: Seq[tp.MediaEntity]): Boolean = { - for { - mediaEntity <- mediaEntities - metadata <- mediaEntity.additionalMetadata.toSeq - info360 <- metadata.info360 - } yield { - info360.is360 - } - }.contains(Some(true)) - - private def getViewCount(mediaEntities: Seq[tp.MediaEntity]): Option[Long] = { - for { - mediaEntity <- mediaEntities - metadata <- mediaEntity.additionalMetadata.toSeq - engagementInfo <- metadata.engagementInfo - viewCounts <- engagementInfo.viewCount - } yield { - viewCounts - } - }.reduceOption(_ max _) - - // metadata defined by the user when uploading the image - private def getUserDefinedProductMetadataFeatures( - mediaEntities: Seq[tp.MediaEntity] - ): Seq[UserDefinedProductMetadataFeatures] = - for { - mediaEntity <- mediaEntities - userDefinedMetadata <- mediaEntity.metadata - } yield { - UserDefinedProductMetadataFeatures( - isMonetizable = userDefinedMetadata.monetizable, - isEmbeddable = userDefinedMetadata.embeddable, - hasSelectedPreviewImage = Some(userDefinedMetadata.previewImage.nonEmpty), - hasTitle = userDefinedMetadata.title.map(_.nonEmpty), - hasDescription = userDefinedMetadata.description.map(_.nonEmpty), - hasVisitSiteCallToAction = userDefinedMetadata.callToActions.map(_.visitSite.nonEmpty), - hasAppInstallCallToAction = userDefinedMetadata.callToActions.map(_.appInstall.nonEmpty), - hasWatchNowCallToAction = userDefinedMetadata.callToActions.map(_.watchNow.nonEmpty) - ) - } - - private def getOptBooleanFromSeqOpt( - seqOpt: Seq[Option[Boolean]] - ): Option[Boolean] = Some( - seqOpt.exists(boolOpt => boolOpt.contains(true)) - ) -} - -case class MediaSizeFeatures(width: Int, height: Int, resizeMethod: Int) - -case class PlaybackFeatures( - durationMs: Option[Int], - bitRate: Option[Int], - aspectRatioNum: Option[Short], - aspectRatioDen: Option[Short]) - -case class UserDefinedProductMetadataFeatures( - isMonetizable: Option[Boolean], - isEmbeddable: Option[Boolean], - hasSelectedPreviewImage: Option[Boolean], - hasTitle: Option[Boolean], - hasDescription: Option[Boolean], - hasVisitSiteCallToAction: Option[Boolean], - hasAppInstallCallToAction: Option[Boolean], - hasWatchNowCallToAction: Option[Boolean]) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content/TweetTextFeaturesExtractor.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content/TweetTextFeaturesExtractor.scala deleted file mode 100644 index 0a403d98a..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content/TweetTextFeaturesExtractor.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.twitter.home_mixer.util.tweetypie.content - -import com.twitter.home_mixer.model.ContentFeatures -import com.twitter.tweetypie.{thriftscala => tp} - -object TweetTextFeaturesExtractor { - - private val QUESTION_MARK_CHARS = Set( - '\u003F', '\u00BF', '\u037E', '\u055E', '\u061F', '\u1367', '\u1945', '\u2047', '\u2048', - '\u2049', '\u2753', '\u2754', '\u2CFA', '\u2CFB', '\u2E2E', '\uA60F', '\uA6F7', '\uFE16', - '\uFE56', '\uFF1F', '\u1114', '\u1E95' - ) - private val NEW_LINE_REGEX = "\r\n|\r|\n".r - - def addTextFeaturesFromTweet( - inputFeatures: ContentFeatures, - tweet: tp.Tweet - ): ContentFeatures = { - tweet.coreData - .map { coreData => - val tweetText = coreData.text - - inputFeatures.copy( - hasQuestion = hasQuestionCharacter(tweetText), - length = getLength(tweetText).toShort, - numCaps = getCaps(tweetText).toShort, - numWhiteSpaces = getSpaces(tweetText).toShort, - numNewlines = Some(getNumNewlines(tweetText)), - ) - } - .getOrElse(inputFeatures) - } - - def getLength(text: String): Int = - text.codePointCount(0, text.length()) - - def getCaps(text: String): Int = text.count(Character.isUpperCase) - - def getSpaces(text: String): Int = text.count(Character.isWhitespace) - - def hasQuestionCharacter(text: String): Boolean = text.exists(QUESTION_MARK_CHARS.contains) - - def getNumNewlines(text: String): Short = NEW_LINE_REGEX.findAllIn(text).length.toShort -}