Merge 624bbf5f94 into 72eda9a24f

improve readability

home-mixer/README.md

@@ -74,7 +74,7 @@ Timeline tabs powered by Home Mixer.
             - ScoredTweetsRecommendationPipelineConfig (main Tweet recommendation layer)
                 - Fetch Tweet Candidates
                     - ScoredTweetsInNetworkCandidatePipelineConfig
-                    - ScoredTweetsCrMixerCandidatePipelineConfig
+                    - ScoredTweetsTweetMixerCandidatePipelineConfig
                     - ScoredTweetsUtegCandidatePipelineConfig
                     - ScoredTweetsFrsCandidatePipelineConfig
                 - Feature Hydration and Scoring
@@ -99,4 +99,3 @@ Timeline tabs powered by Home Mixer.
         - ListTweetsTimelineServiceCandidatePipelineConfig (fetch tweets from timeline service)
         - ConversationServiceCandidatePipelineConfig (fetch ancestors for conversation modules)
         - ListTweetsAdsCandidatePipelineConfig (fetch ads)
-

home-mixer/server/src/main/scala/com/twitter/home_mixer/BUILD.bazel

@@ -21,6 +21,7 @@ scala_library(
         "finatra/inject/inject-utils/src/main/scala",
         "home-mixer/server/src/main/resources",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/controller",
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/federated",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/module",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/product",
@@ -31,6 +32,10 @@ scala_library(
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter",
         "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala",
         "src/thrift/com/twitter/timelines/render:thrift-scala",
+        "strato/config/columns/auth-context:auth-context-strato-client",
+        "strato/config/columns/gizmoduck:gizmoduck-strato-client",
+        "strato/src/main/scala/com/twitter/strato/fed",
+        "strato/src/main/scala/com/twitter/strato/fed/server",
         "stringcenter/client",
         "stringcenter/client/src/main/java",
         "stringcenter/client/src/main/scala/com/twitter/stringcenter/client",

home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerHttpServerWarmupHandler.scala

@@ -2,7 +2,7 @@ package com.twitter.home_mixer
 
 import com.twitter.finatra.http.routing.HttpWarmup
 import com.twitter.finatra.httpclient.RequestBuilder._
-import com.twitter.inject.Logging
+import com.twitter.util.logging.Logging
 import com.twitter.inject.utils.Handler
 import com.twitter.util.Try
 import javax.inject.Inject

home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerServer.scala

@@ -12,57 +12,63 @@ import com.twitter.finatra.thrift.ThriftServer
 import com.twitter.finatra.thrift.filters._
 import com.twitter.finatra.thrift.routing.ThriftRouter
 import com.twitter.home_mixer.controller.HomeThriftController
+import com.twitter.home_mixer.federated.HomeMixerColumn
 import com.twitter.home_mixer.module._
 import com.twitter.home_mixer.param.GlobalParamConfigModule
 import com.twitter.home_mixer.product.HomeMixerProductModule
 import com.twitter.home_mixer.{thriftscala => st}
 import com.twitter.product_mixer.component_library.module.AccountRecommendationsMixerModule
-import com.twitter.product_mixer.component_library.module.CrMixerClientModule
 import com.twitter.product_mixer.component_library.module.DarkTrafficFilterModule
 import com.twitter.product_mixer.component_library.module.EarlybirdModule
 import com.twitter.product_mixer.component_library.module.ExploreRankerClientModule
 import com.twitter.product_mixer.component_library.module.GizmoduckClientModule
 import com.twitter.product_mixer.component_library.module.OnboardingTaskServiceModule
 import com.twitter.product_mixer.component_library.module.SocialGraphServiceModule
-import com.twitter.product_mixer.component_library.module.TimelineMixerClientModule
 import com.twitter.product_mixer.component_library.module.TimelineRankerClientModule
 import com.twitter.product_mixer.component_library.module.TimelineScorerClientModule
 import com.twitter.product_mixer.component_library.module.TimelineServiceClientModule
 import com.twitter.product_mixer.component_library.module.TweetImpressionStoreModule
+import com.twitter.product_mixer.component_library.module.TweetMixerClientModule
 import com.twitter.product_mixer.component_library.module.UserSessionStoreModule
 import com.twitter.product_mixer.core.controllers.ProductMixerController
 import com.twitter.product_mixer.core.module.LoggingThrowableExceptionMapper
 import com.twitter.product_mixer.core.module.ProductMixerModule
-import com.twitter.product_mixer.core.module.StratoClientModule
 import com.twitter.product_mixer.core.module.stringcenter.ProductScopeStringCenterModule
+import com.twitter.strato.fed.StratoFed
+import com.twitter.strato.fed.server.StratoFedServer
 
 object HomeMixerServerMain extends HomeMixerServer
 
-class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMtls {
+class HomeMixerServer
+    extends StratoFedServer
+    with ThriftServer
+    with Mtls
+    with HttpServer
+    with HttpMtls {
   override val name = "home-mixer-server"
 
   override val modules: Seq[Module] = Seq(
     AccountRecommendationsMixerModule,
     AdvertiserBrandSafetySettingsStoreModule,
+    BlenderClientModule,
     ClientSentImpressionsPublisherModule,
     ConversationServiceModule,
-    CrMixerClientModule,
     EarlybirdModule,
     ExploreRankerClientModule,
+    FeedbackHistoryClientModule,
     GizmoduckClientModule,
     GlobalParamConfigModule,
     HomeAdsCandidateSourceModule,
     HomeMixerFlagsModule,
     HomeMixerProductModule,
     HomeMixerResourcesModule,
-    HomeNaviModelClientModule,
     ImpressionBloomFilterModule,
     InjectionHistoryClientModule,
-    FeedbackHistoryClientModule,
     ManhattanClientsModule,
     ManhattanFeatureRepositoryModule,
     ManhattanTweetImpressionStoreModule,
     MemcachedFeatureRepositoryModule,
+    NaviModelClientModule,
     OnboardingTaskServiceModule,
     OptimizedStratoClientModule,
     PeopleDiscoveryServiceModule,
@@ -74,24 +80,23 @@ class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMt
     SimClustersRecentEngagementsClientModule,
     SocialGraphServiceModule,
     StaleTweetsCacheModule,
-    StratoClientModule,
     ThriftFeatureRepositoryModule,
-    TimelineMixerClientModule,
     TimelineRankerClientModule,
     TimelineScorerClientModule,
     TimelineServiceClientModule,
     TimelinesPersistenceStoreClientModule,
+    TopicSocialProofClientModule,
     TweetImpressionStoreModule,
-    TweetyPieClientModule,
+    TweetMixerClientModule,
+    TweetypieClientModule,
     TweetypieStaticEntitiesCacheClientModule,
-    UserMetadataStoreModule,
     UserSessionStoreModule,
     new DarkTrafficFilterModule[st.HomeMixer.ReqRepServicePerEndpoint](),
     new MtlsThriftWebFormsModule[st.HomeMixer.MethodPerEndpoint](this),
     new ProductScopeStringCenterModule()
   )
 
-  def configureThrift(router: ThriftRouter): Unit = {
+  override def configureThrift(router: ThriftRouter): Unit = {
     router
       .filter[LoggingMDCFilter]
       .filter[TraceIdMDCFilter]
@@ -111,6 +116,11 @@ class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMt
         this.injector,
         st.HomeMixer.ExecutePipeline))
 
+  override val dest: String = "/s/home-mixer/home-mixer:strato"
+
+  override val columns: Seq[Class[_ <: StratoFed.Column]] =
+    Seq(classOf[HomeMixerColumn])
+
   override protected def warmup(): Unit = {
     handle[HomeMixerThriftServerWarmupHandler]()
     handle[HomeMixerHttpServerWarmupHandler]()

home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerThriftServerWarmupHandler.scala

@@ -3,7 +3,7 @@ package com.twitter.home_mixer
 import com.twitter.finagle.thrift.ClientId
 import com.twitter.finatra.thrift.routing.ThriftWarmup
 import com.twitter.home_mixer.{thriftscala => st}
-import com.twitter.inject.Logging
+import com.twitter.util.logging.Logging
 import com.twitter.inject.utils.Handler
 import com.twitter.product_mixer.core.{thriftscala => pt}
 import com.twitter.scrooge.Request

home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/BUILD.bazel

@@ -5,19 +5,13 @@ scala_library(
     tags = ["bazel-compatible"],
     dependencies = [
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
@@ -25,10 +19,6 @@ scala_library(
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer",
-        "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
-        "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
-        "timelines/src/main/scala/com/twitter/timelines/injection/scribe",
-        "timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
     ],
     exports = [
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc",

home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfig.scala

@@ -1,18 +1,23 @@
 package com.twitter.home_mixer.candidate_pipeline
 
+import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator
 import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
-import com.twitter.home_mixer.functional_component.feature_hydrator.SocialGraphServiceFeatureHydrator
 import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
 import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter
-import com.twitter.home_mixer.functional_component.filter.PredicateFeatureFilter
+import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter
 import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter
+import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
+import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
 import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature
 import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature
+import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
+import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
 import com.twitter.home_mixer.service.HomeMixerAlertConfig
 import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
 import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSourceRequest
 import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata
 import com.twitter.product_mixer.component_library.filter.FeatureFilter
+import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter
 import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
 import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource
 import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
@@ -33,8 +38,8 @@ import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipel
 class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
   conversationServiceCandidateSource: ConversationServiceCandidateSource,
   tweetypieFeatureHydrator: TweetypieFeatureHydrator,
-  socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator,
   namesFeatureHydrator: NamesFeatureHydrator,
+  invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter,
   override val gates: Seq[BaseGate[Query]],
   override val decorator: Option[CandidateDecorator[Query, TweetCandidate]])
     extends DependentCandidatePipelineConfig[
@@ -62,10 +67,10 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
     val tweetsWithConversationMetadata = candidates.map { candidate =>
       TweetWithConversationMetadata(
         tweetId = candidate.candidateIdLong,
-        userId = None,
+        userId = candidate.features.getOrElse(AuthorIdFeature, None),
-        sourceTweetId = None,
+        sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None),
-        sourceUserId = None,
+        sourceUserId = candidate.features.getOrElse(SourceUserIdFeature, None),
-        inReplyToTweetId = None,
+        inReplyToTweetId = candidate.features.getOrElse(InReplyToTweetIdFeature, None),
         conversationId = None,
         ancestors = Seq.empty
       )
@@ -84,7 +89,10 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
 
   override val preFilterFeatureHydrationPhase1: Seq[
     BaseCandidateFeatureHydrator[Query, TweetCandidate, _]
-  ] = Seq(tweetypieFeatureHydrator, socialGraphServiceFeatureHydrator)
+  ] = Seq(
+    tweetypieFeatureHydrator,
+    InNetworkFeatureHydrator,
+  )
 
   override def filters: Seq[Filter[Query, TweetCandidate]] = Seq(
     RetweetDeduplicationFilter,
@@ -93,6 +101,7 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
       FilterIdentifier(QuotedTweetDroppedFilterId),
       shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) }
     ),
+    invalidSubscriptionTweetFilter,
     InvalidConversationModuleFilter
   )
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfigBuilder.scala

@@ -1,9 +1,9 @@
 package com.twitter.home_mixer.candidate_pipeline
 
-import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
 import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
-import com.twitter.home_mixer.functional_component.feature_hydrator.SocialGraphServiceFeatureHydrator
 import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
+import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter
+import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
 import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
 import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
 import com.twitter.product_mixer.core.functional_component.gate.BaseGate
@@ -15,7 +15,7 @@ import javax.inject.Singleton
 class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery] @Inject() (
   conversationServiceCandidateSource: ConversationServiceCandidateSource,
   tweetypieFeatureHydrator: TweetypieFeatureHydrator,
-  socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator,
+  invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter,
   namesFeatureHydrator: NamesFeatureHydrator) {
 
   def build(
@@ -25,8 +25,8 @@ class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery]
     new ConversationServiceCandidatePipelineConfig(
       conversationServiceCandidateSource,
       tweetypieFeatureHydrator,
-      socialGraphServiceFeatureHydrator,
       namesFeatureHydrator,
+      invalidSubscriptionTweetFilter,
       gates,
       decorator
     )

home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/EditedTweetsCandidatePipelineConfig.scala

@@ -1,7 +1,7 @@
 package com.twitter.home_mixer.candidate_pipeline
 
 import com.twitter.home_mixer.functional_component.candidate_source.StaleTweetsCacheCandidateSource
-import com.twitter.home_mixer.functional_component.decorator.HomeFeedbackActionInfoBuilder
+import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder
 import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
 import com.twitter.home_mixer.functional_component.query_transformer.EditedTweetsCandidatePipelineQueryTransformer
 import com.twitter.home_mixer.service.HomeMixerAlertConfig

home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/BUILD.bazel

@@ -13,8 +13,5 @@ scala_library(
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt",
         "snowflake/src/main/scala/com/twitter/snowflake/id",
-        "src/thrift/com/twitter/context:twitter-context-scala",
-        "src/thrift/com/twitter/timelines/render:thrift-scala",
-        "twitter-context/src/main/scala",
     ],
 )

home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/HomeThriftController.scala

@@ -7,6 +7,7 @@ import com.twitter.home_mixer.service.ScoredTweetsService
 import com.twitter.home_mixer.{thriftscala => t}
 import com.twitter.product_mixer.core.controllers.DebugTwitterContext
 import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder
+import com.twitter.product_mixer.core.service.debug_query.DebugQueryService
 import com.twitter.product_mixer.core.service.urt.UrtService
 import com.twitter.snowflake.id.SnowflakeId
 import com.twitter.stitch.Stitch

home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/BUILD.bazel

@@ -0,0 +1,24 @@
+scala_library(
+    sources = ["*.scala"],
+    compiler_option_sets = ["fatal_warnings"],
+    strict_deps = True,
+    tags = ["bazel-compatible"],
+    dependencies = [
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request",
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
+        "home-mixer/thrift/src/main/thrift:thrift-scala",
+        "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi",
+        "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product",
+        "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry",
+        "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala",
+        "src/thrift/com/twitter/gizmoduck:thrift-scala",
+        "src/thrift/com/twitter/timelines/render:thrift-scala",
+        "stitch/stitch-repo/src/main/scala",
+        "strato/config/columns/auth-context:auth-context-strato-client",
+        "strato/config/columns/gizmoduck:gizmoduck-strato-client",
+        "strato/config/src/thrift/com/twitter/strato/graphql/timelines:graphql-timelines-scala",
+        "strato/src/main/scala/com/twitter/strato/callcontext",
+        "strato/src/main/scala/com/twitter/strato/fed",
+        "strato/src/main/scala/com/twitter/strato/fed/server",
+    ],
+)

home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/HomeMixerColumn.scala

@@ -0,0 +1,217 @@
+package com.twitter.home_mixer.federated
+
+import com.twitter.gizmoduck.{thriftscala => gd}
+import com.twitter.home_mixer.marshaller.request.HomeMixerRequestUnmarshaller
+import com.twitter.home_mixer.model.request.HomeMixerRequest
+import com.twitter.home_mixer.{thriftscala => hm}
+import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder
+import com.twitter.product_mixer.core.pipeline.product.ProductPipelineRequest
+import com.twitter.product_mixer.core.pipeline.product.ProductPipelineResult
+import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry
+import com.twitter.product_mixer.core.{thriftscala => pm}
+import com.twitter.stitch.Arrow
+import com.twitter.stitch.Stitch
+import com.twitter.strato.callcontext.CallContext
+import com.twitter.strato.catalog.OpMetadata
+import com.twitter.strato.config._
+import com.twitter.strato.data._
+import com.twitter.strato.fed.StratoFed
+import com.twitter.strato.generated.client.auth_context.AuditIpClientColumn
+import com.twitter.strato.generated.client.gizmoduck.CompositeOnUserClientColumn
+import com.twitter.strato.graphql.timelines.{thriftscala => gql}
+import com.twitter.strato.thrift.ScroogeConv
+import com.twitter.timelines.render.{thriftscala => tr}
+import com.twitter.util.Try
+import javax.inject.Inject
+import javax.inject.Singleton
+
+@Singleton
+class HomeMixerColumn @Inject() (
+  homeMixerRequestUnmarshaller: HomeMixerRequestUnmarshaller,
+  compositeOnUserClientColumn: CompositeOnUserClientColumn,
+  auditIpClientColumn: AuditIpClientColumn,
+  paramsBuilder: ParamsBuilder,
+  productPipelineRegistry: ProductPipelineRegistry)
+    extends StratoFed.Column(HomeMixerColumn.Path)
+    with StratoFed.Fetch.Arrow {
+
+  override val contactInfo: ContactInfo = ContactInfo(
+    contactEmail = "",
+    ldapGroup = "",
+    slackRoomId = ""
+  )
+
+  override val metadata: OpMetadata =
+    OpMetadata(
+      lifecycle = Some(Lifecycle.Production),
+      description =
+        Some(Description.PlainText("Federated Strato column for Timelines served via Home Mixer"))
+    )
+
+  private val bouncerAccess: Seq[Policy] = Seq(BouncerAccess())
+  private val finatraTestServiceIdentifiers: Seq[Policy] = Seq(
+    ServiceIdentifierPattern(
+      role = "",
+      service = "",
+      env = "",
+      zone = Seq(""))
+  )
+
+  override val policy: Policy = AnyOf(bouncerAccess ++ finatraTestServiceIdentifiers)
+
+  override type Key = gql.TimelineKey
+  override type View = gql.HomeTimelineView
+  override type Value = tr.Timeline
+
+  override val keyConv: Conv[Key] = ScroogeConv.fromStruct[gql.TimelineKey]
+  override val viewConv: Conv[View] = ScroogeConv.fromStruct[gql.HomeTimelineView]
+  override val valueConv: Conv[Value] = ScroogeConv.fromStruct[tr.Timeline]
+
+  private def createHomeMixerRequestArrow(
+    compositeOnUserClientColumn: CompositeOnUserClientColumn,
+    auditIpClientColumn: AuditIpClientColumn
+  ): Arrow[(Key, View), hm.HomeMixerRequest] = {
+
+    val populateUserRolesAndIp: Arrow[(Key, View), (Option[Set[String]], Option[String])] = {
+      val gizmoduckView: (gd.LookupContext, Set[gd.QueryFields]) =
+        (gd.LookupContext(), Set(gd.QueryFields.Roles))
+
+      val populateUserRoles = Arrow
+        .flatMap[(Key, View), Option[Set[String]]] { _ =>
+          Stitch.collect {
+            CallContext.twitterUserId.map { userId =>
+              compositeOnUserClientColumn.fetcher
+                .callStack(HomeMixerColumn.FetchCallstack)
+                .fetch(userId, gizmoduckView).map(_.v)
+                .map {
+                  _.flatMap(_.roles.map(_.roles.toSet)).getOrElse(Set.empty)
+                }
+            }
+          }
+        }
+
+      val populateIpAddress = Arrow
+        .flatMap[(Key, View), Option[String]](_ =>
+          auditIpClientColumn.fetcher
+            .callStack(HomeMixerColumn.FetchCallstack)
+            .fetch((), ()).map(_.v))
+
+      Arrow.join(
+        populateUserRoles,
+        populateIpAddress
+      )
+    }
+
+    Arrow.zipWithArg(populateUserRolesAndIp).map {
+      case ((key, view), (roles, ipAddress)) =>
+        val deviceContextOpt = Some(
+          hm.DeviceContext(
+            isPolling = CallContext.isPolling,
+            requestContext = view.requestContext,
+            latestControlAvailable = view.latestControlAvailable,
+            autoplayEnabled = view.autoplayEnabled
+          ))
+        val seenTweetIds = view.seenTweetIds.filter(_.nonEmpty)
+
+        val (product, productContext) = key match {
+          case gql.TimelineKey.HomeTimeline(_) | gql.TimelineKey.HomeTimelineV2(_) =>
+            (
+              hm.Product.ForYou,
+              hm.ProductContext.ForYou(
+                hm.ForYou(
+                  deviceContextOpt,
+                  seenTweetIds,
+                  view.dspClientContext,
+                  view.pushToHomeTweetId
+                )
+              ))
+          case gql.TimelineKey.HomeLatestTimeline(_) | gql.TimelineKey.HomeLatestTimelineV2(_) =>
+            (
+              hm.Product.Following,
+              hm.ProductContext.Following(
+                hm.Following(deviceContextOpt, seenTweetIds, view.dspClientContext)))
+          case gql.TimelineKey.CreatorSubscriptionsTimeline(_) =>
+            (
+              hm.Product.Subscribed,
+              hm.ProductContext.Subscribed(hm.Subscribed(deviceContextOpt, seenTweetIds)))
+          case _ => throw new UnsupportedOperationException(s"Unknown product: $key")
+        }
+
+        val clientContext = pm.ClientContext(
+          userId = CallContext.twitterUserId,
+          guestId = CallContext.guestId,
+          guestIdAds = CallContext.guestIdAds,
+          guestIdMarketing = CallContext.guestIdMarketing,
+          appId = CallContext.clientApplicationId,
+          ipAddress = ipAddress,
+          userAgent = CallContext.userAgent,
+          countryCode = CallContext.requestCountryCode,
+          languageCode = CallContext.requestLanguageCode,
+          isTwoffice = CallContext.isInternalOrTwoffice,
+          userRoles = roles,
+          deviceId = CallContext.deviceId,
+          mobileDeviceId = CallContext.mobileDeviceId,
+          mobileDeviceAdId = CallContext.adId,
+          limitAdTracking = CallContext.limitAdTracking
+        )
+
+        hm.HomeMixerRequest(
+          clientContext = clientContext,
+          product = product,
+          productContext = Some(productContext),
+          maxResults = Try(view.count.get.toInt).toOption.orElse(HomeMixerColumn.MaxCount),
+          cursor = view.cursor.filter(_.nonEmpty)
+        )
+    }
+  }
+
+  override val fetch: Arrow[(Key, View), Result[Value]] = {
+    val transformThriftIntoPipelineRequest: Arrow[
+      (Key, View),
+      ProductPipelineRequest[HomeMixerRequest]
+    ] = {
+      Arrow
+        .identity[(Key, View)]
+        .andThen {
+          createHomeMixerRequestArrow(compositeOnUserClientColumn, auditIpClientColumn)
+        }
+        .map {
+          case thriftRequest =>
+            val request = homeMixerRequestUnmarshaller(thriftRequest)
+            val params = paramsBuilder.build(
+              clientContext = request.clientContext,
+              product = request.product,
+              featureOverrides =
+                request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty),
+            )
+            ProductPipelineRequest(request, params)
+        }
+    }
+
+    val underlyingProduct: Arrow[
+      ProductPipelineRequest[HomeMixerRequest],
+      ProductPipelineResult[tr.TimelineResponse]
+    ] = Arrow
+      .identity[ProductPipelineRequest[HomeMixerRequest]]
+      .map { pipelineRequest =>
+        val pipelineArrow = productPipelineRegistry
+          .getProductPipeline[HomeMixerRequest, tr.TimelineResponse](
+            pipelineRequest.request.product)
+          .arrow
+        (pipelineArrow, pipelineRequest)
+      }.applyArrow
+
+    transformThriftIntoPipelineRequest.andThen(underlyingProduct).map {
+      _.result match {
+        case Some(result) => found(result.timeline)
+        case _ => missing
+      }
+    }
+  }
+}
+
+object HomeMixerColumn {
+  val Path = "home-mixer/homeMixer.Timeline"
+  private val FetchCallstack = s"$Path:fetch"
+  private val MaxCount: Option[Int] = Some(100)
+}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/BUILD.bazel

@@ -10,11 +10,8 @@ scala_library(
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
-        "src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala",
         "src/thrift/com/twitter/search:earlybird-scala",
         "stitch/stitch-timelineservice/src/main/scala",
-        "strato/config/columns/recommendations/similarity:similarity-strato-client",
-        "strato/src/main/scala/com/twitter/strato/client",
     ],
     exports = [
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/BUILD.bazel

@@ -6,13 +6,11 @@ scala_library(
     dependencies = [
         "finagle/finagle-core/src/main",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
-        "joinkey/src/main/scala/com/twitter/joinkey/context",
-        "joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
         "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope",
@@ -25,8 +23,6 @@ scala_library(
         "src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala",
         "stringcenter/client",
         "stringcenter/client/src/main/java",
-        "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
-        "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/translation",
         "timelines/src/main/scala/com/twitter/timelines/injection/scribe",
     ],
 )

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeConversationServiceCandidateDecorator.scala

@@ -1,6 +1,8 @@
 package com.twitter.home_mixer.functional_component.decorator
 
 import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder
+import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder
+import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder
 import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature
 import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator
 import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/BUILD.bazel

@@ -8,7 +8,9 @@ scala_library(
         "finagle/finagle-core/src/main",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
+        "joinkey/src/main/scala/com/twitter/joinkey/context",
+        "joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt",
@@ -18,6 +20,7 @@ scala_library(
         "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala",
         "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
         "src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala",
+        "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
         "timelines/src/main/scala/com/twitter/timelines/injection/scribe",
     ],
 )

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeAdsClientEventDetailsBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.builder
 
 import com.twitter.finagle.tracing.Trace
 import com.twitter.product_mixer.core.feature.featuremap.FeatureMap

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventDetailsBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.builder
 
 import com.twitter.bijection.Base64String
 import com.twitter.bijection.scrooge.BinaryScalaCodec

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTimelinesScoreInfoBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
 import com.twitter.home_mixer.param.HomeGlobalParams.EnableSendScoresToClient

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTweetTypePredicates.scala

@@ -1,8 +1,10 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.builder
 
 import com.twitter.conversions.DurationOps._
 import com.twitter.home_mixer.model.HomeFeatures._
 import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
+import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType
+import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType
 import com.twitter.timelinemixer.injection.model.candidate.SemanticCoreFeatures
 import com.twitter.tweetypie.{thriftscala => tpt}
 
@@ -25,74 +27,76 @@ object HomeTweetTypePredicates {
     (
       "has_exclusive_conversation_author_id",
       _.getOrElse(ExclusiveConversationAuthorIdFeature, None).nonEmpty),
-    ("is_eligible_for_connect_boost", _.getOrElse(AuthorIsEligibleForConnectBoostFeature, false)),
+    ("is_eligible_for_connect_boost", _ => false),
     ("hashtag", _.getOrElse(EarlybirdFeature, None).exists(_.numHashtags > 0)),
     ("has_scheduled_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isScheduled)),
     ("has_recorded_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isRecorded)),
     ("is_read_from_cache", _.getOrElse(IsReadFromCacheFeature, false)),
-    (
-      "is_self_thread_tweet",
-      _.getOrElse(ConversationFeature, None).exists(_.isSelfThreadTweet.contains(true))),
     ("get_initial", _.getOrElse(GetInitialFeature, false)),
     ("get_newer", _.getOrElse(GetNewerFeature, false)),
     ("get_middle", _.getOrElse(GetMiddleFeature, false)),
     ("get_older", _.getOrElse(GetOlderFeature, false)),
     ("pull_to_refresh", _.getOrElse(PullToRefreshFeature, false)),
     ("polling", _.getOrElse(PollingFeature, false)),
-    ("tls_size_20_plus", _ => false),
+    ("near_empty", _.getOrElse(ServedSizeFeature, None).exists(_ < 3)),
-    ("near_empty", _ => false),
+    ("is_request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)),
-    ("ranked_request", _ => false),
     ("mutual_follow", _.getOrElse(EarlybirdFeature, None).exists(_.fromMutualFollow)),
+    (
+      "less_than_10_mins_since_lnpt",
+      _.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 10.minutes)),
+    ("served_in_conversation_module", _.getOrElse(ServedInConversationModuleFeature, false)),
     ("has_ticketed_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasTickets)),
     ("in_utis_top5", _.getOrElse(PositionFeature, None).exists(_ < 5)),
-    ("is_utis_pos0", _.getOrElse(PositionFeature, None).exists(_ == 0)),
-    ("is_utis_pos1", _.getOrElse(PositionFeature, None).exists(_ == 1)),
-    ("is_utis_pos2", _.getOrElse(PositionFeature, None).exists(_ == 2)),
-    ("is_utis_pos3", _.getOrElse(PositionFeature, None).exists(_ == 3)),
-    ("is_utis_pos4", _.getOrElse(PositionFeature, None).exists(_ == 4)),
     (
-      "is_signup_request",
+      "conversation_module_has_2_displayed_tweets",
-      candidate => candidate.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)),
+      _.getOrElse(ConversationModule2DisplayedTweetsFeature, false)),
-    ("empty_request", _ => false),
+    ("empty_request", _.getOrElse(ServedSizeFeature, None).exists(_ == 0)),
-    ("served_size_less_than_5", _.getOrElse(ServedSizeFeature, None).exists(_ < 5)),
-    ("served_size_less_than_10", _.getOrElse(ServedSizeFeature, None).exists(_ < 10)),
-    ("served_size_less_than_20", _.getOrElse(ServedSizeFeature, None).exists(_ < 20)),
     ("served_size_less_than_50", _.getOrElse(ServedSizeFeature, None).exists(_ < 50)),
     (
       "served_size_between_50_and_100",
       _.getOrElse(ServedSizeFeature, None).exists(size => size >= 50 && size < 100)),
     ("authored_by_contextual_user", _.getOrElse(AuthoredByContextualUserFeature, false)),
+    (
+      "is_self_thread_tweet",
+      _.getOrElse(ConversationFeature, None).exists(_.isSelfThreadTweet.contains(true))),
     ("has_ancestors", _.getOrElse(AncestorsFeature, Seq.empty).nonEmpty),
     ("full_scoring_succeeded", _.getOrElse(FullScoringSucceededFeature, false)),
+    ("served_size_less_than_20", _.getOrElse(ServedSizeFeature, None).exists(_ < 20)),
+    ("served_size_less_than_10", _.getOrElse(ServedSizeFeature, None).exists(_ < 10)),
+    ("served_size_less_than_5", _.getOrElse(ServedSizeFeature, None).exists(_ < 5)),
     (
       "account_age_less_than_30_minutes",
       _.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)),
+    ("conversation_module_has_gap", _.getOrElse(ConversationModuleHasGapFeature, false)),
+    (
+      "directed_at_user_is_in_first_degree",
+      _.getOrElse(EarlybirdFeature, None).exists(_.directedAtUserIdIsInFirstDegree.contains(true))),
+    (
+      "has_semantic_core_annotation",
+      _.getOrElse(EarlybirdFeature, None).exists(_.semanticCoreAnnotations.nonEmpty)),
+    ("is_request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false)),
     (
       "account_age_less_than_1_day",
       _.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 1.day)),
     (
       "account_age_less_than_7_days",
       _.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 7.days)),
-    (
-      "directed_at_user_is_in_first_degree",
-      _.getOrElse(EarlybirdFeature, None).exists(_.directedAtUserIdIsInFirstDegree.contains(true))),
-    ("root_user_is_in_first_degree", _ => false),
-    (
-      "has_semantic_core_annotation",
-      _.getOrElse(EarlybirdFeature, None).exists(_.semanticCoreAnnotations.nonEmpty)),
-    ("is_request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false)),
     (
       "part_of_utt",
       _.getOrElse(EarlybirdFeature, None)
         .exists(_.semanticCoreAnnotations.exists(_.exists(annotation =>
           annotation.domainId == SemanticCoreFeatures.UnifiedTwitterTaxonomy)))),
+    (
+      "has_home_latest_request_past_week",
+      _.getOrElse(FollowingLastNonPollingTimeFeature, None).exists(_.untilNow < 7.days)),
+    ("is_utis_pos0", _.getOrElse(PositionFeature, None).exists(_ == 0)),
+    ("is_utis_pos1", _.getOrElse(PositionFeature, None).exists(_ == 1)),
+    ("is_utis_pos2", _.getOrElse(PositionFeature, None).exists(_ == 2)),
+    ("is_utis_pos3", _.getOrElse(PositionFeature, None).exists(_ == 3)),
+    ("is_utis_pos4", _.getOrElse(PositionFeature, None).exists(_ == 4)),
     ("is_random_tweet", _.getOrElse(IsRandomTweetFeature, false)),
     ("has_random_tweet_in_response", _.getOrElse(HasRandomTweetFeature, false)),
     ("is_random_tweet_above_in_utis", _.getOrElse(IsRandomTweetAboveFeature, false)),
-    ("is_request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)),
-    ("viewer_is_employee", _ => false),
-    ("viewer_is_timelines_employee", _ => false),
-    ("viewer_follows_any_topics", _.getOrElse(UserFollowedTopicsCountFeature, None).exists(_ > 0)),
     (
       "has_ancestor_authored_by_viewer",
       candidate =>
@@ -100,11 +104,6 @@ object HomeTweetTypePredicates {
           .getOrElse(AncestorsFeature, Seq.empty).exists(ancestor =>
             candidate.getOrElse(ViewerIdFeature, 0L) == ancestor.userId)),
     ("ancestor", _.getOrElse(IsAncestorCandidateFeature, false)),
-    (
-      "root_ancestor",
-      candidate =>
-        candidate.getOrElse(IsAncestorCandidateFeature, false) && candidate
-          .getOrElse(InReplyToTweetIdFeature, None).isEmpty),
     (
       "deep_reply",
       candidate =>
@@ -119,23 +118,22 @@ object HomeTweetTypePredicates {
       "tweet_age_less_than_15_seconds",
       _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
         .exists(_.untilNow <= 15.seconds)),
-    ("is_followed_topic_tweet", _ => false),
+    (
-    ("is_recommended_topic_tweet", _ => false),
+      "less_than_1_hour_since_lnpt",
-    ("is_topic_tweet", _ => false),
+      _.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 1.hour)),
-    ("preferred_language_matches_tweet_language", _ => false),
+    ("has_gte_10_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 10))),
     (
       "device_language_matches_tweet_language",
       candidate =>
         candidate.getOrElse(TweetLanguageFeature, None) ==
           candidate.getOrElse(DeviceLanguageFeature, None)),
+    (
+      "root_ancestor",
+      candidate =>
+        candidate.getOrElse(IsAncestorCandidateFeature, false) && candidate
+          .getOrElse(InReplyToTweetIdFeature, None).isEmpty),
     ("question", _.getOrElse(EarlybirdFeature, None).exists(_.hasQuestion.contains(true))),
-    ("in_network", _.getOrElse(FromInNetworkSourceFeature, true)),
+    ("in_network", _.getOrElse(InNetworkFeature, true)),
-    ("viewer_follows_original_author", _ => false),
-    ("has_account_follow_prompt", _ => false),
-    ("has_relevance_prompt", _ => false),
-    ("has_topic_annotation_haug_prompt", _ => false),
-    ("has_topic_annotation_random_precision_prompt", _ => false),
-    ("has_topic_annotation_prompt", _ => false),
     (
       "has_political_annotation",
       _.getOrElse(EarlybirdFeature, None).exists(
@@ -153,9 +151,14 @@ object HomeTweetTypePredicates {
       _.getOrElse(EarlybirdFeature, None)
         .exists(_.conversationControl.exists(_.isInstanceOf[tpt.ConversationControl.Community]))),
     ("has_zero_score", _.getOrElse(ScoreFeature, None).exists(_ == 0.0)),
-    ("is_viewer_not_invited_to_reply", _ => false),
+    (
-    ("is_viewer_invited_to_reply", _ => false),
+      "is_followed_topic_tweet",
-    ("has_gte_10_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 10))),
+      _.getOrElse(TopicContextFunctionalityTypeFeature, None)
+        .exists(_ == BasicTopicContextFunctionalityType)),
+    (
+      "is_recommended_topic_tweet",
+      _.getOrElse(TopicContextFunctionalityTypeFeature, None)
+        .exists(_ == RecommendationTopicContextFunctionalityType)),
     ("has_gte_100_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100))),
     ("has_gte_1k_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 1000))),
     (
@@ -164,8 +167,6 @@ object HomeTweetTypePredicates {
     (
       "has_gte_100k_favs",
       _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100000))),
-    ("above_neighbor_is_topic_tweet", _ => false),
-    ("is_topic_tweet_with_neighbor_below", _ => false),
     ("has_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasSpace)),
     ("has_live_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isLive)),
     (
@@ -187,6 +188,7 @@ object HomeTweetTypePredicates {
     (
       "has_toxicity_score_above_threshold",
       _.getOrElse(EarlybirdFeature, None).exists(_.toxicityScore.exists(_ > 0.91))),
+    ("is_topic_tweet", _.getOrElse(TopicIdSocialContextFeature, None).isDefined),
     (
       "text_only",
       candidate =>
@@ -204,23 +206,50 @@ object HomeTweetTypePredicates {
     ("has_3_images", _.getOrElse(NumImagesFeature, None).exists(_ == 3)),
     ("has_4_images", _.getOrElse(NumImagesFeature, None).exists(_ == 4)),
     ("has_card", _.getOrElse(EarlybirdFeature, None).exists(_.hasCard)),
-    ("3_or_more_consecutive_not_in_network", _ => false),
-    ("2_or_more_consecutive_not_in_network", _ => false),
-    ("5_out_of_7_not_in_network", _ => false),
-    ("7_out_of_7_not_in_network", _ => false),
-    ("5_out_of_5_not_in_network", _ => false),
     ("user_follow_count_gte_50", _.getOrElse(UserFollowingCountFeature, None).exists(_ > 50)),
-    ("has_liked_by_social_context", _ => false),
-    ("has_followed_by_social_context", _ => false),
-    ("has_topic_social_context", _ => false),
-    ("timeline_entry_has_banner", _ => false),
-    ("served_in_conversation_module", _.getOrElse(ServedInConversationModuleFeature, false)),
     (
-      "conversation_module_has_2_displayed_tweets",
+      "has_liked_by_social_context",
-      _.getOrElse(ConversationModule2DisplayedTweetsFeature, false)),
+      candidateFeatures =>
-    ("conversation_module_has_gap", _.getOrElse(ConversationModuleHasGapFeature, false)),
+        candidateFeatures
-    ("served_in_recap_tweet_candidate_module_injection", _ => false),
+          .getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty)
-    ("served_in_threaded_conversation_module", _ => false)
+          .exists(candidateFeatures
+            .getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Seq.empty).toSet.contains)),
+    (
+      "has_followed_by_social_context",
+      _.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty).nonEmpty),
+    (
+      "has_topic_social_context",
+      candidateFeatures =>
+        candidateFeatures
+          .getOrElse(TopicIdSocialContextFeature, None)
+          .isDefined &&
+          candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None).isDefined),
+    ("video_lte_10_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ <= 10000)),
+    (
+      "video_bt_10_60_sec",
+      _.getOrElse(VideoDurationMsFeature, None).exists(duration =>
+        duration > 10000 && duration <= 60000)),
+    ("video_gt_60_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ > 60000)),
+    (
+      "tweet_age_lte_30_minutes",
+      _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
+        .exists(_.untilNow <= 30.minutes)),
+    (
+      "tweet_age_lte_1_hour",
+      _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
+        .exists(_.untilNow <= 1.hour)),
+    (
+      "tweet_age_lte_6_hours",
+      _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
+        .exists(_.untilNow <= 6.hours)),
+    (
+      "tweet_age_lte_12_hours",
+      _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
+        .exists(_.untilNow <= 12.hours)),
+    (
+      "tweet_age_gte_24_hours",
+      _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
+        .exists(_.untilNow >= 24.hours)),
   )
 
   val PredicateMap = CandidatePredicates.toMap

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/ListClientEventDetailsBuilder.scala

@@ -8,7 +8,7 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ti
 import com.twitter.product_mixer.core.pipeline.PipelineQuery
 import com.twitter.timelineservice.suggests.{thriftscala => st}
 
-object ListClientEventDetailsBuilder
+case class ListClientEventDetailsBuilder(suggestType: st.SuggestType)
     extends BaseClientEventDetailsBuilder[PipelineQuery, UniversalNoun[Any]] {
 
   override def apply(
@@ -20,7 +20,7 @@ object ListClientEventDetailsBuilder
       conversationDetails = None,
       timelinesDetails = Some(
         TimelinesDetails(
-          injectionType = Some(st.SuggestType.OrganicListTweet.name),
+          injectionType = Some(suggestType.name),
           controllerData = None,
           sourceData = None)),
       articleDetails = None,

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AuthorChildFeedbackActionBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature
 import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
@@ -9,7 +9,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ch
 import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
 import com.twitter.stringcenter.client.StringCenter
 import com.twitter.timelines.service.{thriftscala => t}
-
 import javax.inject.Inject
 import javax.inject.Singleton
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BUILD.bazel

@@ -4,9 +4,16 @@ scala_library(
     strict_deps = True,
     tags = ["bazel-compatible"],
     dependencies = [
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder",
+        "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt",
         "src/thrift/com/twitter/timelines/service:thrift-scala",
         "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
+        "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
     ],
 )

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BlockUserChildFeedbackActionBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
 import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
@@ -13,7 +13,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
 import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorBlockUser
 import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
 import com.twitter.stringcenter.client.StringCenter
-
 import javax.inject.Inject
 import javax.inject.Singleton
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/DontLikeFeedbackActionBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.conversions.DurationOps._
 import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
@@ -17,7 +17,6 @@ import com.twitter.timelines.common.{thriftscala => tlc}
 import com.twitter.timelineservice.model.FeedbackInfo
 import com.twitter.timelineservice.model.FeedbackMetadata
 import com.twitter.timelineservice.{thriftscala => tls}
-
 import javax.inject.Inject
 import javax.inject.Singleton
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/EngagerSocialContextBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.RealNamesFeature
 import com.twitter.product_mixer.core.feature.featuremap.FeatureMap

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ExtendedReplySocialContextBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.FocalTweetAuthorIdFeature
 import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackStrings.scala

@@ -0,0 +1,18 @@
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
+
+import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
+import com.twitter.stringcenter.client.ExternalStringRegistry
+import javax.inject.Inject
+import javax.inject.Provider
+import javax.inject.Singleton
+
+@Singleton
+class FeedbackStrings @Inject() (
+  @ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry]) {
+  private val externalStringRegistry = externalStringRegistryProvider.get()
+
+  val seeLessOftenFeedbackString =
+    externalStringRegistry.createProdString("Feedback.seeLessOften")
+  val seeLessOftenConfirmationFeedbackString =
+    externalStringRegistry.createProdString("Feedback.seeLessOftenConfirmation")
+}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackUtil.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.conversions.DurationOps._
 import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FollowedBySocialContextBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
 import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeFeedbackActionInfoBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
 import com.twitter.home_mixer.model.request.FollowingProduct
@@ -12,7 +12,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Fe
 import com.twitter.product_mixer.core.pipeline.PipelineQuery
 import com.twitter.timelines.service.{thriftscala => t}
 import com.twitter.timelines.util.FeedbackMetadataSerializer
-
 import javax.inject.Inject
 import javax.inject.Singleton
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeTweetSocialContextBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature
 import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleIdFeature
@@ -14,10 +14,13 @@ import javax.inject.Singleton
 @Singleton
 case class HomeTweetSocialContextBuilder @Inject() (
   likedBySocialContextBuilder: LikedBySocialContextBuilder,
+  listsSocialContextBuilder: ListsSocialContextBuilder,
   followedBySocialContextBuilder: FollowedBySocialContextBuilder,
   topicSocialContextBuilder: TopicSocialContextBuilder,
   extendedReplySocialContextBuilder: ExtendedReplySocialContextBuilder,
-  receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder)
+  receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder,
+  popularVideoSocialContextBuilder: PopularVideoSocialContextBuilder,
+  popularInYourAreaSocialContextBuilder: PopularInYourAreaSocialContextBuilder)
     extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
 
   def apply(
@@ -31,6 +34,9 @@ case class HomeTweetSocialContextBuilder @Inject() (
           likedBySocialContextBuilder(query, candidate, features)
             .orElse(followedBySocialContextBuilder(query, candidate, features))
             .orElse(topicSocialContextBuilder(query, candidate, features))
+            .orElse(popularVideoSocialContextBuilder(query, candidate, features))
+            .orElse(listsSocialContextBuilder(query, candidate, features))
+            .orElse(popularInYourAreaSocialContextBuilder(query, candidate, features))
         case Some(_) =>
           val conversationId = features.getOrElse(ConversationModuleIdFeature, None)
           // Only hydrate the social context into the root tweet in a conversation module

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToFollowFeedbackActionInfoBuilder.scala

@@ -5,7 +5,6 @@ import com.twitter.product_mixer.component_library.model.candidate.UserCandidate
 import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
 import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder
 import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
-import com.twitter.stringcenter.client.ExternalStringRegistry
 import com.twitter.stringcenter.client.StringCenter
 import javax.inject.Inject
 import javax.inject.Provider
@@ -32,12 +31,13 @@ object HomeWhoToFollowFeedbackActionInfoBuilder {
 
 @Singleton
 case class HomeWhoToFollowFeedbackActionInfoBuilder @Inject() (
-  @ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry],
+  feedbackStrings: FeedbackStrings,
   @ProductScoped stringCenterProvider: Provider[StringCenter])
     extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] {
 
   private val whoToFollowFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder(
-    externalStringRegistry = externalStringRegistryProvider.get(),
+    seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString,
+    seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenConfirmationFeedbackString,
     stringCenter = stringCenterProvider.get(),
     encodedFeedbackRequest = Some(HomeWhoToFollowFeedbackActionInfoBuilder.EncodedFeedbackRequest)
   )

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToSubscribeFeedbackActionInfoBuilder.scala

@@ -0,0 +1,52 @@
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
+
+import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.WhoToFollowFeedbackActionInfoBuilder
+import com.twitter.product_mixer.component_library.model.candidate.UserCandidate
+import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
+import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder
+import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
+import com.twitter.stringcenter.client.StringCenter
+import javax.inject.Inject
+import javax.inject.Provider
+import javax.inject.Singleton
+import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackActionInfo
+import com.twitter.product_mixer.core.pipeline.PipelineQuery
+import com.twitter.timelines.service.{thriftscala => tl}
+import com.twitter.timelines.util.FeedbackRequestSerializer
+import com.twitter.timelineservice.suggests.thriftscala.SuggestType
+import com.twitter.timelineservice.thriftscala.FeedbackType
+
+object HomeWhoToSubscribeFeedbackActionInfoBuilder {
+  private val FeedbackMetadata = tl.FeedbackMetadata(
+    injectionType = Some(SuggestType.WhoToSubscribe),
+    engagementType = None,
+    entityIds = Seq.empty,
+    ttlMs = None
+  )
+  private val FeedbackRequest =
+    tl.DefaultFeedbackRequest2(FeedbackType.SeeFewer, FeedbackMetadata)
+  private val EncodedFeedbackRequest =
+    FeedbackRequestSerializer.serialize(tl.FeedbackRequest.DefaultFeedbackRequest2(FeedbackRequest))
+}
+
+@Singleton
+case class HomeWhoToSubscribeFeedbackActionInfoBuilder @Inject() (
+  feedbackStrings: FeedbackStrings,
+  @ProductScoped stringCenterProvider: Provider[StringCenter])
+    extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] {
+
+  private val whoToSubscribeFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder(
+    seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString,
+    seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenConfirmationFeedbackString,
+    stringCenter = stringCenterProvider.get(),
+    encodedFeedbackRequest =
+      Some(HomeWhoToSubscribeFeedbackActionInfoBuilder.EncodedFeedbackRequest)
+  )
+
+  override def apply(
+    query: PipelineQuery,
+    candidate: UserCandidate,
+    candidateFeatures: FeatureMap
+  ): Option[FeedbackActionInfo] =
+    whoToSubscribeFeedbackActionInfoBuilder.apply(query, candidate, candidateFeatures)
+}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/LikedBySocialContextBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature
 import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ListsSocialContextBuilder.scala

@@ -0,0 +1,50 @@
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
+
+import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
+import com.twitter.home_mixer.model.HomeFeatures.UserScreenNameFeature
+import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
+import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
+import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
+import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
+import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext
+import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
+import com.twitter.product_mixer.core.pipeline.PipelineQuery
+import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
+import com.twitter.stringcenter.client.StringCenter
+import com.twitter.timelineservice.suggests.{thriftscala => t}
+import javax.inject.Inject
+import javax.inject.Provider
+import javax.inject.Singleton
+
+/**
+ * "Your Lists" will be rendered for the context and a url link for your lists.
+ */
+@Singleton
+case class ListsSocialContextBuilder @Inject() (
+  externalStrings: HomeMixerExternalStrings,
+  @ProductScoped stringCenterProvider: Provider[StringCenter])
+    extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
+
+  private val stringCenter = stringCenterProvider.get()
+  private val listString = externalStrings.ownedSubscribedListsModuleHeaderString
+
+  def apply(
+    query: PipelineQuery,
+    candidate: TweetCandidate,
+    candidateFeatures: FeatureMap
+  ): Option[SocialContext] = {
+    candidateFeatures.get(SuggestTypeFeature) match {
+      case Some(suggestType) if suggestType == t.SuggestType.RankedListTweet =>
+        val userName = query.features.flatMap(_.getOrElse(UserScreenNameFeature, None))
+        Some(
+          GeneralContext(
+            contextType = ListGeneralContextType,
+            text = stringCenter.prepare(listString),
+            url = userName.map(name => ""),
+            contextImageUrls = None,
+            landingUrl = None
+          ))
+      case _ => None
+    }
+  }
+}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/MuteUserChildFeedbackActionBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
 import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
@@ -12,7 +12,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
 import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleMuteUser
 import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
 import com.twitter.stringcenter.client.StringCenter
-
 import javax.inject.Inject
 import javax.inject.Singleton
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotInterestedTopicFeedbackActionBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
 import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature
@@ -15,7 +15,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
 import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorMarkNotInterestedTopic
 import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
 import com.twitter.stringcenter.client.StringCenter
-
 import javax.inject.Inject
 import javax.inject.Singleton
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotRelevantChildFeedbackActionBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
 import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
@@ -12,7 +12,6 @@ import com.twitter.timelines.common.{thriftscala => tlc}
 import com.twitter.timelineservice.model.FeedbackInfo
 import com.twitter.timelineservice.model.FeedbackMetadata
 import com.twitter.timelineservice.{thriftscala => tlst}
-
 import javax.inject.Inject
 import javax.inject.Singleton
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularInYourAreaSocialContextBuilder.scala

@@ -0,0 +1,43 @@
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
+
+import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
+import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
+import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
+import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
+import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
+import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
+import com.twitter.product_mixer.core.pipeline.PipelineQuery
+import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
+import com.twitter.stringcenter.client.StringCenter
+import com.twitter.timelineservice.suggests.{thriftscala => st}
+import javax.inject.Inject
+import javax.inject.Provider
+import javax.inject.Singleton
+
+@Singleton
+case class PopularInYourAreaSocialContextBuilder @Inject() (
+  externalStrings: HomeMixerExternalStrings,
+  @ProductScoped stringCenterProvider: Provider[StringCenter])
+    extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
+
+  private val stringCenter = stringCenterProvider.get()
+  private val popularInYourAreaString = externalStrings.socialContextPopularInYourAreaString
+
+  def apply(
+    query: PipelineQuery,
+    candidate: TweetCandidate,
+    candidateFeatures: FeatureMap
+  ): Option[SocialContext] = {
+    val suggestTypeOpt = candidateFeatures.getOrElse(SuggestTypeFeature, None)
+    if (suggestTypeOpt.contains(st.SuggestType.RecommendedTrendTweet)) {
+      Some(
+        GeneralContext(
+          contextType = LocationGeneralContextType,
+          text = stringCenter.prepare(popularInYourAreaString),
+          url = None,
+          contextImageUrls = None,
+          landingUrl = None
+        ))
+    } else None
+  }
+}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularVideoSocialContextBuilder.scala

@@ -1,50 +1,48 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
-import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
+import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
-import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
 import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
 import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
 import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
 import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
-import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext
 import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
 import com.twitter.product_mixer.core.pipeline.PipelineQuery
 import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
 import com.twitter.stringcenter.client.StringCenter
+import com.twitter.timelineservice.suggests.{thriftscala => st}
 import javax.inject.Inject
 import javax.inject.Provider
 import javax.inject.Singleton
 
-/**
- * Renders a fixed 'You Might Like' string above all OON Tweets.
- */
 @Singleton
-case class YouMightLikeSocialContextBuilder @Inject() (
+case class PopularVideoSocialContextBuilder @Inject() (
   externalStrings: HomeMixerExternalStrings,
   @ProductScoped stringCenterProvider: Provider[StringCenter])
     extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
 
   private val stringCenter = stringCenterProvider.get()
-  private val youMightLikeString = externalStrings.socialContextYouMightLikeString
+  private val popularVideoString = externalStrings.socialContextPopularVideoString
 
   def apply(
     query: PipelineQuery,
     candidate: TweetCandidate,
     candidateFeatures: FeatureMap
   ): Option[SocialContext] = {
-    val isInNetwork = candidateFeatures.getOrElse(InNetworkFeature, true)
+    val suggestTypeOpt = candidateFeatures.getOrElse(SuggestTypeFeature, None)
-    val isRetweet = candidateFeatures.getOrElse(IsRetweetFeature, false)
+    if (suggestTypeOpt.contains(st.SuggestType.MediaTweet)) {
-    if (!isInNetwork && !isRetweet) {
       Some(
         GeneralContext(
           contextType = SparkleGeneralContextType,
-          text = stringCenter.prepare(youMightLikeString),
+          text = stringCenter.prepare(popularVideoString),
           url = None,
           contextImageUrls = None,
-          landingUrl = None
+          landingUrl = Some(
+            Url(
+              urlType = DeepLink,
+              url = ""
+            )
+          )
         ))
-    } else {
+    } else None
-      None
-    }
   }
 }

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReceivedReplySocialContextBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
 import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReportTweetChildFeedbackActionBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
 import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
@@ -8,7 +8,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
 import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorReportTweet
 import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
 import com.twitter.stringcenter.client.StringCenter
-
 import javax.inject.Inject
 import javax.inject.Singleton
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/RetweeterChildFeedbackActionBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
 import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
@@ -10,7 +10,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ch
 import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
 import com.twitter.stringcenter.client.StringCenter
 import com.twitter.timelines.service.{thriftscala => t}
-
 import javax.inject.Inject
 import javax.inject.Singleton
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/TopicSocialContextBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
 import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/UnfollowUserChildFeedbackActionBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
 import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
@@ -11,7 +11,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
 import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleFollowUser
 import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
 import com.twitter.stringcenter.client.StringCenter
-
 import javax.inject.Inject
 import javax.inject.Singleton
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BUILD.bazel

@@ -4,95 +4,56 @@ scala_library(
     strict_deps = True,
     tags = ["bazel-compatible"],
     dependencies = [
-        "3rdparty/jvm/com/twitter/storehaus:core",
         "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
         "configapi/configapi-decider",
         "finatra/inject/inject-core/src/main/scala",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/earlybird",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/inferred_topic",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content",
         "joinkey/src/main/scala/com/twitter/joinkey/context",
         "joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
-        "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store",
-        "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/topics",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_visibility_reason",
+        "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
-        "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord",
-        "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util",
-        "representation-scorer/server/src/main/scala/com/twitter/representationscorer/common",
-        "representation-scorer/server/src/main/thrift:thrift-scala",
-        "servo/repo/src/main/scala",
         "snowflake/src/main/scala/com/twitter/snowflake/id",
-        "src/java/com/twitter/ml/api/constant",
         "src/java/com/twitter/search/common/util/lang",
-        "src/scala/com/twitter/ml/api/util",
+        "src/scala/com/twitter/timelines/prediction/adapters/request_context",
-        "src/scala/com/twitter/timelines/prediction/adapters/real_graph",
-        "src/scala/com/twitter/timelines/prediction/adapters/realtime_interaction_graph",
-        "src/scala/com/twitter/timelines/prediction/adapters/twistly",
-        "src/scala/com/twitter/timelines/prediction/adapters/two_hop_features",
-        "src/scala/com/twitter/timelines/prediction/common/util",
-        "src/scala/com/twitter/timelines/prediction/features/common",
-        "src/scala/com/twitter/timelines/prediction/features/realtime_interaction_graph",
-        "src/scala/com/twitter/timelines/prediction/features/recap",
-        "src/scala/com/twitter/timelines/prediction/features/time_features",
         "src/thrift/com/twitter/gizmoduck:thrift-scala",
-        "src/thrift/com/twitter/ml/api:data-java",
-        "src/thrift/com/twitter/ml/api:embedding-java",
-        "src/thrift/com/twitter/onboarding/relevance/features:features-java",
         "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala",
         "src/thrift/com/twitter/search:earlybird-scala",
         "src/thrift/com/twitter/search/common:constants-java",
         "src/thrift/com/twitter/socialgraph:thrift-scala",
         "src/thrift/com/twitter/spam/rtf:safety-result-scala",
         "src/thrift/com/twitter/timelineranker:thrift-scala",
-        "src/thrift/com/twitter/timelines/author_features:thrift-java",
-        "src/thrift/com/twitter/timelines/conversation_features:conversation_features-scala",
         "src/thrift/com/twitter/timelines/impression:thrift-scala",
         "src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala",
         "src/thrift/com/twitter/timelines/real_graph:real_graph-scala",
         "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala",
-        "src/thrift/com/twitter/topic_recos:topic_recos-thrift-java",
         "src/thrift/com/twitter/tweetypie:service-scala",
         "src/thrift/com/twitter/tweetypie:tweet-scala",
         "src/thrift/com/twitter/user_session_store:thrift-java",
         "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala",
-        "src/thrift/com/twitter/wtf/real_time_interaction_graph:wtf-real_time_interaction_graph-thrift-java",
         "stitch/stitch-core",
         "stitch/stitch-gizmoduck",
         "stitch/stitch-socialgraph",
         "stitch/stitch-timelineservice",
         "stitch/stitch-tweetypie",
-        "strato/config/columns/topic-signals/tsp",
-        "strato/config/columns/topic-signals/tsp:tsp-strato-client",
         "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/feedback",
         "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
         "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/persistence",
-        "timelines/src/main/scala/com/twitter/timelines/clients/strato/twistly",
+        "timelines/src/main/scala/com/twitter/timelines/clients/manhattan/store",
-        "timelines/src/main/scala/com/twitter/timelines/clients/user_tweet_entity_graph",
         "timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter",
         "timelines/src/main/scala/com/twitter/timelines/impressionstore/store",
         "timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
-        "topic-social-proof/server/src/main/thrift:thrift-scala",
-        "topiclisting/topiclisting-core/src/main/scala/com/twitter/topiclisting",
-        "tweetconvosvc/thrift/src/main/thrift:thrift-scala",
-        "twitter-config/yaml",
         "user_session_store/src/main/scala/com/twitter/user_session_store",
         "util/util-core",
     ],

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FeedbackHistoryQueryFeatureHydrator.scala

@@ -1,12 +1,10 @@
 package com.twitter.home_mixer.functional_component.feature_hydrator
 
 import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature
-import com.twitter.home_mixer.param.HomeGlobalParams.EnableFeedbackFatigueParam
 import com.twitter.product_mixer.core.feature.Feature
 import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
 import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
 import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
-import com.twitter.product_mixer.core.model.common.Conditionally
 import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
 import com.twitter.product_mixer.core.pipeline.PipelineQuery
 import com.twitter.stitch.Stitch
@@ -17,16 +15,12 @@ import javax.inject.Singleton
 @Singleton
 case class FeedbackHistoryQueryFeatureHydrator @Inject() (
   feedbackHistoryClient: FeedbackHistoryManhattanClient)
-    extends QueryFeatureHydrator[PipelineQuery]
+    extends QueryFeatureHydrator[PipelineQuery] {
-    with Conditionally[PipelineQuery] {
 
   override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FeedbackHistory")
 
   override val features: Set[Feature[_, _]] = Set(FeedbackHistoryFeature)
 
-  override def onlyIf(query: PipelineQuery): Boolean =
-    query.params(EnableFeedbackFatigueParam)
-
   override def hydrate(
     query: PipelineQuery
   ): Stitch[FeatureMap] =

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FollowedTopicsQueryFeatureHydrator.scala

@@ -1,41 +0,0 @@
-package com.twitter.home_mixer.functional_component.feature_hydrator
-
-import com.twitter.conversions.DurationOps._
-import com.twitter.home_mixer.model.HomeFeatures.UserFollowedTopicsCountFeature
-import com.twitter.home_mixer.service.HomeMixerAlertConfig
-import com.twitter.product_mixer.component_library.candidate_source.topics.FollowedTopicsCandidateSource
-import com.twitter.product_mixer.core.feature.Feature
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
-import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyView
-import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
-import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
-import com.twitter.product_mixer.core.pipeline.PipelineQuery
-import com.twitter.stitch.Stitch
-import javax.inject.Inject
-import javax.inject.Singleton
-
-@Singleton
-case class FollowedTopicsQueryFeatureHydrator @Inject() (
-  followedTopicsCandidateSource: FollowedTopicsCandidateSource)
-    extends QueryFeatureHydrator[PipelineQuery] {
-
-  override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FollowedTopics")
-
-  override val features: Set[Feature[_, _]] = Set(UserFollowedTopicsCountFeature)
-
-  override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = {
-    val request: StratoKeyView[Long, Unit] = StratoKeyView(query.getRequiredUserId, Unit)
-    followedTopicsCandidateSource(request)
-      .map { topics =>
-        FeatureMapBuilder().add(UserFollowedTopicsCountFeature, Some(topics.size)).build()
-      }.handle {
-        case _ => FeatureMapBuilder().add(UserFollowedTopicsCountFeature, None).build()
-      }
-  }
-
-  override val alerts = Seq(
-    HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.9),
-    HomeMixerAlertConfig.BusinessHours.defaultLatencyAlert(1500.millis)
-  )
-}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GizmoduckAuthorSafetyFeatureHydrator.scala

@@ -1,58 +0,0 @@
-package com.twitter.home_mixer.functional_component.feature_hydrator
-
-import com.twitter.gizmoduck.{thriftscala => gt}
-import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
-import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature
-import com.twitter.home_mixer.param.HomeGlobalParams.EnableGizmoduckAuthorSafetyFeatureHydratorParam
-import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
-import com.twitter.product_mixer.core.feature.Feature
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
-import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
-import com.twitter.product_mixer.core.model.common.Conditionally
-import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
-import com.twitter.product_mixer.core.pipeline.PipelineQuery
-import com.twitter.stitch.Stitch
-import com.twitter.stitch.gizmoduck.Gizmoduck
-import javax.inject.Inject
-import javax.inject.Singleton
-
-@Singleton
-class GizmoduckAuthorSafetyFeatureHydrator @Inject() (gizmoduck: Gizmoduck)
-    extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
-    with Conditionally[PipelineQuery] {
-
-  override val identifier: FeatureHydratorIdentifier =
-    FeatureHydratorIdentifier("GizmoduckAuthorSafety")
-
-  override val features: Set[Feature[_, _]] = Set(AuthorIsBlueVerifiedFeature)
-
-  override def onlyIf(query: PipelineQuery): Boolean =
-    query.params(EnableGizmoduckAuthorSafetyFeatureHydratorParam)
-
-  private val queryFields: Set[gt.QueryFields] = Set(gt.QueryFields.Safety)
-
-  override def apply(
-    query: PipelineQuery,
-    candidate: TweetCandidate,
-    existingFeatures: FeatureMap
-  ): Stitch[FeatureMap] = {
-    val authorIdOption = existingFeatures.getOrElse(AuthorIdFeature, None)
-
-    val blueVerifiedStitch = authorIdOption
-      .map { authorId =>
-        gizmoduck
-          .getUserById(
-            userId = authorId,
-            queryFields = queryFields
-          )
-          .map { _.safety.flatMap(_.isBlueVerified).getOrElse(false) }
-      }.getOrElse(Stitch.False)
-
-    blueVerifiedStitch.map { isBlueVerified =>
-      FeatureMapBuilder()
-        .add(AuthorIsBlueVerifiedFeature, isBlueVerified)
-        .build()
-    }
-  }
-}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressionBloomFilterQueryFeatureHydrator.scala

@@ -3,6 +3,7 @@ package com.twitter.home_mixer.functional_component.feature_hydrator
 import com.twitter.conversions.DurationOps._
 import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature
 import com.twitter.home_mixer.model.request.HasSeenTweetIds
+import com.twitter.home_mixer.param.HomeGlobalParams.ImpressionBloomFilterFalsePositiveRateParam
 import com.twitter.home_mixer.service.HomeMixerAlertConfig
 import com.twitter.product_mixer.core.feature.Feature
 import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
@@ -11,7 +12,8 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Quer
 import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
 import com.twitter.product_mixer.core.pipeline.PipelineQuery
 import com.twitter.stitch.Stitch
-import com.twitter.timelines.impressionbloomfilter.{thriftscala => t}
+import com.twitter.timelines.clients.manhattan.store.ManhattanStoreClient
+import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm}
 import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilter
 import javax.inject.Inject
 import javax.inject.Singleton
@@ -19,36 +21,39 @@ import javax.inject.Singleton
 @Singleton
 case class ImpressionBloomFilterQueryFeatureHydrator[
   Query <: PipelineQuery with HasSeenTweetIds] @Inject() (
-  bloomFilter: ImpressionBloomFilter)
+  bloomFilterClient: ManhattanStoreClient[
-    extends QueryFeatureHydrator[Query] {
+    blm.ImpressionBloomFilterKey,
+    blm.ImpressionBloomFilterSeq
+  ]) extends QueryFeatureHydrator[Query] {
 
   override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier(
     "ImpressionBloomFilter")
 
   private val ImpressionBloomFilterTTL = 7.day
-  private val ImpressionBloomFilterFalsePositiveRate = 0.002
 
   override val features: Set[Feature[_, _]] = Set(ImpressionBloomFilterFeature)
 
-  private val SurfaceArea = t.SurfaceArea.HomeTimeline
+  private val SurfaceArea = blm.SurfaceArea.HomeTimeline
 
   override def hydrate(query: Query): Stitch[FeatureMap] = {
     val userId = query.getRequiredUserId
-    bloomFilter.getBloomFilterSeq(userId, SurfaceArea).map { bloomFilterSeq =>
+    bloomFilterClient
-      val updatedBloomFilterSeq =
+      .get(blm.ImpressionBloomFilterKey(userId, SurfaceArea))
-        if (query.seenTweetIds.forall(_.isEmpty)) bloomFilterSeq
+      .map(_.getOrElse(blm.ImpressionBloomFilterSeq(Seq.empty)))
-        else {
+      .map { bloomFilterSeq =>
-          bloomFilter.addElements(
+        val updatedBloomFilterSeq =
-            userId = userId,
+          if (query.seenTweetIds.forall(_.isEmpty)) bloomFilterSeq
-            surfaceArea = SurfaceArea,
+          else {
-            tweetIds = query.seenTweetIds.get,
+            ImpressionBloomFilter.addSeenTweetIds(
-            bloomFilterEntrySeq = bloomFilterSeq,
+              surfaceArea = SurfaceArea,
-            timeToLive = ImpressionBloomFilterTTL,
+              tweetIds = query.seenTweetIds.get,
-            falsePositiveRate = ImpressionBloomFilterFalsePositiveRate
+              bloomFilterSeq = bloomFilterSeq,
-          )
+              timeToLive = ImpressionBloomFilterTTL,
-        }
+              falsePositiveRate = query.params(ImpressionBloomFilterFalsePositiveRateParam)
-      FeatureMapBuilder().add(ImpressionBloomFilterFeature, updatedBloomFilterSeq).build()
+            )
-    }
+          }
+        FeatureMapBuilder().add(ImpressionBloomFilterFeature, updatedBloomFilterSeq).build()
+      }
   }
 
   override val alerts = Seq(

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/InNetworkFeatureHydrator.scala

@@ -0,0 +1,41 @@
+package com.twitter.home_mixer.functional_component.feature_hydrator
+
+import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
+import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
+import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature
+import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
+import com.twitter.product_mixer.core.feature.Feature
+import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
+import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
+import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
+import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
+import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
+import com.twitter.product_mixer.core.pipeline.PipelineQuery
+import com.twitter.stitch.Stitch
+
+object InNetworkFeatureHydrator
+    extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
+
+  override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("InNetwork")
+
+  override val features: Set[Feature[_, _]] = Set(InNetworkFeature)
+
+  override def apply(
+    query: PipelineQuery,
+    candidates: Seq[CandidateWithFeatures[TweetCandidate]]
+  ): Stitch[Seq[FeatureMap]] = {
+    val viewerId = query.getRequiredUserId
+    val followedUserIds = query.features.get.get(SGSFollowedUsersFeature).toSet
+
+    val featureMaps = candidates.map { candidate =>
+      // We use authorId and not sourceAuthorId here so that retweets are defined as in network
+      val isInNetworkOpt = candidate.features.getOrElse(AuthorIdFeature, None).map { authorId =>
+        // Users cannot follow themselves but this is in network by definition
+        val isSelfTweet = authorId == viewerId
+        isSelfTweet || followedUserIds.contains(authorId)
+      }
+      FeatureMapBuilder().add(InNetworkFeature, isInNetworkOpt.getOrElse(true)).build()
+    }
+    Stitch.value(featureMaps)
+  }
+}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PersistenceStoreQueryFeatureHydrator.scala

@@ -2,8 +2,10 @@ package com.twitter.home_mixer.functional_component.feature_hydrator
 
 import com.twitter.conversions.DurationOps._
 import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent
+import com.twitter.finagle.stats.StatsReceiver
 import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature
 import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature
+import com.twitter.home_mixer.model.HomeFeatures.ServedTweetPreviewIdsFeature
 import com.twitter.home_mixer.model.HomeFeatures.WhoToFollowExcludedUserIdsFeature
 import com.twitter.home_mixer.model.request.FollowingProduct
 import com.twitter.home_mixer.model.request.ForYouProduct
@@ -27,17 +29,27 @@ import javax.inject.Singleton
 
 @Singleton
 case class PersistenceStoreQueryFeatureHydrator @Inject() (
-  timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3])
+  timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3],
+  statsReceiver: StatsReceiver)
     extends QueryFeatureHydrator[PipelineQuery] {
 
   override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("PersistenceStore")
 
+  private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
+  private val servedTweetIdsSizeStat = scopedStatsReceiver.stat("ServedTweetIdsSize")
+
   private val WhoToFollowExcludedUserIdsLimit = 1000
-  private val ServedTweetIdsDuration = 1.hour
+  private val ServedTweetIdsDuration = 10.minutes
   private val ServedTweetIdsLimit = 100
+  private val ServedTweetPreviewIdsDuration = 10.hours
+  private val ServedTweetPreviewIdsLimit = 10
 
   override val features: Set[Feature[_, _]] =
-    Set(ServedTweetIdsFeature, PersistenceEntriesFeature, WhoToFollowExcludedUserIdsFeature)
+    Set(
+      ServedTweetIdsFeature,
+      ServedTweetPreviewIdsFeature,
+      PersistenceEntriesFeature,
+      WhoToFollowExcludedUserIdsFeature)
 
   private val supportedClients = Seq(
     ClientPlatform.IPhone,
@@ -80,8 +92,19 @@ case class PersistenceStoreQueryFeatureHydrator @Inject() (
             .flatMap(
               _.entries.flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetIdsLimit))
 
+          servedTweetIdsSizeStat.add(servedTweetIds.size)
+
+          val servedTweetPreviewIds = timelineResponses
+            .filter(_.clientPlatform == clientPlatform)
+            .filter(_.servedTime >= Time.now - ServedTweetPreviewIdsDuration)
+            .sortBy(-_.servedTime.inMilliseconds)
+            .flatMap(_.entries
+              .filter(_.entityIdType == EntityIdType.TweetPreview)
+              .flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetPreviewIdsLimit))
+
           FeatureMapBuilder()
             .add(ServedTweetIdsFeature, servedTweetIds)
+            .add(ServedTweetPreviewIdsFeature, servedTweetPreviewIds)
             .add(PersistenceEntriesFeature, timelineResponses)
             .add(WhoToFollowExcludedUserIdsFeature, whoToFollowUserIds)
             .build()

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PerspectiveFilteredSocialContextFeatureHydrator.scala

@@ -10,6 +10,7 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
 import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
 import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
 import com.twitter.product_mixer.core.pipeline.PipelineQuery
+import com.twitter.product_mixer.core.util.OffloadFuturePools
 import com.twitter.stitch.Stitch
 import com.twitter.stitch.timelineservice.TimelineService
 import com.twitter.stitch.timelineservice.TimelineService.GetPerspectives
@@ -37,10 +38,10 @@ class PerspectiveFilteredSocialContextFeatureHydrator @Inject() (timelineService
   override def apply(
     query: PipelineQuery,
     candidates: Seq[CandidateWithFeatures[TweetCandidate]]
-  ): Stitch[Seq[FeatureMap]] = {
+  ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch {
     val engagingUserIdtoTweetId = candidates.flatMap { candidate =>
       candidate.features
-        .get(FavoritedByUserIdsFeature).take(MaxCountUsers)
+        .getOrElse(FavoritedByUserIdsFeature, Seq.empty).take(MaxCountUsers)
         .map(favoritedBy => favoritedBy -> candidate.candidate.id)
     }
 
@@ -59,7 +60,7 @@ class PerspectiveFilteredSocialContextFeatureHydrator @Inject() (timelineService
 
       candidates.map { candidate =>
         val perspectiveFilteredFavoritedByUserIds: Seq[Long] = candidate.features
-          .get(FavoritedByUserIdsFeature).take(MaxCountUsers)
+          .getOrElse(FavoritedByUserIdsFeature, Seq.empty).take(MaxCountUsers)
           .filter { userId => validUserIdTweetIds.contains((userId, candidate.candidate.id)) }
 
         FeatureMapBuilder()

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphInNetworkScoresQueryFeatureHydrator.scala

@@ -32,11 +32,15 @@ case class RealGraphInNetworkScoresQueryFeatureHydrator @Inject() (
       val realGraphScoresFeatures = realGraphFollowedUsers
         .getOrElse(Seq.empty)
         .sortBy(-_.score)
-        .map(candidate => candidate.userId -> candidate.score)
+        .map(candidate => candidate.userId -> scaleScore(candidate.score))
         .take(RealGraphCandidateCount)
         .toMap
 
       FeatureMapBuilder().add(RealGraphInNetworkScoresFeature, realGraphScoresFeatures).build()
     }
   }
+
+  // Rescale Real Graph v2 scores from [0,1] to the v1 scores distribution [1,2.97]
+  private def scaleScore(score: Double): Double =
+    if (score >= 0.0 && score <= 1.0) score * 1.97 + 1.0 else score
 }

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RequestQueryFeatureHydrator.scala

@@ -17,9 +17,13 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.G
 import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor
 import com.twitter.product_mixer.core.pipeline.HasPipelineCursor
 import com.twitter.product_mixer.core.pipeline.PipelineQuery
+import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest
+import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure
 import com.twitter.search.common.util.lang.ThriftLanguageUtil
 import com.twitter.snowflake.id.SnowflakeId
 import com.twitter.stitch.Stitch
+import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter.dowFromTimestamp
+import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter.hourFromTimestamp
 import java.util.UUID
 import javax.inject.Inject
 import javax.inject.Singleton
@@ -45,6 +49,9 @@ class RequestQueryFeatureHydrator[
     PullToRefreshFeature,
     RequestJoinIdFeature,
     ServedRequestIdFeature,
+    TimestampFeature,
+    TimestampGMTDowFeature,
+    TimestampGMTHourFeature,
     ViewerIdFeature
   )
 
@@ -67,6 +74,7 @@ class RequestQueryFeatureHydrator[
   override def hydrate(query: Query): Stitch[FeatureMap] = {
     val requestContext = query.deviceContext.flatMap(_.requestContextValue)
     val servedRequestId = UUID.randomUUID.getMostSignificantBits
+    val timestamp = query.queryTime.inMilliseconds
 
     val featureMap = FeatureMapBuilder()
       .add(AccountAgeFeature, query.getOptionalUserId.flatMap(SnowflakeId.timeFromIdOpt))
@@ -97,8 +105,15 @@ class RequestQueryFeatureHydrator[
       .add(PullToRefreshFeature, requestContext.contains(RequestContext.PullToRefresh))
       .add(ServedRequestIdFeature, Some(servedRequestId))
       .add(RequestJoinIdFeature, getRequestJoinId(servedRequestId))
+      .add(TimestampFeature, timestamp)
+      .add(TimestampGMTDowFeature, dowFromTimestamp(timestamp))
+      .add(TimestampGMTHourFeature, hourFromTimestamp(timestamp))
       .add(HasDarkRequestFeature, hasDarkRequest)
-      .add(ViewerIdFeature, query.getRequiredUserId)
+      .add(
+        ViewerIdFeature,
+        query.getOptionalUserId
+          .orElse(query.getGuestId).getOrElse(
+            throw PipelineFailure(BadRequest, "Missing viewer id")))
       .build()
 
     Stitch.value(featureMap)

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSFollowedUsersQueryFeatureHydrator.scala

@@ -1,46 +0,0 @@
-package com.twitter.home_mixer.functional_component.feature_hydrator
-
-import com.twitter.product_mixer.core.feature.Feature
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
-import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
-import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
-import com.twitter.product_mixer.core.pipeline.PipelineQuery
-import com.twitter.socialgraph.{thriftscala => sg}
-import com.twitter.stitch.Stitch
-import com.twitter.stitch.socialgraph.{SocialGraph => SocialGraphStitchClient}
-import javax.inject.Inject
-import javax.inject.Singleton
-
-object SGSFollowedUsersFeature extends Feature[PipelineQuery, Seq[Long]]
-
-@Singleton
-case class SGSFollowedUsersQueryFeatureHydrator @Inject() (
-  socialGraphStitchClient: SocialGraphStitchClient)
-    extends QueryFeatureHydrator[PipelineQuery] {
-
-  override val identifier: FeatureHydratorIdentifier =
-    FeatureHydratorIdentifier("SGSFollowedUsers")
-
-  override val features: Set[Feature[_, _]] = Set(SGSFollowedUsersFeature)
-
-  private val SocialGraphLimit = 14999
-
-  override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = {
-    val userId = query.getRequiredUserId
-
-    val request = sg.IdsRequest(
-      relationships = Seq(
-        sg.SrcRelationship(userId, sg.RelationshipType.Following, hasRelationship = true),
-        sg.SrcRelationship(userId, sg.RelationshipType.Muting, hasRelationship = false)
-      ),
-      pageRequest = Some(sg.PageRequest(count = Some(SocialGraphLimit)))
-    )
-
-    socialGraphStitchClient
-      .ids(request).map(_.ids)
-      .map { followedUsers =>
-        FeatureMapBuilder().add(SGSFollowedUsersFeature, followedUsers).build()
-      }
-  }
-}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSValidSocialContextFeatureHydrator.scala

@@ -12,6 +12,7 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
 import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
 import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
 import com.twitter.product_mixer.core.pipeline.PipelineQuery
+import com.twitter.product_mixer.core.util.OffloadFuturePools
 import com.twitter.socialgraph.{thriftscala => sg}
 import com.twitter.stitch.Stitch
 import com.twitter.stitch.socialgraph.SocialGraph
@@ -41,8 +42,7 @@ class SGSValidSocialContextFeatureHydrator @Inject() (
   override def apply(
     query: PipelineQuery,
     candidates: Seq[CandidateWithFeatures[TweetCandidate]]
-  ): Stitch[Seq[FeatureMap]] = {
+  ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch {
-
     val allSocialContextUserIds =
       candidates.flatMap { candidate =>
         candidate.features.getOrElse(FavoritedByUserIdsFeature, Nil).take(MaxCountUsers) ++

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SocialGraphServiceFeatureHydrator.scala

@@ -1,67 +0,0 @@
-package com.twitter.home_mixer.functional_component.feature_hydrator
-
-import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
-import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
-import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
-import com.twitter.product_mixer.core.feature.Feature
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
-import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
-import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
-import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
-import com.twitter.product_mixer.core.pipeline.PipelineQuery
-import com.twitter.socialgraph.{thriftscala => sg}
-import com.twitter.stitch.Stitch
-import com.twitter.stitch.socialgraph.{SocialGraph => SocialGraphStitchClient}
-import javax.inject.Inject
-import javax.inject.Singleton
-
-@Singleton
-class SocialGraphServiceFeatureHydrator @Inject() (socialGraphStitchClient: SocialGraphStitchClient)
-    extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
-
-  override val identifier: FeatureHydratorIdentifier =
-    FeatureHydratorIdentifier("SocialGraphService")
-
-  override val features: Set[Feature[_, _]] = Set(InNetworkFeature)
-
-  override def apply(
-    query: PipelineQuery,
-    candidates: Seq[CandidateWithFeatures[TweetCandidate]]
-  ): Stitch[Seq[FeatureMap]] = {
-    val viewerId = query.getRequiredUserId
-
-    // We use authorId and not sourceAuthorId here so that retweets are defined as in network
-    val authorIds = candidates.map(_.features.getOrElse(AuthorIdFeature, None).getOrElse(0L))
-    val distinctNonSelfAuthorIds = authorIds.filter(_ != viewerId).distinct
-
-    val idsRequest = createIdsRequest(
-      userId = viewerId,
-      relationshipTypes = Set(sg.RelationshipType.Following),
-      targetIds = Some(distinctNonSelfAuthorIds)
-    )
-
-    socialGraphStitchClient
-      .ids(request = idsRequest, requestContext = None)
-      .map { idResult =>
-        authorIds.map { authorId =>
-          // Users cannot follow themselves but this is in network by definition
-          val isSelfTweet = authorId == viewerId
-          val inNetworkAuthorIds = idResult.ids.toSet
-          val isInNetwork = isSelfTweet || inNetworkAuthorIds.contains(authorId) || authorId == 0L
-          FeatureMapBuilder().add(InNetworkFeature, isInNetwork).build()
-        }
-      }
-  }
-
-  private def createIdsRequest(
-    userId: Long,
-    relationshipTypes: Set[sg.RelationshipType],
-    targetIds: Option[Seq[Long]] = None
-  ): sg.IdsRequest = sg.IdsRequest(
-    relationshipTypes.map { relationshipType =>
-      sg.SrcRelationship(userId, relationshipType, targets = targetIds)
-    }.toSeq,
-    Some(sg.PageRequest(selectAll = Some(true)))
-  )
-}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TSPInferredTopicFeatureHydrator.scala

@@ -1,162 +0,0 @@
-package com.twitter.home_mixer.functional_component.feature_hydrator
-
-import com.twitter.contentrecommender.{thriftscala => cr}
-import com.twitter.finagle.stats.StatsReceiver
-import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.inferred_topic.InferredTopicAdapter
-import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature
-import com.twitter.home_mixer.model.HomeFeatures.TSPMetricTagFeature
-import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature
-import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature
-import com.twitter.ml.api.DataRecord
-import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
-import com.twitter.product_mixer.core.feature.Feature
-import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
-import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
-import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
-import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
-import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
-import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType
-import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType
-import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContextFunctionalityType
-import com.twitter.product_mixer.core.pipeline.PipelineQuery
-import com.twitter.stitch.Stitch
-import com.twitter.strato.generated.client.topic_signals.tsp.TopicSocialProofClientColumn
-import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => sid}
-import com.twitter.topiclisting.TopicListingViewerContext
-import com.twitter.tsp.{thriftscala => tsp}
-
-import javax.inject.Inject
-import javax.inject.Singleton
-import scala.collection.JavaConverters._
-
-object TSPInferredTopicFeature extends Feature[TweetCandidate, Map[Long, Double]]
-object TSPInferredTopicDataRecordFeature
-    extends DataRecordInAFeature[TweetCandidate]
-    with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
-  override def defaultValue: DataRecord = new DataRecord()
-}
-
-@Singleton
-class TSPInferredTopicFeatureHydrator @Inject() (
-  topicSocialProofClientColumn: TopicSocialProofClientColumn,
-  statsReceiver: StatsReceiver,
-) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
-
-  override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TSPInferredTopic")
-
-  override val features: Set[Feature[_, _]] =
-    Set(
-      TSPInferredTopicFeature,
-      TSPInferredTopicDataRecordFeature,
-      TopicIdSocialContextFeature,
-      TopicContextFunctionalityTypeFeature)
-
-  private val topK = 3
-
-  private val sourcesToSetSocialProof: Set[sid.CandidateTweetSourceId] = Set(
-    sid.CandidateTweetSourceId.Simcluster,
-    sid.CandidateTweetSourceId.CroonTweet
-  )
-
-  private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
-  private val keyFoundCounter = scopedStatsReceiver.counter("key/found")
-  private val keyLossCounter = scopedStatsReceiver.counter("key/loss")
-  private val requestFailCounter = scopedStatsReceiver.counter("request/fail")
-
-  private val DefaultFeatureMap = FeatureMapBuilder()
-    .add(TSPInferredTopicFeature, Map.empty[Long, Double])
-    .add(TSPInferredTopicDataRecordFeature, new DataRecord())
-    .add(TopicIdSocialContextFeature, None)
-    .add(TopicContextFunctionalityTypeFeature, None)
-    .build()
-
-  override def apply(
-    query: PipelineQuery,
-    candidates: Seq[CandidateWithFeatures[TweetCandidate]]
-  ): Stitch[Seq[FeatureMap]] = {
-    val tags = candidates.collect {
-      case candidate if candidate.features.getTry(TSPMetricTagFeature).isReturn =>
-        candidate.candidate.id -> candidate.features
-          .getOrElse(TSPMetricTagFeature, Set.empty[tsp.MetricTag])
-    }.toMap
-
-    val topicSocialProofRequest =
-      tsp.TopicSocialProofRequest(
-        userId = query.getRequiredUserId,
-        tweetIds = candidates.map(_.candidate.id).toSet,
-        displayLocation = cr.DisplayLocation.HomeTimeline,
-        topicListingSetting = tsp.TopicListingSetting.Followable,
-        context = TopicListingViewerContext.fromClientContext(query.clientContext).toThrift,
-        bypassModes = None,
-        // Only CRMixer source has this data. Convert the CRMixer metric tag to tsp metric tag.
-        tags = if (tags.isEmpty) None else Some(tags)
-      )
-
-    topicSocialProofClientColumn.fetcher
-      .fetch(topicSocialProofRequest)
-      .map(_.v)
-      .map {
-        case Some(response) =>
-          candidates.map { candidate =>
-            val topicWithScores = response.socialProofs.getOrElse(candidate.candidate.id, Seq.empty)
-            if (topicWithScores.nonEmpty) {
-              keyFoundCounter.incr()
-              val (socialProofId, socialProofFunctionalityType) =
-                if (candidate.features
-                    .getOrElse(CandidateSourceIdFeature, None)
-                    .exists(sourcesToSetSocialProof.contains)) {
-                  getSocialProof(topicWithScores)
-                } else {
-                  (None, None)
-                }
-              val inferredTopicFeatures = convertTopicWithScores(topicWithScores)
-              val inferredTopicDataRecord =
-                InferredTopicAdapter.adaptToDataRecords(inferredTopicFeatures).asScala.head
-              FeatureMapBuilder()
-                .add(TSPInferredTopicFeature, inferredTopicFeatures)
-                .add(TSPInferredTopicDataRecordFeature, inferredTopicDataRecord)
-                .add(TopicIdSocialContextFeature, socialProofId)
-                .add(TopicContextFunctionalityTypeFeature, socialProofFunctionalityType)
-                .build()
-            } else {
-              keyLossCounter.incr()
-              DefaultFeatureMap
-            }
-          }
-        case _ =>
-          requestFailCounter.incr()
-          candidates.map { _ =>
-            DefaultFeatureMap
-          }
-      }
-  }
-
-  private def getSocialProof(
-    topicWithScores: Seq[tsp.TopicWithScore]
-  ): (Option[Long], Option[TopicContextFunctionalityType]) = {
-    val followingTopicId = topicWithScores
-      .collectFirst {
-        case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.Following)) =>
-          topicId
-      }
-    if (followingTopicId.nonEmpty) {
-      return (followingTopicId, Some(BasicTopicContextFunctionalityType))
-    }
-    val implicitFollowingId = topicWithScores.collectFirst {
-      case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.ImplicitFollow)) =>
-        topicId
-    }
-    if (implicitFollowingId.nonEmpty) {
-      return (implicitFollowingId, Some(RecommendationTopicContextFunctionalityType))
-    }
-    (None, None)
-  }
-
-  private def convertTopicWithScores(
-    topicWithScores: Seq[tsp.TopicWithScore],
-  ): Map[Long, Double] = {
-    topicWithScores.sortBy(-_.score).take(topK).map(a => (a.topicId, a.score)).toMap
-  }
-}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TimeFeaturesHydrator.scala

@@ -1,251 +0,0 @@
-package com.twitter.home_mixer.functional_component.feature_hydrator
-
-import com.twitter.conversions.DurationOps._
-import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature
-import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature
-import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
-import com.twitter.ml.api.DataRecord
-import com.twitter.ml.api.RichDataRecord
-import com.twitter.ml.api.util.FDsl._
-import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
-import com.twitter.product_mixer.core.feature.Feature
-import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
-import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
-import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
-import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
-import com.twitter.product_mixer.core.pipeline.PipelineQuery
-import com.twitter.search.common.features.{thriftscala => sc}
-import com.twitter.snowflake.id.SnowflakeId
-import com.twitter.stitch.Stitch
-import com.twitter.timelines.prediction.features.time_features.AccountAgeInterval
-import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures._
-import com.twitter.timelines.prediction.features.time_features.TimeFeatures
-import com.twitter.util.Duration
-import scala.collection.Searching._
-
-object TimeFeaturesDataRecordFeature
-    extends DataRecordInAFeature[TweetCandidate]
-    with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
-  override def defaultValue: DataRecord = new DataRecord()
-}
-
-object TimeFeaturesHydrator extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
-
-  override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TimeFeatures")
-
-  override val features: Set[Feature[_, _]] = Set(TimeFeaturesDataRecordFeature)
-
-  override def apply(
-    query: PipelineQuery,
-    candidate: TweetCandidate,
-    existingFeatures: FeatureMap
-  ): Stitch[FeatureMap] = {
-    Stitch.value {
-      val richDataRecord = new RichDataRecord()
-      setTimeFeatures(richDataRecord, candidate, existingFeatures, query)
-      FeatureMapBuilder()
-        .add(TimeFeaturesDataRecordFeature, richDataRecord.getRecord)
-        .build()
-    }
-  }
-
-  private def setTimeFeatures(
-    richDataRecord: RichDataRecord,
-    candidate: TweetCandidate,
-    existingFeatures: FeatureMap,
-    query: PipelineQuery,
-  ): Unit = {
-    val timeFeaturesOpt = getTimeFeatures(query, candidate, existingFeatures)
-    timeFeaturesOpt.foreach(timeFeatures => setFeatures(timeFeatures, richDataRecord))
-  }
-
-  private[feature_hydrator] def getTimeFeatures(
-    query: PipelineQuery,
-    candidate: TweetCandidate,
-    existingFeatures: FeatureMap,
-  ): Option[TimeFeatures] = {
-    for {
-      requestTimestampMs <- Some(query.queryTime.inMilliseconds)
-      tweetId <- Some(candidate.id)
-      viewerId <- query.getOptionalUserId
-      tweetCreationTimeMs <- timeFromTweetOrUserId(tweetId)
-      timeSinceTweetCreation = requestTimestampMs - tweetCreationTimeMs
-      accountAgeDurationOpt = timeFromTweetOrUserId(viewerId).map { viewerAccountCreationTimeMs =>
-        Duration.fromMilliseconds(requestTimestampMs - viewerAccountCreationTimeMs)
-      }
-      timeSinceSourceTweetCreation =
-        existingFeatures
-          .getOrElse(SourceTweetIdFeature, None)
-          .flatMap { sourceTweetId =>
-            timeFromTweetOrUserId(sourceTweetId).map { sourceTweetCreationTimeMs =>
-              requestTimestampMs - sourceTweetCreationTimeMs
-            }
-          }
-          .getOrElse(timeSinceTweetCreation)
-      if (timeSinceTweetCreation > 0 && timeSinceSourceTweetCreation > 0)
-    } yield {
-      val timeFeatures = TimeFeatures(
-        timeSinceTweetCreation = timeSinceTweetCreation,
-        timeSinceSourceTweetCreation = timeSinceSourceTweetCreation,
-        timeSinceViewerAccountCreationSecs = accountAgeDurationOpt.map(_.inSeconds),
-        isDay30NewUser = accountAgeDurationOpt.map(_ < 30.days).getOrElse(false),
-        isMonth12NewUser = accountAgeDurationOpt.map(_ < 365.days).getOrElse(false),
-        accountAgeInterval = accountAgeDurationOpt.flatMap(AccountAgeInterval.fromDuration),
-        isTweetRecycled = false // only set in RecyclableTweetCandidateFilter, but it's not used
-      )
-
-      val timeFeaturesWithLastEngagement = addLastEngagementTimeFeatures(
-        existingFeatures.getOrElse(EarlybirdFeature, None),
-        timeFeatures,
-        timeSinceSourceTweetCreation
-      ).getOrElse(timeFeatures)
-
-      val nonPollingTimestampsMs =
-        query.features.map(_.getOrElse(NonPollingTimesFeature, Seq.empty))
-      val timeFeaturesWithNonPollingOpt = addNonPollingTimeFeatures(
-        timeFeaturesWithLastEngagement,
-        requestTimestampMs,
-        tweetCreationTimeMs,
-        nonPollingTimestampsMs
-      )
-      timeFeaturesWithNonPollingOpt.getOrElse(timeFeaturesWithLastEngagement)
-    }
-  }
-
-  private def timeFromTweetOrUserId(tweetOrUserId: Long): Option[Long] = {
-    if (SnowflakeId.isSnowflakeId(tweetOrUserId))
-      Some(SnowflakeId(tweetOrUserId).time.inMilliseconds)
-    else None
-  }
-
-  private def addLastEngagementTimeFeatures(
-    tweetFeaturesOpt: Option[sc.ThriftTweetFeatures],
-    timeFeatures: TimeFeatures,
-    timeSinceSourceTweetCreation: Long
-  ): Option[TimeFeatures] = {
-    tweetFeaturesOpt.map { tweetFeatures =>
-      val lastFavSinceCreationHrs = tweetFeatures.lastFavSinceCreationHrs.map(_.toDouble)
-      val lastRetweetSinceCreationHrs = tweetFeatures.lastRetweetSinceCreationHrs.map(_.toDouble)
-      val lastReplySinceCreationHrs = tweetFeatures.lastReplySinceCreationHrs.map(_.toDouble)
-      val lastQuoteSinceCreationHrs = tweetFeatures.lastQuoteSinceCreationHrs.map(_.toDouble)
-
-      timeFeatures.copy(
-        lastFavSinceCreationHrs = lastFavSinceCreationHrs,
-        lastRetweetSinceCreationHrs = lastRetweetSinceCreationHrs,
-        lastReplySinceCreationHrs = lastReplySinceCreationHrs,
-        lastQuoteSinceCreationHrs = lastQuoteSinceCreationHrs,
-        timeSinceLastFavoriteHrs = getTimeSinceLastEngagementHrs(
-          lastFavSinceCreationHrs,
-          timeSinceSourceTweetCreation
-        ),
-        timeSinceLastRetweetHrs = getTimeSinceLastEngagementHrs(
-          lastRetweetSinceCreationHrs,
-          timeSinceSourceTweetCreation
-        ),
-        timeSinceLastReplyHrs = getTimeSinceLastEngagementHrs(
-          lastReplySinceCreationHrs,
-          timeSinceSourceTweetCreation
-        ),
-        timeSinceLastQuoteHrs = getTimeSinceLastEngagementHrs(
-          lastQuoteSinceCreationHrs,
-          timeSinceSourceTweetCreation
-        )
-      )
-    }
-  }
-
-  private def addNonPollingTimeFeatures(
-    timeFeatures: TimeFeatures,
-    requestTimestampMs: Long,
-    creationTimeMs: Long,
-    nonPollingTimestampsMs: Option[Seq[Long]]
-  ): Option[TimeFeatures] = {
-    for {
-      nonPollingTimestampsMs <- nonPollingTimestampsMs
-      lastNonPollingTimestampMs <- nonPollingTimestampsMs.headOption
-      earliestNonPollingTimestampMs <- nonPollingTimestampsMs.lastOption
-    } yield {
-      val timeSinceLastNonPollingRequest = requestTimestampMs - lastNonPollingTimestampMs
-      val tweetAgeRatio = timeSinceLastNonPollingRequest / math.max(
-        1.0,
-        timeFeatures.timeSinceTweetCreation
-      )
-      /*
-       * Non-polling timestamps are stored in chronological order.
-       * The latest timestamps occur first, therefore we need to explicitly search in reverse order.
-       */
-      val nonPollingRequestsSinceTweetCreation =
-        if (nonPollingTimestampsMs.nonEmpty) {
-          nonPollingTimestampsMs.search(creationTimeMs)(Ordering[Long].reverse).insertionPoint
-        } else {
-          0
-        }
-      /*
-       * Calculate the average time between non-polling requests; include
-       * request time in this calculation as latest timestamp.
-       */
-      val timeBetweenNonPollingRequestsAvg =
-        (requestTimestampMs - earliestNonPollingTimestampMs) / math
-          .max(1.0, nonPollingTimestampsMs.size)
-      val timeFeaturesWithNonPolling = timeFeatures.copy(
-        timeBetweenNonPollingRequestsAvg = Some(timeBetweenNonPollingRequestsAvg),
-        timeSinceLastNonPollingRequest = Some(timeSinceLastNonPollingRequest),
-        nonPollingRequestsSinceTweetCreation = Some(nonPollingRequestsSinceTweetCreation),
-        tweetAgeRatio = Some(tweetAgeRatio)
-      )
-      timeFeaturesWithNonPolling
-    }
-  }
-
-  private[this] def getTimeSinceLastEngagementHrs(
-    lastEngagementTimeSinceCreationHrsOpt: Option[Double],
-    timeSinceTweetCreation: Long
-  ): Option[Double] = {
-    lastEngagementTimeSinceCreationHrsOpt.map { lastEngagementTimeSinceCreationHrs =>
-      val timeSinceTweetCreationHrs = (timeSinceTweetCreation / (60 * 60 * 1000)).toInt
-      timeSinceTweetCreationHrs - lastEngagementTimeSinceCreationHrs
-    }
-  }
-
-  private def setFeatures(features: TimeFeatures, richDataRecord: RichDataRecord): Unit = {
-    val record = richDataRecord.getRecord
-      .setFeatureValue(IS_TWEET_RECYCLED, features.isTweetRecycled)
-      .setFeatureValue(TIME_SINCE_TWEET_CREATION, features.timeSinceTweetCreation)
-      .setFeatureValueFromOption(
-        TIME_SINCE_VIEWER_ACCOUNT_CREATION_SECS,
-        features.timeSinceViewerAccountCreationSecs)
-      .setFeatureValue(
-        USER_ID_IS_SNOWFLAKE_ID,
-        features.timeSinceViewerAccountCreationSecs.isDefined
-      )
-      .setFeatureValueFromOption(ACCOUNT_AGE_INTERVAL, features.accountAgeInterval.map(_.id.toLong))
-      .setFeatureValue(IS_30_DAY_NEW_USER, features.isDay30NewUser)
-      .setFeatureValue(IS_12_MONTH_NEW_USER, features.isMonth12NewUser)
-      .setFeatureValueFromOption(LAST_FAVORITE_SINCE_CREATION_HRS, features.lastFavSinceCreationHrs)
-      .setFeatureValueFromOption(
-        LAST_RETWEET_SINCE_CREATION_HRS,
-        features.lastRetweetSinceCreationHrs
-      )
-      .setFeatureValueFromOption(LAST_REPLY_SINCE_CREATION_HRS, features.lastReplySinceCreationHrs)
-      .setFeatureValueFromOption(LAST_QUOTE_SINCE_CREATION_HRS, features.lastQuoteSinceCreationHrs)
-      .setFeatureValueFromOption(TIME_SINCE_LAST_FAVORITE_HRS, features.timeSinceLastFavoriteHrs)
-      .setFeatureValueFromOption(TIME_SINCE_LAST_RETWEET_HRS, features.timeSinceLastRetweetHrs)
-      .setFeatureValueFromOption(TIME_SINCE_LAST_REPLY_HRS, features.timeSinceLastReplyHrs)
-      .setFeatureValueFromOption(TIME_SINCE_LAST_QUOTE_HRS, features.timeSinceLastQuoteHrs)
-    /*
-     * set features whose values are optional as some users do not have non-polling timestamps
-     */
-    features.timeBetweenNonPollingRequestsAvg.foreach(
-      record.setFeatureValue(TIME_BETWEEN_NON_POLLING_REQUESTS_AVG, _)
-    )
-    features.timeSinceLastNonPollingRequest.foreach(
-      record.setFeatureValue(TIME_SINCE_LAST_NON_POLLING_REQUEST, _)
-    )
-    features.nonPollingRequestsSinceTweetCreation.foreach(
-      record.setFeatureValue(NON_POLLING_REQUESTS_SINCE_TWEET_CREATION, _)
-    )
-    features.tweetAgeRatio.foreach(record.setFeatureValue(TWEET_AGE_RATIO, _))
-  }
-}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetImpressionsQueryFeatureHydrator.scala

@@ -24,8 +24,8 @@ case class TweetImpressionsQueryFeatureHydrator[
   manhattanTweetImpressionStoreClient: ManhattanTweetImpressionStoreClient)
     extends QueryFeatureHydrator[Query] {
 
-  private val TweetImpressionTTL = 1.day
+  private val TweetImpressionTTL = 2.days
-  private val TweetImpressionCap = 3000
+  private val TweetImpressionCap = 5000
 
   override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetImpressions")
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetypieFeatureHydrator.scala

@@ -1,6 +1,9 @@
 package com.twitter.home_mixer.functional_component.feature_hydrator
 
+import com.twitter.finagle.stats.StatsReceiver
 import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
+import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature
+import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
 import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
 import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature
 import com.twitter.home_mixer.model.HomeFeatures.IsNsfwFeature
@@ -10,11 +13,13 @@ import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature
 import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature
 import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
 import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
+import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature
 import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature
 import com.twitter.home_mixer.model.request.FollowingProduct
 import com.twitter.home_mixer.model.request.ForYouProduct
 import com.twitter.home_mixer.model.request.ListTweetsProduct
 import com.twitter.home_mixer.model.request.ScoredTweetsProduct
+import com.twitter.home_mixer.model.request.SubscribedProduct
 import com.twitter.home_mixer.util.tweetypie.RequestFields
 import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_is_nsfw.IsNsfw
 import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason.VisibilityReason
@@ -29,17 +34,22 @@ import com.twitter.spam.rtf.{thriftscala => rtf}
 import com.twitter.stitch.Stitch
 import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient}
 import com.twitter.tweetypie.{thriftscala => tp}
+import com.twitter.util.logging.Logging
 import javax.inject.Inject
 import javax.inject.Singleton
 
 @Singleton
-class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitchClient)
+class TweetypieFeatureHydrator @Inject() (
-    extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
+  tweetypieStitchClient: TweetypieStitchClient,
+  statsReceiver: StatsReceiver)
+    extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
+    with Logging {
 
   override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Tweetypie")
 
   override val features: Set[Feature[_, _]] = Set(
     AuthorIdFeature,
+    ExclusiveConversationAuthorIdFeature,
     InReplyToTweetIdFeature,
     IsHydratedFeature,
     IsNsfw,
@@ -51,6 +61,7 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
     SourceTweetIdFeature,
     SourceUserIdFeature,
     TweetTextFeature,
+    TweetLanguageFeature,
     VisibilityReason
   )
 
@@ -70,9 +81,12 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
   ): Stitch[FeatureMap] = {
     val safetyLevel = query.product match {
       case FollowingProduct => rtf.SafetyLevel.TimelineHomeLatest
-      case ForYouProduct => rtf.SafetyLevel.TimelineHome
+      case ForYouProduct =>
+        val inNetwork = existingFeatures.getOrElse(InNetworkFeature, true)
+        if (inNetwork) rtf.SafetyLevel.TimelineHome else rtf.SafetyLevel.TimelineHomeRecommendations
       case ScoredTweetsProduct => rtf.SafetyLevel.TimelineHome
       case ListTweetsProduct => rtf.SafetyLevel.TimelineLists
+      case SubscribedProduct => rtf.SafetyLevel.TimelineHomeSubscribed
       case unknown => throw new UnsupportedOperationException(s"Unknown product: $unknown")
     }
 
@@ -82,9 +96,12 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
       includeQuotedTweet = true,
       visibilityPolicy = tp.TweetVisibilityPolicy.UserVisible,
       safetyLevel = Some(safetyLevel),
-      forUserId = Some(query.getRequiredUserId)
+      forUserId = query.getOptionalUserId
     )
 
+    val exclusiveAuthorIdOpt =
+      existingFeatures.getOrElse(ExclusiveConversationAuthorIdFeature, None)
+
     tweetypieStitchClient.getTweetFields(tweetId = candidate.id, options = tweetFieldsOptions).map {
       case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), quoteOpt, _) =>
         val coreData = found.tweet.coreData
@@ -106,6 +123,7 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
           found.retweetedTweet.exists(_.coreData.exists(data => data.nsfwAdmin || data.nsfwUser))
 
         val tweetText = coreData.map(_.text)
+        val tweetLanguage = found.tweet.language.map(_.language)
 
         val tweetAuthorId = coreData.map(_.userId)
         val inReplyToTweetId = coreData.flatMap(_.reply.flatMap(_.inReplyToStatusId))
@@ -127,6 +145,7 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
 
         FeatureMapBuilder()
           .add(AuthorIdFeature, tweetAuthorId)
+          .add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt)
           .add(InReplyToTweetIdFeature, inReplyToTweetId)
           .add(IsHydratedFeature, true)
           .add(IsNsfw, Some(isNsfw))
@@ -137,20 +156,24 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
           .add(QuotedUserIdFeature, quotedTweetUserId)
           .add(SourceTweetIdFeature, retweetedTweetId)
           .add(SourceUserIdFeature, retweetedTweetUserId)
+          .add(TweetLanguageFeature, tweetLanguage)
           .add(TweetTextFeature, tweetText)
           .add(VisibilityReason, found.suppressReason)
           .build()
 
       // If no tweet result found, return default and pre-existing features
       case _ =>
-        DefaultFeatureMap +
+        DefaultFeatureMap ++ FeatureMapBuilder()
-          (AuthorIdFeature, existingFeatures.getOrElse(AuthorIdFeature, None)) +
+          .add(AuthorIdFeature, existingFeatures.getOrElse(AuthorIdFeature, None))
-          (InReplyToTweetIdFeature, existingFeatures.getOrElse(InReplyToTweetIdFeature, None)) +
+          .add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt)
-          (IsRetweetFeature, existingFeatures.getOrElse(IsRetweetFeature, false)) +
+          .add(InReplyToTweetIdFeature, existingFeatures.getOrElse(InReplyToTweetIdFeature, None))
-          (QuotedTweetIdFeature, existingFeatures.getOrElse(QuotedTweetIdFeature, None)) +
+          .add(IsRetweetFeature, existingFeatures.getOrElse(IsRetweetFeature, false))
-          (QuotedUserIdFeature, existingFeatures.getOrElse(QuotedUserIdFeature, None)) +
+          .add(QuotedTweetIdFeature, existingFeatures.getOrElse(QuotedTweetIdFeature, None))
-          (SourceTweetIdFeature, existingFeatures.getOrElse(SourceTweetIdFeature, None)) +
+          .add(QuotedUserIdFeature, existingFeatures.getOrElse(QuotedUserIdFeature, None))
-          (SourceUserIdFeature, existingFeatures.getOrElse(SourceUserIdFeature, None))
+          .add(SourceTweetIdFeature, existingFeatures.getOrElse(SourceTweetIdFeature, None))
+          .add(SourceUserIdFeature, existingFeatures.getOrElse(SourceUserIdFeature, None))
+          .add(TweetLanguageFeature, existingFeatures.getOrElse(TweetLanguageFeature, None))
+          .build()
     }
   }
 }

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features/AuthorFeaturesAdapter.scala

@@ -1,59 +0,0 @@
-package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.author_features
-
-import com.twitter.ml.api.DataRecordMerger
-import com.twitter.ml.api.Feature
-import com.twitter.ml.api.FeatureContext
-import com.twitter.ml.api.RichDataRecord
-import com.twitter.ml.api.util.CompactDataRecordConverter
-import com.twitter.ml.api.util.FDsl._
-import com.twitter.timelines.author_features.v1.{thriftjava => af}
-import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase
-import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig
-import com.twitter.timelines.prediction.features.user_health.UserHealthFeatures
-
-object AuthorFeaturesAdapter extends TimelinesMutatingAdapterBase[Option[af.AuthorFeatures]] {
-
-  private val originalAuthorAggregatesFeatures =
-    TimelinesAggregationConfig.originalAuthorReciprocalEngagementAggregates
-      .buildTypedAggregateGroups().flatMap(_.allOutputFeatures)
-  private val authorFeatures = originalAuthorAggregatesFeatures ++
-    Seq(
-      UserHealthFeatures.AuthorState,
-      UserHealthFeatures.NumAuthorFollowers,
-      UserHealthFeatures.NumAuthorConnectDays,
-      UserHealthFeatures.NumAuthorConnect)
-  private val featureContext = new FeatureContext(authorFeatures: _*)
-
-  override def getFeatureContext: FeatureContext = featureContext
-
-  override val commonFeatures: Set[Feature[_]] = Set.empty
-
-  private val compactDataRecordConverter = new CompactDataRecordConverter()
-  private val drMerger = new DataRecordMerger()
-
-  override def setFeatures(
-    authorFeaturesOpt: Option[af.AuthorFeatures],
-    richDataRecord: RichDataRecord
-  ): Unit = {
-    authorFeaturesOpt.foreach { authorFeatures =>
-      val dataRecord = richDataRecord.getRecord
-
-      dataRecord.setFeatureValue(
-        UserHealthFeatures.AuthorState,
-        authorFeatures.user_health.user_state.getValue.toLong)
-      dataRecord.setFeatureValue(
-        UserHealthFeatures.NumAuthorFollowers,
-        authorFeatures.user_health.num_followers.toDouble)
-      dataRecord.setFeatureValue(
-        UserHealthFeatures.NumAuthorConnectDays,
-        authorFeatures.user_health.num_connect_days.toDouble)
-      dataRecord.setFeatureValue(
-        UserHealthFeatures.NumAuthorConnect,
-        authorFeatures.user_health.num_connect.toDouble)
-
-      val originalAuthorAggregatesDataRecord =
-        compactDataRecordConverter.compactDataRecordToDataRecord(authorFeatures.aggregates)
-      drMerger.merge(dataRecord, originalAuthorAggregatesDataRecord)
-    }
-  }
-}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/Phase2EdgeAggregateFeatureHydrator.scala

@@ -1,22 +0,0 @@
-package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates
-
-import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures._
-import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
-import javax.inject.Inject
-import javax.inject.Singleton
-
-@Singleton
-class Phase2EdgeAggregateFeatureHydrator @Inject() extends BaseEdgeAggregateFeatureHydrator {
-
-  override val identifier: FeatureHydratorIdentifier =
-    FeatureHydratorIdentifier("Phase2EdgeAggregate")
-
-  override val aggregateFeatures: Set[BaseEdgeAggregateFeature] =
-    Set(
-      UserEngagerAggregateFeature,
-      UserEngagerGoodClickAggregateFeature,
-      UserInferredTopicAggregateFeature,
-      UserTopicAggregateFeature,
-      UserMediaUnderstandingAnnotationAggregateFeature
-    )
-}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/BUILD.bazel

@@ -4,6 +4,7 @@ scala_library(
     strict_deps = True,
     tags = ["bazel-compatible"],
     dependencies = [
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param",
@@ -16,9 +17,11 @@ scala_library(
         "src/thrift/com/twitter/tweetypie:service-scala",
         "src/thrift/com/twitter/tweetypie:tweet-scala",
         "stitch/stitch-core",
+        "stitch/stitch-socialgraph",
         "stitch/stitch-tweetypie",
         "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
         "timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
+        "util/util-slf4j-api/src/main/scala",
     ],
     exports = [
         "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/FeedbackFatigueFilter.scala

@@ -72,7 +72,7 @@ object FeedbackFatigueFilter
 
       originalAuthorId.exists(authorsToFilter.contains) ||
       (likers.nonEmpty && eligibleLikers.isEmpty) ||
-      (followers.nonEmpty && eligibleFollowers.isEmpty) ||
+      (followers.nonEmpty && eligibleFollowers.isEmpty && likers.isEmpty) ||
       (candidate.features.getOrElse(IsRetweetFeature, false) &&
       authorId.exists(retweetersToFilter.contains))
     }

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidSubscriptionTweetFilter.scala

@@ -0,0 +1,70 @@
+package com.twitter.home_mixer.functional_component.filter
+
+import com.twitter.finagle.stats.StatsReceiver
+import com.twitter.finagle.tracing.Trace
+import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature
+import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
+import com.twitter.product_mixer.core.functional_component.filter.Filter
+import com.twitter.product_mixer.core.functional_component.filter.FilterResult
+import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
+import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
+import com.twitter.product_mixer.core.pipeline.PipelineQuery
+import com.twitter.socialgraph.{thriftscala => sg}
+import com.twitter.stitch.Stitch
+import com.twitter.stitch.socialgraph.SocialGraph
+import com.twitter.util.logging.Logging
+
+import javax.inject.Inject
+import javax.inject.Singleton
+
+/**
+ * Exclude invalid subscription tweets - cases where the viewer is not subscribed to the author
+ *
+ * If SGS hydration fails, `SGSInvalidSubscriptionTweetFeature` will be set to None for
+ * subscription tweets, so we explicitly filter those tweets out.
+ */
+@Singleton
+case class InvalidSubscriptionTweetFilter @Inject() (
+  socialGraphClient: SocialGraph,
+  statsReceiver: StatsReceiver)
+    extends Filter[PipelineQuery, TweetCandidate]
+    with Logging {
+
+  override val identifier: FilterIdentifier = FilterIdentifier("InvalidSubscriptionTweet")
+
+  private val scopedStatsReceiver = statsReceiver.scope(identifier.toString)
+  private val validCounter = scopedStatsReceiver.counter("validExclusiveTweet")
+  private val invalidCounter = scopedStatsReceiver.counter("invalidExclusiveTweet")
+
+  override def apply(
+    query: PipelineQuery,
+    candidates: Seq[CandidateWithFeatures[TweetCandidate]]
+  ): Stitch[FilterResult[TweetCandidate]] = Stitch
+    .traverse(candidates) { candidate =>
+      val exclusiveAuthorId =
+        candidate.features.getOrElse(ExclusiveConversationAuthorIdFeature, None)
+
+      if (exclusiveAuthorId.isDefined) {
+        val request = sg.ExistsRequest(
+          source = query.getRequiredUserId,
+          target = exclusiveAuthorId.get,
+          relationships =
+            Seq(sg.Relationship(sg.RelationshipType.TierOneSuperFollowing, hasRelationship = true)),
+        )
+        socialGraphClient.exists(request).map(_.exists).map { valid =>
+          if (!valid) invalidCounter.incr() else validCounter.incr()
+          valid
+        }
+      } else Stitch.value(true)
+    }.map { validResults =>
+      val (kept, removed) = candidates
+        .map(_.candidate)
+        .zip(validResults)
+        .partition { case (candidate, valid) => valid }
+
+      val keptCandidates = kept.map { case (candidate, _) => candidate }
+      val removedCandidates = removed.map { case (candidate, _) => candidate }
+
+      FilterResult(kept = keptCandidates, removed = removedCandidates)
+    }
+}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PredicateFeatureFilter.scala

@@ -1,59 +0,0 @@
-package com.twitter.home_mixer.functional_component.filter
-
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
-import com.twitter.product_mixer.core.functional_component.filter.Filter
-import com.twitter.product_mixer.core.functional_component.filter.FilterResult
-import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
-import com.twitter.product_mixer.core.model.common.UniversalNoun
-import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
-import com.twitter.product_mixer.core.pipeline.PipelineQuery
-import com.twitter.stitch.Stitch
-
-/**
- * Predicate which will be applied to each candidate. True indicates that the candidate will be
- * @tparam Candidate - the type of the candidate
- */
-trait ShouldKeepCandidate {
-  def apply(features: FeatureMap): Boolean
-}
-
-object PredicateFeatureFilter {
-
-  /**
-   * Builds a simple Filter out of a predicate function from the candidate to a boolean. For clarity,
-   * we recommend including the name of the shouldKeepCandidate parameter.
-   *
-   * @param identifier A FilterIdentifier for the new filter
-   * @param shouldKeepCandidate A predicate function. Candidates will be kept when
-   *                            this function returns True.
-   */
-  def fromPredicate[Candidate <: UniversalNoun[Any]](
-    identifier: FilterIdentifier,
-    shouldKeepCandidate: ShouldKeepCandidate
-  ): Filter[PipelineQuery, Candidate] = {
-    val i = identifier
-
-    new Filter[PipelineQuery, Candidate] {
-      override val identifier: FilterIdentifier = i
-
-      /**
-       * Filter the list of candidates
-       *
-       * @return a FilterResult including both the list of kept candidate and the list of removed candidates
-       */
-      override def apply(
-        query: PipelineQuery,
-        candidates: Seq[CandidateWithFeatures[Candidate]]
-      ): Stitch[FilterResult[Candidate]] = {
-        val allowedIds = candidates
-          .filter(candidate => shouldKeepCandidate(candidate.features)).map(_.candidate.id).toSet
-
-        val (keptCandidates, removedCandidates) = candidates.map(_.candidate).partition {
-          candidate => allowedIds.contains(candidate.id)
-        }
-
-        Stitch.value(FilterResult(kept = keptCandidates, removed = removedCandidates))
-      }
-    }
-  }
-}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetPreviewsFilter.scala

@@ -1,8 +1,6 @@
 package com.twitter.home_mixer.functional_component.filter
 
-import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
+import com.twitter.home_mixer.model.HomeFeatures.ServedTweetPreviewIdsFeature
-import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
-import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
 import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
 import com.twitter.product_mixer.core.functional_component.filter.Filter
 import com.twitter.product_mixer.core.functional_component.filter.FilterResult
@@ -11,24 +9,20 @@ import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
 import com.twitter.product_mixer.core.pipeline.PipelineQuery
 import com.twitter.stitch.Stitch
 
-object KeepBestOutOfNetworkTweetPerAuthorFilter extends Filter[PipelineQuery, TweetCandidate] {
+object PreviouslyServedTweetPreviewsFilter extends Filter[PipelineQuery, TweetCandidate] {
 
-  override val identifier: FilterIdentifier = FilterIdentifier("KeepBestOutOfNetworkTweetPerAuthor")
+  override val identifier: FilterIdentifier = FilterIdentifier("PreviouslyServedTweetPreviews")
 
   override def apply(
     query: PipelineQuery,
     candidates: Seq[CandidateWithFeatures[TweetCandidate]]
   ): Stitch[FilterResult[TweetCandidate]] = {
-    // Set containing best OON tweet for each authorId
+
-    val bestCandidatesForAuthorId = candidates
+    val servedTweetPreviewIds =
-      .filter(!_.features.getOrElse(InNetworkFeature, true))
+      query.features.map(_.getOrElse(ServedTweetPreviewIdsFeature, Seq.empty)).toSeq.flatten.toSet
-      .groupBy(_.features.getOrElse(AuthorIdFeature, None))
-      .values.map(_.maxBy(_.features.getOrElse(ScoreFeature, None)))
-      .toSet
 
     val (removed, kept) = candidates.partition { candidate =>
-      !candidate.features.getOrElse(InNetworkFeature, true) &&
+      servedTweetPreviewIds.contains(candidate.candidate.id)
-      !bestCandidatesForAuthorId.contains(candidate)
     }
 
     Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate)))

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ReplyFilter.scala

@@ -0,0 +1,32 @@
+package com.twitter.home_mixer.functional_component.filter
+
+import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
+import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
+import com.twitter.product_mixer.core.functional_component.filter.Filter
+import com.twitter.product_mixer.core.functional_component.filter.FilterResult
+import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
+import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
+import com.twitter.product_mixer.core.pipeline.PipelineQuery
+import com.twitter.stitch.Stitch
+
+object ReplyFilter extends Filter[PipelineQuery, TweetCandidate] {
+  override val identifier: FilterIdentifier = FilterIdentifier("Reply")
+
+  override def apply(
+    query: PipelineQuery,
+    candidates: Seq[CandidateWithFeatures[TweetCandidate]]
+  ): Stitch[FilterResult[TweetCandidate]] = {
+
+    val (kept, removed) = candidates
+      .partition { candidate =>
+        candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty
+      }
+
+    val filterResult = FilterResult(
+      kept = kept.map(_.candidate),
+      removed = removed.map(_.candidate)
+    )
+
+    Stitch.value(filterResult)
+  }
+}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetFilter.scala

@@ -0,0 +1,32 @@
+package com.twitter.home_mixer.functional_component.filter
+
+import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
+import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
+import com.twitter.product_mixer.core.functional_component.filter.Filter
+import com.twitter.product_mixer.core.functional_component.filter.FilterResult
+import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
+import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
+import com.twitter.product_mixer.core.pipeline.PipelineQuery
+import com.twitter.stitch.Stitch
+
+object RetweetFilter extends Filter[PipelineQuery, TweetCandidate] {
+  override val identifier: FilterIdentifier = FilterIdentifier("Retweet")
+
+  override def apply(
+    query: PipelineQuery,
+    candidates: Seq[CandidateWithFeatures[TweetCandidate]]
+  ): Stitch[FilterResult[TweetCandidate]] = {
+
+    val (kept, removed) = candidates
+      .partition { candidate =>
+        !candidate.features.getOrElse(IsRetweetFeature, false)
+      }
+
+    val filterResult = FilterResult(
+      kept = kept.map(_.candidate),
+      removed = removed.map(_.candidate)
+    )
+
+    Stitch.value(filterResult)
+  }
+}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/BUILD.bazel

@@ -14,7 +14,6 @@ scala_library(
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate",
         "src/thrift/com/twitter/gizmoduck:thrift-scala",
         "stitch/stitch-socialgraph",
-        "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
         "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
         "timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
     ],

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/DismissFatigueGate.scala

@@ -1,14 +1,14 @@
 package com.twitter.home_mixer.functional_component.gate
 
+import com.twitter.conversions.DurationOps._
+import com.twitter.product_mixer.core.feature.Feature
 import com.twitter.product_mixer.core.functional_component.gate.Gate
 import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier
 import com.twitter.product_mixer.core.pipeline.PipelineQuery
-import com.twitter.timelinemixer.clients.manhattan.DismissInfo
 import com.twitter.stitch.Stitch
-import com.twitter.util.Duration
+import com.twitter.timelinemixer.clients.manhattan.DismissInfo
-import com.twitter.conversions.DurationOps._
-import com.twitter.product_mixer.core.feature.Feature
 import com.twitter.timelineservice.suggests.thriftscala.SuggestType
+import com.twitter.util.Duration
 
 object DismissFatigueGate {
   // how long a dismiss action from user needs to be respected

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/NonEmptySeqFeatureGate.scala

@@ -1,18 +0,0 @@
-package com.twitter.home_mixer.functional_component.gate
-
-import com.twitter.product_mixer.core.feature.Feature
-import com.twitter.product_mixer.core.functional_component.gate.Gate
-import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier
-import com.twitter.product_mixer.core.pipeline.PipelineQuery
-import com.twitter.stitch.Stitch
-import scala.reflect.runtime.universe._
-
-case class NonEmptySeqFeatureGate[T: TypeTag](
-  feature: Feature[PipelineQuery, Seq[T]])
-    extends Gate[PipelineQuery] {
-
-  override val identifier: GateIdentifier = GateIdentifier(s"NonEmptySeq$feature")
-
-  override def shouldContinue(query: PipelineQuery): Stitch[Boolean] =
-    Stitch.value(query.features.exists(_.get(feature).nonEmpty))
-}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer/EditedTweetsCandidatePipelineQueryTransformer.scala

@@ -19,7 +19,7 @@ object EditedTweetsCandidatePipelineQueryTransformer
   override val identifier: TransformerIdentifier = TransformerIdentifier("EditedTweets")
 
   // The time window for which a tweet remains editable after creation.
-  private val EditTimeWindow = 30.minutes
+  private val EditTimeWindow = 60.minutes
 
   override def transform(query: PipelineQuery): Seq[Long] = {
     val applicableCandidates = getApplicableCandidates(query)
@@ -29,8 +29,8 @@ object EditedTweetsCandidatePipelineQueryTransformer
       // Any tweets in it could have become stale since being served.
       val previousTimelineLoadTime = applicableCandidates.head.servedTime
 
-      // The time window for editing a tweet is 30 minutes,
+      // The time window for editing a tweet is 60 minutes,
-      // so we ignore responses older than (PTL Time - 30 mins).
+      // so we ignore responses older than (PTL Time - 60 mins).
       val inWindowCandidates: Seq[PersistenceStoreEntry] = applicableCandidates
         .takeWhile(_.servedTime.until(previousTimelineLoadTime) < EditTimeWindow)
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/BUILD.bazel

@@ -6,7 +6,6 @@ scala_library(
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product",

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/FeedbackFatigueScorer.scala

@@ -35,8 +35,8 @@ object FeedbackFatigueScorer
   override def onlyIf(query: PipelineQuery): Boolean =
     query.features.exists(_.getOrElse(FeedbackHistoryFeature, Seq.empty).nonEmpty)
 
-  private val DurationForFiltering = 14.days
+  val DurationForFiltering = 14.days
-  private val DurationForDiscounting = 140.days
+  val DurationForDiscounting = 140.days
   private val ScoreMultiplierLowerBound = 0.2
   private val ScoreMultiplierUpperBound = 1.0
   private val ScoreMultiplierIncrementsCount = 4
@@ -76,42 +76,56 @@ object FeedbackFatigueScorer
         feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Retweet, Seq.empty))
 
     val featureMaps = candidates.map { candidate =>
+      val multiplier = getScoreMultiplier(
+        candidate,
+        authorsToDiscount,
+        likersToDiscount,
+        followersToDiscount,
+        retweetersToDiscount
+      )
       val score = candidate.features.getOrElse(ScoreFeature, None)
-
-      val originalAuthorId =
-        CandidatesUtil.getOriginalAuthorId(candidate.features).getOrElse(0L)
-      val originalAuthorMultiplier = authorsToDiscount.getOrElse(originalAuthorId, 1.0)
-
-      val likers = candidate.features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty)
-      val likerMultipliers = likers.flatMap(likersToDiscount.get)
-      val likerMultiplier =
-        if (likerMultipliers.nonEmpty && likers.size == likerMultipliers.size)
-          likerMultipliers.max
-        else 1.0
-
-      val followers = candidate.features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty)
-      val followerMultipliers = followers.flatMap(followersToDiscount.get)
-      val followerMultiplier =
-        if (followerMultipliers.nonEmpty && followers.size == followerMultipliers.size)
-          followerMultipliers.max
-        else 1.0
-
-      val authorId = candidate.features.getOrElse(AuthorIdFeature, None).getOrElse(0L)
-      val retweeterMultiplier =
-        if (candidate.features.getOrElse(IsRetweetFeature, false))
-          retweetersToDiscount.getOrElse(authorId, 1.0)
-        else 1.0
-
-      val multiplier =
-        originalAuthorMultiplier * likerMultiplier * followerMultiplier * retweeterMultiplier
-
       FeatureMapBuilder().add(ScoreFeature, score.map(_ * multiplier)).build()
     }
 
     Stitch.value(featureMaps)
   }
 
-  private def getUserDiscounts(
+  def getScoreMultiplier(
+    candidate: CandidateWithFeatures[TweetCandidate],
+    authorsToDiscount: Map[Long, Double],
+    likersToDiscount: Map[Long, Double],
+    followersToDiscount: Map[Long, Double],
+    retweetersToDiscount: Map[Long, Double],
+  ): Double = {
+    val originalAuthorId =
+      CandidatesUtil.getOriginalAuthorId(candidate.features).getOrElse(0L)
+    val originalAuthorMultiplier = authorsToDiscount.getOrElse(originalAuthorId, 1.0)
+
+    val likers = candidate.features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty)
+    val likerMultipliers = likers.flatMap(likersToDiscount.get)
+    val likerMultiplier =
+      if (likerMultipliers.nonEmpty && likers.size == likerMultipliers.size)
+        likerMultipliers.max
+      else 1.0
+
+    val followers = candidate.features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty)
+    val followerMultipliers = followers.flatMap(followersToDiscount.get)
+    val followerMultiplier =
+      if (followerMultipliers.nonEmpty && followers.size == followerMultipliers.size &&
+        likers.isEmpty)
+        followerMultipliers.max
+      else 1.0
+
+    val authorId = candidate.features.getOrElse(AuthorIdFeature, None).getOrElse(0L)
+    val retweeterMultiplier =
+      if (candidate.features.getOrElse(IsRetweetFeature, false))
+        retweetersToDiscount.getOrElse(authorId, 1.0)
+      else 1.0
+
+    originalAuthorMultiplier * likerMultiplier * followerMultiplier * retweeterMultiplier
+  }
+
+  def getUserDiscounts(
     queryTime: Time,
     feedbackEntries: Seq[FeedbackEntry],
   ): Map[Long, Double] = {

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/VerifiedAuthorScalingScorer.scala

@@ -1,61 +0,0 @@
-package com.twitter.home_mixer.functional_component.scorer
-
-import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature
-import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
-import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
-import com.twitter.home_mixer.param.HomeGlobalParams.BlueVerifiedAuthorInNetworkMultiplierParam
-import com.twitter.home_mixer.param.HomeGlobalParams.BlueVerifiedAuthorOutOfNetworkMultiplierParam
-import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
-import com.twitter.product_mixer.core.feature.Feature
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
-import com.twitter.product_mixer.core.functional_component.scorer.Scorer
-import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
-import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier
-import com.twitter.product_mixer.core.pipeline.PipelineQuery
-import com.twitter.stitch.Stitch
-
-/**
- * Scales scores of tweets whose author is Blue Verified by the provided scale factor
- */
-object VerifiedAuthorScalingScorer extends Scorer[PipelineQuery, TweetCandidate] {
-
-  override val identifier: ScorerIdentifier = ScorerIdentifier("VerifiedAuthorScaling")
-
-  override val features: Set[Feature[_, _]] = Set(ScoreFeature)
-
-  override def apply(
-    query: PipelineQuery,
-    candidates: Seq[CandidateWithFeatures[TweetCandidate]]
-  ): Stitch[Seq[FeatureMap]] = {
-    Stitch.value {
-      candidates.map { candidate =>
-        val score = candidate.features.getOrElse(ScoreFeature, None)
-        val updatedScore = getUpdatedScore(score, candidate, query)
-        FeatureMapBuilder().add(ScoreFeature, updatedScore).build()
-      }
-    }
-  }
-
-  /**
-   * We should only be applying this multiplier if the author of the candidate is Blue Verified.
-   * We also treat In-Network vs Out-of-Network differently.
-   */
-  private def getUpdatedScore(
-    score: Option[Double],
-    candidate: CandidateWithFeatures[TweetCandidate],
-    query: PipelineQuery
-  ): Option[Double] = {
-    val isAuthorBlueVerified = candidate.features.getOrElse(AuthorIsBlueVerifiedFeature, false)
-
-    if (isAuthorBlueVerified) {
-      val isCandidateInNetwork = candidate.features.getOrElse(InNetworkFeature, false)
-
-      val scaleFactor =
-        if (isCandidateInNetwork) query.params(BlueVerifiedAuthorInNetworkMultiplierParam)
-        else query.params(BlueVerifiedAuthorOutOfNetworkMultiplierParam)
-
-      score.map(_ * scaleFactor)
-    } else score
-  }
-}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/BUILD.bazel

@@ -5,6 +5,7 @@ scala_library(
     tags = ["bazel-compatible"],
     dependencies = [
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator",
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateHomeClientEventDetails.scala

@@ -1,6 +1,6 @@
 package com.twitter.home_mixer.functional_component.selector
 
-import com.twitter.home_mixer.functional_component.decorator.HomeClientEventDetailsBuilder
+import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventDetailsBuilder
 import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature
 import com.twitter.home_mixer.model.HomeFeatures.ConversationModule2DisplayedTweetsFeature
 import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleHasGapFeature

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/BUILD.bazel

@@ -7,14 +7,12 @@ scala_library(
         "3rdparty/jvm/javax/inject:javax.inject",
         "eventbus/client/src/main/scala/com/twitter/eventbus/client",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator",
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
         "kafka/finagle-kafka/finatra-kafka/src/main/scala",
@@ -22,6 +20,7 @@ scala_library(
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/urt",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module",
+        "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_subscribe_module",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product",
         "src/scala/com/twitter/timelines/prediction/common/adapters",
@@ -33,13 +32,12 @@ scala_library(
         "src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java",
         "src/thrift/com/twitter/timelines/timeline_logging:thrift-scala",
         "src/thrift/com/twitter/user_session_store:thrift-scala",
-        "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/core",
         "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
-        "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/uss",
         "timelines/ml:kafka",
         "timelines/ml/cont_train/common/client/src/main/scala/com/twitter/timelines/ml/cont_train/common/client/kafka",
         "timelines/ml/cont_train/common/domain/src/main/scala/com/twitter/timelines/ml/cont_train/common/domain/non_scalding",
         "timelines/src/main/scala/com/twitter/timelines/clientconfig",
+        "timelines/src/main/scala/com/twitter/timelines/clients/manhattan/store",
         "timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter",
         "timelines/src/main/scala/com/twitter/timelines/impressionstore/store",
         "timelines/src/main/scala/com/twitter/timelines/injection/scribe",

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ClientEventsBuilder.scala

@@ -2,13 +2,14 @@ package com.twitter.home_mixer.functional_component.side_effect
 
 import com.twitter.conversions.DurationOps._
 import com.twitter.home_mixer.functional_component.decorator.HomeQueryTypePredicates
-import com.twitter.home_mixer.functional_component.decorator.HomeTweetTypePredicates
+import com.twitter.home_mixer.functional_component.decorator.builder.HomeTweetTypePredicates
 import com.twitter.home_mixer.model.HomeFeatures.AccountAgeFeature
 import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
 import com.twitter.home_mixer.model.HomeFeatures.VideoDurationMsFeature
 import com.twitter.home_mixer.model.request.FollowingProduct
 import com.twitter.home_mixer.model.request.ForYouProduct
 import com.twitter.home_mixer.model.request.ListTweetsProduct
+import com.twitter.home_mixer.model.request.SubscribedProduct
 import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.ClientEvent
 import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.EventNamespace
 import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
@@ -18,15 +19,17 @@ import com.twitter.product_mixer.core.pipeline.PipelineQuery
 import com.twitter.timelines.injection.scribe.InjectionScribeUtil
 
 private[side_effect] sealed trait ClientEventsBuilder {
-  private val FollowingSection = Some("home_latest")
+  private val FollowingSection = Some("latest")
   private val ForYouSection = Some("home")
   private val ListTweetsSection = Some("list")
+  private val SubscribedSection = Some("subscribed")
 
   protected def section(query: PipelineQuery): Option[String] = {
     query.product match {
       case FollowingProduct => FollowingSection
       case ForYouProduct => ForYouSection
       case ListTweetsProduct => ListTweetsSection
+      case SubscribedProduct => SubscribedSection
       case other => throw new UnsupportedOperationException(s"Unknown product: $other")
     }
   }
@@ -52,6 +55,7 @@ private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder {
   private val InjectedComponent = Some("injected")
   private val PromotedComponent = Some("promoted")
   private val WhoToFollowComponent = Some("who_to_follow")
+  private val WhoToSubscribeComponent = Some("who_to_subscribe")
   private val WithVideoDurationComponent = Some("with_video_duration")
   private val VideoDurationSumElement = Some("video_duration_sum")
   private val NumVideosElement = Some("num_videos")
@@ -60,7 +64,8 @@ private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder {
     query: PipelineQuery,
     injectedTweets: Seq[ItemCandidateWithDetails],
     promotedTweets: Seq[ItemCandidateWithDetails],
-    whoToFollowUsers: Seq[ItemCandidateWithDetails]
+    whoToFollowUsers: Seq[ItemCandidateWithDetails],
+    whoToSubscribeUsers: Seq[ItemCandidateWithDetails]
   ): Seq[ClientEvent] = {
     val baseEventNamespace = EventNamespace(
       section = section(query),
@@ -77,6 +82,9 @@ private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder {
       ClientEvent(
         baseEventNamespace.copy(component = WhoToFollowComponent, action = ServedUsersAction),
         eventValue = count(whoToFollowUsers)),
+      ClientEvent(
+        baseEventNamespace.copy(component = WhoToSubscribeComponent, action = ServedUsersAction),
+        eventValue = count(whoToSubscribeUsers)),
     )
 
     val tweetTypeServedEvents = HomeTweetTypePredicates.PredicateMap.map {

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeClientEventSideEffect.scala

@@ -5,6 +5,7 @@ import com.twitter.home_mixer.service.HomeMixerAlertConfig
 import com.twitter.home_mixer.util.CandidatesUtil
 import com.twitter.logpipeline.client.common.EventPublisher
 import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect
+import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
 import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier
 import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
 import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
@@ -15,16 +16,30 @@ import com.twitter.product_mixer.core.pipeline.PipelineQuery
  * Side effect that logs served tweet metrics to Scribe as client events.
  */
 case class HomeScribeClientEventSideEffect(
+  enableScribeClientEvents: Boolean,
   override val logPipelinePublisher: EventPublisher[LogEvent],
   injectedTweetsCandidatePipelineIdentifiers: Seq[CandidatePipelineIdentifier],
-  adsCandidatePipelineIdentifier: CandidatePipelineIdentifier,
+  adsCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None,
   whoToFollowCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None,
-) extends ScribeClientEventSideEffect[PipelineQuery, Timeline] {
+  whoToSubscribeCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None)
+    extends ScribeClientEventSideEffect[PipelineQuery, Timeline]
+    with PipelineResultSideEffect.Conditionally[
+      PipelineQuery,
+      Timeline
+    ] {
 
   override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeClientEvent")
 
   override val page = "timelinemixer"
 
+  override def onlyIf(
+    query: PipelineQuery,
+    selectedCandidates: Seq[CandidateWithDetails],
+    remainingCandidates: Seq[CandidateWithDetails],
+    droppedCandidates: Seq[CandidateWithDetails],
+    response: Timeline
+  ): Boolean = enableScribeClientEvents
+
   override def buildClientEvents(
     query: PipelineQuery,
     selectedCandidates: Seq[CandidateWithDetails],
@@ -37,13 +52,15 @@ case class HomeScribeClientEventSideEffect(
     val sources = itemCandidates.groupBy(_.source)
     val injectedTweets =
       injectedTweetsCandidatePipelineIdentifiers.flatMap(sources.getOrElse(_, Seq.empty))
-    val promotedTweets = sources.getOrElse(adsCandidatePipelineIdentifier, Seq.empty)
+    val promotedTweets = adsCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten
 
-    // WhoToFollow module is not required for all home-mixer products, e.g. list tweets timeline.
+    // WhoToFollow and WhoToSubscribe modules are not required for all home-mixer products, e.g. list tweets timeline.
     val whoToFollowUsers = whoToFollowCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten
+    val whoToSubscribeUsers =
+      whoToSubscribeCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten
 
     val servedEvents = ServedEventsBuilder
-      .build(query, injectedTweets, promotedTweets, whoToFollowUsers)
+      .build(query, injectedTweets, promotedTweets, whoToFollowUsers, whoToSubscribeUsers)
 
     val emptyTimelineEvents = EmptyTimelineEventsBuilder.build(query, injectedTweets)
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeServedCandidatesSideEffect.scala

@@ -0,0 +1,245 @@
+package com.twitter.home_mixer.functional_component.side_effect
+
+import com.twitter.finagle.tracing.Trace
+import com.twitter.home_mixer.marshaller.timeline_logging.PromotedTweetDetailsMarshaller
+import com.twitter.home_mixer.marshaller.timeline_logging.TweetDetailsMarshaller
+import com.twitter.home_mixer.marshaller.timeline_logging.WhoToFollowDetailsMarshaller
+import com.twitter.home_mixer.model.HomeFeatures.GetInitialFeature
+import com.twitter.home_mixer.model.HomeFeatures.GetMiddleFeature
+import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature
+import com.twitter.home_mixer.model.HomeFeatures.GetOlderFeature
+import com.twitter.home_mixer.model.HomeFeatures.HasDarkRequestFeature
+import com.twitter.home_mixer.model.HomeFeatures.RequestJoinIdFeature
+import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
+import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature
+import com.twitter.home_mixer.model.request.DeviceContext.RequestContext
+import com.twitter.home_mixer.model.request.HasDeviceContext
+import com.twitter.home_mixer.model.request.HasSeenTweetIds
+import com.twitter.home_mixer.model.request.FollowingProduct
+import com.twitter.home_mixer.model.request.ForYouProduct
+import com.twitter.home_mixer.model.request.SubscribedProduct
+import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeServedCandidatesFlag
+import com.twitter.home_mixer.param.HomeGlobalParams.EnableScribeServedCandidatesParam
+import com.twitter.home_mixer.service.HomeMixerAlertConfig
+import com.twitter.inject.annotations.Flag
+import com.twitter.logpipeline.client.common.EventPublisher
+import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate
+import com.twitter.product_mixer.component_library.model.candidate.BaseUserCandidate
+import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator
+import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidateDecorator
+import com.twitter.product_mixer.component_library.side_effect.ScribeLogEventSideEffect
+import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
+import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
+import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
+import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
+import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails
+import com.twitter.product_mixer.core.model.marshalling.response.urt.AddEntriesTimelineInstruction
+import com.twitter.product_mixer.core.model.marshalling.response.urt.ModuleItem
+import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline
+import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule
+import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
+import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem
+import com.twitter.product_mixer.core.pipeline.PipelineQuery
+import com.twitter.timelines.timeline_logging.{thriftscala => thrift}
+import com.twitter.util.Time
+import javax.inject.Inject
+import javax.inject.Singleton
+
+/**
+ * Side effect that logs home timeline served candidates to Scribe.
+ */
+@Singleton
+class HomeScribeServedCandidatesSideEffect @Inject() (
+  @Flag(ScribeServedCandidatesFlag) enableScribeServedCandidates: Boolean,
+  scribeEventPublisher: EventPublisher[thrift.ServedEntry])
+    extends ScribeLogEventSideEffect[
+      thrift.ServedEntry,
+      PipelineQuery with HasSeenTweetIds with HasDeviceContext,
+      Timeline
+    ]
+    with PipelineResultSideEffect.Conditionally[
+      PipelineQuery with HasSeenTweetIds with HasDeviceContext,
+      Timeline
+    ] {
+
+  override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeServedCandidates")
+
+  override def onlyIf(
+    query: PipelineQuery with HasSeenTweetIds with HasDeviceContext,
+    selectedCandidates: Seq[CandidateWithDetails],
+    remainingCandidates: Seq[CandidateWithDetails],
+    droppedCandidates: Seq[CandidateWithDetails],
+    response: Timeline
+  ): Boolean = enableScribeServedCandidates && query.params(EnableScribeServedCandidatesParam)
+
+  override def buildLogEvents(
+    query: PipelineQuery with HasSeenTweetIds with HasDeviceContext,
+    selectedCandidates: Seq[CandidateWithDetails],
+    remainingCandidates: Seq[CandidateWithDetails],
+    droppedCandidates: Seq[CandidateWithDetails],
+    response: Timeline
+  ): Seq[thrift.ServedEntry] = {
+    val timelineType = query.product match {
+      case FollowingProduct => thrift.TimelineType.HomeLatest
+      case ForYouProduct => thrift.TimelineType.Home
+      case SubscribedProduct => thrift.TimelineType.HomeSubscribed
+      case other => throw new UnsupportedOperationException(s"Unknown product: $other")
+    }
+    val requestProvenance = query.deviceContext.map { deviceContext =>
+      deviceContext.requestContextValue match {
+        case RequestContext.Foreground => thrift.RequestProvenance.Foreground
+        case RequestContext.Launch => thrift.RequestProvenance.Launch
+        case RequestContext.PullToRefresh => thrift.RequestProvenance.Ptr
+        case _ => thrift.RequestProvenance.Other
+      }
+    }
+    val queryType = query.features.map { featureMap =>
+      if (featureMap.getOrElse(GetOlderFeature, false)) thrift.QueryType.GetOlder
+      else if (featureMap.getOrElse(GetNewerFeature, false)) thrift.QueryType.GetNewer
+      else if (featureMap.getOrElse(GetMiddleFeature, false)) thrift.QueryType.GetMiddle
+      else if (featureMap.getOrElse(GetInitialFeature, false)) thrift.QueryType.GetInitial
+      else thrift.QueryType.Other
+    }
+    val requestInfo = thrift.RequestInfo(
+      requestTimeMs = query.queryTime.inMilliseconds,
+      traceId = Trace.id.traceId.toLong,
+      userId = query.getOptionalUserId,
+      clientAppId = query.clientContext.appId,
+      hasDarkRequest = query.features.flatMap(_.getOrElse(HasDarkRequestFeature, None)),
+      parentId = Some(Trace.id.parentId.toLong),
+      spanId = Some(Trace.id.spanId.toLong),
+      timelineType = Some(timelineType),
+      ipAddress = query.clientContext.ipAddress,
+      userAgent = query.clientContext.userAgent,
+      queryType = queryType,
+      requestProvenance = requestProvenance,
+      languageCode = query.clientContext.languageCode,
+      countryCode = query.clientContext.countryCode,
+      requestEndTimeMs = Some(Time.now.inMilliseconds),
+      servedRequestId = query.features.flatMap(_.getOrElse(ServedRequestIdFeature, None)),
+      requestJoinId = query.features.flatMap(_.getOrElse(RequestJoinIdFeature, None))
+    )
+
+    val tweetIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] =
+      selectedCandidates.flatMap {
+        case item: ItemCandidateWithDetails if item.candidate.isInstanceOf[BaseTweetCandidate] =>
+          Seq((item.candidateIdLong, item))
+        case module: ModuleCandidateWithDetails
+            if module.candidates.headOption.exists(_.candidate.isInstanceOf[BaseTweetCandidate]) =>
+          module.candidates.map(item => (item.candidateIdLong, item))
+        case _ => Seq.empty
+      }.toMap
+
+    val userIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] =
+      selectedCandidates.flatMap {
+        case module: ModuleCandidateWithDetails
+            if module.candidates.forall(_.candidate.isInstanceOf[BaseUserCandidate]) =>
+          module.candidates.map { item =>
+            (item.candidateIdLong, item)
+          }
+        case _ => Seq.empty
+      }.toMap
+
+    response.instructions.zipWithIndex
+      .collect {
+        case (AddEntriesTimelineInstruction(entries), index) =>
+          entries.collect {
+            case entry: TweetItem if entry.promotedMetadata.isDefined =>
+              val promotedTweetDetails = PromotedTweetDetailsMarshaller(entry, index)
+              Seq(
+                thrift.EntryInfo(
+                  id = entry.id,
+                  position = index.shortValue(),
+                  entryId = entry.entryIdentifier,
+                  entryType = thrift.EntryType.PromotedTweet,
+                  sortIndex = entry.sortIndex,
+                  verticalSize = Some(1),
+                  displayType = Some(entry.displayType.toString),
+                  details = Some(thrift.ItemDetails.PromotedTweetDetails(promotedTweetDetails))
+                )
+              )
+            case entry: TweetItem =>
+              val candidate = tweetIdToItemCandidateMap(entry.id)
+              val tweetDetails = TweetDetailsMarshaller(entry, candidate)
+              Seq(
+                thrift.EntryInfo(
+                  id = candidate.candidateIdLong,
+                  position = index.shortValue(),
+                  entryId = entry.entryIdentifier,
+                  entryType = thrift.EntryType.Tweet,
+                  sortIndex = entry.sortIndex,
+                  verticalSize = Some(1),
+                  score = candidate.features.getOrElse(ScoreFeature, None),
+                  displayType = Some(entry.displayType.toString),
+                  details = Some(thrift.ItemDetails.TweetDetails(tweetDetails))
+                )
+              )
+            case module: TimelineModule
+                if module.entryNamespace.toString == WhoToFollowCandidateDecorator.EntryNamespaceString =>
+              module.items.collect {
+                case ModuleItem(entry: UserItem, _, _) =>
+                  val candidate = userIdToItemCandidateMap(entry.id)
+                  val whoToFollowDetails = WhoToFollowDetailsMarshaller(entry, candidate)
+                  thrift.EntryInfo(
+                    id = entry.id,
+                    position = index.shortValue(),
+                    entryId = module.entryIdentifier,
+                    entryType = thrift.EntryType.WhoToFollowModule,
+                    sortIndex = module.sortIndex,
+                    score = candidate.features.getOrElse(ScoreFeature, None),
+                    displayType = Some(entry.displayType.toString),
+                    details = Some(thrift.ItemDetails.WhoToFollowDetails(whoToFollowDetails))
+                  )
+              }
+            case module: TimelineModule
+                if module.entryNamespace.toString == WhoToSubscribeCandidateDecorator.EntryNamespaceString =>
+              module.items.collect {
+                case ModuleItem(entry: UserItem, _, _) =>
+                  val candidate = userIdToItemCandidateMap(entry.id)
+                  val whoToSubscribeDetails = WhoToFollowDetailsMarshaller(entry, candidate)
+                  thrift.EntryInfo(
+                    id = entry.id,
+                    position = index.shortValue(),
+                    entryId = module.entryIdentifier,
+                    entryType = thrift.EntryType.WhoToSubscribeModule,
+                    sortIndex = module.sortIndex,
+                    score = candidate.features.getOrElse(ScoreFeature, None),
+                    displayType = Some(entry.displayType.toString),
+                    details = Some(thrift.ItemDetails.WhoToFollowDetails(whoToSubscribeDetails))
+                  )
+              }
+            case module: TimelineModule
+                if module.sortIndex.isDefined && module.items.headOption.exists(
+                  _.item.isInstanceOf[TweetItem]) =>
+              module.items.collect {
+                case ModuleItem(entry: TweetItem, _, _) =>
+                  val candidate = tweetIdToItemCandidateMap(entry.id)
+                  thrift.EntryInfo(
+                    id = entry.id,
+                    position = index.shortValue(),
+                    entryId = module.entryIdentifier,
+                    entryType = thrift.EntryType.ConversationModule,
+                    sortIndex = module.sortIndex,
+                    score = candidate.features.getOrElse(ScoreFeature, None),
+                    displayType = Some(entry.displayType.toString)
+                  )
+              }
+            case _ => Seq.empty
+          }.flatten
+        // Other instructions
+        case _ => Seq.empty[thrift.EntryInfo]
+      }.flatten.map { entryInfo =>
+        thrift.ServedEntry(
+          entry = Some(entryInfo),
+          request = requestInfo
+        )
+      }
+  }
+
+  override val logPipelinePublisher: EventPublisher[thrift.ServedEntry] =
+    scribeEventPublisher
+
+  override val alerts = Seq(
+    HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert()
+  )
+}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeServedEntriesSideEffect.scala

@@ -1,212 +0,0 @@
-package com.twitter.home_mixer.functional_component.side_effect
-
-import com.twitter.finagle.tracing.Trace
-import com.twitter.home_mixer.marshaller.timeline_logging.ConversationEntryMarshaller
-import com.twitter.home_mixer.marshaller.timeline_logging.PromotedTweetEntryMarshaller
-import com.twitter.home_mixer.marshaller.timeline_logging.TweetEntryMarshaller
-import com.twitter.home_mixer.marshaller.timeline_logging.WhoToFollowEntryMarshaller
-import com.twitter.home_mixer.model.HomeFeatures.GetInitialFeature
-import com.twitter.home_mixer.model.HomeFeatures.GetMiddleFeature
-import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature
-import com.twitter.home_mixer.model.HomeFeatures.GetOlderFeature
-import com.twitter.home_mixer.model.HomeFeatures.HasDarkRequestFeature
-import com.twitter.home_mixer.model.HomeFeatures.RequestJoinIdFeature
-import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature
-import com.twitter.home_mixer.model.request.DeviceContext.RequestContext
-import com.twitter.home_mixer.model.request.HasDeviceContext
-import com.twitter.home_mixer.model.request.HasSeenTweetIds
-import com.twitter.home_mixer.model.request.FollowingProduct
-import com.twitter.home_mixer.model.request.ForYouProduct
-import com.twitter.home_mixer.service.HomeMixerAlertConfig
-import com.twitter.logpipeline.client.common.EventPublisher
-import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate
-import com.twitter.product_mixer.component_library.model.candidate.BaseUserCandidate
-import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator
-import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
-import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
-import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
-import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails
-import com.twitter.product_mixer.core.model.marshalling.response.urt.AddEntriesTimelineInstruction
-import com.twitter.product_mixer.core.model.marshalling.response.urt.ModuleItem
-import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline
-import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule
-import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
-import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem
-import com.twitter.product_mixer.core.pipeline.PipelineQuery
-import com.twitter.stitch.Stitch
-import com.twitter.timelines.timeline_logging.{thriftscala => thrift}
-import com.twitter.util.Time
-import javax.inject.Inject
-import javax.inject.Singleton
-
-/**
- * Side effect that logs home timeline served entries to Scribe.
- */
-@Singleton
-class HomeScribeServedEntriesSideEffect @Inject() (
-  scribeEventPublisher: EventPublisher[thrift.Timeline])
-    extends PipelineResultSideEffect[
-      PipelineQuery with HasSeenTweetIds with HasDeviceContext,
-      Timeline
-    ] {
-
-  override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeServedEntries")
-
-  final override def apply(
-    inputs: PipelineResultSideEffect.Inputs[
-      PipelineQuery with HasSeenTweetIds with HasDeviceContext,
-      Timeline
-    ]
-  ): Stitch[Unit] = {
-    val timelineThrift = buildTimeline(inputs)
-    Stitch.callFuture(scribeEventPublisher.publish(timelineThrift)).unit
-  }
-
-  def buildTimeline(
-    inputs: PipelineResultSideEffect.Inputs[
-      PipelineQuery with HasSeenTweetIds with HasDeviceContext,
-      Timeline
-    ]
-  ): thrift.Timeline = {
-    val timelineType = inputs.query.product match {
-      case FollowingProduct => thrift.TimelineType.HomeLatest
-      case ForYouProduct => thrift.TimelineType.Home
-      case other => throw new UnsupportedOperationException(s"Unknown product: $other")
-    }
-    val requestProvenance = inputs.query.deviceContext.map { deviceContext =>
-      deviceContext.requestContextValue match {
-        case RequestContext.Foreground => thrift.RequestProvenance.Foreground
-        case RequestContext.Launch => thrift.RequestProvenance.Launch
-        case RequestContext.PullToRefresh => thrift.RequestProvenance.Ptr
-        case _ => thrift.RequestProvenance.Other
-      }
-    }
-    val queryType = inputs.query.features.map { featureMap =>
-      if (featureMap.getOrElse(GetOlderFeature, false)) thrift.QueryType.GetOlder
-      else if (featureMap.getOrElse(GetNewerFeature, false)) thrift.QueryType.GetNewer
-      else if (featureMap.getOrElse(GetMiddleFeature, false)) thrift.QueryType.GetMiddle
-      else if (featureMap.getOrElse(GetInitialFeature, false)) thrift.QueryType.GetInitial
-      else thrift.QueryType.Other
-    }
-
-    val tweetIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] =
-      inputs.selectedCandidates.flatMap {
-        case item: ItemCandidateWithDetails if item.candidate.isInstanceOf[BaseTweetCandidate] =>
-          Seq((item.candidateIdLong, item))
-        case module: ModuleCandidateWithDetails
-            if module.candidates.headOption.exists(_.candidate.isInstanceOf[BaseTweetCandidate]) =>
-          module.candidates.map(item => (item.candidateIdLong, item))
-        case _ => Seq.empty
-      }.toMap
-
-    val userIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] =
-      inputs.selectedCandidates.flatMap {
-        case module: ModuleCandidateWithDetails
-            if module.candidates.forall(_.candidate.isInstanceOf[BaseUserCandidate]) =>
-          module.candidates.map { item =>
-            (item.candidateIdLong, item)
-          }
-        case _ => Seq.empty
-      }.toMap
-
-    val timelineEntries = inputs.response.instructions.zipWithIndex.collect {
-      case (AddEntriesTimelineInstruction(entries), index) =>
-        entries.collect {
-          case entry: TweetItem if entry.promotedMetadata.isDefined =>
-            val promotedTweetEntry = PromotedTweetEntryMarshaller(entry, index)
-            Seq(
-              thrift.TimelineEntry(
-                content = thrift.Content.PromotedTweetEntry(promotedTweetEntry),
-                position = index.shortValue(),
-                entryId = entry.entryIdentifier,
-                entryType = thrift.EntryType.PromotedTweet,
-                sortIndex = entry.sortIndex,
-                verticalSize = Some(1)
-              )
-            )
-          case entry: TweetItem =>
-            val candidate = tweetIdToItemCandidateMap(entry.id)
-            val tweetEntry = TweetEntryMarshaller(entry, candidate)
-            Seq(
-              thrift.TimelineEntry(
-                content = thrift.Content.TweetEntry(tweetEntry),
-                position = index.shortValue(),
-                entryId = entry.entryIdentifier,
-                entryType = thrift.EntryType.Tweet,
-                sortIndex = entry.sortIndex,
-                verticalSize = Some(1)
-              )
-            )
-          case module: TimelineModule
-              if module.entryNamespace.toString == WhoToFollowCandidateDecorator.EntryNamespaceString =>
-            val whoToFollowEntries = module.items.collect {
-              case ModuleItem(entry: UserItem, _, _) =>
-                val candidate = userIdToItemCandidateMap(entry.id)
-                val whoToFollowEntry = WhoToFollowEntryMarshaller(entry, candidate)
-                thrift.AtomicEntry.WtfEntry(whoToFollowEntry)
-            }
-            Seq(
-              thrift.TimelineEntry(
-                content = thrift.Content.Entries(whoToFollowEntries),
-                position = index.shortValue(),
-                entryId = module.entryIdentifier,
-                entryType = thrift.EntryType.WhoToFollowModule,
-                sortIndex = module.sortIndex
-              )
-            )
-          case module: TimelineModule
-              if module.sortIndex.isDefined && module.items.headOption.exists(
-                _.item.isInstanceOf[TweetItem]) =>
-            val conversationTweetEntries = module.items.collect {
-              case ModuleItem(entry: TweetItem, _, _) =>
-                val candidate = tweetIdToItemCandidateMap(entry.id)
-                val conversationEntry = ConversationEntryMarshaller(entry, candidate)
-                thrift.AtomicEntry.ConversationEntry(conversationEntry)
-            }
-            Seq(
-              thrift.TimelineEntry(
-                content = thrift.Content.Entries(conversationTweetEntries),
-                position = index.shortValue(),
-                entryId = module.entryIdentifier,
-                entryType = thrift.EntryType.ConversationModule,
-                sortIndex = module.sortIndex
-              )
-            )
-          case _ => Seq.empty
-        }.flatten
-      // Other instructions
-      case _ => Seq.empty[thrift.TimelineEntry]
-    }.flatten
-
-    thrift.Timeline(
-      timelineEntries = timelineEntries,
-      requestTimeMs = inputs.query.queryTime.inMilliseconds,
-      traceId = Trace.id.traceId.toLong,
-      userId = inputs.query.getOptionalUserId,
-      clientAppId = inputs.query.clientContext.appId,
-      sourceJobInstance = None,
-      hasDarkRequest = inputs.query.features.flatMap(_.getOrElse(HasDarkRequestFeature, None)),
-      parentId = Some(Trace.id.parentId.toLong),
-      spanId = Some(Trace.id.spanId.toLong),
-      timelineType = Some(timelineType),
-      ipAddress = inputs.query.clientContext.ipAddress,
-      userAgent = inputs.query.clientContext.userAgent,
-      queryType = queryType,
-      requestProvenance = requestProvenance,
-      sessionId = None,
-      timeZone = None,
-      browserNotificationPermission = None,
-      lastNonePollingTimeMs = None,
-      languageCode = inputs.query.clientContext.languageCode,
-      countryCode = inputs.query.clientContext.countryCode,
-      requestEndTimeMs = Some(Time.now.inMilliseconds),
-      servedRequestId = inputs.query.features.flatMap(_.getOrElse(ServedRequestIdFeature, None)),
-      requestJoinId = inputs.query.features.flatMap(_.getOrElse(RequestJoinIdFeature, None)),
-      requestSeenTweetIds = inputs.query.seenTweetIds
-    )
-  }
-
-  override val alerts = Seq(
-    HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert()
-  )
-}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsEventBusSideEffect.scala

@@ -3,6 +3,7 @@ package com.twitter.home_mixer.functional_component.side_effect
 import com.twitter.eventbus.client.EventBusPublisher
 import com.twitter.home_mixer.model.request.FollowingProduct
 import com.twitter.home_mixer.model.request.ForYouProduct
+import com.twitter.home_mixer.model.request.SubscribedProduct
 import com.twitter.home_mixer.model.request.HasSeenTweetIds
 import com.twitter.home_mixer.service.HomeMixerAlertConfig
 import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
@@ -22,6 +23,7 @@ import javax.inject.Singleton
 object PublishClientSentImpressionsEventBusSideEffect {
   val HomeSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeTimeline))
   val HomeLatestSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeLatestTimeline))
+  val HomeSubscribedSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeSubscribed))
 }
 
 /**
@@ -56,6 +58,7 @@ class PublishClientSentImpressionsEventBusSideEffect @Inject() (
     val surfaceArea = query.product match {
       case ForYouProduct => HomeSurfaceArea
       case FollowingProduct => HomeLatestSurfaceArea
+      case SubscribedProduct => HomeSubscribedSurfaceArea
       case _ => None
     }
     query.seenTweetIds.map { seenTweetIds =>

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishImpressionBloomFilterSideEffect.scala

@@ -1,39 +1,46 @@
 package com.twitter.home_mixer.functional_component.side_effect
 
-import com.twitter.conversions.DurationOps._
 import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature
 import com.twitter.home_mixer.model.request.HasSeenTweetIds
+import com.twitter.home_mixer.param.HomeGlobalParams.EnableImpressionBloomFilter
 import com.twitter.home_mixer.service.HomeMixerAlertConfig
 import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
 import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
 import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
-import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline
+import com.twitter.product_mixer.core.model.marshalling.HasMarshalling
 import com.twitter.product_mixer.core.pipeline.PipelineQuery
 import com.twitter.stitch.Stitch
-import com.twitter.timelines.impressionbloomfilter.{thriftscala => t}
+import com.twitter.timelines.clients.manhattan.store.ManhattanStoreClient
-import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilter
+import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm}
 import javax.inject.Inject
 import javax.inject.Singleton
 
 @Singleton
-class UpdateImpressionBloomFilterSideEffect @Inject() (bloomFilter: ImpressionBloomFilter)
+class PublishImpressionBloomFilterSideEffect @Inject() (
-    extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, Timeline]
+  bloomFilterClient: ManhattanStoreClient[
-    with PipelineResultSideEffect.Conditionally[PipelineQuery with HasSeenTweetIds, Timeline] {
+    blm.ImpressionBloomFilterKey,
-
+    blm.ImpressionBloomFilterSeq
-  private val SurfaceArea = t.SurfaceArea.HomeTimeline
+  ]) extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, HasMarshalling]
+    with PipelineResultSideEffect.Conditionally[
+      PipelineQuery with HasSeenTweetIds,
+      HasMarshalling
+    ] {
 
   override val identifier: SideEffectIdentifier =
-    SideEffectIdentifier("UpdateImpressionBloomFilter")
+    SideEffectIdentifier("PublishImpressionBloomFilter")
+
+  private val SurfaceArea = blm.SurfaceArea.HomeTimeline
 
   override def onlyIf(
     query: PipelineQuery with HasSeenTweetIds,
     selectedCandidates: Seq[CandidateWithDetails],
     remainingCandidates: Seq[CandidateWithDetails],
     droppedCandidates: Seq[CandidateWithDetails],
-    response: Timeline
+    response: HasMarshalling
-  ): Boolean = query.seenTweetIds.exists(_.nonEmpty)
+  ): Boolean =
+    query.params.getBoolean(EnableImpressionBloomFilter) && query.seenTweetIds.exists(_.nonEmpty)
 
-  def buildEvents(query: PipelineQuery): Option[t.ImpressionBloomFilterSeq] = {
+  def buildEvents(query: PipelineQuery): Option[blm.ImpressionBloomFilterSeq] = {
     query.features.flatMap { featureMap =>
       val impressionBloomFilterSeq = featureMap.get(ImpressionBloomFilterFeature)
       if (impressionBloomFilterSeq.entries.nonEmpty) Some(impressionBloomFilterSeq)
@@ -42,19 +49,17 @@ class UpdateImpressionBloomFilterSideEffect @Inject() (bloomFilter: ImpressionBl
   }
 
   override def apply(
-    inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, Timeline]
+    inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, HasMarshalling]
   ): Stitch[Unit] = {
     buildEvents(inputs.query)
-      .map { updatedBloomFilter =>
+      .map { updatedBloomFilterSeq =>
-        bloomFilter.writeBloomFilterSeq(
+        bloomFilterClient.write(
-          userId = inputs.query.getRequiredUserId,
+          blm.ImpressionBloomFilterKey(inputs.query.getRequiredUserId, SurfaceArea),
-          surfaceArea = SurfaceArea,
+          updatedBloomFilterSeq)
-          impressionBloomFilterSeq = updatedBloomFilter)
       }.getOrElse(Stitch.Unit)
   }
 
   override val alerts = Seq(
-    HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8),
+    HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8)
-    HomeMixerAlertConfig.BusinessHours.defaultLatencyAlert(30.millis)
   )
 }

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ServedStatsSideEffect.scala

@@ -1,80 +0,0 @@
-package com.twitter.home_mixer.functional_component.side_effect
-
-import com.twitter.finagle.stats.StatsReceiver
-import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
-import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature
-import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
-import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
-import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
-import com.twitter.home_mixer.param.HomeGlobalParams.AuthorListForStatsParam
-import com.twitter.home_mixer.util.CandidatesUtil
-import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
-import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
-import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
-import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
-import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
-import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline
-import com.twitter.product_mixer.core.pipeline.PipelineQuery
-import com.twitter.stitch.Stitch
-import javax.inject.Inject
-import javax.inject.Singleton
-
-@Singleton
-class ServedStatsSideEffect @Inject() (statsReceiver: StatsReceiver)
-    extends PipelineResultSideEffect[PipelineQuery, Timeline] {
-
-  override val identifier: SideEffectIdentifier = SideEffectIdentifier("ServedStats")
-
-  private val baseStatsReceiver = statsReceiver.scope(identifier.toString)
-  private val authorStatsReceiver = baseStatsReceiver.scope("Author")
-  private val candidateSourceStatsReceiver = baseStatsReceiver.scope("CandidateSource")
-  private val contentBalanceStatsReceiver = baseStatsReceiver.scope("ContentBalance")
-  private val inNetworkStatsCounter = contentBalanceStatsReceiver.counter("InNetwork")
-  private val outOfNetworkStatsCounter = contentBalanceStatsReceiver.counter("OutOfNetwork")
-
-  override def apply(
-    inputs: PipelineResultSideEffect.Inputs[PipelineQuery, Timeline]
-  ): Stitch[Unit] = {
-    val tweetCandidates = CandidatesUtil
-      .getItemCandidates(inputs.selectedCandidates).filter(_.isCandidateType[TweetCandidate]())
-
-    recordAuthorStats(tweetCandidates, inputs.query.params(AuthorListForStatsParam))
-    recordCandidateSourceStats(tweetCandidates)
-    recordContentBalanceStats(tweetCandidates)
-    Stitch.Unit
-  }
-
-  def recordAuthorStats(candidates: Seq[CandidateWithDetails], authors: Set[Long]): Unit = {
-    candidates
-      .filter { candidate =>
-        candidate.features.getOrElse(AuthorIdFeature, None).exists(authors.contains) &&
-        // Only include original tweets
-        (!candidate.features.getOrElse(IsRetweetFeature, false)) &&
-        candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty
-      }
-      .groupBy { candidate =>
-        (getCandidateSourceId(candidate), candidate.features.get(AuthorIdFeature).get)
-      }
-      .foreach {
-        case ((candidateSourceId, authorId), authorCandidates) =>
-          authorStatsReceiver
-            .scope(authorId.toString).counter(candidateSourceId).incr(authorCandidates.size)
-      }
-  }
-
-  def recordCandidateSourceStats(candidates: Seq[ItemCandidateWithDetails]): Unit = {
-    candidates.groupBy(getCandidateSourceId).foreach {
-      case (candidateSourceId, candidateSourceCandidates) =>
-        candidateSourceStatsReceiver.counter(candidateSourceId).incr(candidateSourceCandidates.size)
-    }
-  }
-
-  def recordContentBalanceStats(candidates: Seq[ItemCandidateWithDetails]): Unit = {
-    val (in, oon) = candidates.partition(_.features.getOrElse(InNetworkFeature, true))
-    inNetworkStatsCounter.incr(in.size)
-    outOfNetworkStatsCounter.incr(oon.size)
-  }
-
-  private def getCandidateSourceId(candidate: CandidateWithDetails): String =
-    candidate.features.getOrElse(CandidateSourceIdFeature, None).map(_.name).getOrElse("None")
-}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateTimelinesPersistenceStoreSideEffect.scala

@@ -3,8 +3,10 @@ package com.twitter.home_mixer.functional_component.side_effect
 import com.twitter.home_mixer.model.HomeFeatures._
 import com.twitter.home_mixer.model.request.FollowingProduct
 import com.twitter.home_mixer.model.request.ForYouProduct
+import com.twitter.home_mixer.model.HomeFeatures.IsTweetPreviewFeature
 import com.twitter.home_mixer.service.HomeMixerAlertConfig
 import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator
+import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidateDecorator
 import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
 import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
 import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
@@ -97,12 +99,14 @@ class UpdateTimelinesPersistenceStoreSideEffect @Inject() (
       val entries = inputs.response.instructions.collect {
         case AddEntriesTimelineInstruction(entries) =>
           entries.collect {
-            // includes both tweets and promoted tweets
+            // includes tweets, tweet previews, and promoted tweets
-            case entry: TweetItem if entry.sortIndex.isDefined =>
+            case entry: TweetItem if entry.sortIndex.isDefined => {
               Seq(
                 buildTweetEntryWithItemIds(
                   tweetIdToItemCandidateMap(entry.id),
-                  entry.sortIndex.get))
+                  entry.sortIndex.get
+                ))
+            }
             // tweet conversation modules are flattened to individual tweets in the persistence store
             case module: TimelineModule
                 if module.sortIndex.isDefined && module.items.headOption.exists(
@@ -125,6 +129,19 @@ class UpdateTimelinesPersistenceStoreSideEffect @Inject() (
                   size = module.items.size.toShort,
                   itemIds = Some(userIds)
                 ))
+            case module: TimelineModule
+                if module.sortIndex.isDefined && module.entryNamespace.toString == WhoToSubscribeCandidateDecorator.EntryNamespaceString =>
+              val userIds = module.items
+                .map(item =>
+                  UpdateTimelinesPersistenceStoreSideEffect.EmptyItemIds.copy(userId =
+                    Some(item.item.id.asInstanceOf[Long])))
+              Seq(
+                EntryWithItemIds(
+                  entityIdType = EntityIdType.WhoToSubscribe,
+                  sortIndex = module.sortIndex.get,
+                  size = module.items.size.toShort,
+                  itemIds = Some(userIds)
+                ))
           }.flatten
         case ShowCoverInstruction(cover) =>
           Seq(
@@ -216,8 +233,11 @@ class UpdateTimelinesPersistenceStoreSideEffect @Inject() (
       userId = None
     )
 
+    val isPreview = features.getOrElse(IsTweetPreviewFeature, default = false)
+    val entityType = if (isPreview) EntityIdType.TweetPreview else EntityIdType.Tweet
+
     EntryWithItemIds(
-      entityIdType = EntityIdType.Tweet,
+      entityIdType = entityType,
       sortIndex = sortIndex,
       size = 1.toShort,
       itemIds = Some(Seq(itemIds))

home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/BUILD.bazel

@@ -4,14 +4,12 @@ scala_library(
     strict_deps = True,
     tags = ["bazel-compatible"],
     dependencies = [
-        "dspbidder/thrift/src/main/thrift/com/twitter/dspbidder/commons:thrift-scala",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
         "home-mixer/thrift/src/main/thrift:thrift-scala",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common",
     ],
     exports = [
-        "dspbidder/thrift/src/main/thrift/com/twitter/dspbidder/commons:thrift-scala",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
         "home-mixer/thrift/src/main/thrift:thrift-scala",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request",

home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductContextUnmarshaller.scala

@@ -5,9 +5,9 @@ import com.twitter.home_mixer.model.request.ForYouProductContext
 import com.twitter.home_mixer.model.request.ListRecommendedUsersProductContext
 import com.twitter.home_mixer.model.request.ListTweetsProductContext
 import com.twitter.home_mixer.model.request.ScoredTweetsProductContext
+import com.twitter.home_mixer.model.request.SubscribedProductContext
 import com.twitter.home_mixer.{thriftscala => t}
 import com.twitter.product_mixer.core.model.marshalling.request.ProductContext
-
 import javax.inject.Inject
 import javax.inject.Singleton
 
@@ -26,15 +26,17 @@ class HomeMixerProductContextUnmarshaller @Inject() (
       ForYouProductContext(
         deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)),
         seenTweetIds = p.seenTweetIds,
-        dspClientContext = p.dspClientContext
+        dspClientContext = p.dspClientContext,
+        pushToHomeTweetId = p.pushToHomeTweetId
       )
-    case t.ProductContext.Realtime(p) =>
+    case t.ProductContext.ListManagement(p) =>
       throw new UnsupportedOperationException(s"This product is no longer used")
     case t.ProductContext.ScoredTweets(p) =>
       ScoredTweetsProductContext(
         deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)),
         seenTweetIds = p.seenTweetIds,
-        servedTweetIds = p.servedTweetIds
+        servedTweetIds = p.servedTweetIds,
+        backfillTweetIds = p.backfillTweetIds
       )
     case t.ProductContext.ListTweets(p) =>
       ListTweetsProductContext(
@@ -46,7 +48,13 @@ class HomeMixerProductContextUnmarshaller @Inject() (
       ListRecommendedUsersProductContext(
         listId = p.listId,
         selectedUserIds = p.selectedUserIds,
-        excludedUserIds = p.excludedUserIds
+        excludedUserIds = p.excludedUserIds,
+        listName = p.listName
+      )
+    case t.ProductContext.Subscribed(p) =>
+      SubscribedProductContext(
+        deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)),
+        seenTweetIds = p.seenTweetIds,
       )
     case t.ProductContext.UnknownUnionField(field) =>
       throw new UnsupportedOperationException(s"Unknown display context: ${field.field.name}")

home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductUnmarshaller.scala

@@ -5,9 +5,9 @@ import com.twitter.home_mixer.model.request.ForYouProduct
 import com.twitter.home_mixer.model.request.ListRecommendedUsersProduct
 import com.twitter.home_mixer.model.request.ListTweetsProduct
 import com.twitter.home_mixer.model.request.ScoredTweetsProduct
+import com.twitter.home_mixer.model.request.SubscribedProduct
 import com.twitter.home_mixer.{thriftscala => t}
 import com.twitter.product_mixer.core.model.marshalling.request.Product
-
 import javax.inject.Inject
 import javax.inject.Singleton
 
@@ -17,11 +17,12 @@ class HomeMixerProductUnmarshaller @Inject() () {
   def apply(product: t.Product): Product = product match {
     case t.Product.Following => FollowingProduct
     case t.Product.ForYou => ForYouProduct
-    case t.Product.Realtime =>
+    case t.Product.ListManagement =>
       throw new UnsupportedOperationException(s"This product is no longer used")
     case t.Product.ScoredTweets => ScoredTweetsProduct
     case t.Product.ListTweets => ListTweetsProduct
     case t.Product.ListRecommendedUsers => ListRecommendedUsersProduct
+    case t.Product.Subscribed => SubscribedProduct
     case t.Product.EnumUnknownProduct(value) =>
       throw new UnsupportedOperationException(s"Unknown product: $value")
   }

home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/ConversationEntryMarshaller.scala

@@ -1,16 +0,0 @@
-package com.twitter.home_mixer.marshaller.timeline_logging
-
-import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
-import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
-import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
-import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
-
-object ConversationEntryMarshaller {
-
-  def apply(entry: TweetItem, candidate: ItemCandidateWithDetails): thriftlog.ConversationEntry =
-    thriftlog.ConversationEntry(
-      displayedTweetId = entry.id,
-      displayType = Some(entry.displayType.toString),
-      score = candidate.features.getOrElse(ScoreFeature, None)
-    )
-}

home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/PromotedTweetDetailsMarshaller.scala

@@ -3,15 +3,13 @@ package com.twitter.home_mixer.marshaller.timeline_logging
 import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
 import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
 
-object PromotedTweetEntryMarshaller {
+object PromotedTweetDetailsMarshaller {
 
-  def apply(entry: TweetItem, position: Int): thriftlog.PromotedTweetEntry = {
+  def apply(entry: TweetItem, position: Int): thriftlog.PromotedTweetDetails = {
-    thriftlog.PromotedTweetEntry(
+    thriftlog.PromotedTweetDetails(
-      id = entry.id,
+      advertiserId = Some(entry.promotedMetadata.map(_.advertiserId).getOrElse(0L)),
-      advertiserId = entry.promotedMetadata.map(_.advertiserId).getOrElse(0L),
+      insertPosition = Some(position),
-      insertPosition = position,
+      impressionId = entry.promotedMetadata.flatMap(_.impressionString)
-      impressionId = entry.promotedMetadata.flatMap(_.impressionString),
-      displayType = Some(entry.displayType.toString)
     )
   }
 }

home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/TweetDetailsMarshaller.scala

@@ -0,0 +1,47 @@
+package com.twitter.home_mixer.marshaller.timeline_logging
+
+import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
+import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
+import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
+import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
+import com.twitter.product_mixer.component_library.model.presentation.urt.UrtItemPresentation
+import com.twitter.product_mixer.component_library.model.presentation.urt.UrtModulePresentation
+import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.metadata.GeneralContextTypeMarshaller
+import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
+import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
+import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ConversationGeneralContextType
+import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.GeneralContext
+import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContext
+import com.twitter.timelines.service.{thriftscala => tst}
+import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
+
+object TweetDetailsMarshaller {
+
+  private val generalContextTypeMarshaller = new GeneralContextTypeMarshaller()
+
+  def apply(entry: TweetItem, candidate: CandidateWithDetails): thriftlog.TweetDetails = {
+    val socialContext = candidate.presentation.flatMap {
+      case _ @UrtItemPresentation(timelineItem: TweetItem, _) => timelineItem.socialContext
+      case _ @UrtModulePresentation(timelineModule) =>
+        timelineModule.items.head.item match {
+          case timelineItem: TweetItem => timelineItem.socialContext
+          case _ => Some(ConversationGeneralContextType)
+        }
+    }
+
+    val socialContextType = socialContext match {
+      case Some(GeneralContext(contextType, _, _, _, _)) =>
+        Some(generalContextTypeMarshaller(contextType).value.toShort)
+      case Some(TopicContext(_, _)) => Some(tst.ContextType.Topic.value.toShort)
+      case _ => None
+    }
+
+    thriftlog.TweetDetails(
+      sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None),
+      socialContextType = socialContextType,
+      suggestType = candidate.features.getOrElse(SuggestTypeFeature, None).map(_.name),
+      authorId = candidate.features.getOrElse(AuthorIdFeature, None),
+      sourceAuthorId = candidate.features.getOrElse(SourceUserIdFeature, None)
+    )
+  }
+}

home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/TweetEntryMarshaller.scala

@@ -1,29 +0,0 @@
-package com.twitter.home_mixer.marshaller.timeline_logging
-
-import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
-import com.twitter.home_mixer.model.HomeFeatures.SocialContextFeature
-import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
-import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
-import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
-import com.twitter.timelines.service.{thriftscala => tst}
-import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
-
-object TweetEntryMarshaller {
-
-  def apply(entry: TweetItem, candidate: CandidateWithDetails): thriftlog.TweetEntry = {
-    val socialContextType = candidate.features.getOrElse(SocialContextFeature, None) match {
-      case Some(tst.SocialContext.GeneralContext(tst.GeneralContext(contextType, _, _, _, _))) =>
-        Some(contextType.value.toShort)
-      case Some(tst.SocialContext.TopicContext(_)) =>
-        Some(tst.ContextType.Topic.value.toShort)
-      case _ => None
-    }
-    thriftlog.TweetEntry(
-      id = candidate.candidateIdLong,
-      sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None),
-      displayType = Some(entry.displayType.toString),
-      score = candidate.features.getOrElse(ScoreFeature, None),
-      socialContextType = socialContextType
-    )
-  }
-}

home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/WhoToFollowDetailsMarshaller.scala

@@ -1,17 +1,13 @@
 package com.twitter.home_mixer.marshaller.timeline_logging
 
-import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.ScoreFeature
 import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
 import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem
 import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
 
-object WhoToFollowEntryMarshaller {
+object WhoToFollowDetailsMarshaller {
 
-  def apply(entry: UserItem, candidate: ItemCandidateWithDetails): thriftlog.WhoToFollowEntry =
+  def apply(entry: UserItem, candidate: ItemCandidateWithDetails): thriftlog.WhoToFollowDetails =
-    thriftlog.WhoToFollowEntry(
+    thriftlog.WhoToFollowDetails(
-      userId = entry.id,
-      displayType = Some(entry.displayType.toString),
-      score = candidate.features.getOrElse(ScoreFeature, None),
       enableReactiveBlending = entry.enableReactiveBlending,
       impressionId = entry.promotedMetadata.flatMap(_.impressionString),
       advertiserId = entry.promotedMetadata.map(_.advertiserId)

home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/BUILD.bazel

@@ -4,13 +4,8 @@ scala_library(
     strict_deps = True,
     tags = ["bazel-compatible"],
     dependencies = [
-        "3rdparty/jvm/javax/inject:javax.inject",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor",
-        "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request",
-        "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt",
-        "src/thrift/com/twitter/timelines/render:thrift-scala",
         "src/thrift/com/twitter/timelineservice:thrift-scala",
-        "timelineservice/common:model",
     ],
 )