Merge eaae3e286e into 72eda9a24f

Setting up the different tools for development can be hard for new
contributors. This commit adds a Nix flake that provides a reproducible
development environment which is easy to set up and ensures consistent
code style across all subprojects.

The initial flake contains `git`, `pre-commit`, `which` and `bazelisk`
wrapped as a `bazel` executable. Dependencies like Rust, Scala etc may
be added in the `packages` attribute in the future.

The current pre-commit configuration is a no-op. Future commits may
enable the currently disabled parts of `pre-commit-hooks.nix` for a
gradual transition.

Supports `(x86_64|aarch64) (Linux|MacOS)` systems.

.envrc

@@ -0,0 +1 @@
+use flake

.gitignore

@@ -1,2 +1,6 @@
 .DS_Store
 
+# Autogenerated by the Nix flake.
+.devenv/
+.direnv/
+.pre-commit-config.yaml

README.md

@@ -62,6 +62,23 @@ The core components of Recommended Notifications included in this repository are
 
 We include Bazel BUILD files for most components, but not a top-level BUILD or WORKSPACE file. We plan to add a more complete build and test system in the future.
 
+## Development environment
+
+The unified developement environment is still in an experimental state.
+Contributions are welcome!
+
+You can set up a development environment with various tooling via the nix flake:
+
+1. Install the [nix package manager](https://nixos.org/download.html) and enable
+   [flakes](https://nixos.wiki/wiki/Flakes).
+
+2. Enter the development shell by running `nix develop` in the root directory of
+   the repository.
+
+3. Optionally, install [direnv](https://github.com/direnv/direnv/blob/master/docs/installation.md)
+   which will automatically drop you into the development environment when you
+   `cd` into your local clone.
+
 ## Contributing
 
 We invite the community to submit GitHub issues and pull requests for suggestions on improving the recommendation algorithm. We are working on tools to manage these suggestions and sync changes to our internal repository. Any security concerns or issues should be routed to our official [bug bounty program](https://hackerone.com/twitter) through HackerOne. We hope to benefit from the collective intelligence and expertise of the global community in helping us identify issues and suggest improvements, ultimately leading to a better Twitter.

flake.lock

@@ -0,0 +1,341 @@
+{
+  "nodes": {
+    "devenv": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "nix": "nix",
+        "nixpkgs": "nixpkgs",
+        "pre-commit-hooks": "pre-commit-hooks"
+      },
+      "locked": {
+        "lastModified": 1678113758,
+        "narHash": "sha256-mD3SkN43b1s5CJ8Rx3l2oK3Dqgs+6Ze0FfWrdMcrrYk=",
+        "owner": "cachix",
+        "repo": "devenv",
+        "rev": "6455f319fc90e0be2071327093c5458f9afc61bf",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "ref": "latest",
+        "repo": "devenv",
+        "type": "github"
+      }
+    },
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1673956053,
+        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-compat_2": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1673956053,
+        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "locked": {
+        "lastModified": 1667395993,
+        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_2": {
+      "locked": {
+        "lastModified": 1678901627,
+        "narHash": "sha256-U02riOqrKKzwjsxc/400XnElV+UtPUQWpANPlyazjH0=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "93a2b84fc4b70d9e089d029deacc3583435c2ed6",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_3": {
+      "locked": {
+        "lastModified": 1667395993,
+        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "gitignore": {
+      "inputs": {
+        "nixpkgs": [
+          "devenv",
+          "pre-commit-hooks",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1660459072,
+        "narHash": "sha256-8DFJjXG8zqoONA1vXtgeKXy68KdJL5UaXR8NtVMUbx8=",
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "rev": "a20de23b925fd8264fd7fad6454652e142fd7f73",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "type": "github"
+      }
+    },
+    "gitignore_2": {
+      "inputs": {
+        "nixpkgs": [
+          "pre-commit-hooks-nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1660459072,
+        "narHash": "sha256-8DFJjXG8zqoONA1vXtgeKXy68KdJL5UaXR8NtVMUbx8=",
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "rev": "a20de23b925fd8264fd7fad6454652e142fd7f73",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "type": "github"
+      }
+    },
+    "lowdown-src": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1633514407,
+        "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=",
+        "owner": "kristapsdz",
+        "repo": "lowdown",
+        "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "kristapsdz",
+        "repo": "lowdown",
+        "type": "github"
+      }
+    },
+    "nix": {
+      "inputs": {
+        "lowdown-src": "lowdown-src",
+        "nixpkgs": [
+          "devenv",
+          "nixpkgs"
+        ],
+        "nixpkgs-regression": "nixpkgs-regression"
+      },
+      "locked": {
+        "lastModified": 1676545802,
+        "narHash": "sha256-EK4rZ+Hd5hsvXnzSzk2ikhStJnD63odF7SzsQ8CuSPU=",
+        "owner": "domenkozar",
+        "repo": "nix",
+        "rev": "7c91803598ffbcfe4a55c44ac6d49b2cf07a527f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "domenkozar",
+        "ref": "relaxed-flakes",
+        "repo": "nix",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1677534593,
+        "narHash": "sha256-PuZSAHeq4/9pP/uYH1FcagQ3nLm/DrDrvKi/xC9glvw=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "3ad64d9e2d5bf80c877286102355b1625891ae9a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs-regression": {
+      "locked": {
+        "lastModified": 1643052045,
+        "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
+        "type": "github"
+      }
+    },
+    "nixpkgs-stable": {
+      "locked": {
+        "lastModified": 1673800717,
+        "narHash": "sha256-SFHraUqLSu5cC6IxTprex/nTsI81ZQAtDvlBvGDWfnA=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "2f9fd351ec37f5d479556cd48be4ca340da59b8f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-22.11",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs-stable_2": {
+      "locked": {
+        "lastModified": 1678872516,
+        "narHash": "sha256-/E1YwtMtFAu2KUQKV/1+KFuReYPANM2Rzehk84VxVoc=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "9b8e5abb18324c7fe9f07cb100c3cd4a29cda8b8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-22.11",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1680296362,
+        "narHash": "sha256-zI8rKfSoRf261clT5Fi5M+buLyN1z5FgWFgnPHgJd1Y=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "ff40c8086733f2104d416b6ffd4770068a1fd43f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_3": {
+      "locked": {
+        "lastModified": 1678898370,
+        "narHash": "sha256-xTICr1j+uat5hk9FyuPOFGxpWHdJRibwZC+ATi0RbtE=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "ac718d02867a84b42522a0ece52d841188208f2c",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "pre-commit-hooks": {
+      "inputs": {
+        "flake-compat": [
+          "devenv",
+          "flake-compat"
+        ],
+        "flake-utils": "flake-utils",
+        "gitignore": "gitignore",
+        "nixpkgs": [
+          "devenv",
+          "nixpkgs"
+        ],
+        "nixpkgs-stable": "nixpkgs-stable"
+      },
+      "locked": {
+        "lastModified": 1677160285,
+        "narHash": "sha256-tBzpCjMP+P3Y3nKLYvdBkXBg3KvTMo3gvi8tLQaqXVY=",
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "rev": "2bd861ab81469428d9c823ef72c4bb08372dd2c4",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "type": "github"
+      }
+    },
+    "pre-commit-hooks-nix": {
+      "inputs": {
+        "flake-compat": "flake-compat_2",
+        "flake-utils": "flake-utils_3",
+        "gitignore": "gitignore_2",
+        "nixpkgs": "nixpkgs_3",
+        "nixpkgs-stable": "nixpkgs-stable_2"
+      },
+      "locked": {
+        "lastModified": 1680170909,
+        "narHash": "sha256-FtKU/edv1jFRr/KwUxWTYWXEyj9g8GBrHntC2o8oFI8=",
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "rev": "29dbe1efaa91c3a415d8b45d62d48325a4748816",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "devenv": "devenv",
+        "flake-utils": "flake-utils_2",
+        "nixpkgs": "nixpkgs_2",
+        "pre-commit-hooks-nix": "pre-commit-hooks-nix"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,63 @@
+{
+  description = "Twitter development environment";
+
+  inputs = {
+    nixpkgs.url = "github:nixos/nixpkgs";
+    flake-utils.url = "github:numtide/flake-utils";
+    devenv.url = "github:cachix/devenv/latest";
+    pre-commit-hooks-nix.url = "github:cachix/pre-commit-hooks.nix";
+  };
+
+  outputs = { self, nixpkgs, flake-utils, devenv, pre-commit-hooks-nix, ... } @ inputs:
+    flake-utils.lib.eachSystem [
+      "x86_64-linux"
+      "x86_64-darwin"
+      "aarch64-linux"
+      "aarch64-darwin"
+    ]
+      (system:
+        let
+          pkgs = import nixpkgs { inherit system; };
+        in
+        rec {
+
+          hooks = import ./pre-commit-hooks.nix {
+            inherit pkgs;
+          };
+
+          checks = {
+            pre-commit-check = pre-commit-hooks-nix.lib.${system}.run {
+              src = ./.;
+              hooks = hooks;
+            };
+          };
+
+          devShells = {
+            default = mkTwitterShell;
+          };
+
+          mkTwitterShell = devenv.lib.mkShell {
+            inherit inputs pkgs;
+
+            modules = [{
+              pre-commit.hooks = hooks;
+
+              scripts.bazel.exec = "${pkgs.bazelisk}/bin/bazelisk $@";
+
+              packages = [
+                pkgs.git
+                pkgs.pre-commit
+                pkgs.which
+              ];
+
+              enterShell = ''
+                # Remove potential previous Bazel aliases.
+                [[ $(type -t bazel) == "alias" ]] && unalias bazel
+
+                # Prettier color output for the ls command.
+                alias ls='ls --color=auto'
+              '';
+            }];
+          };
+        });
+}

home-mixer/README.md

@@ -74,7 +74,7 @@ Timeline tabs powered by Home Mixer.
             - ScoredTweetsRecommendationPipelineConfig (main Tweet recommendation layer)
                 - Fetch Tweet Candidates
                     - ScoredTweetsInNetworkCandidatePipelineConfig
-                    - ScoredTweetsCrMixerCandidatePipelineConfig
+                    - ScoredTweetsTweetMixerCandidatePipelineConfig
                     - ScoredTweetsUtegCandidatePipelineConfig
                     - ScoredTweetsFrsCandidatePipelineConfig
                 - Feature Hydration and Scoring
@@ -99,4 +99,3 @@ Timeline tabs powered by Home Mixer.
         - ListTweetsTimelineServiceCandidatePipelineConfig (fetch tweets from timeline service)
         - ConversationServiceCandidatePipelineConfig (fetch ancestors for conversation modules)
         - ListTweetsAdsCandidatePipelineConfig (fetch ads)
-

home-mixer/server/src/main/scala/com/twitter/home_mixer/BUILD.bazel

@@ -21,6 +21,7 @@ scala_library(
         "finatra/inject/inject-utils/src/main/scala",
         "home-mixer/server/src/main/resources",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/controller",
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/federated",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/module",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/product",
@@ -31,6 +32,10 @@ scala_library(
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter",
         "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala",
         "src/thrift/com/twitter/timelines/render:thrift-scala",
+        "strato/config/columns/auth-context:auth-context-strato-client",
+        "strato/config/columns/gizmoduck:gizmoduck-strato-client",
+        "strato/src/main/scala/com/twitter/strato/fed",
+        "strato/src/main/scala/com/twitter/strato/fed/server",
         "stringcenter/client",
         "stringcenter/client/src/main/java",
         "stringcenter/client/src/main/scala/com/twitter/stringcenter/client",

home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerHttpServerWarmupHandler.scala

@@ -2,7 +2,7 @@ package com.twitter.home_mixer
 
 import com.twitter.finatra.http.routing.HttpWarmup
 import com.twitter.finatra.httpclient.RequestBuilder._
-import com.twitter.inject.Logging
+import com.twitter.util.logging.Logging
 import com.twitter.inject.utils.Handler
 import com.twitter.util.Try
 import javax.inject.Inject

home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerServer.scala

@@ -12,57 +12,63 @@ import com.twitter.finatra.thrift.ThriftServer
 import com.twitter.finatra.thrift.filters._
 import com.twitter.finatra.thrift.routing.ThriftRouter
 import com.twitter.home_mixer.controller.HomeThriftController
+import com.twitter.home_mixer.federated.HomeMixerColumn
 import com.twitter.home_mixer.module._
 import com.twitter.home_mixer.param.GlobalParamConfigModule
 import com.twitter.home_mixer.product.HomeMixerProductModule
 import com.twitter.home_mixer.{thriftscala => st}
 import com.twitter.product_mixer.component_library.module.AccountRecommendationsMixerModule
-import com.twitter.product_mixer.component_library.module.CrMixerClientModule
 import com.twitter.product_mixer.component_library.module.DarkTrafficFilterModule
 import com.twitter.product_mixer.component_library.module.EarlybirdModule
 import com.twitter.product_mixer.component_library.module.ExploreRankerClientModule
 import com.twitter.product_mixer.component_library.module.GizmoduckClientModule
 import com.twitter.product_mixer.component_library.module.OnboardingTaskServiceModule
 import com.twitter.product_mixer.component_library.module.SocialGraphServiceModule
-import com.twitter.product_mixer.component_library.module.TimelineMixerClientModule
 import com.twitter.product_mixer.component_library.module.TimelineRankerClientModule
 import com.twitter.product_mixer.component_library.module.TimelineScorerClientModule
 import com.twitter.product_mixer.component_library.module.TimelineServiceClientModule
 import com.twitter.product_mixer.component_library.module.TweetImpressionStoreModule
+import com.twitter.product_mixer.component_library.module.TweetMixerClientModule
 import com.twitter.product_mixer.component_library.module.UserSessionStoreModule
 import com.twitter.product_mixer.core.controllers.ProductMixerController
 import com.twitter.product_mixer.core.module.LoggingThrowableExceptionMapper
 import com.twitter.product_mixer.core.module.ProductMixerModule
-import com.twitter.product_mixer.core.module.StratoClientModule
 import com.twitter.product_mixer.core.module.stringcenter.ProductScopeStringCenterModule
+import com.twitter.strato.fed.StratoFed
+import com.twitter.strato.fed.server.StratoFedServer
 
 object HomeMixerServerMain extends HomeMixerServer
 
-class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMtls {
+class HomeMixerServer
+    extends StratoFedServer
+    with ThriftServer
+    with Mtls
+    with HttpServer
+    with HttpMtls {
   override val name = "home-mixer-server"
 
   override val modules: Seq[Module] = Seq(
     AccountRecommendationsMixerModule,
     AdvertiserBrandSafetySettingsStoreModule,
+    BlenderClientModule,
     ClientSentImpressionsPublisherModule,
     ConversationServiceModule,
-    CrMixerClientModule,
     EarlybirdModule,
     ExploreRankerClientModule,
+    FeedbackHistoryClientModule,
     GizmoduckClientModule,
     GlobalParamConfigModule,
     HomeAdsCandidateSourceModule,
     HomeMixerFlagsModule,
     HomeMixerProductModule,
     HomeMixerResourcesModule,
-    HomeNaviModelClientModule,
     ImpressionBloomFilterModule,
     InjectionHistoryClientModule,
-    FeedbackHistoryClientModule,
     ManhattanClientsModule,
     ManhattanFeatureRepositoryModule,
     ManhattanTweetImpressionStoreModule,
     MemcachedFeatureRepositoryModule,
+    NaviModelClientModule,
     OnboardingTaskServiceModule,
     OptimizedStratoClientModule,
     PeopleDiscoveryServiceModule,
@@ -74,24 +80,23 @@ class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMt
     SimClustersRecentEngagementsClientModule,
     SocialGraphServiceModule,
     StaleTweetsCacheModule,
-    StratoClientModule,
     ThriftFeatureRepositoryModule,
-    TimelineMixerClientModule,
     TimelineRankerClientModule,
     TimelineScorerClientModule,
     TimelineServiceClientModule,
     TimelinesPersistenceStoreClientModule,
+    TopicSocialProofClientModule,
     TweetImpressionStoreModule,
-    TweetyPieClientModule,
+    TweetMixerClientModule,
+    TweetypieClientModule,
     TweetypieStaticEntitiesCacheClientModule,
-    UserMetadataStoreModule,
     UserSessionStoreModule,
     new DarkTrafficFilterModule[st.HomeMixer.ReqRepServicePerEndpoint](),
     new MtlsThriftWebFormsModule[st.HomeMixer.MethodPerEndpoint](this),
     new ProductScopeStringCenterModule()
   )
 
-  def configureThrift(router: ThriftRouter): Unit = {
+  override def configureThrift(router: ThriftRouter): Unit = {
     router
       .filter[LoggingMDCFilter]
       .filter[TraceIdMDCFilter]
@@ -111,6 +116,11 @@ class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMt
         this.injector,
         st.HomeMixer.ExecutePipeline))
 
+  override val dest: String = "/s/home-mixer/home-mixer:strato"
+
+  override val columns: Seq[Class[_ <: StratoFed.Column]] =
+    Seq(classOf[HomeMixerColumn])
+
   override protected def warmup(): Unit = {
     handle[HomeMixerThriftServerWarmupHandler]()
     handle[HomeMixerHttpServerWarmupHandler]()

home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerThriftServerWarmupHandler.scala

@@ -3,7 +3,7 @@ package com.twitter.home_mixer
 import com.twitter.finagle.thrift.ClientId
 import com.twitter.finatra.thrift.routing.ThriftWarmup
 import com.twitter.home_mixer.{thriftscala => st}
-import com.twitter.inject.Logging
+import com.twitter.util.logging.Logging
 import com.twitter.inject.utils.Handler
 import com.twitter.product_mixer.core.{thriftscala => pt}
 import com.twitter.scrooge.Request

home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/BUILD.bazel

@@ -5,19 +5,13 @@ scala_library(
     tags = ["bazel-compatible"],
     dependencies = [
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
@@ -25,10 +19,6 @@ scala_library(
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer",
-        "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
-        "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
-        "timelines/src/main/scala/com/twitter/timelines/injection/scribe",
-        "timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
     ],
     exports = [
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc",

home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfig.scala

@@ -1,18 +1,23 @@
 package com.twitter.home_mixer.candidate_pipeline
 
+import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator
 import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
-import com.twitter.home_mixer.functional_component.feature_hydrator.SocialGraphServiceFeatureHydrator
 import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
 import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter
-import com.twitter.home_mixer.functional_component.filter.PredicateFeatureFilter
+import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter
 import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter
+import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
+import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
 import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature
 import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature
+import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
+import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
 import com.twitter.home_mixer.service.HomeMixerAlertConfig
 import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
 import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSourceRequest
 import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata
 import com.twitter.product_mixer.component_library.filter.FeatureFilter
+import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter
 import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
 import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource
 import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
@@ -33,8 +38,8 @@ import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipel
 class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
   conversationServiceCandidateSource: ConversationServiceCandidateSource,
   tweetypieFeatureHydrator: TweetypieFeatureHydrator,
-  socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator,
   namesFeatureHydrator: NamesFeatureHydrator,
+  invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter,
   override val gates: Seq[BaseGate[Query]],
   override val decorator: Option[CandidateDecorator[Query, TweetCandidate]])
     extends DependentCandidatePipelineConfig[
@@ -62,10 +67,10 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
     val tweetsWithConversationMetadata = candidates.map { candidate =>
       TweetWithConversationMetadata(
         tweetId = candidate.candidateIdLong,
-        userId = None,
-        sourceTweetId = None,
-        sourceUserId = None,
-        inReplyToTweetId = None,
+        userId = candidate.features.getOrElse(AuthorIdFeature, None),
+        sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None),
+        sourceUserId = candidate.features.getOrElse(SourceUserIdFeature, None),
+        inReplyToTweetId = candidate.features.getOrElse(InReplyToTweetIdFeature, None),
         conversationId = None,
         ancestors = Seq.empty
       )
@@ -84,7 +89,10 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
 
   override val preFilterFeatureHydrationPhase1: Seq[
     BaseCandidateFeatureHydrator[Query, TweetCandidate, _]
-  ] = Seq(tweetypieFeatureHydrator, socialGraphServiceFeatureHydrator)
+  ] = Seq(
+    tweetypieFeatureHydrator,
+    InNetworkFeatureHydrator,
+  )
 
   override def filters: Seq[Filter[Query, TweetCandidate]] = Seq(
     RetweetDeduplicationFilter,
@@ -93,6 +101,7 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
       FilterIdentifier(QuotedTweetDroppedFilterId),
       shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) }
     ),
+    invalidSubscriptionTweetFilter,
     InvalidConversationModuleFilter
   )
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfigBuilder.scala

@@ -1,9 +1,9 @@
 package com.twitter.home_mixer.candidate_pipeline
 
-import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
 import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
-import com.twitter.home_mixer.functional_component.feature_hydrator.SocialGraphServiceFeatureHydrator
 import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
+import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter
+import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
 import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
 import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
 import com.twitter.product_mixer.core.functional_component.gate.BaseGate
@@ -15,7 +15,7 @@ import javax.inject.Singleton
 class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery] @Inject() (
   conversationServiceCandidateSource: ConversationServiceCandidateSource,
   tweetypieFeatureHydrator: TweetypieFeatureHydrator,
-  socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator,
+  invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter,
   namesFeatureHydrator: NamesFeatureHydrator) {
 
   def build(
@@ -25,8 +25,8 @@ class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery]
     new ConversationServiceCandidatePipelineConfig(
       conversationServiceCandidateSource,
       tweetypieFeatureHydrator,
-      socialGraphServiceFeatureHydrator,
       namesFeatureHydrator,
+      invalidSubscriptionTweetFilter,
       gates,
       decorator
     )

home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/EditedTweetsCandidatePipelineConfig.scala

@@ -1,7 +1,7 @@
 package com.twitter.home_mixer.candidate_pipeline
 
 import com.twitter.home_mixer.functional_component.candidate_source.StaleTweetsCacheCandidateSource
-import com.twitter.home_mixer.functional_component.decorator.HomeFeedbackActionInfoBuilder
+import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder
 import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
 import com.twitter.home_mixer.functional_component.query_transformer.EditedTweetsCandidatePipelineQueryTransformer
 import com.twitter.home_mixer.service.HomeMixerAlertConfig

home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/BUILD.bazel

@@ -13,8 +13,5 @@ scala_library(
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt",
         "snowflake/src/main/scala/com/twitter/snowflake/id",
-        "src/thrift/com/twitter/context:twitter-context-scala",
-        "src/thrift/com/twitter/timelines/render:thrift-scala",
-        "twitter-context/src/main/scala",
     ],
 )

home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/HomeThriftController.scala

@@ -7,6 +7,7 @@ import com.twitter.home_mixer.service.ScoredTweetsService
 import com.twitter.home_mixer.{thriftscala => t}
 import com.twitter.product_mixer.core.controllers.DebugTwitterContext
 import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder
+import com.twitter.product_mixer.core.service.debug_query.DebugQueryService
 import com.twitter.product_mixer.core.service.urt.UrtService
 import com.twitter.snowflake.id.SnowflakeId
 import com.twitter.stitch.Stitch

home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/BUILD.bazel

@@ -0,0 +1,24 @@
+scala_library(
+    sources = ["*.scala"],
+    compiler_option_sets = ["fatal_warnings"],
+    strict_deps = True,
+    tags = ["bazel-compatible"],
+    dependencies = [
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request",
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
+        "home-mixer/thrift/src/main/thrift:thrift-scala",
+        "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi",
+        "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product",
+        "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry",
+        "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala",
+        "src/thrift/com/twitter/gizmoduck:thrift-scala",
+        "src/thrift/com/twitter/timelines/render:thrift-scala",
+        "stitch/stitch-repo/src/main/scala",
+        "strato/config/columns/auth-context:auth-context-strato-client",
+        "strato/config/columns/gizmoduck:gizmoduck-strato-client",
+        "strato/config/src/thrift/com/twitter/strato/graphql/timelines:graphql-timelines-scala",
+        "strato/src/main/scala/com/twitter/strato/callcontext",
+        "strato/src/main/scala/com/twitter/strato/fed",
+        "strato/src/main/scala/com/twitter/strato/fed/server",
+    ],
+)

home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/HomeMixerColumn.scala

@@ -0,0 +1,217 @@
+package com.twitter.home_mixer.federated
+
+import com.twitter.gizmoduck.{thriftscala => gd}
+import com.twitter.home_mixer.marshaller.request.HomeMixerRequestUnmarshaller
+import com.twitter.home_mixer.model.request.HomeMixerRequest
+import com.twitter.home_mixer.{thriftscala => hm}
+import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder
+import com.twitter.product_mixer.core.pipeline.product.ProductPipelineRequest
+import com.twitter.product_mixer.core.pipeline.product.ProductPipelineResult
+import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry
+import com.twitter.product_mixer.core.{thriftscala => pm}
+import com.twitter.stitch.Arrow
+import com.twitter.stitch.Stitch
+import com.twitter.strato.callcontext.CallContext
+import com.twitter.strato.catalog.OpMetadata
+import com.twitter.strato.config._
+import com.twitter.strato.data._
+import com.twitter.strato.fed.StratoFed
+import com.twitter.strato.generated.client.auth_context.AuditIpClientColumn
+import com.twitter.strato.generated.client.gizmoduck.CompositeOnUserClientColumn
+import com.twitter.strato.graphql.timelines.{thriftscala => gql}
+import com.twitter.strato.thrift.ScroogeConv
+import com.twitter.timelines.render.{thriftscala => tr}
+import com.twitter.util.Try
+import javax.inject.Inject
+import javax.inject.Singleton
+
+@Singleton
+class HomeMixerColumn @Inject() (
+  homeMixerRequestUnmarshaller: HomeMixerRequestUnmarshaller,
+  compositeOnUserClientColumn: CompositeOnUserClientColumn,
+  auditIpClientColumn: AuditIpClientColumn,
+  paramsBuilder: ParamsBuilder,
+  productPipelineRegistry: ProductPipelineRegistry)
+    extends StratoFed.Column(HomeMixerColumn.Path)
+    with StratoFed.Fetch.Arrow {
+
+  override val contactInfo: ContactInfo = ContactInfo(
+    contactEmail = "",
+    ldapGroup = "",
+    slackRoomId = ""
+  )
+
+  override val metadata: OpMetadata =
+    OpMetadata(
+      lifecycle = Some(Lifecycle.Production),
+      description =
+        Some(Description.PlainText("Federated Strato column for Timelines served via Home Mixer"))
+    )
+
+  private val bouncerAccess: Seq[Policy] = Seq(BouncerAccess())
+  private val finatraTestServiceIdentifiers: Seq[Policy] = Seq(
+    ServiceIdentifierPattern(
+      role = "",
+      service = "",
+      env = "",
+      zone = Seq(""))
+  )
+
+  override val policy: Policy = AnyOf(bouncerAccess ++ finatraTestServiceIdentifiers)
+
+  override type Key = gql.TimelineKey
+  override type View = gql.HomeTimelineView
+  override type Value = tr.Timeline
+
+  override val keyConv: Conv[Key] = ScroogeConv.fromStruct[gql.TimelineKey]
+  override val viewConv: Conv[View] = ScroogeConv.fromStruct[gql.HomeTimelineView]
+  override val valueConv: Conv[Value] = ScroogeConv.fromStruct[tr.Timeline]
+
+  private def createHomeMixerRequestArrow(
+    compositeOnUserClientColumn: CompositeOnUserClientColumn,
+    auditIpClientColumn: AuditIpClientColumn
+  ): Arrow[(Key, View), hm.HomeMixerRequest] = {
+
+    val populateUserRolesAndIp: Arrow[(Key, View), (Option[Set[String]], Option[String])] = {
+      val gizmoduckView: (gd.LookupContext, Set[gd.QueryFields]) =
+        (gd.LookupContext(), Set(gd.QueryFields.Roles))
+
+      val populateUserRoles = Arrow
+        .flatMap[(Key, View), Option[Set[String]]] { _ =>
+          Stitch.collect {
+            CallContext.twitterUserId.map { userId =>
+              compositeOnUserClientColumn.fetcher
+                .callStack(HomeMixerColumn.FetchCallstack)
+                .fetch(userId, gizmoduckView).map(_.v)
+                .map {
+                  _.flatMap(_.roles.map(_.roles.toSet)).getOrElse(Set.empty)
+                }
+            }
+          }
+        }
+
+      val populateIpAddress = Arrow
+        .flatMap[(Key, View), Option[String]](_ =>
+          auditIpClientColumn.fetcher
+            .callStack(HomeMixerColumn.FetchCallstack)
+            .fetch((), ()).map(_.v))
+
+      Arrow.join(
+        populateUserRoles,
+        populateIpAddress
+      )
+    }
+
+    Arrow.zipWithArg(populateUserRolesAndIp).map {
+      case ((key, view), (roles, ipAddress)) =>
+        val deviceContextOpt = Some(
+          hm.DeviceContext(
+            isPolling = CallContext.isPolling,
+            requestContext = view.requestContext,
+            latestControlAvailable = view.latestControlAvailable,
+            autoplayEnabled = view.autoplayEnabled
+          ))
+        val seenTweetIds = view.seenTweetIds.filter(_.nonEmpty)
+
+        val (product, productContext) = key match {
+          case gql.TimelineKey.HomeTimeline(_) | gql.TimelineKey.HomeTimelineV2(_) =>
+            (
+              hm.Product.ForYou,
+              hm.ProductContext.ForYou(
+                hm.ForYou(
+                  deviceContextOpt,
+                  seenTweetIds,
+                  view.dspClientContext,
+                  view.pushToHomeTweetId
+                )
+              ))
+          case gql.TimelineKey.HomeLatestTimeline(_) | gql.TimelineKey.HomeLatestTimelineV2(_) =>
+            (
+              hm.Product.Following,
+              hm.ProductContext.Following(
+                hm.Following(deviceContextOpt, seenTweetIds, view.dspClientContext)))
+          case gql.TimelineKey.CreatorSubscriptionsTimeline(_) =>
+            (
+              hm.Product.Subscribed,
+              hm.ProductContext.Subscribed(hm.Subscribed(deviceContextOpt, seenTweetIds)))
+          case _ => throw new UnsupportedOperationException(s"Unknown product: $key")
+        }
+
+        val clientContext = pm.ClientContext(
+          userId = CallContext.twitterUserId,
+          guestId = CallContext.guestId,
+          guestIdAds = CallContext.guestIdAds,
+          guestIdMarketing = CallContext.guestIdMarketing,
+          appId = CallContext.clientApplicationId,
+          ipAddress = ipAddress,
+          userAgent = CallContext.userAgent,
+          countryCode = CallContext.requestCountryCode,
+          languageCode = CallContext.requestLanguageCode,
+          isTwoffice = CallContext.isInternalOrTwoffice,
+          userRoles = roles,
+          deviceId = CallContext.deviceId,
+          mobileDeviceId = CallContext.mobileDeviceId,
+          mobileDeviceAdId = CallContext.adId,
+          limitAdTracking = CallContext.limitAdTracking
+        )
+
+        hm.HomeMixerRequest(
+          clientContext = clientContext,
+          product = product,
+          productContext = Some(productContext),
+          maxResults = Try(view.count.get.toInt).toOption.orElse(HomeMixerColumn.MaxCount),
+          cursor = view.cursor.filter(_.nonEmpty)
+        )
+    }
+  }
+
+  override val fetch: Arrow[(Key, View), Result[Value]] = {
+    val transformThriftIntoPipelineRequest: Arrow[
+      (Key, View),
+      ProductPipelineRequest[HomeMixerRequest]
+    ] = {
+      Arrow
+        .identity[(Key, View)]
+        .andThen {
+          createHomeMixerRequestArrow(compositeOnUserClientColumn, auditIpClientColumn)
+        }
+        .map {
+          case thriftRequest =>
+            val request = homeMixerRequestUnmarshaller(thriftRequest)
+            val params = paramsBuilder.build(
+              clientContext = request.clientContext,
+              product = request.product,
+              featureOverrides =
+                request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty),
+            )
+            ProductPipelineRequest(request, params)
+        }
+    }
+
+    val underlyingProduct: Arrow[
+      ProductPipelineRequest[HomeMixerRequest],
+      ProductPipelineResult[tr.TimelineResponse]
+    ] = Arrow
+      .identity[ProductPipelineRequest[HomeMixerRequest]]
+      .map { pipelineRequest =>
+        val pipelineArrow = productPipelineRegistry
+          .getProductPipeline[HomeMixerRequest, tr.TimelineResponse](
+            pipelineRequest.request.product)
+          .arrow
+        (pipelineArrow, pipelineRequest)
+      }.applyArrow
+
+    transformThriftIntoPipelineRequest.andThen(underlyingProduct).map {
+      _.result match {
+        case Some(result) => found(result.timeline)
+        case _ => missing
+      }
+    }
+  }
+}
+
+object HomeMixerColumn {
+  val Path = "home-mixer/homeMixer.Timeline"
+  private val FetchCallstack = s"$Path:fetch"
+  private val MaxCount: Option[Int] = Some(100)
+}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/BUILD.bazel

@@ -10,11 +10,8 @@ scala_library(
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
-        "src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala",
         "src/thrift/com/twitter/search:earlybird-scala",
         "stitch/stitch-timelineservice/src/main/scala",
-        "strato/config/columns/recommendations/similarity:similarity-strato-client",
-        "strato/src/main/scala/com/twitter/strato/client",
     ],
     exports = [
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/BUILD.bazel

@@ -6,13 +6,11 @@ scala_library(
     dependencies = [
         "finagle/finagle-core/src/main",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
-        "joinkey/src/main/scala/com/twitter/joinkey/context",
-        "joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
         "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope",
@@ -25,8 +23,6 @@ scala_library(
         "src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala",
         "stringcenter/client",
         "stringcenter/client/src/main/java",
-        "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
-        "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/translation",
         "timelines/src/main/scala/com/twitter/timelines/injection/scribe",
     ],
 )

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeConversationServiceCandidateDecorator.scala

@@ -1,6 +1,8 @@
 package com.twitter.home_mixer.functional_component.decorator
 
 import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder
+import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder
+import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder
 import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature
 import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator
 import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/BUILD.bazel

@@ -8,7 +8,9 @@ scala_library(
         "finagle/finagle-core/src/main",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
+        "joinkey/src/main/scala/com/twitter/joinkey/context",
+        "joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt",
@@ -18,6 +20,7 @@ scala_library(
         "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala",
         "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
         "src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala",
+        "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
         "timelines/src/main/scala/com/twitter/timelines/injection/scribe",
     ],
 )

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeAdsClientEventDetailsBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.builder
 
 import com.twitter.finagle.tracing.Trace
 import com.twitter.product_mixer.core.feature.featuremap.FeatureMap

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventDetailsBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.builder
 
 import com.twitter.bijection.Base64String
 import com.twitter.bijection.scrooge.BinaryScalaCodec

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTimelinesScoreInfoBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
 import com.twitter.home_mixer.param.HomeGlobalParams.EnableSendScoresToClient

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTweetTypePredicates.scala

@@ -1,8 +1,10 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.builder
 
 import com.twitter.conversions.DurationOps._
 import com.twitter.home_mixer.model.HomeFeatures._
 import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
+import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType
+import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType
 import com.twitter.timelinemixer.injection.model.candidate.SemanticCoreFeatures
 import com.twitter.tweetypie.{thriftscala => tpt}
 
@@ -25,74 +27,76 @@ object HomeTweetTypePredicates {
     (
       "has_exclusive_conversation_author_id",
       _.getOrElse(ExclusiveConversationAuthorIdFeature, None).nonEmpty),
-    ("is_eligible_for_connect_boost", _.getOrElse(AuthorIsEligibleForConnectBoostFeature, false)),
+    ("is_eligible_for_connect_boost", _ => false),
     ("hashtag", _.getOrElse(EarlybirdFeature, None).exists(_.numHashtags > 0)),
     ("has_scheduled_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isScheduled)),
     ("has_recorded_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isRecorded)),
     ("is_read_from_cache", _.getOrElse(IsReadFromCacheFeature, false)),
-    (
-      "is_self_thread_tweet",
-      _.getOrElse(ConversationFeature, None).exists(_.isSelfThreadTweet.contains(true))),
     ("get_initial", _.getOrElse(GetInitialFeature, false)),
     ("get_newer", _.getOrElse(GetNewerFeature, false)),
     ("get_middle", _.getOrElse(GetMiddleFeature, false)),
     ("get_older", _.getOrElse(GetOlderFeature, false)),
     ("pull_to_refresh", _.getOrElse(PullToRefreshFeature, false)),
     ("polling", _.getOrElse(PollingFeature, false)),
-    ("tls_size_20_plus", _ => false),
-    ("near_empty", _ => false),
-    ("ranked_request", _ => false),
+    ("near_empty", _.getOrElse(ServedSizeFeature, None).exists(_ < 3)),
+    ("is_request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)),
     ("mutual_follow", _.getOrElse(EarlybirdFeature, None).exists(_.fromMutualFollow)),
+    (
+      "less_than_10_mins_since_lnpt",
+      _.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 10.minutes)),
+    ("served_in_conversation_module", _.getOrElse(ServedInConversationModuleFeature, false)),
     ("has_ticketed_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasTickets)),
     ("in_utis_top5", _.getOrElse(PositionFeature, None).exists(_ < 5)),
-    ("is_utis_pos0", _.getOrElse(PositionFeature, None).exists(_ == 0)),
-    ("is_utis_pos1", _.getOrElse(PositionFeature, None).exists(_ == 1)),
-    ("is_utis_pos2", _.getOrElse(PositionFeature, None).exists(_ == 2)),
-    ("is_utis_pos3", _.getOrElse(PositionFeature, None).exists(_ == 3)),
-    ("is_utis_pos4", _.getOrElse(PositionFeature, None).exists(_ == 4)),
     (
-      "is_signup_request",
-      candidate => candidate.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)),
-    ("empty_request", _ => false),
-    ("served_size_less_than_5", _.getOrElse(ServedSizeFeature, None).exists(_ < 5)),
-    ("served_size_less_than_10", _.getOrElse(ServedSizeFeature, None).exists(_ < 10)),
-    ("served_size_less_than_20", _.getOrElse(ServedSizeFeature, None).exists(_ < 20)),
+      "conversation_module_has_2_displayed_tweets",
+      _.getOrElse(ConversationModule2DisplayedTweetsFeature, false)),
+    ("empty_request", _.getOrElse(ServedSizeFeature, None).exists(_ == 0)),
     ("served_size_less_than_50", _.getOrElse(ServedSizeFeature, None).exists(_ < 50)),
     (
       "served_size_between_50_and_100",
       _.getOrElse(ServedSizeFeature, None).exists(size => size >= 50 && size < 100)),
     ("authored_by_contextual_user", _.getOrElse(AuthoredByContextualUserFeature, false)),
+    (
+      "is_self_thread_tweet",
+      _.getOrElse(ConversationFeature, None).exists(_.isSelfThreadTweet.contains(true))),
     ("has_ancestors", _.getOrElse(AncestorsFeature, Seq.empty).nonEmpty),
     ("full_scoring_succeeded", _.getOrElse(FullScoringSucceededFeature, false)),
+    ("served_size_less_than_20", _.getOrElse(ServedSizeFeature, None).exists(_ < 20)),
+    ("served_size_less_than_10", _.getOrElse(ServedSizeFeature, None).exists(_ < 10)),
+    ("served_size_less_than_5", _.getOrElse(ServedSizeFeature, None).exists(_ < 5)),
     (
       "account_age_less_than_30_minutes",
       _.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)),
+    ("conversation_module_has_gap", _.getOrElse(ConversationModuleHasGapFeature, false)),
+    (
+      "directed_at_user_is_in_first_degree",
+      _.getOrElse(EarlybirdFeature, None).exists(_.directedAtUserIdIsInFirstDegree.contains(true))),
+    (
+      "has_semantic_core_annotation",
+      _.getOrElse(EarlybirdFeature, None).exists(_.semanticCoreAnnotations.nonEmpty)),
+    ("is_request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false)),
     (
       "account_age_less_than_1_day",
       _.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 1.day)),
     (
       "account_age_less_than_7_days",
       _.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 7.days)),
-    (
-      "directed_at_user_is_in_first_degree",
-      _.getOrElse(EarlybirdFeature, None).exists(_.directedAtUserIdIsInFirstDegree.contains(true))),
-    ("root_user_is_in_first_degree", _ => false),
-    (
-      "has_semantic_core_annotation",
-      _.getOrElse(EarlybirdFeature, None).exists(_.semanticCoreAnnotations.nonEmpty)),
-    ("is_request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false)),
     (
       "part_of_utt",
       _.getOrElse(EarlybirdFeature, None)
         .exists(_.semanticCoreAnnotations.exists(_.exists(annotation =>
           annotation.domainId == SemanticCoreFeatures.UnifiedTwitterTaxonomy)))),
+    (
+      "has_home_latest_request_past_week",
+      _.getOrElse(FollowingLastNonPollingTimeFeature, None).exists(_.untilNow < 7.days)),
+    ("is_utis_pos0", _.getOrElse(PositionFeature, None).exists(_ == 0)),
+    ("is_utis_pos1", _.getOrElse(PositionFeature, None).exists(_ == 1)),
+    ("is_utis_pos2", _.getOrElse(PositionFeature, None).exists(_ == 2)),
+    ("is_utis_pos3", _.getOrElse(PositionFeature, None).exists(_ == 3)),
+    ("is_utis_pos4", _.getOrElse(PositionFeature, None).exists(_ == 4)),
     ("is_random_tweet", _.getOrElse(IsRandomTweetFeature, false)),
     ("has_random_tweet_in_response", _.getOrElse(HasRandomTweetFeature, false)),
     ("is_random_tweet_above_in_utis", _.getOrElse(IsRandomTweetAboveFeature, false)),
-    ("is_request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)),
-    ("viewer_is_employee", _ => false),
-    ("viewer_is_timelines_employee", _ => false),
-    ("viewer_follows_any_topics", _.getOrElse(UserFollowedTopicsCountFeature, None).exists(_ > 0)),
     (
       "has_ancestor_authored_by_viewer",
       candidate =>
@@ -100,11 +104,6 @@ object HomeTweetTypePredicates {
           .getOrElse(AncestorsFeature, Seq.empty).exists(ancestor =>
             candidate.getOrElse(ViewerIdFeature, 0L) == ancestor.userId)),
     ("ancestor", _.getOrElse(IsAncestorCandidateFeature, false)),
-    (
-      "root_ancestor",
-      candidate =>
-        candidate.getOrElse(IsAncestorCandidateFeature, false) && candidate
-          .getOrElse(InReplyToTweetIdFeature, None).isEmpty),
     (
       "deep_reply",
       candidate =>
@@ -119,23 +118,22 @@ object HomeTweetTypePredicates {
       "tweet_age_less_than_15_seconds",
       _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
         .exists(_.untilNow <= 15.seconds)),
-    ("is_followed_topic_tweet", _ => false),
-    ("is_recommended_topic_tweet", _ => false),
-    ("is_topic_tweet", _ => false),
-    ("preferred_language_matches_tweet_language", _ => false),
+    (
+      "less_than_1_hour_since_lnpt",
+      _.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 1.hour)),
+    ("has_gte_10_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 10))),
     (
       "device_language_matches_tweet_language",
       candidate =>
         candidate.getOrElse(TweetLanguageFeature, None) ==
           candidate.getOrElse(DeviceLanguageFeature, None)),
+    (
+      "root_ancestor",
+      candidate =>
+        candidate.getOrElse(IsAncestorCandidateFeature, false) && candidate
+          .getOrElse(InReplyToTweetIdFeature, None).isEmpty),
     ("question", _.getOrElse(EarlybirdFeature, None).exists(_.hasQuestion.contains(true))),
-    ("in_network", _.getOrElse(FromInNetworkSourceFeature, true)),
-    ("viewer_follows_original_author", _ => false),
-    ("has_account_follow_prompt", _ => false),
-    ("has_relevance_prompt", _ => false),
-    ("has_topic_annotation_haug_prompt", _ => false),
-    ("has_topic_annotation_random_precision_prompt", _ => false),
-    ("has_topic_annotation_prompt", _ => false),
+    ("in_network", _.getOrElse(InNetworkFeature, true)),
     (
       "has_political_annotation",
       _.getOrElse(EarlybirdFeature, None).exists(
@@ -153,9 +151,14 @@ object HomeTweetTypePredicates {
       _.getOrElse(EarlybirdFeature, None)
         .exists(_.conversationControl.exists(_.isInstanceOf[tpt.ConversationControl.Community]))),
     ("has_zero_score", _.getOrElse(ScoreFeature, None).exists(_ == 0.0)),
-    ("is_viewer_not_invited_to_reply", _ => false),
-    ("is_viewer_invited_to_reply", _ => false),
-    ("has_gte_10_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 10))),
+    (
+      "is_followed_topic_tweet",
+      _.getOrElse(TopicContextFunctionalityTypeFeature, None)
+        .exists(_ == BasicTopicContextFunctionalityType)),
+    (
+      "is_recommended_topic_tweet",
+      _.getOrElse(TopicContextFunctionalityTypeFeature, None)
+        .exists(_ == RecommendationTopicContextFunctionalityType)),
     ("has_gte_100_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100))),
     ("has_gte_1k_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 1000))),
     (
@@ -164,8 +167,6 @@ object HomeTweetTypePredicates {
     (
       "has_gte_100k_favs",
       _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100000))),
-    ("above_neighbor_is_topic_tweet", _ => false),
-    ("is_topic_tweet_with_neighbor_below", _ => false),
     ("has_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasSpace)),
     ("has_live_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isLive)),
     (
@@ -187,6 +188,7 @@ object HomeTweetTypePredicates {
     (
       "has_toxicity_score_above_threshold",
       _.getOrElse(EarlybirdFeature, None).exists(_.toxicityScore.exists(_ > 0.91))),
+    ("is_topic_tweet", _.getOrElse(TopicIdSocialContextFeature, None).isDefined),
     (
       "text_only",
       candidate =>
@@ -204,23 +206,50 @@ object HomeTweetTypePredicates {
     ("has_3_images", _.getOrElse(NumImagesFeature, None).exists(_ == 3)),
     ("has_4_images", _.getOrElse(NumImagesFeature, None).exists(_ == 4)),
     ("has_card", _.getOrElse(EarlybirdFeature, None).exists(_.hasCard)),
-    ("3_or_more_consecutive_not_in_network", _ => false),
-    ("2_or_more_consecutive_not_in_network", _ => false),
-    ("5_out_of_7_not_in_network", _ => false),
-    ("7_out_of_7_not_in_network", _ => false),
-    ("5_out_of_5_not_in_network", _ => false),
     ("user_follow_count_gte_50", _.getOrElse(UserFollowingCountFeature, None).exists(_ > 50)),
-    ("has_liked_by_social_context", _ => false),
-    ("has_followed_by_social_context", _ => false),
-    ("has_topic_social_context", _ => false),
-    ("timeline_entry_has_banner", _ => false),
-    ("served_in_conversation_module", _.getOrElse(ServedInConversationModuleFeature, false)),
     (
-      "conversation_module_has_2_displayed_tweets",
-      _.getOrElse(ConversationModule2DisplayedTweetsFeature, false)),
-    ("conversation_module_has_gap", _.getOrElse(ConversationModuleHasGapFeature, false)),
-    ("served_in_recap_tweet_candidate_module_injection", _ => false),
-    ("served_in_threaded_conversation_module", _ => false)
+      "has_liked_by_social_context",
+      candidateFeatures =>
+        candidateFeatures
+          .getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty)
+          .exists(candidateFeatures
+            .getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Seq.empty).toSet.contains)),
+    (
+      "has_followed_by_social_context",
+      _.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty).nonEmpty),
+    (
+      "has_topic_social_context",
+      candidateFeatures =>
+        candidateFeatures
+          .getOrElse(TopicIdSocialContextFeature, None)
+          .isDefined &&
+          candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None).isDefined),
+    ("video_lte_10_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ <= 10000)),
+    (
+      "video_bt_10_60_sec",
+      _.getOrElse(VideoDurationMsFeature, None).exists(duration =>
+        duration > 10000 && duration <= 60000)),
+    ("video_gt_60_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ > 60000)),
+    (
+      "tweet_age_lte_30_minutes",
+      _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
+        .exists(_.untilNow <= 30.minutes)),
+    (
+      "tweet_age_lte_1_hour",
+      _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
+        .exists(_.untilNow <= 1.hour)),
+    (
+      "tweet_age_lte_6_hours",
+      _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
+        .exists(_.untilNow <= 6.hours)),
+    (
+      "tweet_age_lte_12_hours",
+      _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
+        .exists(_.untilNow <= 12.hours)),
+    (
+      "tweet_age_gte_24_hours",
+      _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
+        .exists(_.untilNow >= 24.hours)),
   )
 
   val PredicateMap = CandidatePredicates.toMap

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/ListClientEventDetailsBuilder.scala

@@ -8,7 +8,7 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ti
 import com.twitter.product_mixer.core.pipeline.PipelineQuery
 import com.twitter.timelineservice.suggests.{thriftscala => st}
 
-object ListClientEventDetailsBuilder
+case class ListClientEventDetailsBuilder(suggestType: st.SuggestType)
     extends BaseClientEventDetailsBuilder[PipelineQuery, UniversalNoun[Any]] {
 
   override def apply(
@@ -20,7 +20,7 @@ object ListClientEventDetailsBuilder
       conversationDetails = None,
       timelinesDetails = Some(
         TimelinesDetails(
-          injectionType = Some(st.SuggestType.OrganicListTweet.name),
+          injectionType = Some(suggestType.name),
           controllerData = None,
           sourceData = None)),
       articleDetails = None,

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AuthorChildFeedbackActionBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature
 import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
@@ -9,7 +9,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ch
 import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
 import com.twitter.stringcenter.client.StringCenter
 import com.twitter.timelines.service.{thriftscala => t}
-
 import javax.inject.Inject
 import javax.inject.Singleton
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BUILD.bazel

@@ -4,9 +4,16 @@ scala_library(
     strict_deps = True,
     tags = ["bazel-compatible"],
     dependencies = [
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder",
+        "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt",
         "src/thrift/com/twitter/timelines/service:thrift-scala",
         "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
+        "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
     ],
 )

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BlockUserChildFeedbackActionBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
 import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
@@ -13,7 +13,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
 import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorBlockUser
 import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
 import com.twitter.stringcenter.client.StringCenter
-
 import javax.inject.Inject
 import javax.inject.Singleton
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/DontLikeFeedbackActionBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.conversions.DurationOps._
 import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
@@ -17,7 +17,6 @@ import com.twitter.timelines.common.{thriftscala => tlc}
 import com.twitter.timelineservice.model.FeedbackInfo
 import com.twitter.timelineservice.model.FeedbackMetadata
 import com.twitter.timelineservice.{thriftscala => tls}
-
 import javax.inject.Inject
 import javax.inject.Singleton
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/EngagerSocialContextBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.RealNamesFeature
 import com.twitter.product_mixer.core.feature.featuremap.FeatureMap

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ExtendedReplySocialContextBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.FocalTweetAuthorIdFeature
 import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackStrings.scala

@@ -0,0 +1,18 @@
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
+
+import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
+import com.twitter.stringcenter.client.ExternalStringRegistry
+import javax.inject.Inject
+import javax.inject.Provider
+import javax.inject.Singleton
+
+@Singleton
+class FeedbackStrings @Inject() (
+  @ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry]) {
+  private val externalStringRegistry = externalStringRegistryProvider.get()
+
+  val seeLessOftenFeedbackString =
+    externalStringRegistry.createProdString("Feedback.seeLessOften")
+  val seeLessOftenConfirmationFeedbackString =
+    externalStringRegistry.createProdString("Feedback.seeLessOftenConfirmation")
+}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackUtil.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.conversions.DurationOps._
 import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FollowedBySocialContextBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
 import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeFeedbackActionInfoBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
 import com.twitter.home_mixer.model.request.FollowingProduct
@@ -12,7 +12,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Fe
 import com.twitter.product_mixer.core.pipeline.PipelineQuery
 import com.twitter.timelines.service.{thriftscala => t}
 import com.twitter.timelines.util.FeedbackMetadataSerializer
-
 import javax.inject.Inject
 import javax.inject.Singleton
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeTweetSocialContextBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature
 import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleIdFeature
@@ -14,10 +14,13 @@ import javax.inject.Singleton
 @Singleton
 case class HomeTweetSocialContextBuilder @Inject() (
   likedBySocialContextBuilder: LikedBySocialContextBuilder,
+  listsSocialContextBuilder: ListsSocialContextBuilder,
   followedBySocialContextBuilder: FollowedBySocialContextBuilder,
   topicSocialContextBuilder: TopicSocialContextBuilder,
   extendedReplySocialContextBuilder: ExtendedReplySocialContextBuilder,
-  receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder)
+  receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder,
+  popularVideoSocialContextBuilder: PopularVideoSocialContextBuilder,
+  popularInYourAreaSocialContextBuilder: PopularInYourAreaSocialContextBuilder)
     extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
 
   def apply(
@@ -31,6 +34,9 @@ case class HomeTweetSocialContextBuilder @Inject() (
           likedBySocialContextBuilder(query, candidate, features)
             .orElse(followedBySocialContextBuilder(query, candidate, features))
             .orElse(topicSocialContextBuilder(query, candidate, features))
+            .orElse(popularVideoSocialContextBuilder(query, candidate, features))
+            .orElse(listsSocialContextBuilder(query, candidate, features))
+            .orElse(popularInYourAreaSocialContextBuilder(query, candidate, features))
         case Some(_) =>
           val conversationId = features.getOrElse(ConversationModuleIdFeature, None)
           // Only hydrate the social context into the root tweet in a conversation module

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToFollowFeedbackActionInfoBuilder.scala

@@ -5,7 +5,6 @@ import com.twitter.product_mixer.component_library.model.candidate.UserCandidate
 import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
 import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder
 import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
-import com.twitter.stringcenter.client.ExternalStringRegistry
 import com.twitter.stringcenter.client.StringCenter
 import javax.inject.Inject
 import javax.inject.Provider
@@ -32,12 +31,13 @@ object HomeWhoToFollowFeedbackActionInfoBuilder {
 
 @Singleton
 case class HomeWhoToFollowFeedbackActionInfoBuilder @Inject() (
-  @ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry],
+  feedbackStrings: FeedbackStrings,
   @ProductScoped stringCenterProvider: Provider[StringCenter])
     extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] {
 
   private val whoToFollowFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder(
-    externalStringRegistry = externalStringRegistryProvider.get(),
+    seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString,
+    seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenConfirmationFeedbackString,
     stringCenter = stringCenterProvider.get(),
     encodedFeedbackRequest = Some(HomeWhoToFollowFeedbackActionInfoBuilder.EncodedFeedbackRequest)
   )

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToSubscribeFeedbackActionInfoBuilder.scala

@@ -0,0 +1,52 @@
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
+
+import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.WhoToFollowFeedbackActionInfoBuilder
+import com.twitter.product_mixer.component_library.model.candidate.UserCandidate
+import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
+import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder
+import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
+import com.twitter.stringcenter.client.StringCenter
+import javax.inject.Inject
+import javax.inject.Provider
+import javax.inject.Singleton
+import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackActionInfo
+import com.twitter.product_mixer.core.pipeline.PipelineQuery
+import com.twitter.timelines.service.{thriftscala => tl}
+import com.twitter.timelines.util.FeedbackRequestSerializer
+import com.twitter.timelineservice.suggests.thriftscala.SuggestType
+import com.twitter.timelineservice.thriftscala.FeedbackType
+
+object HomeWhoToSubscribeFeedbackActionInfoBuilder {
+  private val FeedbackMetadata = tl.FeedbackMetadata(
+    injectionType = Some(SuggestType.WhoToSubscribe),
+    engagementType = None,
+    entityIds = Seq.empty,
+    ttlMs = None
+  )
+  private val FeedbackRequest =
+    tl.DefaultFeedbackRequest2(FeedbackType.SeeFewer, FeedbackMetadata)
+  private val EncodedFeedbackRequest =
+    FeedbackRequestSerializer.serialize(tl.FeedbackRequest.DefaultFeedbackRequest2(FeedbackRequest))
+}
+
+@Singleton
+case class HomeWhoToSubscribeFeedbackActionInfoBuilder @Inject() (
+  feedbackStrings: FeedbackStrings,
+  @ProductScoped stringCenterProvider: Provider[StringCenter])
+    extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] {
+
+  private val whoToSubscribeFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder(
+    seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString,
+    seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenConfirmationFeedbackString,
+    stringCenter = stringCenterProvider.get(),
+    encodedFeedbackRequest =
+      Some(HomeWhoToSubscribeFeedbackActionInfoBuilder.EncodedFeedbackRequest)
+  )
+
+  override def apply(
+    query: PipelineQuery,
+    candidate: UserCandidate,
+    candidateFeatures: FeatureMap
+  ): Option[FeedbackActionInfo] =
+    whoToSubscribeFeedbackActionInfoBuilder.apply(query, candidate, candidateFeatures)
+}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/LikedBySocialContextBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature
 import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ListsSocialContextBuilder.scala

@@ -0,0 +1,50 @@
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
+
+import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
+import com.twitter.home_mixer.model.HomeFeatures.UserScreenNameFeature
+import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
+import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
+import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
+import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
+import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext
+import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
+import com.twitter.product_mixer.core.pipeline.PipelineQuery
+import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
+import com.twitter.stringcenter.client.StringCenter
+import com.twitter.timelineservice.suggests.{thriftscala => t}
+import javax.inject.Inject
+import javax.inject.Provider
+import javax.inject.Singleton
+
+/**
+ * "Your Lists" will be rendered for the context and a url link for your lists.
+ */
+@Singleton
+case class ListsSocialContextBuilder @Inject() (
+  externalStrings: HomeMixerExternalStrings,
+  @ProductScoped stringCenterProvider: Provider[StringCenter])
+    extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
+
+  private val stringCenter = stringCenterProvider.get()
+  private val listString = externalStrings.ownedSubscribedListsModuleHeaderString
+
+  def apply(
+    query: PipelineQuery,
+    candidate: TweetCandidate,
+    candidateFeatures: FeatureMap
+  ): Option[SocialContext] = {
+    candidateFeatures.get(SuggestTypeFeature) match {
+      case Some(suggestType) if suggestType == t.SuggestType.RankedListTweet =>
+        val userName = query.features.flatMap(_.getOrElse(UserScreenNameFeature, None))
+        Some(
+          GeneralContext(
+            contextType = ListGeneralContextType,
+            text = stringCenter.prepare(listString),
+            url = userName.map(name => ""),
+            contextImageUrls = None,
+            landingUrl = None
+          ))
+      case _ => None
+    }
+  }
+}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/MuteUserChildFeedbackActionBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
 import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
@@ -12,7 +12,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
 import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleMuteUser
 import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
 import com.twitter.stringcenter.client.StringCenter
-
 import javax.inject.Inject
 import javax.inject.Singleton
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotInterestedTopicFeedbackActionBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
 import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature
@@ -15,7 +15,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
 import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorMarkNotInterestedTopic
 import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
 import com.twitter.stringcenter.client.StringCenter
-
 import javax.inject.Inject
 import javax.inject.Singleton
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotRelevantChildFeedbackActionBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
 import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
@@ -12,7 +12,6 @@ import com.twitter.timelines.common.{thriftscala => tlc}
 import com.twitter.timelineservice.model.FeedbackInfo
 import com.twitter.timelineservice.model.FeedbackMetadata
 import com.twitter.timelineservice.{thriftscala => tlst}
-
 import javax.inject.Inject
 import javax.inject.Singleton
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularInYourAreaSocialContextBuilder.scala

@@ -0,0 +1,43 @@
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
+
+import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
+import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
+import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
+import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
+import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
+import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
+import com.twitter.product_mixer.core.pipeline.PipelineQuery
+import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
+import com.twitter.stringcenter.client.StringCenter
+import com.twitter.timelineservice.suggests.{thriftscala => st}
+import javax.inject.Inject
+import javax.inject.Provider
+import javax.inject.Singleton
+
+@Singleton
+case class PopularInYourAreaSocialContextBuilder @Inject() (
+  externalStrings: HomeMixerExternalStrings,
+  @ProductScoped stringCenterProvider: Provider[StringCenter])
+    extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
+
+  private val stringCenter = stringCenterProvider.get()
+  private val popularInYourAreaString = externalStrings.socialContextPopularInYourAreaString
+
+  def apply(
+    query: PipelineQuery,
+    candidate: TweetCandidate,
+    candidateFeatures: FeatureMap
+  ): Option[SocialContext] = {
+    val suggestTypeOpt = candidateFeatures.getOrElse(SuggestTypeFeature, None)
+    if (suggestTypeOpt.contains(st.SuggestType.RecommendedTrendTweet)) {
+      Some(
+        GeneralContext(
+          contextType = LocationGeneralContextType,
+          text = stringCenter.prepare(popularInYourAreaString),
+          url = None,
+          contextImageUrls = None,
+          landingUrl = None
+        ))
+    } else None
+  }
+}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularVideoSocialContextBuilder.scala

@@ -1,50 +1,48 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
-import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
-import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
+import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
 import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
 import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
 import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
 import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
-import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext
 import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
 import com.twitter.product_mixer.core.pipeline.PipelineQuery
 import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
 import com.twitter.stringcenter.client.StringCenter
+import com.twitter.timelineservice.suggests.{thriftscala => st}
 import javax.inject.Inject
 import javax.inject.Provider
 import javax.inject.Singleton
 
-/**
- * Renders a fixed 'You Might Like' string above all OON Tweets.
- */
 @Singleton
-case class YouMightLikeSocialContextBuilder @Inject() (
+case class PopularVideoSocialContextBuilder @Inject() (
   externalStrings: HomeMixerExternalStrings,
   @ProductScoped stringCenterProvider: Provider[StringCenter])
     extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
 
   private val stringCenter = stringCenterProvider.get()
-  private val youMightLikeString = externalStrings.socialContextYouMightLikeString
+  private val popularVideoString = externalStrings.socialContextPopularVideoString
 
   def apply(
     query: PipelineQuery,
     candidate: TweetCandidate,
     candidateFeatures: FeatureMap
   ): Option[SocialContext] = {
-    val isInNetwork = candidateFeatures.getOrElse(InNetworkFeature, true)
-    val isRetweet = candidateFeatures.getOrElse(IsRetweetFeature, false)
-    if (!isInNetwork && !isRetweet) {
+    val suggestTypeOpt = candidateFeatures.getOrElse(SuggestTypeFeature, None)
+    if (suggestTypeOpt.contains(st.SuggestType.MediaTweet)) {
       Some(
         GeneralContext(
           contextType = SparkleGeneralContextType,
-          text = stringCenter.prepare(youMightLikeString),
+          text = stringCenter.prepare(popularVideoString),
           url = None,
           contextImageUrls = None,
-          landingUrl = None
+          landingUrl = Some(
+            Url(
+              urlType = DeepLink,
+              url = ""
+            )
+          )
         ))
-    } else {
-      None
-    }
+    } else None
   }
 }

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReceivedReplySocialContextBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
 import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReportTweetChildFeedbackActionBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
 import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
@@ -8,7 +8,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
 import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorReportTweet
 import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
 import com.twitter.stringcenter.client.StringCenter
-
 import javax.inject.Inject
 import javax.inject.Singleton
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/RetweeterChildFeedbackActionBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
 import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
@@ -10,7 +10,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ch
 import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
 import com.twitter.stringcenter.client.StringCenter
 import com.twitter.timelines.service.{thriftscala => t}
-
 import javax.inject.Inject
 import javax.inject.Singleton
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/TopicSocialContextBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
 import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/UnfollowUserChildFeedbackActionBuilder.scala

@@ -1,4 +1,4 @@
-package com.twitter.home_mixer.functional_component.decorator
+package com.twitter.home_mixer.functional_component.decorator.urt.builder
 
 import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
 import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
@@ -11,7 +11,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
 import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleFollowUser
 import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
 import com.twitter.stringcenter.client.StringCenter
-
 import javax.inject.Inject
 import javax.inject.Singleton
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BUILD.bazel

@@ -4,95 +4,56 @@ scala_library(
     strict_deps = True,
     tags = ["bazel-compatible"],
     dependencies = [
-        "3rdparty/jvm/com/twitter/storehaus:core",
         "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
         "configapi/configapi-decider",
         "finatra/inject/inject-core/src/main/scala",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/earlybird",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/inferred_topic",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content",
         "joinkey/src/main/scala/com/twitter/joinkey/context",
         "joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
-        "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store",
-        "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/topics",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_visibility_reason",
+        "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
-        "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord",
-        "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util",
-        "representation-scorer/server/src/main/scala/com/twitter/representationscorer/common",
-        "representation-scorer/server/src/main/thrift:thrift-scala",
-        "servo/repo/src/main/scala",
         "snowflake/src/main/scala/com/twitter/snowflake/id",
-        "src/java/com/twitter/ml/api/constant",
         "src/java/com/twitter/search/common/util/lang",
-        "src/scala/com/twitter/ml/api/util",
-        "src/scala/com/twitter/timelines/prediction/adapters/real_graph",
-        "src/scala/com/twitter/timelines/prediction/adapters/realtime_interaction_graph",
-        "src/scala/com/twitter/timelines/prediction/adapters/twistly",
-        "src/scala/com/twitter/timelines/prediction/adapters/two_hop_features",
-        "src/scala/com/twitter/timelines/prediction/common/util",
-        "src/scala/com/twitter/timelines/prediction/features/common",
-        "src/scala/com/twitter/timelines/prediction/features/realtime_interaction_graph",
-        "src/scala/com/twitter/timelines/prediction/features/recap",
-        "src/scala/com/twitter/timelines/prediction/features/time_features",
+        "src/scala/com/twitter/timelines/prediction/adapters/request_context",
         "src/thrift/com/twitter/gizmoduck:thrift-scala",
-        "src/thrift/com/twitter/ml/api:data-java",
-        "src/thrift/com/twitter/ml/api:embedding-java",
-        "src/thrift/com/twitter/onboarding/relevance/features:features-java",
         "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala",
         "src/thrift/com/twitter/search:earlybird-scala",
         "src/thrift/com/twitter/search/common:constants-java",
         "src/thrift/com/twitter/socialgraph:thrift-scala",
         "src/thrift/com/twitter/spam/rtf:safety-result-scala",
         "src/thrift/com/twitter/timelineranker:thrift-scala",
-        "src/thrift/com/twitter/timelines/author_features:thrift-java",
-        "src/thrift/com/twitter/timelines/conversation_features:conversation_features-scala",
         "src/thrift/com/twitter/timelines/impression:thrift-scala",
         "src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala",
         "src/thrift/com/twitter/timelines/real_graph:real_graph-scala",
         "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala",
-        "src/thrift/com/twitter/topic_recos:topic_recos-thrift-java",
         "src/thrift/com/twitter/tweetypie:service-scala",
         "src/thrift/com/twitter/tweetypie:tweet-scala",
         "src/thrift/com/twitter/user_session_store:thrift-java",
         "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala",
-        "src/thrift/com/twitter/wtf/real_time_interaction_graph:wtf-real_time_interaction_graph-thrift-java",
         "stitch/stitch-core",
         "stitch/stitch-gizmoduck",
         "stitch/stitch-socialgraph",
         "stitch/stitch-timelineservice",
         "stitch/stitch-tweetypie",
-        "strato/config/columns/topic-signals/tsp",
-        "strato/config/columns/topic-signals/tsp:tsp-strato-client",
         "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/feedback",
         "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
         "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/persistence",
-        "timelines/src/main/scala/com/twitter/timelines/clients/strato/twistly",
-        "timelines/src/main/scala/com/twitter/timelines/clients/user_tweet_entity_graph",
+        "timelines/src/main/scala/com/twitter/timelines/clients/manhattan/store",
         "timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter",
         "timelines/src/main/scala/com/twitter/timelines/impressionstore/store",
         "timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
-        "topic-social-proof/server/src/main/thrift:thrift-scala",
-        "topiclisting/topiclisting-core/src/main/scala/com/twitter/topiclisting",
-        "tweetconvosvc/thrift/src/main/thrift:thrift-scala",
-        "twitter-config/yaml",
         "user_session_store/src/main/scala/com/twitter/user_session_store",
         "util/util-core",
     ],

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FeedbackHistoryQueryFeatureHydrator.scala

@@ -1,12 +1,10 @@
 package com.twitter.home_mixer.functional_component.feature_hydrator
 
 import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature
-import com.twitter.home_mixer.param.HomeGlobalParams.EnableFeedbackFatigueParam
 import com.twitter.product_mixer.core.feature.Feature
 import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
 import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
 import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
-import com.twitter.product_mixer.core.model.common.Conditionally
 import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
 import com.twitter.product_mixer.core.pipeline.PipelineQuery
 import com.twitter.stitch.Stitch
@@ -17,16 +15,12 @@ import javax.inject.Singleton
 @Singleton
 case class FeedbackHistoryQueryFeatureHydrator @Inject() (
   feedbackHistoryClient: FeedbackHistoryManhattanClient)
-    extends QueryFeatureHydrator[PipelineQuery]
-    with Conditionally[PipelineQuery] {
+    extends QueryFeatureHydrator[PipelineQuery] {
 
   override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FeedbackHistory")
 
   override val features: Set[Feature[_, _]] = Set(FeedbackHistoryFeature)
 
-  override def onlyIf(query: PipelineQuery): Boolean =
-    query.params(EnableFeedbackFatigueParam)
-
   override def hydrate(
     query: PipelineQuery
   ): Stitch[FeatureMap] =

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FollowedTopicsQueryFeatureHydrator.scala

@@ -1,41 +0,0 @@
-package com.twitter.home_mixer.functional_component.feature_hydrator
-
-import com.twitter.conversions.DurationOps._
-import com.twitter.home_mixer.model.HomeFeatures.UserFollowedTopicsCountFeature
-import com.twitter.home_mixer.service.HomeMixerAlertConfig
-import com.twitter.product_mixer.component_library.candidate_source.topics.FollowedTopicsCandidateSource
-import com.twitter.product_mixer.core.feature.Feature
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
-import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyView
-import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
-import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
-import com.twitter.product_mixer.core.pipeline.PipelineQuery
-import com.twitter.stitch.Stitch
-import javax.inject.Inject
-import javax.inject.Singleton
-
-@Singleton
-case class FollowedTopicsQueryFeatureHydrator @Inject() (
-  followedTopicsCandidateSource: FollowedTopicsCandidateSource)
-    extends QueryFeatureHydrator[PipelineQuery] {
-
-  override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FollowedTopics")
-
-  override val features: Set[Feature[_, _]] = Set(UserFollowedTopicsCountFeature)
-
-  override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = {
-    val request: StratoKeyView[Long, Unit] = StratoKeyView(query.getRequiredUserId, Unit)
-    followedTopicsCandidateSource(request)
-      .map { topics =>
-        FeatureMapBuilder().add(UserFollowedTopicsCountFeature, Some(topics.size)).build()
-      }.handle {
-        case _ => FeatureMapBuilder().add(UserFollowedTopicsCountFeature, None).build()
-      }
-  }
-
-  override val alerts = Seq(
-    HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.9),
-    HomeMixerAlertConfig.BusinessHours.defaultLatencyAlert(1500.millis)
-  )
-}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GizmoduckAuthorSafetyFeatureHydrator.scala

@@ -1,58 +0,0 @@
-package com.twitter.home_mixer.functional_component.feature_hydrator
-
-import com.twitter.gizmoduck.{thriftscala => gt}
-import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
-import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature
-import com.twitter.home_mixer.param.HomeGlobalParams.EnableGizmoduckAuthorSafetyFeatureHydratorParam
-import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
-import com.twitter.product_mixer.core.feature.Feature
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
-import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
-import com.twitter.product_mixer.core.model.common.Conditionally
-import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
-import com.twitter.product_mixer.core.pipeline.PipelineQuery
-import com.twitter.stitch.Stitch
-import com.twitter.stitch.gizmoduck.Gizmoduck
-import javax.inject.Inject
-import javax.inject.Singleton
-
-@Singleton
-class GizmoduckAuthorSafetyFeatureHydrator @Inject() (gizmoduck: Gizmoduck)
-    extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
-    with Conditionally[PipelineQuery] {
-
-  override val identifier: FeatureHydratorIdentifier =
-    FeatureHydratorIdentifier("GizmoduckAuthorSafety")
-
-  override val features: Set[Feature[_, _]] = Set(AuthorIsBlueVerifiedFeature)
-
-  override def onlyIf(query: PipelineQuery): Boolean =
-    query.params(EnableGizmoduckAuthorSafetyFeatureHydratorParam)
-
-  private val queryFields: Set[gt.QueryFields] = Set(gt.QueryFields.Safety)
-
-  override def apply(
-    query: PipelineQuery,
-    candidate: TweetCandidate,
-    existingFeatures: FeatureMap
-  ): Stitch[FeatureMap] = {
-    val authorIdOption = existingFeatures.getOrElse(AuthorIdFeature, None)
-
-    val blueVerifiedStitch = authorIdOption
-      .map { authorId =>
-        gizmoduck
-          .getUserById(
-            userId = authorId,
-            queryFields = queryFields
-          )
-          .map { _.safety.flatMap(_.isBlueVerified).getOrElse(false) }
-      }.getOrElse(Stitch.False)
-
-    blueVerifiedStitch.map { isBlueVerified =>
-      FeatureMapBuilder()
-        .add(AuthorIsBlueVerifiedFeature, isBlueVerified)
-        .build()
-    }
-  }
-}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressionBloomFilterQueryFeatureHydrator.scala

@@ -3,6 +3,7 @@ package com.twitter.home_mixer.functional_component.feature_hydrator
 import com.twitter.conversions.DurationOps._
 import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature
 import com.twitter.home_mixer.model.request.HasSeenTweetIds
+import com.twitter.home_mixer.param.HomeGlobalParams.ImpressionBloomFilterFalsePositiveRateParam
 import com.twitter.home_mixer.service.HomeMixerAlertConfig
 import com.twitter.product_mixer.core.feature.Feature
 import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
@@ -11,7 +12,8 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Quer
 import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
 import com.twitter.product_mixer.core.pipeline.PipelineQuery
 import com.twitter.stitch.Stitch
-import com.twitter.timelines.impressionbloomfilter.{thriftscala => t}
+import com.twitter.timelines.clients.manhattan.store.ManhattanStoreClient
+import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm}
 import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilter
 import javax.inject.Inject
 import javax.inject.Singleton
@@ -19,32 +21,35 @@ import javax.inject.Singleton
 @Singleton
 case class ImpressionBloomFilterQueryFeatureHydrator[
   Query <: PipelineQuery with HasSeenTweetIds] @Inject() (
-  bloomFilter: ImpressionBloomFilter)
-    extends QueryFeatureHydrator[Query] {
+  bloomFilterClient: ManhattanStoreClient[
+    blm.ImpressionBloomFilterKey,
+    blm.ImpressionBloomFilterSeq
+  ]) extends QueryFeatureHydrator[Query] {
 
   override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier(
     "ImpressionBloomFilter")
 
   private val ImpressionBloomFilterTTL = 7.day
-  private val ImpressionBloomFilterFalsePositiveRate = 0.002
 
   override val features: Set[Feature[_, _]] = Set(ImpressionBloomFilterFeature)
 
-  private val SurfaceArea = t.SurfaceArea.HomeTimeline
+  private val SurfaceArea = blm.SurfaceArea.HomeTimeline
 
   override def hydrate(query: Query): Stitch[FeatureMap] = {
     val userId = query.getRequiredUserId
-    bloomFilter.getBloomFilterSeq(userId, SurfaceArea).map { bloomFilterSeq =>
+    bloomFilterClient
+      .get(blm.ImpressionBloomFilterKey(userId, SurfaceArea))
+      .map(_.getOrElse(blm.ImpressionBloomFilterSeq(Seq.empty)))
+      .map { bloomFilterSeq =>
         val updatedBloomFilterSeq =
           if (query.seenTweetIds.forall(_.isEmpty)) bloomFilterSeq
           else {
-          bloomFilter.addElements(
-            userId = userId,
+            ImpressionBloomFilter.addSeenTweetIds(
               surfaceArea = SurfaceArea,
               tweetIds = query.seenTweetIds.get,
-            bloomFilterEntrySeq = bloomFilterSeq,
+              bloomFilterSeq = bloomFilterSeq,
               timeToLive = ImpressionBloomFilterTTL,
-            falsePositiveRate = ImpressionBloomFilterFalsePositiveRate
+              falsePositiveRate = query.params(ImpressionBloomFilterFalsePositiveRateParam)
             )
           }
         FeatureMapBuilder().add(ImpressionBloomFilterFeature, updatedBloomFilterSeq).build()

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/InNetworkFeatureHydrator.scala

@@ -0,0 +1,41 @@
+package com.twitter.home_mixer.functional_component.feature_hydrator
+
+import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
+import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
+import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature
+import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
+import com.twitter.product_mixer.core.feature.Feature
+import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
+import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
+import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
+import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
+import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
+import com.twitter.product_mixer.core.pipeline.PipelineQuery
+import com.twitter.stitch.Stitch
+
+object InNetworkFeatureHydrator
+    extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
+
+  override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("InNetwork")
+
+  override val features: Set[Feature[_, _]] = Set(InNetworkFeature)
+
+  override def apply(
+    query: PipelineQuery,
+    candidates: Seq[CandidateWithFeatures[TweetCandidate]]
+  ): Stitch[Seq[FeatureMap]] = {
+    val viewerId = query.getRequiredUserId
+    val followedUserIds = query.features.get.get(SGSFollowedUsersFeature).toSet
+
+    val featureMaps = candidates.map { candidate =>
+      // We use authorId and not sourceAuthorId here so that retweets are defined as in network
+      val isInNetworkOpt = candidate.features.getOrElse(AuthorIdFeature, None).map { authorId =>
+        // Users cannot follow themselves but this is in network by definition
+        val isSelfTweet = authorId == viewerId
+        isSelfTweet || followedUserIds.contains(authorId)
+      }
+      FeatureMapBuilder().add(InNetworkFeature, isInNetworkOpt.getOrElse(true)).build()
+    }
+    Stitch.value(featureMaps)
+  }
+}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PersistenceStoreQueryFeatureHydrator.scala

@@ -2,8 +2,10 @@ package com.twitter.home_mixer.functional_component.feature_hydrator
 
 import com.twitter.conversions.DurationOps._
 import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent
+import com.twitter.finagle.stats.StatsReceiver
 import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature
 import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature
+import com.twitter.home_mixer.model.HomeFeatures.ServedTweetPreviewIdsFeature
 import com.twitter.home_mixer.model.HomeFeatures.WhoToFollowExcludedUserIdsFeature
 import com.twitter.home_mixer.model.request.FollowingProduct
 import com.twitter.home_mixer.model.request.ForYouProduct
@@ -27,17 +29,27 @@ import javax.inject.Singleton
 
 @Singleton
 case class PersistenceStoreQueryFeatureHydrator @Inject() (
-  timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3])
+  timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3],
+  statsReceiver: StatsReceiver)
     extends QueryFeatureHydrator[PipelineQuery] {
 
   override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("PersistenceStore")
 
+  private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
+  private val servedTweetIdsSizeStat = scopedStatsReceiver.stat("ServedTweetIdsSize")
+
   private val WhoToFollowExcludedUserIdsLimit = 1000
-  private val ServedTweetIdsDuration = 1.hour
+  private val ServedTweetIdsDuration = 10.minutes
   private val ServedTweetIdsLimit = 100
+  private val ServedTweetPreviewIdsDuration = 10.hours
+  private val ServedTweetPreviewIdsLimit = 10
 
   override val features: Set[Feature[_, _]] =
-    Set(ServedTweetIdsFeature, PersistenceEntriesFeature, WhoToFollowExcludedUserIdsFeature)
+    Set(
+      ServedTweetIdsFeature,
+      ServedTweetPreviewIdsFeature,
+      PersistenceEntriesFeature,
+      WhoToFollowExcludedUserIdsFeature)
 
   private val supportedClients = Seq(
     ClientPlatform.IPhone,
@@ -80,8 +92,19 @@ case class PersistenceStoreQueryFeatureHydrator @Inject() (
             .flatMap(
               _.entries.flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetIdsLimit))
 
+          servedTweetIdsSizeStat.add(servedTweetIds.size)
+
+          val servedTweetPreviewIds = timelineResponses
+            .filter(_.clientPlatform == clientPlatform)
+            .filter(_.servedTime >= Time.now - ServedTweetPreviewIdsDuration)
+            .sortBy(-_.servedTime.inMilliseconds)
+            .flatMap(_.entries
+              .filter(_.entityIdType == EntityIdType.TweetPreview)
+              .flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetPreviewIdsLimit))
+
           FeatureMapBuilder()
             .add(ServedTweetIdsFeature, servedTweetIds)
+            .add(ServedTweetPreviewIdsFeature, servedTweetPreviewIds)
             .add(PersistenceEntriesFeature, timelineResponses)
             .add(WhoToFollowExcludedUserIdsFeature, whoToFollowUserIds)
             .build()

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PerspectiveFilteredSocialContextFeatureHydrator.scala

@@ -10,6 +10,7 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
 import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
 import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
 import com.twitter.product_mixer.core.pipeline.PipelineQuery
+import com.twitter.product_mixer.core.util.OffloadFuturePools
 import com.twitter.stitch.Stitch
 import com.twitter.stitch.timelineservice.TimelineService
 import com.twitter.stitch.timelineservice.TimelineService.GetPerspectives
@@ -37,10 +38,10 @@ class PerspectiveFilteredSocialContextFeatureHydrator @Inject() (timelineService
   override def apply(
     query: PipelineQuery,
     candidates: Seq[CandidateWithFeatures[TweetCandidate]]
-  ): Stitch[Seq[FeatureMap]] = {
+  ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch {
     val engagingUserIdtoTweetId = candidates.flatMap { candidate =>
       candidate.features
-        .get(FavoritedByUserIdsFeature).take(MaxCountUsers)
+        .getOrElse(FavoritedByUserIdsFeature, Seq.empty).take(MaxCountUsers)
         .map(favoritedBy => favoritedBy -> candidate.candidate.id)
     }
 
@@ -59,7 +60,7 @@ class PerspectiveFilteredSocialContextFeatureHydrator @Inject() (timelineService
 
       candidates.map { candidate =>
         val perspectiveFilteredFavoritedByUserIds: Seq[Long] = candidate.features
-          .get(FavoritedByUserIdsFeature).take(MaxCountUsers)
+          .getOrElse(FavoritedByUserIdsFeature, Seq.empty).take(MaxCountUsers)
           .filter { userId => validUserIdTweetIds.contains((userId, candidate.candidate.id)) }
 
         FeatureMapBuilder()

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphInNetworkScoresQueryFeatureHydrator.scala

@@ -32,11 +32,15 @@ case class RealGraphInNetworkScoresQueryFeatureHydrator @Inject() (
       val realGraphScoresFeatures = realGraphFollowedUsers
         .getOrElse(Seq.empty)
         .sortBy(-_.score)
-        .map(candidate => candidate.userId -> candidate.score)
+        .map(candidate => candidate.userId -> scaleScore(candidate.score))
         .take(RealGraphCandidateCount)
         .toMap
 
       FeatureMapBuilder().add(RealGraphInNetworkScoresFeature, realGraphScoresFeatures).build()
     }
   }
+
+  // Rescale Real Graph v2 scores from [0,1] to the v1 scores distribution [1,2.97]
+  private def scaleScore(score: Double): Double =
+    if (score >= 0.0 && score <= 1.0) score * 1.97 + 1.0 else score
 }

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RequestQueryFeatureHydrator.scala

@@ -17,9 +17,13 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.G
 import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor
 import com.twitter.product_mixer.core.pipeline.HasPipelineCursor
 import com.twitter.product_mixer.core.pipeline.PipelineQuery
+import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest
+import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure
 import com.twitter.search.common.util.lang.ThriftLanguageUtil
 import com.twitter.snowflake.id.SnowflakeId
 import com.twitter.stitch.Stitch
+import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter.dowFromTimestamp
+import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter.hourFromTimestamp
 import java.util.UUID
 import javax.inject.Inject
 import javax.inject.Singleton
@@ -45,6 +49,9 @@ class RequestQueryFeatureHydrator[
     PullToRefreshFeature,
     RequestJoinIdFeature,
     ServedRequestIdFeature,
+    TimestampFeature,
+    TimestampGMTDowFeature,
+    TimestampGMTHourFeature,
     ViewerIdFeature
   )
 
@@ -67,6 +74,7 @@ class RequestQueryFeatureHydrator[
   override def hydrate(query: Query): Stitch[FeatureMap] = {
     val requestContext = query.deviceContext.flatMap(_.requestContextValue)
     val servedRequestId = UUID.randomUUID.getMostSignificantBits
+    val timestamp = query.queryTime.inMilliseconds
 
     val featureMap = FeatureMapBuilder()
       .add(AccountAgeFeature, query.getOptionalUserId.flatMap(SnowflakeId.timeFromIdOpt))
@@ -97,8 +105,15 @@ class RequestQueryFeatureHydrator[
       .add(PullToRefreshFeature, requestContext.contains(RequestContext.PullToRefresh))
       .add(ServedRequestIdFeature, Some(servedRequestId))
       .add(RequestJoinIdFeature, getRequestJoinId(servedRequestId))
+      .add(TimestampFeature, timestamp)
+      .add(TimestampGMTDowFeature, dowFromTimestamp(timestamp))
+      .add(TimestampGMTHourFeature, hourFromTimestamp(timestamp))
       .add(HasDarkRequestFeature, hasDarkRequest)
-      .add(ViewerIdFeature, query.getRequiredUserId)
+      .add(
+        ViewerIdFeature,
+        query.getOptionalUserId
+          .orElse(query.getGuestId).getOrElse(
+            throw PipelineFailure(BadRequest, "Missing viewer id")))
       .build()
 
     Stitch.value(featureMap)

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSFollowedUsersQueryFeatureHydrator.scala

@@ -1,46 +0,0 @@
-package com.twitter.home_mixer.functional_component.feature_hydrator
-
-import com.twitter.product_mixer.core.feature.Feature
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
-import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
-import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
-import com.twitter.product_mixer.core.pipeline.PipelineQuery
-import com.twitter.socialgraph.{thriftscala => sg}
-import com.twitter.stitch.Stitch
-import com.twitter.stitch.socialgraph.{SocialGraph => SocialGraphStitchClient}
-import javax.inject.Inject
-import javax.inject.Singleton
-
-object SGSFollowedUsersFeature extends Feature[PipelineQuery, Seq[Long]]
-
-@Singleton
-case class SGSFollowedUsersQueryFeatureHydrator @Inject() (
-  socialGraphStitchClient: SocialGraphStitchClient)
-    extends QueryFeatureHydrator[PipelineQuery] {
-
-  override val identifier: FeatureHydratorIdentifier =
-    FeatureHydratorIdentifier("SGSFollowedUsers")
-
-  override val features: Set[Feature[_, _]] = Set(SGSFollowedUsersFeature)
-
-  private val SocialGraphLimit = 14999
-
-  override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = {
-    val userId = query.getRequiredUserId
-
-    val request = sg.IdsRequest(
-      relationships = Seq(
-        sg.SrcRelationship(userId, sg.RelationshipType.Following, hasRelationship = true),
-        sg.SrcRelationship(userId, sg.RelationshipType.Muting, hasRelationship = false)
-      ),
-      pageRequest = Some(sg.PageRequest(count = Some(SocialGraphLimit)))
-    )
-
-    socialGraphStitchClient
-      .ids(request).map(_.ids)
-      .map { followedUsers =>
-        FeatureMapBuilder().add(SGSFollowedUsersFeature, followedUsers).build()
-      }
-  }
-}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSValidSocialContextFeatureHydrator.scala

@@ -12,6 +12,7 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
 import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
 import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
 import com.twitter.product_mixer.core.pipeline.PipelineQuery
+import com.twitter.product_mixer.core.util.OffloadFuturePools
 import com.twitter.socialgraph.{thriftscala => sg}
 import com.twitter.stitch.Stitch
 import com.twitter.stitch.socialgraph.SocialGraph
@@ -41,8 +42,7 @@ class SGSValidSocialContextFeatureHydrator @Inject() (
   override def apply(
     query: PipelineQuery,
     candidates: Seq[CandidateWithFeatures[TweetCandidate]]
-  ): Stitch[Seq[FeatureMap]] = {
-
+  ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch {
     val allSocialContextUserIds =
       candidates.flatMap { candidate =>
         candidate.features.getOrElse(FavoritedByUserIdsFeature, Nil).take(MaxCountUsers) ++

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SocialGraphServiceFeatureHydrator.scala

@@ -1,67 +0,0 @@
-package com.twitter.home_mixer.functional_component.feature_hydrator
-
-import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
-import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
-import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
-import com.twitter.product_mixer.core.feature.Feature
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
-import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
-import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
-import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
-import com.twitter.product_mixer.core.pipeline.PipelineQuery
-import com.twitter.socialgraph.{thriftscala => sg}
-import com.twitter.stitch.Stitch
-import com.twitter.stitch.socialgraph.{SocialGraph => SocialGraphStitchClient}
-import javax.inject.Inject
-import javax.inject.Singleton
-
-@Singleton
-class SocialGraphServiceFeatureHydrator @Inject() (socialGraphStitchClient: SocialGraphStitchClient)
-    extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
-
-  override val identifier: FeatureHydratorIdentifier =
-    FeatureHydratorIdentifier("SocialGraphService")
-
-  override val features: Set[Feature[_, _]] = Set(InNetworkFeature)
-
-  override def apply(
-    query: PipelineQuery,
-    candidates: Seq[CandidateWithFeatures[TweetCandidate]]
-  ): Stitch[Seq[FeatureMap]] = {
-    val viewerId = query.getRequiredUserId
-
-    // We use authorId and not sourceAuthorId here so that retweets are defined as in network
-    val authorIds = candidates.map(_.features.getOrElse(AuthorIdFeature, None).getOrElse(0L))
-    val distinctNonSelfAuthorIds = authorIds.filter(_ != viewerId).distinct
-
-    val idsRequest = createIdsRequest(
-      userId = viewerId,
-      relationshipTypes = Set(sg.RelationshipType.Following),
-      targetIds = Some(distinctNonSelfAuthorIds)
-    )
-
-    socialGraphStitchClient
-      .ids(request = idsRequest, requestContext = None)
-      .map { idResult =>
-        authorIds.map { authorId =>
-          // Users cannot follow themselves but this is in network by definition
-          val isSelfTweet = authorId == viewerId
-          val inNetworkAuthorIds = idResult.ids.toSet
-          val isInNetwork = isSelfTweet || inNetworkAuthorIds.contains(authorId) || authorId == 0L
-          FeatureMapBuilder().add(InNetworkFeature, isInNetwork).build()
-        }
-      }
-  }
-
-  private def createIdsRequest(
-    userId: Long,
-    relationshipTypes: Set[sg.RelationshipType],
-    targetIds: Option[Seq[Long]] = None
-  ): sg.IdsRequest = sg.IdsRequest(
-    relationshipTypes.map { relationshipType =>
-      sg.SrcRelationship(userId, relationshipType, targets = targetIds)
-    }.toSeq,
-    Some(sg.PageRequest(selectAll = Some(true)))
-  )
-}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TSPInferredTopicFeatureHydrator.scala

@@ -1,162 +0,0 @@
-package com.twitter.home_mixer.functional_component.feature_hydrator
-
-import com.twitter.contentrecommender.{thriftscala => cr}
-import com.twitter.finagle.stats.StatsReceiver
-import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.inferred_topic.InferredTopicAdapter
-import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature
-import com.twitter.home_mixer.model.HomeFeatures.TSPMetricTagFeature
-import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature
-import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature
-import com.twitter.ml.api.DataRecord
-import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
-import com.twitter.product_mixer.core.feature.Feature
-import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
-import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
-import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
-import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
-import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
-import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType
-import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType
-import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContextFunctionalityType
-import com.twitter.product_mixer.core.pipeline.PipelineQuery
-import com.twitter.stitch.Stitch
-import com.twitter.strato.generated.client.topic_signals.tsp.TopicSocialProofClientColumn
-import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => sid}
-import com.twitter.topiclisting.TopicListingViewerContext
-import com.twitter.tsp.{thriftscala => tsp}
-
-import javax.inject.Inject
-import javax.inject.Singleton
-import scala.collection.JavaConverters._
-
-object TSPInferredTopicFeature extends Feature[TweetCandidate, Map[Long, Double]]
-object TSPInferredTopicDataRecordFeature
-    extends DataRecordInAFeature[TweetCandidate]
-    with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
-  override def defaultValue: DataRecord = new DataRecord()
-}
-
-@Singleton
-class TSPInferredTopicFeatureHydrator @Inject() (
-  topicSocialProofClientColumn: TopicSocialProofClientColumn,
-  statsReceiver: StatsReceiver,
-) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
-
-  override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TSPInferredTopic")
-
-  override val features: Set[Feature[_, _]] =
-    Set(
-      TSPInferredTopicFeature,
-      TSPInferredTopicDataRecordFeature,
-      TopicIdSocialContextFeature,
-      TopicContextFunctionalityTypeFeature)
-
-  private val topK = 3
-
-  private val sourcesToSetSocialProof: Set[sid.CandidateTweetSourceId] = Set(
-    sid.CandidateTweetSourceId.Simcluster,
-    sid.CandidateTweetSourceId.CroonTweet
-  )
-
-  private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
-  private val keyFoundCounter = scopedStatsReceiver.counter("key/found")
-  private val keyLossCounter = scopedStatsReceiver.counter("key/loss")
-  private val requestFailCounter = scopedStatsReceiver.counter("request/fail")
-
-  private val DefaultFeatureMap = FeatureMapBuilder()
-    .add(TSPInferredTopicFeature, Map.empty[Long, Double])
-    .add(TSPInferredTopicDataRecordFeature, new DataRecord())
-    .add(TopicIdSocialContextFeature, None)
-    .add(TopicContextFunctionalityTypeFeature, None)
-    .build()
-
-  override def apply(
-    query: PipelineQuery,
-    candidates: Seq[CandidateWithFeatures[TweetCandidate]]
-  ): Stitch[Seq[FeatureMap]] = {
-    val tags = candidates.collect {
-      case candidate if candidate.features.getTry(TSPMetricTagFeature).isReturn =>
-        candidate.candidate.id -> candidate.features
-          .getOrElse(TSPMetricTagFeature, Set.empty[tsp.MetricTag])
-    }.toMap
-
-    val topicSocialProofRequest =
-      tsp.TopicSocialProofRequest(
-        userId = query.getRequiredUserId,
-        tweetIds = candidates.map(_.candidate.id).toSet,
-        displayLocation = cr.DisplayLocation.HomeTimeline,
-        topicListingSetting = tsp.TopicListingSetting.Followable,
-        context = TopicListingViewerContext.fromClientContext(query.clientContext).toThrift,
-        bypassModes = None,
-        // Only CRMixer source has this data. Convert the CRMixer metric tag to tsp metric tag.
-        tags = if (tags.isEmpty) None else Some(tags)
-      )
-
-    topicSocialProofClientColumn.fetcher
-      .fetch(topicSocialProofRequest)
-      .map(_.v)
-      .map {
-        case Some(response) =>
-          candidates.map { candidate =>
-            val topicWithScores = response.socialProofs.getOrElse(candidate.candidate.id, Seq.empty)
-            if (topicWithScores.nonEmpty) {
-              keyFoundCounter.incr()
-              val (socialProofId, socialProofFunctionalityType) =
-                if (candidate.features
-                    .getOrElse(CandidateSourceIdFeature, None)
-                    .exists(sourcesToSetSocialProof.contains)) {
-                  getSocialProof(topicWithScores)
-                } else {
-                  (None, None)
-                }
-              val inferredTopicFeatures = convertTopicWithScores(topicWithScores)
-              val inferredTopicDataRecord =
-                InferredTopicAdapter.adaptToDataRecords(inferredTopicFeatures).asScala.head
-              FeatureMapBuilder()
-                .add(TSPInferredTopicFeature, inferredTopicFeatures)
-                .add(TSPInferredTopicDataRecordFeature, inferredTopicDataRecord)
-                .add(TopicIdSocialContextFeature, socialProofId)
-                .add(TopicContextFunctionalityTypeFeature, socialProofFunctionalityType)
-                .build()
-            } else {
-              keyLossCounter.incr()
-              DefaultFeatureMap
-            }
-          }
-        case _ =>
-          requestFailCounter.incr()
-          candidates.map { _ =>
-            DefaultFeatureMap
-          }
-      }
-  }
-
-  private def getSocialProof(
-    topicWithScores: Seq[tsp.TopicWithScore]
-  ): (Option[Long], Option[TopicContextFunctionalityType]) = {
-    val followingTopicId = topicWithScores
-      .collectFirst {
-        case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.Following)) =>
-          topicId
-      }
-    if (followingTopicId.nonEmpty) {
-      return (followingTopicId, Some(BasicTopicContextFunctionalityType))
-    }
-    val implicitFollowingId = topicWithScores.collectFirst {
-      case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.ImplicitFollow)) =>
-        topicId
-    }
-    if (implicitFollowingId.nonEmpty) {
-      return (implicitFollowingId, Some(RecommendationTopicContextFunctionalityType))
-    }
-    (None, None)
-  }
-
-  private def convertTopicWithScores(
-    topicWithScores: Seq[tsp.TopicWithScore],
-  ): Map[Long, Double] = {
-    topicWithScores.sortBy(-_.score).take(topK).map(a => (a.topicId, a.score)).toMap
-  }
-}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TimeFeaturesHydrator.scala

@@ -1,251 +0,0 @@
-package com.twitter.home_mixer.functional_component.feature_hydrator
-
-import com.twitter.conversions.DurationOps._
-import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature
-import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature
-import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
-import com.twitter.ml.api.DataRecord
-import com.twitter.ml.api.RichDataRecord
-import com.twitter.ml.api.util.FDsl._
-import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
-import com.twitter.product_mixer.core.feature.Feature
-import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
-import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
-import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
-import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
-import com.twitter.product_mixer.core.pipeline.PipelineQuery
-import com.twitter.search.common.features.{thriftscala => sc}
-import com.twitter.snowflake.id.SnowflakeId
-import com.twitter.stitch.Stitch
-import com.twitter.timelines.prediction.features.time_features.AccountAgeInterval
-import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures._
-import com.twitter.timelines.prediction.features.time_features.TimeFeatures
-import com.twitter.util.Duration
-import scala.collection.Searching._
-
-object TimeFeaturesDataRecordFeature
-    extends DataRecordInAFeature[TweetCandidate]
-    with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
-  override def defaultValue: DataRecord = new DataRecord()
-}
-
-object TimeFeaturesHydrator extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
-
-  override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TimeFeatures")
-
-  override val features: Set[Feature[_, _]] = Set(TimeFeaturesDataRecordFeature)
-
-  override def apply(
-    query: PipelineQuery,
-    candidate: TweetCandidate,
-    existingFeatures: FeatureMap
-  ): Stitch[FeatureMap] = {
-    Stitch.value {
-      val richDataRecord = new RichDataRecord()
-      setTimeFeatures(richDataRecord, candidate, existingFeatures, query)
-      FeatureMapBuilder()
-        .add(TimeFeaturesDataRecordFeature, richDataRecord.getRecord)
-        .build()
-    }
-  }
-
-  private def setTimeFeatures(
-    richDataRecord: RichDataRecord,
-    candidate: TweetCandidate,
-    existingFeatures: FeatureMap,
-    query: PipelineQuery,
-  ): Unit = {
-    val timeFeaturesOpt = getTimeFeatures(query, candidate, existingFeatures)
-    timeFeaturesOpt.foreach(timeFeatures => setFeatures(timeFeatures, richDataRecord))
-  }
-
-  private[feature_hydrator] def getTimeFeatures(
-    query: PipelineQuery,
-    candidate: TweetCandidate,
-    existingFeatures: FeatureMap,
-  ): Option[TimeFeatures] = {
-    for {
-      requestTimestampMs <- Some(query.queryTime.inMilliseconds)
-      tweetId <- Some(candidate.id)
-      viewerId <- query.getOptionalUserId
-      tweetCreationTimeMs <- timeFromTweetOrUserId(tweetId)
-      timeSinceTweetCreation = requestTimestampMs - tweetCreationTimeMs
-      accountAgeDurationOpt = timeFromTweetOrUserId(viewerId).map { viewerAccountCreationTimeMs =>
-        Duration.fromMilliseconds(requestTimestampMs - viewerAccountCreationTimeMs)
-      }
-      timeSinceSourceTweetCreation =
-        existingFeatures
-          .getOrElse(SourceTweetIdFeature, None)
-          .flatMap { sourceTweetId =>
-            timeFromTweetOrUserId(sourceTweetId).map { sourceTweetCreationTimeMs =>
-              requestTimestampMs - sourceTweetCreationTimeMs
-            }
-          }
-          .getOrElse(timeSinceTweetCreation)
-      if (timeSinceTweetCreation > 0 && timeSinceSourceTweetCreation > 0)
-    } yield {
-      val timeFeatures = TimeFeatures(
-        timeSinceTweetCreation = timeSinceTweetCreation,
-        timeSinceSourceTweetCreation = timeSinceSourceTweetCreation,
-        timeSinceViewerAccountCreationSecs = accountAgeDurationOpt.map(_.inSeconds),
-        isDay30NewUser = accountAgeDurationOpt.map(_ < 30.days).getOrElse(false),
-        isMonth12NewUser = accountAgeDurationOpt.map(_ < 365.days).getOrElse(false),
-        accountAgeInterval = accountAgeDurationOpt.flatMap(AccountAgeInterval.fromDuration),
-        isTweetRecycled = false // only set in RecyclableTweetCandidateFilter, but it's not used
-      )
-
-      val timeFeaturesWithLastEngagement = addLastEngagementTimeFeatures(
-        existingFeatures.getOrElse(EarlybirdFeature, None),
-        timeFeatures,
-        timeSinceSourceTweetCreation
-      ).getOrElse(timeFeatures)
-
-      val nonPollingTimestampsMs =
-        query.features.map(_.getOrElse(NonPollingTimesFeature, Seq.empty))
-      val timeFeaturesWithNonPollingOpt = addNonPollingTimeFeatures(
-        timeFeaturesWithLastEngagement,
-        requestTimestampMs,
-        tweetCreationTimeMs,
-        nonPollingTimestampsMs
-      )
-      timeFeaturesWithNonPollingOpt.getOrElse(timeFeaturesWithLastEngagement)
-    }
-  }
-
-  private def timeFromTweetOrUserId(tweetOrUserId: Long): Option[Long] = {
-    if (SnowflakeId.isSnowflakeId(tweetOrUserId))
-      Some(SnowflakeId(tweetOrUserId).time.inMilliseconds)
-    else None
-  }
-
-  private def addLastEngagementTimeFeatures(
-    tweetFeaturesOpt: Option[sc.ThriftTweetFeatures],
-    timeFeatures: TimeFeatures,
-    timeSinceSourceTweetCreation: Long
-  ): Option[TimeFeatures] = {
-    tweetFeaturesOpt.map { tweetFeatures =>
-      val lastFavSinceCreationHrs = tweetFeatures.lastFavSinceCreationHrs.map(_.toDouble)
-      val lastRetweetSinceCreationHrs = tweetFeatures.lastRetweetSinceCreationHrs.map(_.toDouble)
-      val lastReplySinceCreationHrs = tweetFeatures.lastReplySinceCreationHrs.map(_.toDouble)
-      val lastQuoteSinceCreationHrs = tweetFeatures.lastQuoteSinceCreationHrs.map(_.toDouble)
-
-      timeFeatures.copy(
-        lastFavSinceCreationHrs = lastFavSinceCreationHrs,
-        lastRetweetSinceCreationHrs = lastRetweetSinceCreationHrs,
-        lastReplySinceCreationHrs = lastReplySinceCreationHrs,
-        lastQuoteSinceCreationHrs = lastQuoteSinceCreationHrs,
-        timeSinceLastFavoriteHrs = getTimeSinceLastEngagementHrs(
-          lastFavSinceCreationHrs,
-          timeSinceSourceTweetCreation
-        ),
-        timeSinceLastRetweetHrs = getTimeSinceLastEngagementHrs(
-          lastRetweetSinceCreationHrs,
-          timeSinceSourceTweetCreation
-        ),
-        timeSinceLastReplyHrs = getTimeSinceLastEngagementHrs(
-          lastReplySinceCreationHrs,
-          timeSinceSourceTweetCreation
-        ),
-        timeSinceLastQuoteHrs = getTimeSinceLastEngagementHrs(
-          lastQuoteSinceCreationHrs,
-          timeSinceSourceTweetCreation
-        )
-      )
-    }
-  }
-
-  private def addNonPollingTimeFeatures(
-    timeFeatures: TimeFeatures,
-    requestTimestampMs: Long,
-    creationTimeMs: Long,
-    nonPollingTimestampsMs: Option[Seq[Long]]
-  ): Option[TimeFeatures] = {
-    for {
-      nonPollingTimestampsMs <- nonPollingTimestampsMs
-      lastNonPollingTimestampMs <- nonPollingTimestampsMs.headOption
-      earliestNonPollingTimestampMs <- nonPollingTimestampsMs.lastOption
-    } yield {
-      val timeSinceLastNonPollingRequest = requestTimestampMs - lastNonPollingTimestampMs
-      val tweetAgeRatio = timeSinceLastNonPollingRequest / math.max(
-        1.0,
-        timeFeatures.timeSinceTweetCreation
-      )
-      /*
-       * Non-polling timestamps are stored in chronological order.
-       * The latest timestamps occur first, therefore we need to explicitly search in reverse order.
-       */
-      val nonPollingRequestsSinceTweetCreation =
-        if (nonPollingTimestampsMs.nonEmpty) {
-          nonPollingTimestampsMs.search(creationTimeMs)(Ordering[Long].reverse).insertionPoint
-        } else {
-          0
-        }
-      /*
-       * Calculate the average time between non-polling requests; include
-       * request time in this calculation as latest timestamp.
-       */
-      val timeBetweenNonPollingRequestsAvg =
-        (requestTimestampMs - earliestNonPollingTimestampMs) / math
-          .max(1.0, nonPollingTimestampsMs.size)
-      val timeFeaturesWithNonPolling = timeFeatures.copy(
-        timeBetweenNonPollingRequestsAvg = Some(timeBetweenNonPollingRequestsAvg),
-        timeSinceLastNonPollingRequest = Some(timeSinceLastNonPollingRequest),
-        nonPollingRequestsSinceTweetCreation = Some(nonPollingRequestsSinceTweetCreation),
-        tweetAgeRatio = Some(tweetAgeRatio)
-      )
-      timeFeaturesWithNonPolling
-    }
-  }
-
-  private[this] def getTimeSinceLastEngagementHrs(
-    lastEngagementTimeSinceCreationHrsOpt: Option[Double],
-    timeSinceTweetCreation: Long
-  ): Option[Double] = {
-    lastEngagementTimeSinceCreationHrsOpt.map { lastEngagementTimeSinceCreationHrs =>
-      val timeSinceTweetCreationHrs = (timeSinceTweetCreation / (60 * 60 * 1000)).toInt
-      timeSinceTweetCreationHrs - lastEngagementTimeSinceCreationHrs
-    }
-  }
-
-  private def setFeatures(features: TimeFeatures, richDataRecord: RichDataRecord): Unit = {
-    val record = richDataRecord.getRecord
-      .setFeatureValue(IS_TWEET_RECYCLED, features.isTweetRecycled)
-      .setFeatureValue(TIME_SINCE_TWEET_CREATION, features.timeSinceTweetCreation)
-      .setFeatureValueFromOption(
-        TIME_SINCE_VIEWER_ACCOUNT_CREATION_SECS,
-        features.timeSinceViewerAccountCreationSecs)
-      .setFeatureValue(
-        USER_ID_IS_SNOWFLAKE_ID,
-        features.timeSinceViewerAccountCreationSecs.isDefined
-      )
-      .setFeatureValueFromOption(ACCOUNT_AGE_INTERVAL, features.accountAgeInterval.map(_.id.toLong))
-      .setFeatureValue(IS_30_DAY_NEW_USER, features.isDay30NewUser)
-      .setFeatureValue(IS_12_MONTH_NEW_USER, features.isMonth12NewUser)
-      .setFeatureValueFromOption(LAST_FAVORITE_SINCE_CREATION_HRS, features.lastFavSinceCreationHrs)
-      .setFeatureValueFromOption(
-        LAST_RETWEET_SINCE_CREATION_HRS,
-        features.lastRetweetSinceCreationHrs
-      )
-      .setFeatureValueFromOption(LAST_REPLY_SINCE_CREATION_HRS, features.lastReplySinceCreationHrs)
-      .setFeatureValueFromOption(LAST_QUOTE_SINCE_CREATION_HRS, features.lastQuoteSinceCreationHrs)
-      .setFeatureValueFromOption(TIME_SINCE_LAST_FAVORITE_HRS, features.timeSinceLastFavoriteHrs)
-      .setFeatureValueFromOption(TIME_SINCE_LAST_RETWEET_HRS, features.timeSinceLastRetweetHrs)
-      .setFeatureValueFromOption(TIME_SINCE_LAST_REPLY_HRS, features.timeSinceLastReplyHrs)
-      .setFeatureValueFromOption(TIME_SINCE_LAST_QUOTE_HRS, features.timeSinceLastQuoteHrs)
-    /*
-     * set features whose values are optional as some users do not have non-polling timestamps
-     */
-    features.timeBetweenNonPollingRequestsAvg.foreach(
-      record.setFeatureValue(TIME_BETWEEN_NON_POLLING_REQUESTS_AVG, _)
-    )
-    features.timeSinceLastNonPollingRequest.foreach(
-      record.setFeatureValue(TIME_SINCE_LAST_NON_POLLING_REQUEST, _)
-    )
-    features.nonPollingRequestsSinceTweetCreation.foreach(
-      record.setFeatureValue(NON_POLLING_REQUESTS_SINCE_TWEET_CREATION, _)
-    )
-    features.tweetAgeRatio.foreach(record.setFeatureValue(TWEET_AGE_RATIO, _))
-  }
-}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetImpressionsQueryFeatureHydrator.scala

@@ -24,8 +24,8 @@ case class TweetImpressionsQueryFeatureHydrator[
   manhattanTweetImpressionStoreClient: ManhattanTweetImpressionStoreClient)
     extends QueryFeatureHydrator[Query] {
 
-  private val TweetImpressionTTL = 1.day
-  private val TweetImpressionCap = 3000
+  private val TweetImpressionTTL = 2.days
+  private val TweetImpressionCap = 5000
 
   override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetImpressions")
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetypieFeatureHydrator.scala

@@ -1,6 +1,9 @@
 package com.twitter.home_mixer.functional_component.feature_hydrator
 
+import com.twitter.finagle.stats.StatsReceiver
 import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
+import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature
+import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
 import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
 import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature
 import com.twitter.home_mixer.model.HomeFeatures.IsNsfwFeature
@@ -10,11 +13,13 @@ import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature
 import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature
 import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
 import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
+import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature
 import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature
 import com.twitter.home_mixer.model.request.FollowingProduct
 import com.twitter.home_mixer.model.request.ForYouProduct
 import com.twitter.home_mixer.model.request.ListTweetsProduct
 import com.twitter.home_mixer.model.request.ScoredTweetsProduct
+import com.twitter.home_mixer.model.request.SubscribedProduct
 import com.twitter.home_mixer.util.tweetypie.RequestFields
 import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_is_nsfw.IsNsfw
 import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason.VisibilityReason
@@ -29,17 +34,22 @@ import com.twitter.spam.rtf.{thriftscala => rtf}
 import com.twitter.stitch.Stitch
 import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient}
 import com.twitter.tweetypie.{thriftscala => tp}
+import com.twitter.util.logging.Logging
 import javax.inject.Inject
 import javax.inject.Singleton
 
 @Singleton
-class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitchClient)
-    extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
+class TweetypieFeatureHydrator @Inject() (
+  tweetypieStitchClient: TweetypieStitchClient,
+  statsReceiver: StatsReceiver)
+    extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
+    with Logging {
 
   override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Tweetypie")
 
   override val features: Set[Feature[_, _]] = Set(
     AuthorIdFeature,
+    ExclusiveConversationAuthorIdFeature,
     InReplyToTweetIdFeature,
     IsHydratedFeature,
     IsNsfw,
@@ -51,6 +61,7 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
     SourceTweetIdFeature,
     SourceUserIdFeature,
     TweetTextFeature,
+    TweetLanguageFeature,
     VisibilityReason
   )
 
@@ -70,9 +81,12 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
   ): Stitch[FeatureMap] = {
     val safetyLevel = query.product match {
       case FollowingProduct => rtf.SafetyLevel.TimelineHomeLatest
-      case ForYouProduct => rtf.SafetyLevel.TimelineHome
+      case ForYouProduct =>
+        val inNetwork = existingFeatures.getOrElse(InNetworkFeature, true)
+        if (inNetwork) rtf.SafetyLevel.TimelineHome else rtf.SafetyLevel.TimelineHomeRecommendations
       case ScoredTweetsProduct => rtf.SafetyLevel.TimelineHome
       case ListTweetsProduct => rtf.SafetyLevel.TimelineLists
+      case SubscribedProduct => rtf.SafetyLevel.TimelineHomeSubscribed
       case unknown => throw new UnsupportedOperationException(s"Unknown product: $unknown")
     }
 
@@ -82,9 +96,12 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
       includeQuotedTweet = true,
       visibilityPolicy = tp.TweetVisibilityPolicy.UserVisible,
       safetyLevel = Some(safetyLevel),
-      forUserId = Some(query.getRequiredUserId)
+      forUserId = query.getOptionalUserId
     )
 
+    val exclusiveAuthorIdOpt =
+      existingFeatures.getOrElse(ExclusiveConversationAuthorIdFeature, None)
+
     tweetypieStitchClient.getTweetFields(tweetId = candidate.id, options = tweetFieldsOptions).map {
       case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), quoteOpt, _) =>
         val coreData = found.tweet.coreData
@@ -106,6 +123,7 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
           found.retweetedTweet.exists(_.coreData.exists(data => data.nsfwAdmin || data.nsfwUser))
 
         val tweetText = coreData.map(_.text)
+        val tweetLanguage = found.tweet.language.map(_.language)
 
         val tweetAuthorId = coreData.map(_.userId)
         val inReplyToTweetId = coreData.flatMap(_.reply.flatMap(_.inReplyToStatusId))
@@ -127,6 +145,7 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
 
         FeatureMapBuilder()
           .add(AuthorIdFeature, tweetAuthorId)
+          .add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt)
           .add(InReplyToTweetIdFeature, inReplyToTweetId)
           .add(IsHydratedFeature, true)
           .add(IsNsfw, Some(isNsfw))
@@ -137,20 +156,24 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
           .add(QuotedUserIdFeature, quotedTweetUserId)
           .add(SourceTweetIdFeature, retweetedTweetId)
           .add(SourceUserIdFeature, retweetedTweetUserId)
+          .add(TweetLanguageFeature, tweetLanguage)
           .add(TweetTextFeature, tweetText)
           .add(VisibilityReason, found.suppressReason)
           .build()
 
       // If no tweet result found, return default and pre-existing features
       case _ =>
-        DefaultFeatureMap +
-          (AuthorIdFeature, existingFeatures.getOrElse(AuthorIdFeature, None)) +
-          (InReplyToTweetIdFeature, existingFeatures.getOrElse(InReplyToTweetIdFeature, None)) +
-          (IsRetweetFeature, existingFeatures.getOrElse(IsRetweetFeature, false)) +
-          (QuotedTweetIdFeature, existingFeatures.getOrElse(QuotedTweetIdFeature, None)) +
-          (QuotedUserIdFeature, existingFeatures.getOrElse(QuotedUserIdFeature, None)) +
-          (SourceTweetIdFeature, existingFeatures.getOrElse(SourceTweetIdFeature, None)) +
-          (SourceUserIdFeature, existingFeatures.getOrElse(SourceUserIdFeature, None))
+        DefaultFeatureMap ++ FeatureMapBuilder()
+          .add(AuthorIdFeature, existingFeatures.getOrElse(AuthorIdFeature, None))
+          .add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt)
+          .add(InReplyToTweetIdFeature, existingFeatures.getOrElse(InReplyToTweetIdFeature, None))
+          .add(IsRetweetFeature, existingFeatures.getOrElse(IsRetweetFeature, false))
+          .add(QuotedTweetIdFeature, existingFeatures.getOrElse(QuotedTweetIdFeature, None))
+          .add(QuotedUserIdFeature, existingFeatures.getOrElse(QuotedUserIdFeature, None))
+          .add(SourceTweetIdFeature, existingFeatures.getOrElse(SourceTweetIdFeature, None))
+          .add(SourceUserIdFeature, existingFeatures.getOrElse(SourceUserIdFeature, None))
+          .add(TweetLanguageFeature, existingFeatures.getOrElse(TweetLanguageFeature, None))
+          .build()
     }
   }
 }

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features/AuthorFeaturesAdapter.scala

@@ -1,59 +0,0 @@
-package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.author_features
-
-import com.twitter.ml.api.DataRecordMerger
-import com.twitter.ml.api.Feature
-import com.twitter.ml.api.FeatureContext
-import com.twitter.ml.api.RichDataRecord
-import com.twitter.ml.api.util.CompactDataRecordConverter
-import com.twitter.ml.api.util.FDsl._
-import com.twitter.timelines.author_features.v1.{thriftjava => af}
-import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase
-import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig
-import com.twitter.timelines.prediction.features.user_health.UserHealthFeatures
-
-object AuthorFeaturesAdapter extends TimelinesMutatingAdapterBase[Option[af.AuthorFeatures]] {
-
-  private val originalAuthorAggregatesFeatures =
-    TimelinesAggregationConfig.originalAuthorReciprocalEngagementAggregates
-      .buildTypedAggregateGroups().flatMap(_.allOutputFeatures)
-  private val authorFeatures = originalAuthorAggregatesFeatures ++
-    Seq(
-      UserHealthFeatures.AuthorState,
-      UserHealthFeatures.NumAuthorFollowers,
-      UserHealthFeatures.NumAuthorConnectDays,
-      UserHealthFeatures.NumAuthorConnect)
-  private val featureContext = new FeatureContext(authorFeatures: _*)
-
-  override def getFeatureContext: FeatureContext = featureContext
-
-  override val commonFeatures: Set[Feature[_]] = Set.empty
-
-  private val compactDataRecordConverter = new CompactDataRecordConverter()
-  private val drMerger = new DataRecordMerger()
-
-  override def setFeatures(
-    authorFeaturesOpt: Option[af.AuthorFeatures],
-    richDataRecord: RichDataRecord
-  ): Unit = {
-    authorFeaturesOpt.foreach { authorFeatures =>
-      val dataRecord = richDataRecord.getRecord
-
-      dataRecord.setFeatureValue(
-        UserHealthFeatures.AuthorState,
-        authorFeatures.user_health.user_state.getValue.toLong)
-      dataRecord.setFeatureValue(
-        UserHealthFeatures.NumAuthorFollowers,
-        authorFeatures.user_health.num_followers.toDouble)
-      dataRecord.setFeatureValue(
-        UserHealthFeatures.NumAuthorConnectDays,
-        authorFeatures.user_health.num_connect_days.toDouble)
-      dataRecord.setFeatureValue(
-        UserHealthFeatures.NumAuthorConnect,
-        authorFeatures.user_health.num_connect.toDouble)
-
-      val originalAuthorAggregatesDataRecord =
-        compactDataRecordConverter.compactDataRecordToDataRecord(authorFeatures.aggregates)
-      drMerger.merge(dataRecord, originalAuthorAggregatesDataRecord)
-    }
-  }
-}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/Phase2EdgeAggregateFeatureHydrator.scala

@@ -1,22 +0,0 @@
-package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates
-
-import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures._
-import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
-import javax.inject.Inject
-import javax.inject.Singleton
-
-@Singleton
-class Phase2EdgeAggregateFeatureHydrator @Inject() extends BaseEdgeAggregateFeatureHydrator {
-
-  override val identifier: FeatureHydratorIdentifier =
-    FeatureHydratorIdentifier("Phase2EdgeAggregate")
-
-  override val aggregateFeatures: Set[BaseEdgeAggregateFeature] =
-    Set(
-      UserEngagerAggregateFeature,
-      UserEngagerGoodClickAggregateFeature,
-      UserInferredTopicAggregateFeature,
-      UserTopicAggregateFeature,
-      UserMediaUnderstandingAnnotationAggregateFeature
-    )
-}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/BUILD.bazel

@@ -4,6 +4,7 @@ scala_library(
     strict_deps = True,
     tags = ["bazel-compatible"],
     dependencies = [
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param",
@@ -16,9 +17,11 @@ scala_library(
         "src/thrift/com/twitter/tweetypie:service-scala",
         "src/thrift/com/twitter/tweetypie:tweet-scala",
         "stitch/stitch-core",
+        "stitch/stitch-socialgraph",
         "stitch/stitch-tweetypie",
         "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
         "timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
+        "util/util-slf4j-api/src/main/scala",
     ],
     exports = [
         "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/FeedbackFatigueFilter.scala

@@ -72,7 +72,7 @@ object FeedbackFatigueFilter
 
       originalAuthorId.exists(authorsToFilter.contains) ||
       (likers.nonEmpty && eligibleLikers.isEmpty) ||
-      (followers.nonEmpty && eligibleFollowers.isEmpty) ||
+      (followers.nonEmpty && eligibleFollowers.isEmpty && likers.isEmpty) ||
       (candidate.features.getOrElse(IsRetweetFeature, false) &&
       authorId.exists(retweetersToFilter.contains))
     }

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidSubscriptionTweetFilter.scala

@@ -0,0 +1,70 @@
+package com.twitter.home_mixer.functional_component.filter
+
+import com.twitter.finagle.stats.StatsReceiver
+import com.twitter.finagle.tracing.Trace
+import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature
+import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
+import com.twitter.product_mixer.core.functional_component.filter.Filter
+import com.twitter.product_mixer.core.functional_component.filter.FilterResult
+import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
+import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
+import com.twitter.product_mixer.core.pipeline.PipelineQuery
+import com.twitter.socialgraph.{thriftscala => sg}
+import com.twitter.stitch.Stitch
+import com.twitter.stitch.socialgraph.SocialGraph
+import com.twitter.util.logging.Logging
+
+import javax.inject.Inject
+import javax.inject.Singleton
+
+/**
+ * Exclude invalid subscription tweets - cases where the viewer is not subscribed to the author
+ *
+ * If SGS hydration fails, `SGSInvalidSubscriptionTweetFeature` will be set to None for
+ * subscription tweets, so we explicitly filter those tweets out.
+ */
+@Singleton
+case class InvalidSubscriptionTweetFilter @Inject() (
+  socialGraphClient: SocialGraph,
+  statsReceiver: StatsReceiver)
+    extends Filter[PipelineQuery, TweetCandidate]
+    with Logging {
+
+  override val identifier: FilterIdentifier = FilterIdentifier("InvalidSubscriptionTweet")
+
+  private val scopedStatsReceiver = statsReceiver.scope(identifier.toString)
+  private val validCounter = scopedStatsReceiver.counter("validExclusiveTweet")
+  private val invalidCounter = scopedStatsReceiver.counter("invalidExclusiveTweet")
+
+  override def apply(
+    query: PipelineQuery,
+    candidates: Seq[CandidateWithFeatures[TweetCandidate]]
+  ): Stitch[FilterResult[TweetCandidate]] = Stitch
+    .traverse(candidates) { candidate =>
+      val exclusiveAuthorId =
+        candidate.features.getOrElse(ExclusiveConversationAuthorIdFeature, None)
+
+      if (exclusiveAuthorId.isDefined) {
+        val request = sg.ExistsRequest(
+          source = query.getRequiredUserId,
+          target = exclusiveAuthorId.get,
+          relationships =
+            Seq(sg.Relationship(sg.RelationshipType.TierOneSuperFollowing, hasRelationship = true)),
+        )
+        socialGraphClient.exists(request).map(_.exists).map { valid =>
+          if (!valid) invalidCounter.incr() else validCounter.incr()
+          valid
+        }
+      } else Stitch.value(true)
+    }.map { validResults =>
+      val (kept, removed) = candidates
+        .map(_.candidate)
+        .zip(validResults)
+        .partition { case (candidate, valid) => valid }
+
+      val keptCandidates = kept.map { case (candidate, _) => candidate }
+      val removedCandidates = removed.map { case (candidate, _) => candidate }
+
+      FilterResult(kept = keptCandidates, removed = removedCandidates)
+    }
+}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PredicateFeatureFilter.scala

@@ -1,59 +0,0 @@
-package com.twitter.home_mixer.functional_component.filter
-
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
-import com.twitter.product_mixer.core.functional_component.filter.Filter
-import com.twitter.product_mixer.core.functional_component.filter.FilterResult
-import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
-import com.twitter.product_mixer.core.model.common.UniversalNoun
-import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
-import com.twitter.product_mixer.core.pipeline.PipelineQuery
-import com.twitter.stitch.Stitch
-
-/**
- * Predicate which will be applied to each candidate. True indicates that the candidate will be
- * @tparam Candidate - the type of the candidate
- */
-trait ShouldKeepCandidate {
-  def apply(features: FeatureMap): Boolean
-}
-
-object PredicateFeatureFilter {
-
-  /**
-   * Builds a simple Filter out of a predicate function from the candidate to a boolean. For clarity,
-   * we recommend including the name of the shouldKeepCandidate parameter.
-   *
-   * @param identifier A FilterIdentifier for the new filter
-   * @param shouldKeepCandidate A predicate function. Candidates will be kept when
-   *                            this function returns True.
-   */
-  def fromPredicate[Candidate <: UniversalNoun[Any]](
-    identifier: FilterIdentifier,
-    shouldKeepCandidate: ShouldKeepCandidate
-  ): Filter[PipelineQuery, Candidate] = {
-    val i = identifier
-
-    new Filter[PipelineQuery, Candidate] {
-      override val identifier: FilterIdentifier = i
-
-      /**
-       * Filter the list of candidates
-       *
-       * @return a FilterResult including both the list of kept candidate and the list of removed candidates
-       */
-      override def apply(
-        query: PipelineQuery,
-        candidates: Seq[CandidateWithFeatures[Candidate]]
-      ): Stitch[FilterResult[Candidate]] = {
-        val allowedIds = candidates
-          .filter(candidate => shouldKeepCandidate(candidate.features)).map(_.candidate.id).toSet
-
-        val (keptCandidates, removedCandidates) = candidates.map(_.candidate).partition {
-          candidate => allowedIds.contains(candidate.id)
-        }
-
-        Stitch.value(FilterResult(kept = keptCandidates, removed = removedCandidates))
-      }
-    }
-  }
-}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetPreviewsFilter.scala

@@ -1,8 +1,6 @@
 package com.twitter.home_mixer.functional_component.filter
 
-import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
-import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
-import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
+import com.twitter.home_mixer.model.HomeFeatures.ServedTweetPreviewIdsFeature
 import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
 import com.twitter.product_mixer.core.functional_component.filter.Filter
 import com.twitter.product_mixer.core.functional_component.filter.FilterResult
@@ -11,24 +9,20 @@ import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
 import com.twitter.product_mixer.core.pipeline.PipelineQuery
 import com.twitter.stitch.Stitch
 
-object KeepBestOutOfNetworkTweetPerAuthorFilter extends Filter[PipelineQuery, TweetCandidate] {
+object PreviouslyServedTweetPreviewsFilter extends Filter[PipelineQuery, TweetCandidate] {
 
-  override val identifier: FilterIdentifier = FilterIdentifier("KeepBestOutOfNetworkTweetPerAuthor")
+  override val identifier: FilterIdentifier = FilterIdentifier("PreviouslyServedTweetPreviews")
 
   override def apply(
     query: PipelineQuery,
     candidates: Seq[CandidateWithFeatures[TweetCandidate]]
   ): Stitch[FilterResult[TweetCandidate]] = {
-    // Set containing best OON tweet for each authorId
-    val bestCandidatesForAuthorId = candidates
-      .filter(!_.features.getOrElse(InNetworkFeature, true))
-      .groupBy(_.features.getOrElse(AuthorIdFeature, None))
-      .values.map(_.maxBy(_.features.getOrElse(ScoreFeature, None)))
-      .toSet
+
+    val servedTweetPreviewIds =
+      query.features.map(_.getOrElse(ServedTweetPreviewIdsFeature, Seq.empty)).toSeq.flatten.toSet
 
     val (removed, kept) = candidates.partition { candidate =>
-      !candidate.features.getOrElse(InNetworkFeature, true) &&
-      !bestCandidatesForAuthorId.contains(candidate)
+      servedTweetPreviewIds.contains(candidate.candidate.id)
     }
 
     Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate)))

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ReplyFilter.scala

@@ -0,0 +1,32 @@
+package com.twitter.home_mixer.functional_component.filter
+
+import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
+import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
+import com.twitter.product_mixer.core.functional_component.filter.Filter
+import com.twitter.product_mixer.core.functional_component.filter.FilterResult
+import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
+import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
+import com.twitter.product_mixer.core.pipeline.PipelineQuery
+import com.twitter.stitch.Stitch
+
+object ReplyFilter extends Filter[PipelineQuery, TweetCandidate] {
+  override val identifier: FilterIdentifier = FilterIdentifier("Reply")
+
+  override def apply(
+    query: PipelineQuery,
+    candidates: Seq[CandidateWithFeatures[TweetCandidate]]
+  ): Stitch[FilterResult[TweetCandidate]] = {
+
+    val (kept, removed) = candidates
+      .partition { candidate =>
+        candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty
+      }
+
+    val filterResult = FilterResult(
+      kept = kept.map(_.candidate),
+      removed = removed.map(_.candidate)
+    )
+
+    Stitch.value(filterResult)
+  }
+}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetFilter.scala

@@ -0,0 +1,32 @@
+package com.twitter.home_mixer.functional_component.filter
+
+import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
+import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
+import com.twitter.product_mixer.core.functional_component.filter.Filter
+import com.twitter.product_mixer.core.functional_component.filter.FilterResult
+import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
+import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
+import com.twitter.product_mixer.core.pipeline.PipelineQuery
+import com.twitter.stitch.Stitch
+
+object RetweetFilter extends Filter[PipelineQuery, TweetCandidate] {
+  override val identifier: FilterIdentifier = FilterIdentifier("Retweet")
+
+  override def apply(
+    query: PipelineQuery,
+    candidates: Seq[CandidateWithFeatures[TweetCandidate]]
+  ): Stitch[FilterResult[TweetCandidate]] = {
+
+    val (kept, removed) = candidates
+      .partition { candidate =>
+        !candidate.features.getOrElse(IsRetweetFeature, false)
+      }
+
+    val filterResult = FilterResult(
+      kept = kept.map(_.candidate),
+      removed = removed.map(_.candidate)
+    )
+
+    Stitch.value(filterResult)
+  }
+}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/BUILD.bazel

@@ -14,7 +14,6 @@ scala_library(
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate",
         "src/thrift/com/twitter/gizmoduck:thrift-scala",
         "stitch/stitch-socialgraph",
-        "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
         "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
         "timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
     ],

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/DismissFatigueGate.scala

@@ -1,14 +1,14 @@
 package com.twitter.home_mixer.functional_component.gate
 
+import com.twitter.conversions.DurationOps._
+import com.twitter.product_mixer.core.feature.Feature
 import com.twitter.product_mixer.core.functional_component.gate.Gate
 import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier
 import com.twitter.product_mixer.core.pipeline.PipelineQuery
-import com.twitter.timelinemixer.clients.manhattan.DismissInfo
 import com.twitter.stitch.Stitch
-import com.twitter.util.Duration
-import com.twitter.conversions.DurationOps._
-import com.twitter.product_mixer.core.feature.Feature
+import com.twitter.timelinemixer.clients.manhattan.DismissInfo
 import com.twitter.timelineservice.suggests.thriftscala.SuggestType
+import com.twitter.util.Duration
 
 object DismissFatigueGate {
   // how long a dismiss action from user needs to be respected

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/NonEmptySeqFeatureGate.scala

@@ -1,18 +0,0 @@
-package com.twitter.home_mixer.functional_component.gate
-
-import com.twitter.product_mixer.core.feature.Feature
-import com.twitter.product_mixer.core.functional_component.gate.Gate
-import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier
-import com.twitter.product_mixer.core.pipeline.PipelineQuery
-import com.twitter.stitch.Stitch
-import scala.reflect.runtime.universe._
-
-case class NonEmptySeqFeatureGate[T: TypeTag](
-  feature: Feature[PipelineQuery, Seq[T]])
-    extends Gate[PipelineQuery] {
-
-  override val identifier: GateIdentifier = GateIdentifier(s"NonEmptySeq$feature")
-
-  override def shouldContinue(query: PipelineQuery): Stitch[Boolean] =
-    Stitch.value(query.features.exists(_.get(feature).nonEmpty))
-}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer/EditedTweetsCandidatePipelineQueryTransformer.scala

@@ -19,7 +19,7 @@ object EditedTweetsCandidatePipelineQueryTransformer
   override val identifier: TransformerIdentifier = TransformerIdentifier("EditedTweets")
 
   // The time window for which a tweet remains editable after creation.
-  private val EditTimeWindow = 30.minutes
+  private val EditTimeWindow = 60.minutes
 
   override def transform(query: PipelineQuery): Seq[Long] = {
     val applicableCandidates = getApplicableCandidates(query)
@@ -29,8 +29,8 @@ object EditedTweetsCandidatePipelineQueryTransformer
       // Any tweets in it could have become stale since being served.
       val previousTimelineLoadTime = applicableCandidates.head.servedTime
 
-      // The time window for editing a tweet is 30 minutes,
-      // so we ignore responses older than (PTL Time - 30 mins).
+      // The time window for editing a tweet is 60 minutes,
+      // so we ignore responses older than (PTL Time - 60 mins).
       val inWindowCandidates: Seq[PersistenceStoreEntry] = applicableCandidates
         .takeWhile(_.servedTime.until(previousTimelineLoadTime) < EditTimeWindow)
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/BUILD.bazel

@@ -6,7 +6,6 @@ scala_library(
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product",

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/FeedbackFatigueScorer.scala

@@ -35,8 +35,8 @@ object FeedbackFatigueScorer
   override def onlyIf(query: PipelineQuery): Boolean =
     query.features.exists(_.getOrElse(FeedbackHistoryFeature, Seq.empty).nonEmpty)
 
-  private val DurationForFiltering = 14.days
-  private val DurationForDiscounting = 140.days
+  val DurationForFiltering = 14.days
+  val DurationForDiscounting = 140.days
   private val ScoreMultiplierLowerBound = 0.2
   private val ScoreMultiplierUpperBound = 1.0
   private val ScoreMultiplierIncrementsCount = 4
@@ -76,8 +76,27 @@ object FeedbackFatigueScorer
         feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Retweet, Seq.empty))
 
     val featureMaps = candidates.map { candidate =>
+      val multiplier = getScoreMultiplier(
+        candidate,
+        authorsToDiscount,
+        likersToDiscount,
+        followersToDiscount,
+        retweetersToDiscount
+      )
       val score = candidate.features.getOrElse(ScoreFeature, None)
+      FeatureMapBuilder().add(ScoreFeature, score.map(_ * multiplier)).build()
+    }
 
+    Stitch.value(featureMaps)
+  }
+
+  def getScoreMultiplier(
+    candidate: CandidateWithFeatures[TweetCandidate],
+    authorsToDiscount: Map[Long, Double],
+    likersToDiscount: Map[Long, Double],
+    followersToDiscount: Map[Long, Double],
+    retweetersToDiscount: Map[Long, Double],
+  ): Double = {
     val originalAuthorId =
       CandidatesUtil.getOriginalAuthorId(candidate.features).getOrElse(0L)
     val originalAuthorMultiplier = authorsToDiscount.getOrElse(originalAuthorId, 1.0)
@@ -92,7 +111,8 @@ object FeedbackFatigueScorer
     val followers = candidate.features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty)
     val followerMultipliers = followers.flatMap(followersToDiscount.get)
     val followerMultiplier =
-        if (followerMultipliers.nonEmpty && followers.size == followerMultipliers.size)
+      if (followerMultipliers.nonEmpty && followers.size == followerMultipliers.size &&
+        likers.isEmpty)
         followerMultipliers.max
       else 1.0
 
@@ -102,16 +122,10 @@ object FeedbackFatigueScorer
         retweetersToDiscount.getOrElse(authorId, 1.0)
       else 1.0
 
-      val multiplier =
     originalAuthorMultiplier * likerMultiplier * followerMultiplier * retweeterMultiplier
-
-      FeatureMapBuilder().add(ScoreFeature, score.map(_ * multiplier)).build()
   }
 
-    Stitch.value(featureMaps)
-  }
-
-  private def getUserDiscounts(
+  def getUserDiscounts(
     queryTime: Time,
     feedbackEntries: Seq[FeedbackEntry],
   ): Map[Long, Double] = {

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/VerifiedAuthorScalingScorer.scala

@@ -1,61 +0,0 @@
-package com.twitter.home_mixer.functional_component.scorer
-
-import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature
-import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
-import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
-import com.twitter.home_mixer.param.HomeGlobalParams.BlueVerifiedAuthorInNetworkMultiplierParam
-import com.twitter.home_mixer.param.HomeGlobalParams.BlueVerifiedAuthorOutOfNetworkMultiplierParam
-import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
-import com.twitter.product_mixer.core.feature.Feature
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
-import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
-import com.twitter.product_mixer.core.functional_component.scorer.Scorer
-import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
-import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier
-import com.twitter.product_mixer.core.pipeline.PipelineQuery
-import com.twitter.stitch.Stitch
-
-/**
- * Scales scores of tweets whose author is Blue Verified by the provided scale factor
- */
-object VerifiedAuthorScalingScorer extends Scorer[PipelineQuery, TweetCandidate] {
-
-  override val identifier: ScorerIdentifier = ScorerIdentifier("VerifiedAuthorScaling")
-
-  override val features: Set[Feature[_, _]] = Set(ScoreFeature)
-
-  override def apply(
-    query: PipelineQuery,
-    candidates: Seq[CandidateWithFeatures[TweetCandidate]]
-  ): Stitch[Seq[FeatureMap]] = {
-    Stitch.value {
-      candidates.map { candidate =>
-        val score = candidate.features.getOrElse(ScoreFeature, None)
-        val updatedScore = getUpdatedScore(score, candidate, query)
-        FeatureMapBuilder().add(ScoreFeature, updatedScore).build()
-      }
-    }
-  }
-
-  /**
-   * We should only be applying this multiplier if the author of the candidate is Blue Verified.
-   * We also treat In-Network vs Out-of-Network differently.
-   */
-  private def getUpdatedScore(
-    score: Option[Double],
-    candidate: CandidateWithFeatures[TweetCandidate],
-    query: PipelineQuery
-  ): Option[Double] = {
-    val isAuthorBlueVerified = candidate.features.getOrElse(AuthorIsBlueVerifiedFeature, false)
-
-    if (isAuthorBlueVerified) {
-      val isCandidateInNetwork = candidate.features.getOrElse(InNetworkFeature, false)
-
-      val scaleFactor =
-        if (isCandidateInNetwork) query.params(BlueVerifiedAuthorInNetworkMultiplierParam)
-        else query.params(BlueVerifiedAuthorOutOfNetworkMultiplierParam)
-
-      score.map(_ * scaleFactor)
-    } else score
-  }
-}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/BUILD.bazel

@@ -5,6 +5,7 @@ scala_library(
     tags = ["bazel-compatible"],
     dependencies = [
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator",
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateHomeClientEventDetails.scala

@@ -1,6 +1,6 @@
 package com.twitter.home_mixer.functional_component.selector
 
-import com.twitter.home_mixer.functional_component.decorator.HomeClientEventDetailsBuilder
+import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventDetailsBuilder
 import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature
 import com.twitter.home_mixer.model.HomeFeatures.ConversationModule2DisplayedTweetsFeature
 import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleHasGapFeature

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/BUILD.bazel

@@ -7,14 +7,12 @@ scala_library(
         "3rdparty/jvm/javax/inject:javax.inject",
         "eventbus/client/src/main/scala/com/twitter/eventbus/client",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator",
+        "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param",
-        "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
         "kafka/finagle-kafka/finatra-kafka/src/main/scala",
@@ -22,6 +20,7 @@ scala_library(
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/urt",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module",
+        "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_subscribe_module",
         "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product",
         "src/scala/com/twitter/timelines/prediction/common/adapters",
@@ -33,13 +32,12 @@ scala_library(
         "src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java",
         "src/thrift/com/twitter/timelines/timeline_logging:thrift-scala",
         "src/thrift/com/twitter/user_session_store:thrift-scala",
-        "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/core",
         "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
-        "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/uss",
         "timelines/ml:kafka",
         "timelines/ml/cont_train/common/client/src/main/scala/com/twitter/timelines/ml/cont_train/common/client/kafka",
         "timelines/ml/cont_train/common/domain/src/main/scala/com/twitter/timelines/ml/cont_train/common/domain/non_scalding",
         "timelines/src/main/scala/com/twitter/timelines/clientconfig",
+        "timelines/src/main/scala/com/twitter/timelines/clients/manhattan/store",
         "timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter",
         "timelines/src/main/scala/com/twitter/timelines/impressionstore/store",
         "timelines/src/main/scala/com/twitter/timelines/injection/scribe",

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ClientEventsBuilder.scala

@@ -2,13 +2,14 @@ package com.twitter.home_mixer.functional_component.side_effect
 
 import com.twitter.conversions.DurationOps._
 import com.twitter.home_mixer.functional_component.decorator.HomeQueryTypePredicates
-import com.twitter.home_mixer.functional_component.decorator.HomeTweetTypePredicates
+import com.twitter.home_mixer.functional_component.decorator.builder.HomeTweetTypePredicates
 import com.twitter.home_mixer.model.HomeFeatures.AccountAgeFeature
 import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
 import com.twitter.home_mixer.model.HomeFeatures.VideoDurationMsFeature
 import com.twitter.home_mixer.model.request.FollowingProduct
 import com.twitter.home_mixer.model.request.ForYouProduct
 import com.twitter.home_mixer.model.request.ListTweetsProduct
+import com.twitter.home_mixer.model.request.SubscribedProduct
 import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.ClientEvent
 import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.EventNamespace
 import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
@@ -18,15 +19,17 @@ import com.twitter.product_mixer.core.pipeline.PipelineQuery
 import com.twitter.timelines.injection.scribe.InjectionScribeUtil
 
 private[side_effect] sealed trait ClientEventsBuilder {
-  private val FollowingSection = Some("home_latest")
+  private val FollowingSection = Some("latest")
   private val ForYouSection = Some("home")
   private val ListTweetsSection = Some("list")
+  private val SubscribedSection = Some("subscribed")
 
   protected def section(query: PipelineQuery): Option[String] = {
     query.product match {
       case FollowingProduct => FollowingSection
       case ForYouProduct => ForYouSection
       case ListTweetsProduct => ListTweetsSection
+      case SubscribedProduct => SubscribedSection
       case other => throw new UnsupportedOperationException(s"Unknown product: $other")
     }
   }
@@ -52,6 +55,7 @@ private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder {
   private val InjectedComponent = Some("injected")
   private val PromotedComponent = Some("promoted")
   private val WhoToFollowComponent = Some("who_to_follow")
+  private val WhoToSubscribeComponent = Some("who_to_subscribe")
   private val WithVideoDurationComponent = Some("with_video_duration")
   private val VideoDurationSumElement = Some("video_duration_sum")
   private val NumVideosElement = Some("num_videos")
@@ -60,7 +64,8 @@ private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder {
     query: PipelineQuery,
     injectedTweets: Seq[ItemCandidateWithDetails],
     promotedTweets: Seq[ItemCandidateWithDetails],
-    whoToFollowUsers: Seq[ItemCandidateWithDetails]
+    whoToFollowUsers: Seq[ItemCandidateWithDetails],
+    whoToSubscribeUsers: Seq[ItemCandidateWithDetails]
   ): Seq[ClientEvent] = {
     val baseEventNamespace = EventNamespace(
       section = section(query),
@@ -77,6 +82,9 @@ private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder {
       ClientEvent(
         baseEventNamespace.copy(component = WhoToFollowComponent, action = ServedUsersAction),
         eventValue = count(whoToFollowUsers)),
+      ClientEvent(
+        baseEventNamespace.copy(component = WhoToSubscribeComponent, action = ServedUsersAction),
+        eventValue = count(whoToSubscribeUsers)),
     )
 
     val tweetTypeServedEvents = HomeTweetTypePredicates.PredicateMap.map {

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeClientEventSideEffect.scala

@@ -5,6 +5,7 @@ import com.twitter.home_mixer.service.HomeMixerAlertConfig
 import com.twitter.home_mixer.util.CandidatesUtil
 import com.twitter.logpipeline.client.common.EventPublisher
 import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect
+import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
 import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier
 import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
 import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
@@ -15,16 +16,30 @@ import com.twitter.product_mixer.core.pipeline.PipelineQuery
  * Side effect that logs served tweet metrics to Scribe as client events.
  */
 case class HomeScribeClientEventSideEffect(
+  enableScribeClientEvents: Boolean,
   override val logPipelinePublisher: EventPublisher[LogEvent],
   injectedTweetsCandidatePipelineIdentifiers: Seq[CandidatePipelineIdentifier],
-  adsCandidatePipelineIdentifier: CandidatePipelineIdentifier,
+  adsCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None,
   whoToFollowCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None,
-) extends ScribeClientEventSideEffect[PipelineQuery, Timeline] {
+  whoToSubscribeCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None)
+    extends ScribeClientEventSideEffect[PipelineQuery, Timeline]
+    with PipelineResultSideEffect.Conditionally[
+      PipelineQuery,
+      Timeline
+    ] {
 
   override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeClientEvent")
 
   override val page = "timelinemixer"
 
+  override def onlyIf(
+    query: PipelineQuery,
+    selectedCandidates: Seq[CandidateWithDetails],
+    remainingCandidates: Seq[CandidateWithDetails],
+    droppedCandidates: Seq[CandidateWithDetails],
+    response: Timeline
+  ): Boolean = enableScribeClientEvents
+
   override def buildClientEvents(
     query: PipelineQuery,
     selectedCandidates: Seq[CandidateWithDetails],
@@ -37,13 +52,15 @@ case class HomeScribeClientEventSideEffect(
     val sources = itemCandidates.groupBy(_.source)
     val injectedTweets =
       injectedTweetsCandidatePipelineIdentifiers.flatMap(sources.getOrElse(_, Seq.empty))
-    val promotedTweets = sources.getOrElse(adsCandidatePipelineIdentifier, Seq.empty)
+    val promotedTweets = adsCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten
 
-    // WhoToFollow module is not required for all home-mixer products, e.g. list tweets timeline.
+    // WhoToFollow and WhoToSubscribe modules are not required for all home-mixer products, e.g. list tweets timeline.
     val whoToFollowUsers = whoToFollowCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten
+    val whoToSubscribeUsers =
+      whoToSubscribeCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten
 
     val servedEvents = ServedEventsBuilder
-      .build(query, injectedTweets, promotedTweets, whoToFollowUsers)
+      .build(query, injectedTweets, promotedTweets, whoToFollowUsers, whoToSubscribeUsers)
 
     val emptyTimelineEvents = EmptyTimelineEventsBuilder.build(query, injectedTweets)
 

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeServedCandidatesSideEffect.scala

@@ -0,0 +1,245 @@
+package com.twitter.home_mixer.functional_component.side_effect
+
+import com.twitter.finagle.tracing.Trace
+import com.twitter.home_mixer.marshaller.timeline_logging.PromotedTweetDetailsMarshaller
+import com.twitter.home_mixer.marshaller.timeline_logging.TweetDetailsMarshaller
+import com.twitter.home_mixer.marshaller.timeline_logging.WhoToFollowDetailsMarshaller
+import com.twitter.home_mixer.model.HomeFeatures.GetInitialFeature
+import com.twitter.home_mixer.model.HomeFeatures.GetMiddleFeature
+import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature
+import com.twitter.home_mixer.model.HomeFeatures.GetOlderFeature
+import com.twitter.home_mixer.model.HomeFeatures.HasDarkRequestFeature
+import com.twitter.home_mixer.model.HomeFeatures.RequestJoinIdFeature
+import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
+import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature
+import com.twitter.home_mixer.model.request.DeviceContext.RequestContext
+import com.twitter.home_mixer.model.request.HasDeviceContext
+import com.twitter.home_mixer.model.request.HasSeenTweetIds
+import com.twitter.home_mixer.model.request.FollowingProduct
+import com.twitter.home_mixer.model.request.ForYouProduct
+import com.twitter.home_mixer.model.request.SubscribedProduct
+import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeServedCandidatesFlag
+import com.twitter.home_mixer.param.HomeGlobalParams.EnableScribeServedCandidatesParam
+import com.twitter.home_mixer.service.HomeMixerAlertConfig
+import com.twitter.inject.annotations.Flag
+import com.twitter.logpipeline.client.common.EventPublisher
+import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate
+import com.twitter.product_mixer.component_library.model.candidate.BaseUserCandidate
+import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator
+import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidateDecorator
+import com.twitter.product_mixer.component_library.side_effect.ScribeLogEventSideEffect
+import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
+import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
+import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
+import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
+import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails
+import com.twitter.product_mixer.core.model.marshalling.response.urt.AddEntriesTimelineInstruction
+import com.twitter.product_mixer.core.model.marshalling.response.urt.ModuleItem
+import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline
+import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule
+import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
+import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem
+import com.twitter.product_mixer.core.pipeline.PipelineQuery
+import com.twitter.timelines.timeline_logging.{thriftscala => thrift}
+import com.twitter.util.Time
+import javax.inject.Inject
+import javax.inject.Singleton
+
+/**
+ * Side effect that logs home timeline served candidates to Scribe.
+ */
+@Singleton
+class HomeScribeServedCandidatesSideEffect @Inject() (
+  @Flag(ScribeServedCandidatesFlag) enableScribeServedCandidates: Boolean,
+  scribeEventPublisher: EventPublisher[thrift.ServedEntry])
+    extends ScribeLogEventSideEffect[
+      thrift.ServedEntry,
+      PipelineQuery with HasSeenTweetIds with HasDeviceContext,
+      Timeline
+    ]
+    with PipelineResultSideEffect.Conditionally[
+      PipelineQuery with HasSeenTweetIds with HasDeviceContext,
+      Timeline
+    ] {
+
+  override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeServedCandidates")
+
+  override def onlyIf(
+    query: PipelineQuery with HasSeenTweetIds with HasDeviceContext,
+    selectedCandidates: Seq[CandidateWithDetails],
+    remainingCandidates: Seq[CandidateWithDetails],
+    droppedCandidates: Seq[CandidateWithDetails],
+    response: Timeline
+  ): Boolean = enableScribeServedCandidates && query.params(EnableScribeServedCandidatesParam)
+
+  override def buildLogEvents(
+    query: PipelineQuery with HasSeenTweetIds with HasDeviceContext,
+    selectedCandidates: Seq[CandidateWithDetails],
+    remainingCandidates: Seq[CandidateWithDetails],
+    droppedCandidates: Seq[CandidateWithDetails],
+    response: Timeline
+  ): Seq[thrift.ServedEntry] = {
+    val timelineType = query.product match {
+      case FollowingProduct => thrift.TimelineType.HomeLatest
+      case ForYouProduct => thrift.TimelineType.Home
+      case SubscribedProduct => thrift.TimelineType.HomeSubscribed
+      case other => throw new UnsupportedOperationException(s"Unknown product: $other")
+    }
+    val requestProvenance = query.deviceContext.map { deviceContext =>
+      deviceContext.requestContextValue match {
+        case RequestContext.Foreground => thrift.RequestProvenance.Foreground
+        case RequestContext.Launch => thrift.RequestProvenance.Launch
+        case RequestContext.PullToRefresh => thrift.RequestProvenance.Ptr
+        case _ => thrift.RequestProvenance.Other
+      }
+    }
+    val queryType = query.features.map { featureMap =>
+      if (featureMap.getOrElse(GetOlderFeature, false)) thrift.QueryType.GetOlder
+      else if (featureMap.getOrElse(GetNewerFeature, false)) thrift.QueryType.GetNewer
+      else if (featureMap.getOrElse(GetMiddleFeature, false)) thrift.QueryType.GetMiddle
+      else if (featureMap.getOrElse(GetInitialFeature, false)) thrift.QueryType.GetInitial
+      else thrift.QueryType.Other
+    }
+    val requestInfo = thrift.RequestInfo(
+      requestTimeMs = query.queryTime.inMilliseconds,
+      traceId = Trace.id.traceId.toLong,
+      userId = query.getOptionalUserId,
+      clientAppId = query.clientContext.appId,
+      hasDarkRequest = query.features.flatMap(_.getOrElse(HasDarkRequestFeature, None)),
+      parentId = Some(Trace.id.parentId.toLong),
+      spanId = Some(Trace.id.spanId.toLong),
+      timelineType = Some(timelineType),
+      ipAddress = query.clientContext.ipAddress,
+      userAgent = query.clientContext.userAgent,
+      queryType = queryType,
+      requestProvenance = requestProvenance,
+      languageCode = query.clientContext.languageCode,
+      countryCode = query.clientContext.countryCode,
+      requestEndTimeMs = Some(Time.now.inMilliseconds),
+      servedRequestId = query.features.flatMap(_.getOrElse(ServedRequestIdFeature, None)),
+      requestJoinId = query.features.flatMap(_.getOrElse(RequestJoinIdFeature, None))
+    )
+
+    val tweetIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] =
+      selectedCandidates.flatMap {
+        case item: ItemCandidateWithDetails if item.candidate.isInstanceOf[BaseTweetCandidate] =>
+          Seq((item.candidateIdLong, item))
+        case module: ModuleCandidateWithDetails
+            if module.candidates.headOption.exists(_.candidate.isInstanceOf[BaseTweetCandidate]) =>
+          module.candidates.map(item => (item.candidateIdLong, item))
+        case _ => Seq.empty
+      }.toMap
+
+    val userIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] =
+      selectedCandidates.flatMap {
+        case module: ModuleCandidateWithDetails
+            if module.candidates.forall(_.candidate.isInstanceOf[BaseUserCandidate]) =>
+          module.candidates.map { item =>
+            (item.candidateIdLong, item)
+          }
+        case _ => Seq.empty
+      }.toMap
+
+    response.instructions.zipWithIndex
+      .collect {
+        case (AddEntriesTimelineInstruction(entries), index) =>
+          entries.collect {
+            case entry: TweetItem if entry.promotedMetadata.isDefined =>
+              val promotedTweetDetails = PromotedTweetDetailsMarshaller(entry, index)
+              Seq(
+                thrift.EntryInfo(
+                  id = entry.id,
+                  position = index.shortValue(),
+                  entryId = entry.entryIdentifier,
+                  entryType = thrift.EntryType.PromotedTweet,
+                  sortIndex = entry.sortIndex,
+                  verticalSize = Some(1),
+                  displayType = Some(entry.displayType.toString),
+                  details = Some(thrift.ItemDetails.PromotedTweetDetails(promotedTweetDetails))
+                )
+              )
+            case entry: TweetItem =>
+              val candidate = tweetIdToItemCandidateMap(entry.id)
+              val tweetDetails = TweetDetailsMarshaller(entry, candidate)
+              Seq(
+                thrift.EntryInfo(
+                  id = candidate.candidateIdLong,
+                  position = index.shortValue(),
+                  entryId = entry.entryIdentifier,
+                  entryType = thrift.EntryType.Tweet,
+                  sortIndex = entry.sortIndex,
+                  verticalSize = Some(1),
+                  score = candidate.features.getOrElse(ScoreFeature, None),
+                  displayType = Some(entry.displayType.toString),
+                  details = Some(thrift.ItemDetails.TweetDetails(tweetDetails))
+                )
+              )
+            case module: TimelineModule
+                if module.entryNamespace.toString == WhoToFollowCandidateDecorator.EntryNamespaceString =>
+              module.items.collect {
+                case ModuleItem(entry: UserItem, _, _) =>
+                  val candidate = userIdToItemCandidateMap(entry.id)
+                  val whoToFollowDetails = WhoToFollowDetailsMarshaller(entry, candidate)
+                  thrift.EntryInfo(
+                    id = entry.id,
+                    position = index.shortValue(),
+                    entryId = module.entryIdentifier,
+                    entryType = thrift.EntryType.WhoToFollowModule,
+                    sortIndex = module.sortIndex,
+                    score = candidate.features.getOrElse(ScoreFeature, None),
+                    displayType = Some(entry.displayType.toString),
+                    details = Some(thrift.ItemDetails.WhoToFollowDetails(whoToFollowDetails))
+                  )
+              }
+            case module: TimelineModule
+                if module.entryNamespace.toString == WhoToSubscribeCandidateDecorator.EntryNamespaceString =>
+              module.items.collect {
+                case ModuleItem(entry: UserItem, _, _) =>
+                  val candidate = userIdToItemCandidateMap(entry.id)
+                  val whoToSubscribeDetails = WhoToFollowDetailsMarshaller(entry, candidate)
+                  thrift.EntryInfo(
+                    id = entry.id,
+                    position = index.shortValue(),
+                    entryId = module.entryIdentifier,
+                    entryType = thrift.EntryType.WhoToSubscribeModule,
+                    sortIndex = module.sortIndex,
+                    score = candidate.features.getOrElse(ScoreFeature, None),
+                    displayType = Some(entry.displayType.toString),
+                    details = Some(thrift.ItemDetails.WhoToFollowDetails(whoToSubscribeDetails))
+                  )
+              }
+            case module: TimelineModule
+                if module.sortIndex.isDefined && module.items.headOption.exists(
+                  _.item.isInstanceOf[TweetItem]) =>
+              module.items.collect {
+                case ModuleItem(entry: TweetItem, _, _) =>
+                  val candidate = tweetIdToItemCandidateMap(entry.id)
+                  thrift.EntryInfo(
+                    id = entry.id,
+                    position = index.shortValue(),
+                    entryId = module.entryIdentifier,
+                    entryType = thrift.EntryType.ConversationModule,
+                    sortIndex = module.sortIndex,
+                    score = candidate.features.getOrElse(ScoreFeature, None),
+                    displayType = Some(entry.displayType.toString)
+                  )
+              }
+            case _ => Seq.empty
+          }.flatten
+        // Other instructions
+        case _ => Seq.empty[thrift.EntryInfo]
+      }.flatten.map { entryInfo =>
+        thrift.ServedEntry(
+          entry = Some(entryInfo),
+          request = requestInfo
+        )
+      }
+  }
+
+  override val logPipelinePublisher: EventPublisher[thrift.ServedEntry] =
+    scribeEventPublisher
+
+  override val alerts = Seq(
+    HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert()
+  )
+}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeServedEntriesSideEffect.scala

@@ -1,212 +0,0 @@
-package com.twitter.home_mixer.functional_component.side_effect
-
-import com.twitter.finagle.tracing.Trace
-import com.twitter.home_mixer.marshaller.timeline_logging.ConversationEntryMarshaller
-import com.twitter.home_mixer.marshaller.timeline_logging.PromotedTweetEntryMarshaller
-import com.twitter.home_mixer.marshaller.timeline_logging.TweetEntryMarshaller
-import com.twitter.home_mixer.marshaller.timeline_logging.WhoToFollowEntryMarshaller
-import com.twitter.home_mixer.model.HomeFeatures.GetInitialFeature
-import com.twitter.home_mixer.model.HomeFeatures.GetMiddleFeature
-import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature
-import com.twitter.home_mixer.model.HomeFeatures.GetOlderFeature
-import com.twitter.home_mixer.model.HomeFeatures.HasDarkRequestFeature
-import com.twitter.home_mixer.model.HomeFeatures.RequestJoinIdFeature
-import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature
-import com.twitter.home_mixer.model.request.DeviceContext.RequestContext
-import com.twitter.home_mixer.model.request.HasDeviceContext
-import com.twitter.home_mixer.model.request.HasSeenTweetIds
-import com.twitter.home_mixer.model.request.FollowingProduct
-import com.twitter.home_mixer.model.request.ForYouProduct
-import com.twitter.home_mixer.service.HomeMixerAlertConfig
-import com.twitter.logpipeline.client.common.EventPublisher
-import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate
-import com.twitter.product_mixer.component_library.model.candidate.BaseUserCandidate
-import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator
-import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
-import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
-import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
-import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails
-import com.twitter.product_mixer.core.model.marshalling.response.urt.AddEntriesTimelineInstruction
-import com.twitter.product_mixer.core.model.marshalling.response.urt.ModuleItem
-import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline
-import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule
-import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
-import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem
-import com.twitter.product_mixer.core.pipeline.PipelineQuery
-import com.twitter.stitch.Stitch
-import com.twitter.timelines.timeline_logging.{thriftscala => thrift}
-import com.twitter.util.Time
-import javax.inject.Inject
-import javax.inject.Singleton
-
-/**
- * Side effect that logs home timeline served entries to Scribe.
- */
-@Singleton
-class HomeScribeServedEntriesSideEffect @Inject() (
-  scribeEventPublisher: EventPublisher[thrift.Timeline])
-    extends PipelineResultSideEffect[
-      PipelineQuery with HasSeenTweetIds with HasDeviceContext,
-      Timeline
-    ] {
-
-  override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeServedEntries")
-
-  final override def apply(
-    inputs: PipelineResultSideEffect.Inputs[
-      PipelineQuery with HasSeenTweetIds with HasDeviceContext,
-      Timeline
-    ]
-  ): Stitch[Unit] = {
-    val timelineThrift = buildTimeline(inputs)
-    Stitch.callFuture(scribeEventPublisher.publish(timelineThrift)).unit
-  }
-
-  def buildTimeline(
-    inputs: PipelineResultSideEffect.Inputs[
-      PipelineQuery with HasSeenTweetIds with HasDeviceContext,
-      Timeline
-    ]
-  ): thrift.Timeline = {
-    val timelineType = inputs.query.product match {
-      case FollowingProduct => thrift.TimelineType.HomeLatest
-      case ForYouProduct => thrift.TimelineType.Home
-      case other => throw new UnsupportedOperationException(s"Unknown product: $other")
-    }
-    val requestProvenance = inputs.query.deviceContext.map { deviceContext =>
-      deviceContext.requestContextValue match {
-        case RequestContext.Foreground => thrift.RequestProvenance.Foreground
-        case RequestContext.Launch => thrift.RequestProvenance.Launch
-        case RequestContext.PullToRefresh => thrift.RequestProvenance.Ptr
-        case _ => thrift.RequestProvenance.Other
-      }
-    }
-    val queryType = inputs.query.features.map { featureMap =>
-      if (featureMap.getOrElse(GetOlderFeature, false)) thrift.QueryType.GetOlder
-      else if (featureMap.getOrElse(GetNewerFeature, false)) thrift.QueryType.GetNewer
-      else if (featureMap.getOrElse(GetMiddleFeature, false)) thrift.QueryType.GetMiddle
-      else if (featureMap.getOrElse(GetInitialFeature, false)) thrift.QueryType.GetInitial
-      else thrift.QueryType.Other
-    }
-
-    val tweetIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] =
-      inputs.selectedCandidates.flatMap {
-        case item: ItemCandidateWithDetails if item.candidate.isInstanceOf[BaseTweetCandidate] =>
-          Seq((item.candidateIdLong, item))
-        case module: ModuleCandidateWithDetails
-            if module.candidates.headOption.exists(_.candidate.isInstanceOf[BaseTweetCandidate]) =>
-          module.candidates.map(item => (item.candidateIdLong, item))
-        case _ => Seq.empty
-      }.toMap
-
-    val userIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] =
-      inputs.selectedCandidates.flatMap {
-        case module: ModuleCandidateWithDetails
-            if module.candidates.forall(_.candidate.isInstanceOf[BaseUserCandidate]) =>
-          module.candidates.map { item =>
-            (item.candidateIdLong, item)
-          }
-        case _ => Seq.empty
-      }.toMap
-
-    val timelineEntries = inputs.response.instructions.zipWithIndex.collect {
-      case (AddEntriesTimelineInstruction(entries), index) =>
-        entries.collect {
-          case entry: TweetItem if entry.promotedMetadata.isDefined =>
-            val promotedTweetEntry = PromotedTweetEntryMarshaller(entry, index)
-            Seq(
-              thrift.TimelineEntry(
-                content = thrift.Content.PromotedTweetEntry(promotedTweetEntry),
-                position = index.shortValue(),
-                entryId = entry.entryIdentifier,
-                entryType = thrift.EntryType.PromotedTweet,
-                sortIndex = entry.sortIndex,
-                verticalSize = Some(1)
-              )
-            )
-          case entry: TweetItem =>
-            val candidate = tweetIdToItemCandidateMap(entry.id)
-            val tweetEntry = TweetEntryMarshaller(entry, candidate)
-            Seq(
-              thrift.TimelineEntry(
-                content = thrift.Content.TweetEntry(tweetEntry),
-                position = index.shortValue(),
-                entryId = entry.entryIdentifier,
-                entryType = thrift.EntryType.Tweet,
-                sortIndex = entry.sortIndex,
-                verticalSize = Some(1)
-              )
-            )
-          case module: TimelineModule
-              if module.entryNamespace.toString == WhoToFollowCandidateDecorator.EntryNamespaceString =>
-            val whoToFollowEntries = module.items.collect {
-              case ModuleItem(entry: UserItem, _, _) =>
-                val candidate = userIdToItemCandidateMap(entry.id)
-                val whoToFollowEntry = WhoToFollowEntryMarshaller(entry, candidate)
-                thrift.AtomicEntry.WtfEntry(whoToFollowEntry)
-            }
-            Seq(
-              thrift.TimelineEntry(
-                content = thrift.Content.Entries(whoToFollowEntries),
-                position = index.shortValue(),
-                entryId = module.entryIdentifier,
-                entryType = thrift.EntryType.WhoToFollowModule,
-                sortIndex = module.sortIndex
-              )
-            )
-          case module: TimelineModule
-              if module.sortIndex.isDefined && module.items.headOption.exists(
-                _.item.isInstanceOf[TweetItem]) =>
-            val conversationTweetEntries = module.items.collect {
-              case ModuleItem(entry: TweetItem, _, _) =>
-                val candidate = tweetIdToItemCandidateMap(entry.id)
-                val conversationEntry = ConversationEntryMarshaller(entry, candidate)
-                thrift.AtomicEntry.ConversationEntry(conversationEntry)
-            }
-            Seq(
-              thrift.TimelineEntry(
-                content = thrift.Content.Entries(conversationTweetEntries),
-                position = index.shortValue(),
-                entryId = module.entryIdentifier,
-                entryType = thrift.EntryType.ConversationModule,
-                sortIndex = module.sortIndex
-              )
-            )
-          case _ => Seq.empty
-        }.flatten
-      // Other instructions
-      case _ => Seq.empty[thrift.TimelineEntry]
-    }.flatten
-
-    thrift.Timeline(
-      timelineEntries = timelineEntries,
-      requestTimeMs = inputs.query.queryTime.inMilliseconds,
-      traceId = Trace.id.traceId.toLong,
-      userId = inputs.query.getOptionalUserId,
-      clientAppId = inputs.query.clientContext.appId,
-      sourceJobInstance = None,
-      hasDarkRequest = inputs.query.features.flatMap(_.getOrElse(HasDarkRequestFeature, None)),
-      parentId = Some(Trace.id.parentId.toLong),
-      spanId = Some(Trace.id.spanId.toLong),
-      timelineType = Some(timelineType),
-      ipAddress = inputs.query.clientContext.ipAddress,
-      userAgent = inputs.query.clientContext.userAgent,
-      queryType = queryType,
-      requestProvenance = requestProvenance,
-      sessionId = None,
-      timeZone = None,
-      browserNotificationPermission = None,
-      lastNonePollingTimeMs = None,
-      languageCode = inputs.query.clientContext.languageCode,
-      countryCode = inputs.query.clientContext.countryCode,
-      requestEndTimeMs = Some(Time.now.inMilliseconds),
-      servedRequestId = inputs.query.features.flatMap(_.getOrElse(ServedRequestIdFeature, None)),
-      requestJoinId = inputs.query.features.flatMap(_.getOrElse(RequestJoinIdFeature, None)),
-      requestSeenTweetIds = inputs.query.seenTweetIds
-    )
-  }
-
-  override val alerts = Seq(
-    HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert()
-  )
-}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsEventBusSideEffect.scala

@@ -3,6 +3,7 @@ package com.twitter.home_mixer.functional_component.side_effect
 import com.twitter.eventbus.client.EventBusPublisher
 import com.twitter.home_mixer.model.request.FollowingProduct
 import com.twitter.home_mixer.model.request.ForYouProduct
+import com.twitter.home_mixer.model.request.SubscribedProduct
 import com.twitter.home_mixer.model.request.HasSeenTweetIds
 import com.twitter.home_mixer.service.HomeMixerAlertConfig
 import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
@@ -22,6 +23,7 @@ import javax.inject.Singleton
 object PublishClientSentImpressionsEventBusSideEffect {
   val HomeSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeTimeline))
   val HomeLatestSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeLatestTimeline))
+  val HomeSubscribedSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeSubscribed))
 }
 
 /**
@@ -56,6 +58,7 @@ class PublishClientSentImpressionsEventBusSideEffect @Inject() (
     val surfaceArea = query.product match {
       case ForYouProduct => HomeSurfaceArea
       case FollowingProduct => HomeLatestSurfaceArea
+      case SubscribedProduct => HomeSubscribedSurfaceArea
       case _ => None
     }
     query.seenTweetIds.map { seenTweetIds =>

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishImpressionBloomFilterSideEffect.scala

@@ -1,39 +1,46 @@
 package com.twitter.home_mixer.functional_component.side_effect
 
-import com.twitter.conversions.DurationOps._
 import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature
 import com.twitter.home_mixer.model.request.HasSeenTweetIds
+import com.twitter.home_mixer.param.HomeGlobalParams.EnableImpressionBloomFilter
 import com.twitter.home_mixer.service.HomeMixerAlertConfig
 import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
 import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
 import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
-import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline
+import com.twitter.product_mixer.core.model.marshalling.HasMarshalling
 import com.twitter.product_mixer.core.pipeline.PipelineQuery
 import com.twitter.stitch.Stitch
-import com.twitter.timelines.impressionbloomfilter.{thriftscala => t}
-import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilter
+import com.twitter.timelines.clients.manhattan.store.ManhattanStoreClient
+import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm}
 import javax.inject.Inject
 import javax.inject.Singleton
 
 @Singleton
-class UpdateImpressionBloomFilterSideEffect @Inject() (bloomFilter: ImpressionBloomFilter)
-    extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, Timeline]
-    with PipelineResultSideEffect.Conditionally[PipelineQuery with HasSeenTweetIds, Timeline] {
-
-  private val SurfaceArea = t.SurfaceArea.HomeTimeline
+class PublishImpressionBloomFilterSideEffect @Inject() (
+  bloomFilterClient: ManhattanStoreClient[
+    blm.ImpressionBloomFilterKey,
+    blm.ImpressionBloomFilterSeq
+  ]) extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, HasMarshalling]
+    with PipelineResultSideEffect.Conditionally[
+      PipelineQuery with HasSeenTweetIds,
+      HasMarshalling
+    ] {
 
   override val identifier: SideEffectIdentifier =
-    SideEffectIdentifier("UpdateImpressionBloomFilter")
+    SideEffectIdentifier("PublishImpressionBloomFilter")
+
+  private val SurfaceArea = blm.SurfaceArea.HomeTimeline
 
   override def onlyIf(
     query: PipelineQuery with HasSeenTweetIds,
     selectedCandidates: Seq[CandidateWithDetails],
     remainingCandidates: Seq[CandidateWithDetails],
     droppedCandidates: Seq[CandidateWithDetails],
-    response: Timeline
-  ): Boolean = query.seenTweetIds.exists(_.nonEmpty)
+    response: HasMarshalling
+  ): Boolean =
+    query.params.getBoolean(EnableImpressionBloomFilter) && query.seenTweetIds.exists(_.nonEmpty)
 
-  def buildEvents(query: PipelineQuery): Option[t.ImpressionBloomFilterSeq] = {
+  def buildEvents(query: PipelineQuery): Option[blm.ImpressionBloomFilterSeq] = {
     query.features.flatMap { featureMap =>
       val impressionBloomFilterSeq = featureMap.get(ImpressionBloomFilterFeature)
       if (impressionBloomFilterSeq.entries.nonEmpty) Some(impressionBloomFilterSeq)
@@ -42,19 +49,17 @@ class UpdateImpressionBloomFilterSideEffect @Inject() (bloomFilter: ImpressionBl
   }
 
   override def apply(
-    inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, Timeline]
+    inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, HasMarshalling]
   ): Stitch[Unit] = {
     buildEvents(inputs.query)
-      .map { updatedBloomFilter =>
-        bloomFilter.writeBloomFilterSeq(
-          userId = inputs.query.getRequiredUserId,
-          surfaceArea = SurfaceArea,
-          impressionBloomFilterSeq = updatedBloomFilter)
+      .map { updatedBloomFilterSeq =>
+        bloomFilterClient.write(
+          blm.ImpressionBloomFilterKey(inputs.query.getRequiredUserId, SurfaceArea),
+          updatedBloomFilterSeq)
       }.getOrElse(Stitch.Unit)
   }
 
   override val alerts = Seq(
-    HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8),
-    HomeMixerAlertConfig.BusinessHours.defaultLatencyAlert(30.millis)
+    HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8)
   )
 }

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ServedStatsSideEffect.scala

@@ -1,80 +0,0 @@
-package com.twitter.home_mixer.functional_component.side_effect
-
-import com.twitter.finagle.stats.StatsReceiver
-import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
-import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature
-import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
-import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
-import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
-import com.twitter.home_mixer.param.HomeGlobalParams.AuthorListForStatsParam
-import com.twitter.home_mixer.util.CandidatesUtil
-import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
-import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
-import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
-import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
-import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
-import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline
-import com.twitter.product_mixer.core.pipeline.PipelineQuery
-import com.twitter.stitch.Stitch
-import javax.inject.Inject
-import javax.inject.Singleton
-
-@Singleton
-class ServedStatsSideEffect @Inject() (statsReceiver: StatsReceiver)
-    extends PipelineResultSideEffect[PipelineQuery, Timeline] {
-
-  override val identifier: SideEffectIdentifier = SideEffectIdentifier("ServedStats")
-
-  private val baseStatsReceiver = statsReceiver.scope(identifier.toString)
-  private val authorStatsReceiver = baseStatsReceiver.scope("Author")
-  private val candidateSourceStatsReceiver = baseStatsReceiver.scope("CandidateSource")
-  private val contentBalanceStatsReceiver = baseStatsReceiver.scope("ContentBalance")
-  private val inNetworkStatsCounter = contentBalanceStatsReceiver.counter("InNetwork")
-  private val outOfNetworkStatsCounter = contentBalanceStatsReceiver.counter("OutOfNetwork")
-
-  override def apply(
-    inputs: PipelineResultSideEffect.Inputs[PipelineQuery, Timeline]
-  ): Stitch[Unit] = {
-    val tweetCandidates = CandidatesUtil
-      .getItemCandidates(inputs.selectedCandidates).filter(_.isCandidateType[TweetCandidate]())
-
-    recordAuthorStats(tweetCandidates, inputs.query.params(AuthorListForStatsParam))
-    recordCandidateSourceStats(tweetCandidates)
-    recordContentBalanceStats(tweetCandidates)
-    Stitch.Unit
-  }
-
-  def recordAuthorStats(candidates: Seq[CandidateWithDetails], authors: Set[Long]): Unit = {
-    candidates
-      .filter { candidate =>
-        candidate.features.getOrElse(AuthorIdFeature, None).exists(authors.contains) &&
-        // Only include original tweets
-        (!candidate.features.getOrElse(IsRetweetFeature, false)) &&
-        candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty
-      }
-      .groupBy { candidate =>
-        (getCandidateSourceId(candidate), candidate.features.get(AuthorIdFeature).get)
-      }
-      .foreach {
-        case ((candidateSourceId, authorId), authorCandidates) =>
-          authorStatsReceiver
-            .scope(authorId.toString).counter(candidateSourceId).incr(authorCandidates.size)
-      }
-  }
-
-  def recordCandidateSourceStats(candidates: Seq[ItemCandidateWithDetails]): Unit = {
-    candidates.groupBy(getCandidateSourceId).foreach {
-      case (candidateSourceId, candidateSourceCandidates) =>
-        candidateSourceStatsReceiver.counter(candidateSourceId).incr(candidateSourceCandidates.size)
-    }
-  }
-
-  def recordContentBalanceStats(candidates: Seq[ItemCandidateWithDetails]): Unit = {
-    val (in, oon) = candidates.partition(_.features.getOrElse(InNetworkFeature, true))
-    inNetworkStatsCounter.incr(in.size)
-    outOfNetworkStatsCounter.incr(oon.size)
-  }
-
-  private def getCandidateSourceId(candidate: CandidateWithDetails): String =
-    candidate.features.getOrElse(CandidateSourceIdFeature, None).map(_.name).getOrElse("None")
-}

home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateTimelinesPersistenceStoreSideEffect.scala

@@ -3,8 +3,10 @@ package com.twitter.home_mixer.functional_component.side_effect
 import com.twitter.home_mixer.model.HomeFeatures._
 import com.twitter.home_mixer.model.request.FollowingProduct
 import com.twitter.home_mixer.model.request.ForYouProduct
+import com.twitter.home_mixer.model.HomeFeatures.IsTweetPreviewFeature
 import com.twitter.home_mixer.service.HomeMixerAlertConfig
 import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator
+import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidateDecorator
 import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
 import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
 import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
@@ -97,12 +99,14 @@ class UpdateTimelinesPersistenceStoreSideEffect @Inject() (
       val entries = inputs.response.instructions.collect {
         case AddEntriesTimelineInstruction(entries) =>
           entries.collect {
-            // includes both tweets and promoted tweets
-            case entry: TweetItem if entry.sortIndex.isDefined =>
+            // includes tweets, tweet previews, and promoted tweets
+            case entry: TweetItem if entry.sortIndex.isDefined => {
               Seq(
                 buildTweetEntryWithItemIds(
                   tweetIdToItemCandidateMap(entry.id),
-                  entry.sortIndex.get))
+                  entry.sortIndex.get
+                ))
+            }
             // tweet conversation modules are flattened to individual tweets in the persistence store
             case module: TimelineModule
                 if module.sortIndex.isDefined && module.items.headOption.exists(
@@ -125,6 +129,19 @@ class UpdateTimelinesPersistenceStoreSideEffect @Inject() (
                   size = module.items.size.toShort,
                   itemIds = Some(userIds)
                 ))
+            case module: TimelineModule
+                if module.sortIndex.isDefined && module.entryNamespace.toString == WhoToSubscribeCandidateDecorator.EntryNamespaceString =>
+              val userIds = module.items
+                .map(item =>
+                  UpdateTimelinesPersistenceStoreSideEffect.EmptyItemIds.copy(userId =
+                    Some(item.item.id.asInstanceOf[Long])))
+              Seq(
+                EntryWithItemIds(
+                  entityIdType = EntityIdType.WhoToSubscribe,
+                  sortIndex = module.sortIndex.get,
+                  size = module.items.size.toShort,
+                  itemIds = Some(userIds)
+                ))
           }.flatten
         case ShowCoverInstruction(cover) =>
           Seq(
@@ -216,8 +233,11 @@ class UpdateTimelinesPersistenceStoreSideEffect @Inject() (
       userId = None
     )
 
+    val isPreview = features.getOrElse(IsTweetPreviewFeature, default = false)
+    val entityType = if (isPreview) EntityIdType.TweetPreview else EntityIdType.Tweet
+
     EntryWithItemIds(
-      entityIdType = EntityIdType.Tweet,
+      entityIdType = entityType,
       sortIndex = sortIndex,
       size = 1.toShort,
       itemIds = Some(Seq(itemIds))

home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/BUILD.bazel

@@ -4,14 +4,12 @@ scala_library(
     strict_deps = True,
     tags = ["bazel-compatible"],
     dependencies = [
-        "dspbidder/thrift/src/main/thrift/com/twitter/dspbidder/commons:thrift-scala",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
         "home-mixer/thrift/src/main/thrift:thrift-scala",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common",
     ],
     exports = [
-        "dspbidder/thrift/src/main/thrift/com/twitter/dspbidder/commons:thrift-scala",
         "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
         "home-mixer/thrift/src/main/thrift:thrift-scala",
         "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request",

home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductContextUnmarshaller.scala

@@ -5,9 +5,9 @@ import com.twitter.home_mixer.model.request.ForYouProductContext
 import com.twitter.home_mixer.model.request.ListRecommendedUsersProductContext
 import com.twitter.home_mixer.model.request.ListTweetsProductContext
 import com.twitter.home_mixer.model.request.ScoredTweetsProductContext
+import com.twitter.home_mixer.model.request.SubscribedProductContext
 import com.twitter.home_mixer.{thriftscala => t}
 import com.twitter.product_mixer.core.model.marshalling.request.ProductContext
-
 import javax.inject.Inject
 import javax.inject.Singleton
 
@@ -26,15 +26,17 @@ class HomeMixerProductContextUnmarshaller @Inject() (
       ForYouProductContext(
         deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)),
         seenTweetIds = p.seenTweetIds,
-        dspClientContext = p.dspClientContext
+        dspClientContext = p.dspClientContext,
+        pushToHomeTweetId = p.pushToHomeTweetId
       )
-    case t.ProductContext.Realtime(p) =>
+    case t.ProductContext.ListManagement(p) =>
       throw new UnsupportedOperationException(s"This product is no longer used")
     case t.ProductContext.ScoredTweets(p) =>
       ScoredTweetsProductContext(
         deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)),
         seenTweetIds = p.seenTweetIds,
-        servedTweetIds = p.servedTweetIds
+        servedTweetIds = p.servedTweetIds,
+        backfillTweetIds = p.backfillTweetIds
       )
     case t.ProductContext.ListTweets(p) =>
       ListTweetsProductContext(
@@ -46,7 +48,13 @@ class HomeMixerProductContextUnmarshaller @Inject() (
       ListRecommendedUsersProductContext(
         listId = p.listId,
         selectedUserIds = p.selectedUserIds,
-        excludedUserIds = p.excludedUserIds
+        excludedUserIds = p.excludedUserIds,
+        listName = p.listName
+      )
+    case t.ProductContext.Subscribed(p) =>
+      SubscribedProductContext(
+        deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)),
+        seenTweetIds = p.seenTweetIds,
       )
     case t.ProductContext.UnknownUnionField(field) =>
       throw new UnsupportedOperationException(s"Unknown display context: ${field.field.name}")

home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductUnmarshaller.scala

@@ -5,9 +5,9 @@ import com.twitter.home_mixer.model.request.ForYouProduct
 import com.twitter.home_mixer.model.request.ListRecommendedUsersProduct
 import com.twitter.home_mixer.model.request.ListTweetsProduct
 import com.twitter.home_mixer.model.request.ScoredTweetsProduct
+import com.twitter.home_mixer.model.request.SubscribedProduct
 import com.twitter.home_mixer.{thriftscala => t}
 import com.twitter.product_mixer.core.model.marshalling.request.Product
-
 import javax.inject.Inject
 import javax.inject.Singleton
 
@@ -17,11 +17,12 @@ class HomeMixerProductUnmarshaller @Inject() () {
   def apply(product: t.Product): Product = product match {
     case t.Product.Following => FollowingProduct
     case t.Product.ForYou => ForYouProduct
-    case t.Product.Realtime =>
+    case t.Product.ListManagement =>
       throw new UnsupportedOperationException(s"This product is no longer used")
     case t.Product.ScoredTweets => ScoredTweetsProduct
     case t.Product.ListTweets => ListTweetsProduct
     case t.Product.ListRecommendedUsers => ListRecommendedUsersProduct
+    case t.Product.Subscribed => SubscribedProduct
     case t.Product.EnumUnknownProduct(value) =>
       throw new UnsupportedOperationException(s"Unknown product: $value")
   }

home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/ConversationEntryMarshaller.scala

@@ -1,16 +0,0 @@
-package com.twitter.home_mixer.marshaller.timeline_logging
-
-import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
-import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
-import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
-import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
-
-object ConversationEntryMarshaller {
-
-  def apply(entry: TweetItem, candidate: ItemCandidateWithDetails): thriftlog.ConversationEntry =
-    thriftlog.ConversationEntry(
-      displayedTweetId = entry.id,
-      displayType = Some(entry.displayType.toString),
-      score = candidate.features.getOrElse(ScoreFeature, None)
-    )
-}