diff --git a/pushservice/BUILD.bazel b/pushservice/BUILD.bazel deleted file mode 100644 index 12efdb2e6..000000000 --- a/pushservice/BUILD.bazel +++ /dev/null @@ -1,48 +0,0 @@ -alias( - name = "frigate-pushservice", - target = ":frigate-pushservice_lib", -) - -target( - name = "frigate-pushservice_lib", - dependencies = [ - "frigate/frigate-pushservice-opensource/src/main/scala/com/twitter/frigate/pushservice", - ], -) - -jvm_binary( - name = "bin", - basename = "frigate-pushservice", - main = "com.twitter.frigate.pushservice.PushServiceMain", - runtime_platform = "java11", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/ch/qos/logback:logback-classic", - "finatra/inject/inject-logback/src/main/scala", - "frigate/frigate-pushservice-opensource/src/main/scala/com/twitter/frigate/pushservice", - "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", - "twitter-server/logback-classic/src/main/scala", - ], - excludes = [ - exclude("com.twitter.translations", "translations-twitter"), - exclude("org.apache.hadoop", "hadoop-aws"), - exclude("org.tensorflow"), - scala_exclude("com.twitter", "ckoia-scala"), - ], -) - -jvm_app( - name = "bundle", - basename = "frigate-pushservice-package-dist", - archive = "zip", - binary = ":bin", - tags = ["bazel-compatible"], -) - -python3_library( - name = "mr_model_constants", - sources = [ - "config/deepbird/constants.py", - ], - tags = ["bazel-compatible"], -) diff --git a/pushservice/README.md b/pushservice/README.md deleted file mode 100644 index b1bad0a57..000000000 --- a/pushservice/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# Pushservice - -Pushservice is the main push recommendation service at Twitter used to generate recommendation-based notifications for users. It currently powers two functionalities: - -- RefreshForPushHandler: This handler determines whether to send a recommendation push to a user based on their ID. It generates the best push recommendation item and coordinates with downstream services to deliver it -- SendHandler: This handler determines and manage whether send the push to users based on the given target user details and the provided push recommendation item - -## Overview - -### RefreshForPushHandler - -RefreshForPushHandler follows these steps: - -- Building Target and checking eligibility - - Builds a target user object based on the given user ID - - Performs target-level filterings to determine if the target is eligible for a recommendation push -- Fetch Candidates - - Retrieves a list of potential candidates for the push by querying various candidate sources using the target -- Candidate Hydration - - Hydrates the candidate details with batch calls to different downstream services -- Pre-rank Filtering, also called Light Filtering - - Filters the hydrated candidates with lightweight RPC calls -- Rank - - Perform feature hydration for candidates and target user - - Performs light ranking on candidates - - Performs heavy ranking on candidates -- Take Step, also called Heavy Filtering - - Takes the top-ranked candidates one by one and applies heavy filtering until one candidate passes all filter steps -- Send - - Calls the appropriate downstream service to deliver the eligible candidate as a push and in-app notification to the target user - -### SendHandler - -SendHandler follows these steps: - -- Building Target - - Builds a target user object based on the given user ID -- Candidate Hydration - - Hydrates the candidate details with batch calls to different downstream services -- Feature Hydration - - Perform feature hydration for candidates and target user -- Take Step, also called Heavy Filtering - - Perform filterings and validation checking for the given candidate -- Send - - Calls the appropriate downstream service to deliver the given candidate as a push and/or in-app notification to the target user \ No newline at end of file diff --git a/pushservice/src/main/python/models/heavy_ranking/BUILD b/pushservice/src/main/python/models/heavy_ranking/BUILD deleted file mode 100644 index 2c25693a9..000000000 --- a/pushservice/src/main/python/models/heavy_ranking/BUILD +++ /dev/null @@ -1,169 +0,0 @@ -python37_binary( - name = "update_warm_start_checkpoint", - source = "update_warm_start_checkpoint.py", - tags = ["no-mypy"], - dependencies = [ - ":deep_norm_lib", - "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/heavy_ranking:update_warm_start_checkpoint", - ], -) - -python3_library( - name = "params_lib", - sources = ["params.py"], - tags = ["no-mypy"], - dependencies = [ - "3rdparty/python/pydantic:default", - "src/python/twitter/deepbird/projects/magic_recs/v11/lib:params_lib", - ], -) - -python3_library( - name = "features_lib", - sources = ["features.py"], - tags = ["no-mypy"], - dependencies = [ - ":params_lib", - "src/python/twitter/deepbird/projects/magic_recs/libs", - "twml:twml-nodeps", - ], -) - -python3_library( - name = "model_pools_lib", - sources = ["model_pools.py"], - tags = ["no-mypy"], - dependencies = [ - ":features_lib", - ":params_lib", - "src/python/twitter/deepbird/projects/magic_recs/v11/lib:model_lib", - ], -) - -python3_library( - name = "graph_lib", - sources = ["graph.py"], - tags = ["no-mypy"], - dependencies = [ - ":params_lib", - "src/python/twitter/deepbird/projects/magic_recs/libs", - ], -) - -python3_library( - name = "run_args_lib", - sources = ["run_args.py"], - tags = ["no-mypy"], - dependencies = [ - ":features_lib", - ":params_lib", - "twml:twml-nodeps", - ], -) - -python3_library( - name = "deep_norm_lib", - sources = ["deep_norm.py"], - tags = ["no-mypy"], - dependencies = [ - ":features_lib", - ":graph_lib", - ":model_pools_lib", - ":params_lib", - ":run_args_lib", - "src/python/twitter/deepbird/projects/magic_recs/libs", - "src/python/twitter/deepbird/util/data", - "twml:twml-nodeps", - ], -) - -python3_library( - name = "eval_lib", - sources = ["eval.py"], - tags = ["no-mypy"], - dependencies = [ - ":features_lib", - ":graph_lib", - ":model_pools_lib", - ":params_lib", - ":run_args_lib", - "src/python/twitter/deepbird/projects/magic_recs/libs", - "twml:twml-nodeps", - ], -) - -python37_binary( - name = "deep_norm", - source = "deep_norm.py", - dependencies = [ - ":deep_norm_lib", - "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/heavy_ranking:deep_norm", - "twml", - ], -) - -python37_binary( - name = "eval", - source = "eval.py", - dependencies = [ - ":eval_lib", - "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/heavy_ranking:eval", - "twml", - ], -) - -python3_library( - name = "mlwf_libs", - tags = ["no-mypy"], - dependencies = [ - ":deep_norm_lib", - "twml", - ], -) - -python37_binary( - name = "train_model", - source = "deep_norm.py", - dependencies = [ - ":deep_norm_lib", - "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/heavy_ranking:train_model", - ], -) - -python37_binary( - name = "train_model_local", - source = "deep_norm.py", - dependencies = [ - ":deep_norm_lib", - "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/heavy_ranking:train_model_local", - "twml", - ], -) - -python37_binary( - name = "eval_model_local", - source = "eval.py", - dependencies = [ - ":eval_lib", - "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/heavy_ranking:eval_model_local", - "twml", - ], -) - -python37_binary( - name = "eval_model", - source = "eval.py", - dependencies = [ - ":eval_lib", - "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/heavy_ranking:eval_model", - ], -) - -python37_binary( - name = "mlwf_model", - source = "deep_norm.py", - dependencies = [ - ":mlwf_libs", - "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/heavy_ranking:mlwf_model", - ], -) diff --git a/pushservice/src/main/python/models/heavy_ranking/README.md b/pushservice/src/main/python/models/heavy_ranking/README.md deleted file mode 100644 index 75336a09c..000000000 --- a/pushservice/src/main/python/models/heavy_ranking/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# Notification Heavy Ranker Model - -## Model Context -There are 4 major components of Twitter notifications recommendation system: 1) candidate generation 2) light ranking 3) heavy ranking & 4) quality control. This notification heavy ranker model is the core ranking model for the personalised notifications recommendation. It's a multi-task learning model to predict the probabilities that the target users will open and engage with the sent notifications. - - -## Directory Structure -- BUILD: this file defines python library dependencies -- deep_norm.py: this file contains how to set up continuous training, model evaluation and model exporting for the notification heavy ranker model -- eval.py: the main python entry file to set up the overall model evaluation pipeline -- features.py: this file contains importing feature list and support functions for feature engineering -- graph.py: this file defines how to build the tensorflow graph with specified model architecture, loss function and training configuration -- model_pools.py: this file defines the available model types for the heavy ranker -- params.py: this file defines hyper-parameters used in the notification heavy ranker -- run_args.py: this file defines command line parameters to run model training & evaluation -- update_warm_start_checkpoint.py: this file contains the support to modify checkpoints of the given saved heavy ranker model -- lib/BUILD: this file defines python library dependencies for tensorflow model architecture -- lib/layers.py: this file defines different type of convolution layers to be used in the heavy ranker model -- lib/model.py: this file defines the module containing ClemNet, the heavy ranker model type -- lib/params.py: this file defines parameters used in the heavy ranker model diff --git a/pushservice/src/main/python/models/heavy_ranking/__init__.py b/pushservice/src/main/python/models/heavy_ranking/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/pushservice/src/main/python/models/heavy_ranking/deep_norm.py b/pushservice/src/main/python/models/heavy_ranking/deep_norm.py deleted file mode 100644 index 7db281b4a..000000000 --- a/pushservice/src/main/python/models/heavy_ranking/deep_norm.py +++ /dev/null @@ -1,136 +0,0 @@ -""" -Training job for the heavy ranker of the push notification service. -""" -from datetime import datetime -import json -import os - -import twml - -from ..libs.metric_fn_utils import flip_disliked_labels, get_metric_fn -from ..libs.model_utils import read_config -from ..libs.warm_start_utils import get_feature_list_for_heavy_ranking, warm_start_checkpoint -from .features import get_feature_config -from .model_pools import ALL_MODELS -from .params import load_graph_params -from .run_args import get_training_arg_parser - -import tensorflow.compat.v1 as tf -from tensorflow.compat.v1 import logging - - -def main() -> None: - args, _ = get_training_arg_parser().parse_known_args() - logging.info(f"Parsed args: {args}") - - params = load_graph_params(args) - logging.info(f"Loaded graph params: {params}") - - param_file = os.path.join(args.save_dir, "params.json") - logging.info(f"Saving graph params to: {param_file}") - with tf.io.gfile.GFile(param_file, mode="w") as file: - json.dump(params.json(), file, ensure_ascii=False, indent=4) - - logging.info(f"Get Feature Config: {args.feature_list}") - feature_list = read_config(args.feature_list).items() - feature_config = get_feature_config( - data_spec_path=args.data_spec, - params=params, - feature_list_provided=feature_list, - ) - feature_list_path = args.feature_list - - warm_start_from = args.warm_start_from - if args.warm_start_base_dir: - logging.info(f"Get warm started model from: {args.warm_start_base_dir}.") - - continuous_binary_feat_list_save_path = os.path.join( - args.warm_start_base_dir, "continuous_binary_feat_list.json" - ) - warm_start_folder = os.path.join(args.warm_start_base_dir, "best_checkpoint") - job_name = os.path.basename(args.save_dir) - ws_output_ckpt_folder = os.path.join(args.warm_start_base_dir, f"warm_start_for_{job_name}") - if tf.io.gfile.exists(ws_output_ckpt_folder): - tf.io.gfile.rmtree(ws_output_ckpt_folder) - - tf.io.gfile.mkdir(ws_output_ckpt_folder) - - warm_start_from = warm_start_checkpoint( - warm_start_folder, - continuous_binary_feat_list_save_path, - feature_list_path, - args.data_spec, - ws_output_ckpt_folder, - ) - logging.info(f"Created warm_start_from_ckpt {warm_start_from}.") - - logging.info("Build Trainer.") - metric_fn = get_metric_fn("OONC_Engagement" if len(params.tasks) == 2 else "OONC", False) - - trainer = twml.trainers.DataRecordTrainer( - name="magic_recs", - params=args, - build_graph_fn=lambda *args: ALL_MODELS[params.model.name](params=params)(*args), - save_dir=args.save_dir, - run_config=None, - feature_config=feature_config, - metric_fn=flip_disliked_labels(metric_fn), - warm_start_from=warm_start_from, - ) - - logging.info("Build train and eval input functions.") - train_input_fn = trainer.get_train_input_fn(shuffle=True) - eval_input_fn = trainer.get_eval_input_fn(repeat=False, shuffle=False) - - learn = trainer.learn - if args.distributed or args.num_workers is not None: - learn = trainer.train_and_evaluate - - if not args.directly_export_best: - logging.info("Starting training") - start = datetime.now() - learn( - early_stop_minimize=False, - early_stop_metric="pr_auc_unweighted_OONC", - early_stop_patience=args.early_stop_patience, - early_stop_tolerance=args.early_stop_tolerance, - eval_input_fn=eval_input_fn, - train_input_fn=train_input_fn, - ) - logging.info(f"Total training time: {datetime.now() - start}") - else: - logging.info("Directly exporting the model") - - if not args.export_dir: - args.export_dir = os.path.join(args.save_dir, "exported_models") - - logging.info(f"Exporting the model to {args.export_dir}.") - start = datetime.now() - twml.contrib.export.export_fn.export_all_models( - trainer=trainer, - export_dir=args.export_dir, - parse_fn=feature_config.get_parse_fn(), - serving_input_receiver_fn=feature_config.get_serving_input_receiver_fn(), - export_output_fn=twml.export_output_fns.batch_prediction_continuous_output_fn, - ) - - logging.info(f"Total model export time: {datetime.now() - start}") - logging.info(f"The MLP directory is: {args.save_dir}") - - continuous_binary_feat_list_save_path = os.path.join( - args.save_dir, "continuous_binary_feat_list.json" - ) - logging.info( - f"Saving the list of continuous and binary features to {continuous_binary_feat_list_save_path}." - ) - continuous_binary_feat_list = get_feature_list_for_heavy_ranking( - feature_list_path, args.data_spec - ) - twml.util.write_file( - continuous_binary_feat_list_save_path, continuous_binary_feat_list, encode="json" - ) - - -if __name__ == "__main__": - main() - logging.info("Done.") diff --git a/pushservice/src/main/python/models/heavy_ranking/eval.py b/pushservice/src/main/python/models/heavy_ranking/eval.py deleted file mode 100644 index 7f74472fb..000000000 --- a/pushservice/src/main/python/models/heavy_ranking/eval.py +++ /dev/null @@ -1,59 +0,0 @@ -""" -Evaluation job for the heavy ranker of the push notification service. -""" - -from datetime import datetime - -import twml - -from ..libs.metric_fn_utils import get_metric_fn -from ..libs.model_utils import read_config -from .features import get_feature_config -from .model_pools import ALL_MODELS -from .params import load_graph_params -from .run_args import get_eval_arg_parser - -from tensorflow.compat.v1 import logging - - -def main(): - args, _ = get_eval_arg_parser().parse_known_args() - logging.info(f"Parsed args: {args}") - - params = load_graph_params(args) - logging.info(f"Loaded graph params: {params}") - - logging.info(f"Get Feature Config: {args.feature_list}") - feature_list = read_config(args.feature_list).items() - feature_config = get_feature_config( - data_spec_path=args.data_spec, - params=params, - feature_list_provided=feature_list, - ) - - logging.info("Build DataRecordTrainer.") - metric_fn = get_metric_fn("OONC_Engagement" if len(params.tasks) == 2 else "OONC", False) - - trainer = twml.trainers.DataRecordTrainer( - name="magic_recs", - params=args, - build_graph_fn=lambda *args: ALL_MODELS[params.model.name](params=params)(*args), - save_dir=args.save_dir, - run_config=None, - feature_config=feature_config, - metric_fn=metric_fn, - ) - - logging.info("Run the evaluation.") - start = datetime.now() - trainer._estimator.evaluate( - input_fn=trainer.get_eval_input_fn(repeat=False, shuffle=False), - steps=None if (args.eval_steps is not None and args.eval_steps < 0) else args.eval_steps, - checkpoint_path=args.eval_checkpoint, - ) - logging.info(f"Evaluating time: {datetime.now() - start}.") - - -if __name__ == "__main__": - main() - logging.info("Job done.") diff --git a/pushservice/src/main/python/models/heavy_ranking/features.py b/pushservice/src/main/python/models/heavy_ranking/features.py deleted file mode 100644 index ce6a2686a..000000000 --- a/pushservice/src/main/python/models/heavy_ranking/features.py +++ /dev/null @@ -1,138 +0,0 @@ -import os -from typing import Dict - -from twitter.deepbird.projects.magic_recs.libs.model_utils import filter_nans_and_infs -import twml -from twml.layers import full_sparse, sparse_max_norm - -from .params import FeaturesParams, GraphParams, SparseFeaturesParams - -import tensorflow as tf -from tensorflow import Tensor -import tensorflow.compat.v1 as tf1 - - -FEAT_CONFIG_DEFAULT_VAL = 0 -DEFAULT_FEATURE_LIST_PATH = "./feature_list_default.yaml" -FEATURE_LIST_DEFAULT_PATH = os.path.join( - os.path.dirname(os.path.realpath(__file__)), DEFAULT_FEATURE_LIST_PATH -) - - -def get_feature_config(data_spec_path=None, feature_list_provided=[], params: GraphParams = None): - - a_string_feat_list = [feat for feat, feat_type in feature_list_provided if feat_type != "S"] - - builder = twml.contrib.feature_config.FeatureConfigBuilder( - data_spec_path=data_spec_path, debug=False - ) - - builder = builder.extract_feature_group( - feature_regexes=a_string_feat_list, - group_name="continuous_features", - default_value=FEAT_CONFIG_DEFAULT_VAL, - type_filter=["CONTINUOUS"], - ) - - builder = builder.extract_feature_group( - feature_regexes=a_string_feat_list, - group_name="binary_features", - type_filter=["BINARY"], - ) - - if params.model.features.sparse_features: - builder = builder.extract_features_as_hashed_sparse( - feature_regexes=a_string_feat_list, - hash_space_size_bits=params.model.features.sparse_features.bits, - type_filter=["DISCRETE", "STRING", "SPARSE_BINARY"], - output_tensor_name="sparse_not_continuous", - ) - - builder = builder.extract_features_as_hashed_sparse( - feature_regexes=[feat for feat, feat_type in feature_list_provided if feat_type == "S"], - hash_space_size_bits=params.model.features.sparse_features.bits, - type_filter=["SPARSE_CONTINUOUS"], - output_tensor_name="sparse_continuous", - ) - - builder = builder.add_labels([task.label for task in params.tasks] + ["label.ntabDislike"]) - - if params.weight: - builder = builder.define_weight(params.weight) - - return builder.build() - - -def dense_features(features: Dict[str, Tensor], training: bool) -> Tensor: - """ - Performs feature transformations on the raw dense features (continuous and binary). - """ - with tf.name_scope("dense_features"): - x = filter_nans_and_infs(features["continuous_features"]) - - x = tf.sign(x) * tf.math.log(tf.abs(x) + 1) - x = tf1.layers.batch_normalization( - x, momentum=0.9999, training=training, renorm=training, axis=1 - ) - x = tf.clip_by_value(x, -5, 5) - - transformed_continous_features = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x) - - binary_features = filter_nans_and_infs(features["binary_features"]) - binary_features = tf.dtypes.cast(binary_features, tf.float32) - - output = tf.concat([transformed_continous_features, binary_features], axis=1) - - return output - - -def sparse_features( - features: Dict[str, Tensor], training: bool, params: SparseFeaturesParams -) -> Tensor: - """ - Performs feature transformations on the raw sparse features. - """ - - with tf.name_scope("sparse_features"): - with tf.name_scope("sparse_not_continuous"): - sparse_not_continuous = full_sparse( - inputs=features["sparse_not_continuous"], - output_size=params.embedding_size, - use_sparse_grads=training, - use_binary_values=False, - ) - - with tf.name_scope("sparse_continuous"): - shape_enforced_input = twml.util.limit_sparse_tensor_size( - sparse_tf=features["sparse_continuous"], input_size_bits=params.bits, mask_indices=False - ) - - normalized_continuous_sparse = sparse_max_norm( - inputs=shape_enforced_input, is_training=training - ) - - sparse_continuous = full_sparse( - inputs=normalized_continuous_sparse, - output_size=params.embedding_size, - use_sparse_grads=training, - use_binary_values=False, - ) - - output = tf.concat([sparse_not_continuous, sparse_continuous], axis=1) - - return output - - -def get_features(features: Dict[str, Tensor], training: bool, params: FeaturesParams) -> Tensor: - """ - Performs feature transformations on the dense and sparse features and combine the resulting - tensors into a single one. - """ - with tf.name_scope("features"): - x = dense_features(features, training) - tf1.logging.info(f"Dense features: {x.shape}") - - if params.sparse_features: - x = tf.concat([x, sparse_features(features, training, params.sparse_features)], axis=1) - - return x diff --git a/pushservice/src/main/python/models/heavy_ranking/graph.py b/pushservice/src/main/python/models/heavy_ranking/graph.py deleted file mode 100644 index 4188736ac..000000000 --- a/pushservice/src/main/python/models/heavy_ranking/graph.py +++ /dev/null @@ -1,129 +0,0 @@ -""" -Graph class defining methods to obtain key quantities such as: - * the logits - * the probabilities - * the final score - * the loss function - * the training operator -""" -from __future__ import annotations - -from abc import ABC, abstractmethod -from typing import Any, Dict - -from twitter.deepbird.hparam import HParams -import twml - -from ..libs.model_utils import generate_disliked_mask -from .params import GraphParams - -import tensorflow as tf -import tensorflow.compat.v1 as tf1 - - -class Graph(ABC): - def __init__(self, params: GraphParams): - self.params = params - - @abstractmethod - def get_logits(self, features: Dict[str, tf.Tensor], mode: tf.estimator.ModeKeys) -> tf.Tensor: - pass - - def get_probabilities(self, logits: tf.Tensor) -> tf.Tensor: - return tf.math.cumprod(tf.nn.sigmoid(logits), axis=1, name="probabilities") - - def get_task_weights(self, labels: tf.Tensor) -> tf.Tensor: - oonc_label = tf.reshape(labels[:, 0], shape=(-1, 1)) - task_weights = tf.concat([tf.ones_like(oonc_label), oonc_label], axis=1) - - n_labels = len(self.params.tasks) - task_weights = tf.reshape(task_weights[:, 0:n_labels], shape=(-1, n_labels)) - - return task_weights - - def get_loss(self, labels: tf.Tensor, logits: tf.Tensor, **kwargs: Any) -> tf.Tensor: - with tf.name_scope("weights"): - disliked_mask = generate_disliked_mask(labels) - - labels = tf.reshape(labels[:, 0:2], shape=[-1, 2]) - - labels = labels * tf.cast(tf.logical_not(disliked_mask), dtype=labels.dtype) - - with tf.name_scope("task_weight"): - task_weights = self.get_task_weights(labels) - - with tf.name_scope("batch_size"): - batch_size = tf.cast(tf.shape(labels)[0], dtype=tf.float32, name="batch_size") - - weights = task_weights / batch_size - - with tf.name_scope("loss"): - loss = tf.reduce_sum( - tf.nn.sigmoid_cross_entropy_with_logits(labels=labels, logits=logits) * weights, - ) - - return loss - - def get_score(self, probabilities: tf.Tensor) -> tf.Tensor: - with tf.name_scope("score_weight"): - score_weights = tf.constant([task.score_weight for task in self.params.tasks]) - score_weights = score_weights / tf.reduce_sum(score_weights, axis=0) - - with tf.name_scope("score"): - score = tf.reshape(tf.reduce_sum(probabilities * score_weights, axis=1), shape=[-1, 1]) - - return score - - def get_train_op(self, loss: tf.Tensor, twml_params) -> Any: - with tf.name_scope("optimizer"): - learning_rate = twml_params.learning_rate - optimizer = tf1.train.GradientDescentOptimizer(learning_rate=learning_rate) - - update_ops = set(tf1.get_collection(tf1.GraphKeys.UPDATE_OPS)) - with tf.control_dependencies(update_ops): - train_op = twml.optimizers.optimize_loss( - loss=loss, - variables=tf1.trainable_variables(), - global_step=tf1.train.get_global_step(), - optimizer=optimizer, - learning_rate=None, - ) - - return train_op - - def __call__( - self, - features: Dict[str, tf.Tensor], - labels: tf.Tensor, - mode: tf.estimator.ModeKeys, - params: HParams, - config=None, - ) -> Dict[str, tf.Tensor]: - training = mode == tf.estimator.ModeKeys.TRAIN - logits = self.get_logits(features=features, training=training) - probabilities = self.get_probabilities(logits=logits) - score = None - loss = None - train_op = None - - if mode == tf.estimator.ModeKeys.PREDICT: - score = self.get_score(probabilities=probabilities) - output = {"loss": loss, "train_op": train_op, "prediction": score} - - elif mode in (tf.estimator.ModeKeys.TRAIN, tf.estimator.ModeKeys.EVAL): - loss = self.get_loss(labels=labels, logits=logits) - - if mode == tf.estimator.ModeKeys.TRAIN: - train_op = self.get_train_op(loss=loss, twml_params=params) - - output = {"loss": loss, "train_op": train_op, "output": probabilities} - - else: - raise ValueError( - f""" - Invalid mode. Possible values are: {tf.estimator.ModeKeys.PREDICT}, {tf.estimator.ModeKeys.TRAIN}, and {tf.estimator.ModeKeys.EVAL} - . Passed: {mode} - """ - ) - - return output diff --git a/pushservice/src/main/python/models/heavy_ranking/lib/BUILD b/pushservice/src/main/python/models/heavy_ranking/lib/BUILD deleted file mode 100644 index a0ed713c4..000000000 --- a/pushservice/src/main/python/models/heavy_ranking/lib/BUILD +++ /dev/null @@ -1,42 +0,0 @@ -python3_library( - name = "params_lib", - sources = [ - "params.py", - ], - tags = [ - "bazel-compatible", - "no-mypy", - ], - dependencies = [ - "3rdparty/python/pydantic:default", - ], -) - -python3_library( - name = "layers_lib", - sources = [ - "layers.py", - ], - tags = [ - "bazel-compatible", - "no-mypy", - ], - dependencies = [ - ], -) - -python3_library( - name = "model_lib", - sources = [ - "model.py", - ], - tags = [ - "bazel-compatible", - "no-mypy", - ], - dependencies = [ - ":layers_lib", - ":params_lib", - "3rdparty/python/absl-py:default", - ], -) diff --git a/pushservice/src/main/python/models/heavy_ranking/lib/layers.py b/pushservice/src/main/python/models/heavy_ranking/lib/layers.py deleted file mode 100644 index 33dd6f012..000000000 --- a/pushservice/src/main/python/models/heavy_ranking/lib/layers.py +++ /dev/null @@ -1,128 +0,0 @@ -""" -Different type of convolution layers to be used in the ClemNet. -""" -from typing import Any - -import tensorflow as tf - - -class KerasConv1D(tf.keras.layers.Layer): - """ - Basic Conv1D layer in a wrapper to be compatible with ClemNet. - """ - - def __init__( - self, - kernel_size: int, - filters: int, - strides: int, - padding: str, - use_bias: bool = True, - kernel_initializer: str = "glorot_uniform", - bias_initializer: str = "zeros", - **kwargs: Any, - ): - super(KerasConv1D, self).__init__(**kwargs) - self.kernel_size = kernel_size - self.filters = filters - self.use_bias = use_bias - self.kernel_initializer = kernel_initializer - self.bias_initializer = bias_initializer - self.strides = strides - self.padding = padding - - def build(self, input_shape: tf.TensorShape) -> None: - assert ( - len(input_shape) == 3 - ), f"Tensor shape must be of length 3. Passed tensor of shape {input_shape}." - - self.features = input_shape[1] - - self.w = tf.keras.layers.Conv1D( - kernel_size=self.kernel_size, - filters=self.filters, - strides=self.strides, - padding=self.padding, - use_bias=self.use_bias, - kernel_initializer=self.kernel_initializer, - bias_initializer=self.bias_initializer, - name=self.name, - ) - - def call(self, inputs: tf.Tensor, **kwargs: Any) -> tf.Tensor: - return self.w(inputs) - - -class ChannelWiseDense(tf.keras.layers.Layer): - """ - Dense layer is applied to each channel separately. This is more memory and computationally - efficient than flattening the channels and performing single dense layers over it which is the - default behavior in tf1. - """ - - def __init__( - self, - output_size: int, - use_bias: bool, - kernel_initializer: str = "uniform_glorot", - bias_initializer: str = "zeros", - **kwargs: Any, - ): - super(ChannelWiseDense, self).__init__(**kwargs) - self.output_size = output_size - self.use_bias = use_bias - self.kernel_initializer = kernel_initializer - self.bias_initializer = bias_initializer - - def build(self, input_shape: tf.TensorShape) -> None: - assert ( - len(input_shape) == 3 - ), f"Tensor shape must be of length 3. Passed tensor of shape {input_shape}." - - input_size = input_shape[1] - channels = input_shape[2] - - self.kernel = self.add_weight( - name="kernel", - shape=(channels, input_size, self.output_size), - initializer=self.kernel_initializer, - trainable=True, - ) - - self.bias = self.add_weight( - name="bias", - shape=(channels, self.output_size), - initializer=self.bias_initializer, - trainable=self.use_bias, - ) - - def call(self, inputs: tf.Tensor, **kwargs: Any) -> tf.Tensor: - x = inputs - - transposed_x = tf.transpose(x, perm=[2, 0, 1]) - transposed_residual = ( - tf.transpose(tf.matmul(transposed_x, self.kernel), perm=[1, 0, 2]) + self.bias - ) - output = tf.transpose(transposed_residual, perm=[0, 2, 1]) - - return output - - -class ResidualLayer(tf.keras.layers.Layer): - """ - Layer implementing a 3D-residual connection. - """ - - def build(self, input_shape: tf.TensorShape) -> None: - assert ( - len(input_shape) == 3 - ), f"Tensor shape must be of length 3. Passed tensor of shape {input_shape}." - - def call(self, inputs: tf.Tensor, residual: tf.Tensor, **kwargs: Any) -> tf.Tensor: - shortcut = tf.keras.layers.Conv1D( - filters=int(residual.shape[2]), strides=1, kernel_size=1, padding="SAME", use_bias=False - )(inputs) - - output = tf.add(shortcut, residual) - - return output diff --git a/pushservice/src/main/python/models/heavy_ranking/lib/model.py b/pushservice/src/main/python/models/heavy_ranking/lib/model.py deleted file mode 100644 index c6c8b1c6b..000000000 --- a/pushservice/src/main/python/models/heavy_ranking/lib/model.py +++ /dev/null @@ -1,76 +0,0 @@ -""" -Module containing ClemNet. -""" -from typing import Any - -from .layers import ChannelWiseDense, KerasConv1D, ResidualLayer -from .params import BlockParams, ClemNetParams - -import tensorflow as tf -import tensorflow.compat.v1 as tf1 - - -class Block2(tf.keras.layers.Layer): - """ - Possible ClemNet block. Architecture is as follow: - Optional(DenseLayer + BN + Act) - Optional(ConvLayer + BN + Act) - Optional(Residual Layer) - - """ - - def __init__(self, params: BlockParams, **kwargs: Any): - super(Block2, self).__init__(**kwargs) - self.params = params - - def build(self, input_shape: tf.TensorShape) -> None: - assert ( - len(input_shape) == 3 - ), f"Tensor shape must be of length 3. Passed tensor of shape {input_shape}." - - def call(self, inputs: tf.Tensor, training: bool) -> tf.Tensor: - x = inputs - if self.params.dense: - x = ChannelWiseDense(**self.params.dense.dict())(inputs=x, training=training) - x = tf1.layers.batch_normalization(x, momentum=0.9999, training=training, axis=1) - x = tf.keras.layers.Activation(self.params.activation)(x) - - if self.params.conv: - x = KerasConv1D(**self.params.conv.dict())(inputs=x, training=training) - x = tf1.layers.batch_normalization(x, momentum=0.9999, training=training, axis=1) - x = tf.keras.layers.Activation(self.params.activation)(x) - - if self.params.residual: - x = ResidualLayer()(inputs=inputs, residual=x) - - return x - - -class ClemNet(tf.keras.layers.Layer): - """ - A residual network stacking residual blocks composed of dense layers and convolutions. - """ - - def __init__(self, params: ClemNetParams, **kwargs: Any): - super(ClemNet, self).__init__(**kwargs) - self.params = params - - def build(self, input_shape: tf.TensorShape) -> None: - assert len(input_shape) in ( - 2, - 3, - ), f"Tensor shape must be of length 3. Passed tensor of shape {input_shape}." - - def call(self, inputs: tf.Tensor, training: bool) -> tf.Tensor: - if len(inputs.shape) < 3: - inputs = tf.expand_dims(inputs, axis=-1) - - x = inputs - for block_params in self.params.blocks: - x = Block2(block_params)(inputs=x, training=training) - - x = tf.keras.layers.Flatten(name="flattened")(x) - if self.params.top: - x = tf.keras.layers.Dense(units=self.params.top.n_labels, name="logits")(x) - - return x diff --git a/pushservice/src/main/python/models/heavy_ranking/lib/params.py b/pushservice/src/main/python/models/heavy_ranking/lib/params.py deleted file mode 100644 index 721d6ed95..000000000 --- a/pushservice/src/main/python/models/heavy_ranking/lib/params.py +++ /dev/null @@ -1,49 +0,0 @@ -""" -Parameters used in ClemNet. -""" -from typing import List, Optional - -from pydantic import BaseModel, Extra, Field, PositiveInt - - -# checkstyle: noqa - - -class ExtendedBaseModel(BaseModel): - class Config: - extra = Extra.forbid - - -class DenseParams(ExtendedBaseModel): - name: Optional[str] - bias_initializer: str = "zeros" - kernel_initializer: str = "glorot_uniform" - output_size: PositiveInt - use_bias: bool = Field(True) - - -class ConvParams(ExtendedBaseModel): - name: Optional[str] - bias_initializer: str = "zeros" - filters: PositiveInt - kernel_initializer: str = "glorot_uniform" - kernel_size: PositiveInt - padding: str = "SAME" - strides: PositiveInt = 1 - use_bias: bool = Field(True) - - -class BlockParams(ExtendedBaseModel): - activation: Optional[str] - conv: Optional[ConvParams] - dense: Optional[DenseParams] - residual: Optional[bool] - - -class TopLayerParams(ExtendedBaseModel): - n_labels: PositiveInt - - -class ClemNetParams(ExtendedBaseModel): - blocks: List[BlockParams] = [] - top: Optional[TopLayerParams] diff --git a/pushservice/src/main/python/models/heavy_ranking/model_pools.py b/pushservice/src/main/python/models/heavy_ranking/model_pools.py deleted file mode 100644 index de59ee1a6..000000000 --- a/pushservice/src/main/python/models/heavy_ranking/model_pools.py +++ /dev/null @@ -1,34 +0,0 @@ -""" -Candidate architectures for each task's. -""" - -from __future__ import annotations - -from typing import Dict - -from .features import get_features -from .graph import Graph -from .lib.model import ClemNet -from .params import ModelTypeEnum - -import tensorflow as tf - - -class MagicRecsClemNet(Graph): - def get_logits(self, features: Dict[str, tf.Tensor], training: bool) -> tf.Tensor: - - with tf.name_scope("logits"): - inputs = get_features(features=features, training=training, params=self.params.model.features) - - with tf.name_scope("OONC_logits"): - model = ClemNet(params=self.params.model.architecture) - oonc_logit = model(inputs=inputs, training=training) - - with tf.name_scope("EngagementGivenOONC_logits"): - model = ClemNet(params=self.params.model.architecture) - eng_logits = model(inputs=inputs, training=training) - - return tf.concat([oonc_logit, eng_logits], axis=1) - - -ALL_MODELS = {ModelTypeEnum.clemnet: MagicRecsClemNet} diff --git a/pushservice/src/main/python/models/heavy_ranking/params.py b/pushservice/src/main/python/models/heavy_ranking/params.py deleted file mode 100644 index 64a7de2b1..000000000 --- a/pushservice/src/main/python/models/heavy_ranking/params.py +++ /dev/null @@ -1,89 +0,0 @@ -import enum -import json -from typing import List, Optional - -from .lib.params import BlockParams, ClemNetParams, ConvParams, DenseParams, TopLayerParams - -from pydantic import BaseModel, Extra, NonNegativeFloat -import tensorflow.compat.v1 as tf - - -# checkstyle: noqa - - -class ExtendedBaseModel(BaseModel): - class Config: - extra = Extra.forbid - - -class SparseFeaturesParams(ExtendedBaseModel): - bits: int - embedding_size: int - - -class FeaturesParams(ExtendedBaseModel): - sparse_features: Optional[SparseFeaturesParams] - - -class ModelTypeEnum(str, enum.Enum): - clemnet: str = "clemnet" - - -class ModelParams(ExtendedBaseModel): - name: ModelTypeEnum - features: FeaturesParams - architecture: ClemNetParams - - -class TaskNameEnum(str, enum.Enum): - oonc: str = "OONC" - engagement: str = "Engagement" - - -class Task(ExtendedBaseModel): - name: TaskNameEnum - label: str - score_weight: NonNegativeFloat - - -DEFAULT_TASKS = [ - Task(name=TaskNameEnum.oonc, label="label", score_weight=0.9), - Task(name=TaskNameEnum.engagement, label="label.engagement", score_weight=0.1), -] - - -class GraphParams(ExtendedBaseModel): - tasks: List[Task] = DEFAULT_TASKS - model: ModelParams - weight: Optional[str] - - -DEFAULT_ARCHITECTURE_PARAMS = ClemNetParams( - blocks=[ - BlockParams( - activation="relu", - conv=ConvParams(kernel_size=3, filters=5), - dense=DenseParams(output_size=output_size), - residual=False, - ) - for output_size in [1024, 512, 256, 128] - ], - top=TopLayerParams(n_labels=1), -) - -DEFAULT_GRAPH_PARAMS = GraphParams( - model=ModelParams( - name=ModelTypeEnum.clemnet, - architecture=DEFAULT_ARCHITECTURE_PARAMS, - features=FeaturesParams(sparse_features=SparseFeaturesParams(bits=18, embedding_size=50)), - ), -) - - -def load_graph_params(args) -> GraphParams: - params = DEFAULT_GRAPH_PARAMS - if args.param_file: - with tf.io.gfile.GFile(args.param_file, mode="r+") as file: - params = GraphParams.parse_obj(json.load(file)) - - return params diff --git a/pushservice/src/main/python/models/heavy_ranking/run_args.py b/pushservice/src/main/python/models/heavy_ranking/run_args.py deleted file mode 100644 index 1cc33a8e0..000000000 --- a/pushservice/src/main/python/models/heavy_ranking/run_args.py +++ /dev/null @@ -1,59 +0,0 @@ -from twml.trainers import DataRecordTrainer - -from .features import FEATURE_LIST_DEFAULT_PATH - - -def get_training_arg_parser(): - parser = DataRecordTrainer.add_parser_arguments() - - parser.add_argument( - "--feature_list", - default=FEATURE_LIST_DEFAULT_PATH, - type=str, - help="Which features to use for training", - ) - - parser.add_argument( - "--param_file", - default=None, - type=str, - help="Path to JSON file containing the graph parameters. If None, model will load default parameters.", - ) - - parser.add_argument( - "--directly_export_best", - default=False, - action="store_true", - help="whether to directly_export best_checkpoint", - ) - - parser.add_argument( - "--warm_start_from", default=None, type=str, help="model dir to warm start from" - ) - - parser.add_argument( - "--warm_start_base_dir", - default=None, - type=str, - help="latest ckpt in this folder will be used to ", - ) - - parser.add_argument( - "--model_type", - default=None, - type=str, - help="Which type of model to train.", - ) - return parser - - -def get_eval_arg_parser(): - parser = get_training_arg_parser() - parser.add_argument( - "--eval_checkpoint", - default=None, - type=str, - help="Which checkpoint to use for evaluation", - ) - - return parser diff --git a/pushservice/src/main/python/models/heavy_ranking/update_warm_start_checkpoint.py b/pushservice/src/main/python/models/heavy_ranking/update_warm_start_checkpoint.py deleted file mode 100644 index 04887b9cf..000000000 --- a/pushservice/src/main/python/models/heavy_ranking/update_warm_start_checkpoint.py +++ /dev/null @@ -1,146 +0,0 @@ -""" -Model for modifying the checkpoints of the magic recs cnn Model with addition, deletion, and reordering -of continuous and binary features. -""" - -import os - -from twitter.deepbird.projects.magic_recs.libs.get_feat_config import FEATURE_LIST_DEFAULT_PATH -from twitter.deepbird.projects.magic_recs.libs.warm_start_utils_v11 import ( - get_feature_list_for_heavy_ranking, - mkdirp, - rename_dir, - rmdir, - warm_start_checkpoint, -) -import twml -from twml.trainers import DataRecordTrainer - -import tensorflow.compat.v1 as tf -from tensorflow.compat.v1 import logging - - -def get_arg_parser(): - parser = DataRecordTrainer.add_parser_arguments() - parser.add_argument( - "--model_type", - default="deepnorm_gbdt_inputdrop2_rescale", - type=str, - help="specify the model type to use.", - ) - - parser.add_argument( - "--model_trainer_name", - default="None", - type=str, - help="deprecated, added here just for api compatibility.", - ) - - parser.add_argument( - "--warm_start_base_dir", - default="none", - type=str, - help="latest ckpt in this folder will be used.", - ) - - parser.add_argument( - "--output_checkpoint_dir", - default="none", - type=str, - help="Output folder for warm started ckpt. If none, it will move warm_start_base_dir to backup, and overwrite it", - ) - - parser.add_argument( - "--feature_list", - default="none", - type=str, - help="Which features to use for training", - ) - - parser.add_argument( - "--old_feature_list", - default="none", - type=str, - help="Which features to use for training", - ) - - return parser - - -def get_params(args=None): - parser = get_arg_parser() - if args is None: - return parser.parse_args() - else: - return parser.parse_args(args) - - -def _main(): - opt = get_params() - logging.info("parse is: ") - logging.info(opt) - - if opt.feature_list == "none": - feature_list_path = FEATURE_LIST_DEFAULT_PATH - else: - feature_list_path = opt.feature_list - - if opt.warm_start_base_dir != "none" and tf.io.gfile.exists(opt.warm_start_base_dir): - if opt.output_checkpoint_dir == "none" or opt.output_checkpoint_dir == opt.warm_start_base_dir: - _warm_start_base_dir = os.path.normpath(opt.warm_start_base_dir) + "_backup_warm_start" - _output_folder_dir = opt.warm_start_base_dir - - rename_dir(opt.warm_start_base_dir, _warm_start_base_dir) - tf.logging.info(f"moved {opt.warm_start_base_dir} to {_warm_start_base_dir}") - else: - _warm_start_base_dir = opt.warm_start_base_dir - _output_folder_dir = opt.output_checkpoint_dir - - continuous_binary_feat_list_save_path = os.path.join( - _warm_start_base_dir, "continuous_binary_feat_list.json" - ) - - if opt.old_feature_list != "none": - tf.logging.info("getting old continuous_binary_feat_list") - continuous_binary_feat_list = get_feature_list_for_heavy_ranking( - opt.old_feature_list, opt.data_spec - ) - rmdir(continuous_binary_feat_list_save_path) - twml.util.write_file( - continuous_binary_feat_list_save_path, continuous_binary_feat_list, encode="json" - ) - tf.logging.info(f"Finish writting files to {continuous_binary_feat_list_save_path}") - - warm_start_folder = os.path.join(_warm_start_base_dir, "best_checkpoint") - if not tf.io.gfile.exists(warm_start_folder): - warm_start_folder = _warm_start_base_dir - - rmdir(_output_folder_dir) - mkdirp(_output_folder_dir) - - new_ckpt = warm_start_checkpoint( - warm_start_folder, - continuous_binary_feat_list_save_path, - feature_list_path, - opt.data_spec, - _output_folder_dir, - opt.model_type, - ) - logging.info(f"Created new ckpt {new_ckpt} from {warm_start_folder}") - - tf.logging.info("getting new continuous_binary_feat_list") - new_continuous_binary_feat_list_save_path = os.path.join( - _output_folder_dir, "continuous_binary_feat_list.json" - ) - continuous_binary_feat_list = get_feature_list_for_heavy_ranking( - feature_list_path, opt.data_spec - ) - rmdir(new_continuous_binary_feat_list_save_path) - twml.util.write_file( - new_continuous_binary_feat_list_save_path, continuous_binary_feat_list, encode="json" - ) - tf.logging.info(f"Finish writting files to {new_continuous_binary_feat_list_save_path}") - - -if __name__ == "__main__": - _main() diff --git a/pushservice/src/main/python/models/libs/BUILD b/pushservice/src/main/python/models/libs/BUILD deleted file mode 100644 index 82a014ba5..000000000 --- a/pushservice/src/main/python/models/libs/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -python3_library( - name = "libs", - sources = ["*.py"], - tags = [ - "bazel-compatible", - "no-mypy", - ], - dependencies = [ - "cortex/recsys/src/python/twitter/cortex/recsys/utils", - "magicpony/common/file_access/src/python/twitter/magicpony/common/file_access", - "src/python/twitter/cortex/ml/embeddings/deepbird", - "src/python/twitter/cortex/ml/embeddings/deepbird/grouped_metrics", - "src/python/twitter/deepbird/util/data", - "twml:twml-nodeps", - ], -) diff --git a/pushservice/src/main/python/models/libs/__init__.py b/pushservice/src/main/python/models/libs/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/pushservice/src/main/python/models/libs/customized_full_sparse.py b/pushservice/src/main/python/models/libs/customized_full_sparse.py deleted file mode 100644 index b41f7d694..000000000 --- a/pushservice/src/main/python/models/libs/customized_full_sparse.py +++ /dev/null @@ -1,56 +0,0 @@ -# pylint: disable=no-member, arguments-differ, attribute-defined-outside-init, unused-argument -""" -Implementing Full Sparse Layer, allow specify use_binary_value in call() to -overide default action. -""" - -from twml.layers import FullSparse as defaultFullSparse -from twml.layers.full_sparse import sparse_dense_matmul - -import tensorflow.compat.v1 as tf - - -class FullSparse(defaultFullSparse): - def call(self, inputs, use_binary_values=None, **kwargs): # pylint: disable=unused-argument - """The logic of the layer lives here. - - Arguments: - inputs: - A SparseTensor or a list of SparseTensors. - If `inputs` is a list, all tensors must have same `dense_shape`. - - Returns: - - If `inputs` is `SparseTensor`, then returns `bias + inputs * dense_b`. - - If `inputs` is a `list[SparseTensor`, then returns - `bias + add_n([sp_a * dense_b for sp_a in inputs])`. - """ - - if use_binary_values is not None: - default_use_binary_values = use_binary_values - else: - default_use_binary_values = self.use_binary_values - - if isinstance(default_use_binary_values, (list, tuple)): - raise ValueError( - "use_binary_values can not be %s when inputs is %s" - % (type(default_use_binary_values), type(inputs)) - ) - - outputs = sparse_dense_matmul( - inputs, - self.weight, - self.use_sparse_grads, - default_use_binary_values, - name="sparse_mm", - partition_axis=self.partition_axis, - num_partitions=self.num_partitions, - compress_ids=self._use_compression, - cast_indices_dtype=self._cast_indices_dtype, - ) - - if self.bias is not None: - outputs = tf.nn.bias_add(outputs, self.bias) - - if self.activation is not None: - return self.activation(outputs) # pylint: disable=not-callable - return outputs diff --git a/pushservice/src/main/python/models/libs/get_feat_config.py b/pushservice/src/main/python/models/libs/get_feat_config.py deleted file mode 100644 index 4d8b3e93c..000000000 --- a/pushservice/src/main/python/models/libs/get_feat_config.py +++ /dev/null @@ -1,176 +0,0 @@ -import os - -from twitter.deepbird.projects.magic_recs.libs.metric_fn_utils import USER_AGE_FEATURE_NAME -from twitter.deepbird.projects.magic_recs.libs.model_utils import read_config -from twml.contrib import feature_config as contrib_feature_config - - -# checkstyle: noqa - -FEAT_CONFIG_DEFAULT_VAL = -1.23456789 - -DEFAULT_INPUT_SIZE_BITS = 18 - -DEFAULT_FEATURE_LIST_PATH = "./feature_list_default.yaml" -FEATURE_LIST_DEFAULT_PATH = os.path.join( - os.path.dirname(os.path.realpath(__file__)), DEFAULT_FEATURE_LIST_PATH -) - -DEFAULT_FEATURE_LIST_LIGHT_RANKING_PATH = "./feature_list_light_ranking.yaml" -FEATURE_LIST_DEFAULT_LIGHT_RANKING_PATH = os.path.join( - os.path.dirname(os.path.realpath(__file__)), DEFAULT_FEATURE_LIST_LIGHT_RANKING_PATH -) - -FEATURE_LIST_DEFAULT = read_config(FEATURE_LIST_DEFAULT_PATH).items() -FEATURE_LIST_LIGHT_RANKING_DEFAULT = read_config(FEATURE_LIST_DEFAULT_LIGHT_RANKING_PATH).items() - - -LABELS = ["label"] -LABELS_MTL = {"OONC": ["label"], "OONC_Engagement": ["label", "label.engagement"]} -LABELS_LR = { - "Sent": ["label.sent"], - "HeavyRankPosition": ["meta.ranking.is_top3"], - "HeavyRankProbability": ["meta.ranking.weighted_oonc_model_score"], -} - - -def _get_new_feature_config_base( - data_spec_path, - labels, - add_sparse_continous=True, - add_gbdt=True, - add_user_id=False, - add_timestamp=False, - add_user_age=False, - feature_list_provided=[], - opt=None, - run_light_ranking_group_metrics_in_bq=False, -): - """ - Getter of the feature config based on specification. - - Args: - data_spec_path: A string indicating the path of the data_spec.json file, which could be - either a local path or a hdfs path. - labels: A list of strings indicating the name of the label in the data spec. - add_sparse_continous: A bool indicating if sparse_continuous feature needs to be included. - add_gbdt: A bool indicating if gbdt feature needs to be included. - add_user_id: A bool indicating if user_id feature needs to be included. - add_timestamp: A bool indicating if timestamp feature needs to be included. This will be useful - for sequential models and meta learning models. - add_user_age: A bool indicating if the user age feature needs to be included. - feature_list_provided: A list of features thats need to be included. If not specified, will use - FEATURE_LIST_DEFAULT by default. - opt: A namespace of arguments indicating the hyparameters. - run_light_ranking_group_metrics_in_bq: A bool indicating if heavy ranker score info needs to be included to compute group metrics in BigQuery. - - Returns: - A twml feature config object. - """ - - input_size_bits = DEFAULT_INPUT_SIZE_BITS if opt is None else opt.input_size_bits - - feature_list = feature_list_provided if feature_list_provided != [] else FEATURE_LIST_DEFAULT - a_string_feat_list = [f[0] for f in feature_list if f[1] != "S"] - - builder = contrib_feature_config.FeatureConfigBuilder(data_spec_path=data_spec_path) - - builder = builder.extract_feature_group( - feature_regexes=a_string_feat_list, - group_name="continuous", - default_value=FEAT_CONFIG_DEFAULT_VAL, - type_filter=["CONTINUOUS"], - ) - - builder = builder.extract_features_as_hashed_sparse( - feature_regexes=a_string_feat_list, - output_tensor_name="sparse_no_continuous", - hash_space_size_bits=input_size_bits, - type_filter=["BINARY", "DISCRETE", "STRING", "SPARSE_BINARY"], - ) - - if add_gbdt: - builder = builder.extract_features_as_hashed_sparse( - feature_regexes=["ads\..*"], - output_tensor_name="gbdt_sparse", - hash_space_size_bits=input_size_bits, - ) - - if add_sparse_continous: - s_string_feat_list = [f[0] for f in feature_list if f[1] == "S"] - - builder = builder.extract_features_as_hashed_sparse( - feature_regexes=s_string_feat_list, - output_tensor_name="sparse_continuous", - hash_space_size_bits=input_size_bits, - type_filter=["SPARSE_CONTINUOUS"], - ) - - if add_user_id: - builder = builder.extract_feature("meta.user_id") - if add_timestamp: - builder = builder.extract_feature("meta.timestamp") - if add_user_age: - builder = builder.extract_feature(USER_AGE_FEATURE_NAME) - - if run_light_ranking_group_metrics_in_bq: - builder = builder.extract_feature("meta.trace_id") - builder = builder.extract_feature("meta.ranking.weighted_oonc_model_score") - - builder = builder.add_labels(labels).define_weight("meta.weight") - - return builder.build() - - -def get_feature_config_with_sparse_continuous( - data_spec_path, - feature_list_provided=[], - opt=None, - add_user_id=False, - add_timestamp=False, - add_user_age=False, -): - task_name = opt.task_name if getattr(opt, "task_name", None) is not None else "OONC" - if task_name not in LABELS_MTL: - raise ValueError("Invalid Task Name !") - - return _get_new_feature_config_base( - data_spec_path=data_spec_path, - labels=LABELS_MTL[task_name], - add_sparse_continous=True, - add_user_id=add_user_id, - add_timestamp=add_timestamp, - add_user_age=add_user_age, - feature_list_provided=feature_list_provided, - opt=opt, - ) - - -def get_feature_config_light_ranking( - data_spec_path, - feature_list_provided=[], - opt=None, - add_user_id=True, - add_timestamp=False, - add_user_age=False, - add_gbdt=False, - run_light_ranking_group_metrics_in_bq=False, -): - task_name = opt.task_name if getattr(opt, "task_name", None) is not None else "HeavyRankPosition" - if task_name not in LABELS_LR: - raise ValueError("Invalid Task Name !") - if not feature_list_provided: - feature_list_provided = FEATURE_LIST_LIGHT_RANKING_DEFAULT - - return _get_new_feature_config_base( - data_spec_path=data_spec_path, - labels=LABELS_LR[task_name], - add_sparse_continous=False, - add_gbdt=add_gbdt, - add_user_id=add_user_id, - add_timestamp=add_timestamp, - add_user_age=add_user_age, - feature_list_provided=feature_list_provided, - opt=opt, - run_light_ranking_group_metrics_in_bq=run_light_ranking_group_metrics_in_bq, - ) diff --git a/pushservice/src/main/python/models/libs/graph_utils.py b/pushservice/src/main/python/models/libs/graph_utils.py deleted file mode 100644 index 4a4626a59..000000000 --- a/pushservice/src/main/python/models/libs/graph_utils.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -Utilties that aid in building the magic recs graph. -""" - -import re - -import tensorflow.compat.v1 as tf - - -def get_trainable_variables(all_trainable_variables, trainable_regexes): - """Returns a subset of trainable variables for training. - - Given a collection of trainable variables, this will return all those that match the given regexes. - Will also log those variables. - - Args: - all_trainable_variables (a collection of trainable tf.Variable): The variables to search through. - trainable_regexes (a collection of regexes): Variables that match any regex will be included. - - Returns a list of tf.Variable - """ - if trainable_regexes is None or len(trainable_regexes) == 0: - tf.logging.info("No trainable regexes found. Not using get_trainable_variables behavior.") - return None - - assert any( - tf.is_tensor(var) for var in all_trainable_variables - ), f"Non TF variable found: {all_trainable_variables}" - trainable_variables = list( - filter( - lambda var: any(re.match(regex, var.name, re.IGNORECASE) for regex in trainable_regexes), - all_trainable_variables, - ) - ) - tf.logging.info(f"Using filtered trainable variables: {trainable_variables}") - - assert ( - trainable_variables - ), "Did not find trainable variables after filtering after filtering from {} number of vars originaly. All vars: {} and train regexes: {}".format( - len(all_trainable_variables), all_trainable_variables, trainable_regexes - ) - return trainable_variables diff --git a/pushservice/src/main/python/models/libs/group_metrics.py b/pushservice/src/main/python/models/libs/group_metrics.py deleted file mode 100644 index eeef3c501..000000000 --- a/pushservice/src/main/python/models/libs/group_metrics.py +++ /dev/null @@ -1,114 +0,0 @@ -import os -import time - -from twitter.cortex.ml.embeddings.deepbird.grouped_metrics.computation import ( - write_grouped_metrics_to_mldash, -) -from twitter.cortex.ml.embeddings.deepbird.grouped_metrics.configuration import ( - ClassificationGroupedMetricsConfiguration, - NDCGGroupedMetricsConfiguration, -) -import twml - -from .light_ranking_metrics import ( - CGRGroupedMetricsConfiguration, - ExpectedLossGroupedMetricsConfiguration, - RecallGroupedMetricsConfiguration, -) - -import numpy as np -import tensorflow.compat.v1 as tf -from tensorflow.compat.v1 import logging - - -# checkstyle: noqa - - -def run_group_metrics(trainer, data_dir, model_path, parse_fn, group_feature_name="meta.user_id"): - - start_time = time.time() - logging.info("Evaluating with group metrics.") - - metrics = write_grouped_metrics_to_mldash( - trainer=trainer, - data_dir=data_dir, - model_path=model_path, - group_fn=lambda datarecord: str( - datarecord.discreteFeatures[twml.feature_id(group_feature_name)[0]] - ), - parse_fn=parse_fn, - metric_configurations=[ - ClassificationGroupedMetricsConfiguration(), - NDCGGroupedMetricsConfiguration(k=[5, 10, 20]), - ], - total_records_to_read=1000000000, - shuffle=False, - mldash_metrics_name="grouped_metrics", - ) - - end_time = time.time() - logging.info(f"Evaluated Group Metics: {metrics}.") - logging.info(f"Group metrics evaluation time {end_time - start_time}.") - - -def run_group_metrics_light_ranking( - trainer, data_dir, model_path, parse_fn, group_feature_name="meta.trace_id" -): - - start_time = time.time() - logging.info("Evaluating with group metrics.") - - metrics = write_grouped_metrics_to_mldash( - trainer=trainer, - data_dir=data_dir, - model_path=model_path, - group_fn=lambda datarecord: str( - datarecord.discreteFeatures[twml.feature_id(group_feature_name)[0]] - ), - parse_fn=parse_fn, - metric_configurations=[ - CGRGroupedMetricsConfiguration(lightNs=[50, 100, 200], heavyKs=[1, 3, 10, 20, 50]), - RecallGroupedMetricsConfiguration(n=[50, 100, 200], k=[1, 3, 10, 20, 50]), - ExpectedLossGroupedMetricsConfiguration(lightNs=[50, 100, 200]), - ], - total_records_to_read=10000000, - num_batches_to_load=50, - batch_size=1024, - shuffle=False, - mldash_metrics_name="grouped_metrics_for_light_ranking", - ) - - end_time = time.time() - logging.info(f"Evaluated Group Metics for Light Ranking: {metrics}.") - logging.info(f"Group metrics evaluation time {end_time - start_time}.") - - -def run_group_metrics_light_ranking_in_bq(trainer, params, checkpoint_path): - logging.info("getting Test Predictions for Light Ranking Group Metrics in BigQuery !!!") - eval_input_fn = trainer.get_eval_input_fn(repeat=False, shuffle=False) - info_pool = [] - - for result in trainer.estimator.predict( - eval_input_fn, checkpoint_path=checkpoint_path, yield_single_examples=False - ): - traceID = result["trace_id"] - pred = result["prediction"] - label = result["target"] - info = np.concatenate([traceID, pred, label], axis=1) - info_pool.append(info) - - info_pool = np.concatenate(info_pool) - - locname = "/tmp/000/" - if not os.path.exists(locname): - os.makedirs(locname) - - locfile = locname + params.pred_file_name - columns = ["trace_id", "model_prediction", "meta__ranking__weighted_oonc_model_score"] - np.savetxt(locfile, info_pool, delimiter=",", header=",".join(columns)) - tf.io.gfile.copy(locfile, params.pred_file_path + params.pred_file_name, overwrite=True) - - if os.path.isfile(locfile): - os.remove(locfile) - - logging.info("Done Prediction for Light Ranking Group Metrics in BigQuery.") diff --git a/pushservice/src/main/python/models/libs/initializer.py b/pushservice/src/main/python/models/libs/initializer.py deleted file mode 100644 index 8bba00216..000000000 --- a/pushservice/src/main/python/models/libs/initializer.py +++ /dev/null @@ -1,118 +0,0 @@ -import numpy as np -from tensorflow.keras import backend as K - - -class VarianceScaling(object): - """Initializer capable of adapting its scale to the shape of weights. - With `distribution="normal"`, samples are drawn from a truncated normal - distribution centered on zero, with `stddev = sqrt(scale / n)` where n is: - - number of input units in the weight tensor, if mode = "fan_in" - - number of output units, if mode = "fan_out" - - average of the numbers of input and output units, if mode = "fan_avg" - With `distribution="uniform"`, - samples are drawn from a uniform distribution - within [-limit, limit], with `limit = sqrt(3 * scale / n)`. - # Arguments - scale: Scaling factor (positive float). - mode: One of "fan_in", "fan_out", "fan_avg". - distribution: Random distribution to use. One of "normal", "uniform". - seed: A Python integer. Used to seed the random generator. - # Raises - ValueError: In case of an invalid value for the "scale", mode" or - "distribution" arguments.""" - - def __init__( - self, - scale=1.0, - mode="fan_in", - distribution="normal", - seed=None, - fan_in=None, - fan_out=None, - ): - self.fan_in = fan_in - self.fan_out = fan_out - if scale <= 0.0: - raise ValueError("`scale` must be a positive float. Got:", scale) - mode = mode.lower() - if mode not in {"fan_in", "fan_out", "fan_avg"}: - raise ValueError( - "Invalid `mode` argument: " 'expected on of {"fan_in", "fan_out", "fan_avg"} ' "but got", - mode, - ) - distribution = distribution.lower() - if distribution not in {"normal", "uniform"}: - raise ValueError( - "Invalid `distribution` argument: " 'expected one of {"normal", "uniform"} ' "but got", - distribution, - ) - self.scale = scale - self.mode = mode - self.distribution = distribution - self.seed = seed - - def __call__(self, shape, dtype=None, partition_info=None): - fan_in = shape[-2] if self.fan_in is None else self.fan_in - fan_out = shape[-1] if self.fan_out is None else self.fan_out - - scale = self.scale - if self.mode == "fan_in": - scale /= max(1.0, fan_in) - elif self.mode == "fan_out": - scale /= max(1.0, fan_out) - else: - scale /= max(1.0, float(fan_in + fan_out) / 2) - if self.distribution == "normal": - stddev = np.sqrt(scale) / 0.87962566103423978 - return K.truncated_normal(shape, 0.0, stddev, dtype=dtype, seed=self.seed) - else: - limit = np.sqrt(3.0 * scale) - return K.random_uniform(shape, -limit, limit, dtype=dtype, seed=self.seed) - - def get_config(self): - return { - "scale": self.scale, - "mode": self.mode, - "distribution": self.distribution, - "seed": self.seed, - } - - -def customized_glorot_uniform(seed=None, fan_in=None, fan_out=None): - """Glorot uniform initializer, also called Xavier uniform initializer. - It draws samples from a uniform distribution within [-limit, limit] - where `limit` is `sqrt(6 / (fan_in + fan_out))` - where `fan_in` is the number of input units in the weight tensor - and `fan_out` is the number of output units in the weight tensor. - # Arguments - seed: A Python integer. Used to seed the random generator. - # Returns - An initializer.""" - return VarianceScaling( - scale=1.0, - mode="fan_avg", - distribution="uniform", - seed=seed, - fan_in=fan_in, - fan_out=fan_out, - ) - - -def customized_glorot_norm(seed=None, fan_in=None, fan_out=None): - """Glorot norm initializer, also called Xavier uniform initializer. - It draws samples from a uniform distribution within [-limit, limit] - where `limit` is `sqrt(6 / (fan_in + fan_out))` - where `fan_in` is the number of input units in the weight tensor - and `fan_out` is the number of output units in the weight tensor. - # Arguments - seed: A Python integer. Used to seed the random generator. - # Returns - An initializer.""" - return VarianceScaling( - scale=1.0, - mode="fan_avg", - distribution="normal", - seed=seed, - fan_in=fan_in, - fan_out=fan_out, - ) diff --git a/pushservice/src/main/python/models/libs/light_ranking_metrics.py b/pushservice/src/main/python/models/libs/light_ranking_metrics.py deleted file mode 100644 index b83fcf3ae..000000000 --- a/pushservice/src/main/python/models/libs/light_ranking_metrics.py +++ /dev/null @@ -1,255 +0,0 @@ -from functools import partial - -from twitter.cortex.ml.embeddings.deepbird.grouped_metrics.configuration import ( - GroupedMetricsConfiguration, -) -from twitter.cortex.ml.embeddings.deepbird.grouped_metrics.helpers import ( - extract_prediction_from_prediction_record, -) - - -# checkstyle: noqa - - -def score_loss_at_n(labels, predictions, lightN): - """ - Compute the absolute ScoreLoss ranking metric - Args: - labels (list) : A list of label values (HeavyRanking Reference) - predictions (list): A list of prediction values (LightRanking Predictions) - lightN (int): size of the list at which of Initial candidates to compute ScoreLoss. (LightRanking) - """ - assert len(labels) == len(predictions) - - if lightN <= 0: - return None - - labels_with_predictions = zip(labels, predictions) - labels_with_sorted_predictions = sorted( - labels_with_predictions, key=lambda x: x[1], reverse=True - )[:lightN] - labels_top1_light = max([label for label, _ in labels_with_sorted_predictions]) - labels_top1_heavy = max(labels) - - return labels_top1_heavy - labels_top1_light - - -def cgr_at_nk(labels, predictions, lightN, heavyK): - """ - Compute Cumulative Gain Ratio (CGR) ranking metric - Args: - labels (list) : A list of label values (HeavyRanking Reference) - predictions (list): A list of prediction values (LightRanking Predictions) - lightN (int): size of the list at which of Initial candidates to compute CGR. (LightRanking) - heavyK (int): size of the list at which of Refined candidates to compute CGR. (HeavyRanking) - """ - assert len(labels) == len(predictions) - - if (not lightN) or (not heavyK): - out = None - elif lightN <= 0 or heavyK <= 0: - out = None - else: - - labels_with_predictions = zip(labels, predictions) - labels_with_sorted_predictions = sorted( - labels_with_predictions, key=lambda x: x[1], reverse=True - )[:lightN] - labels_topN_light = [label for label, _ in labels_with_sorted_predictions] - - if lightN <= heavyK: - cg_light = sum(labels_topN_light) - else: - labels_topK_heavy_from_light = sorted(labels_topN_light, reverse=True)[:heavyK] - cg_light = sum(labels_topK_heavy_from_light) - - ideal_ordering = sorted(labels, reverse=True) - cg_heavy = sum(ideal_ordering[: min(lightN, heavyK)]) - - out = 0.0 - if cg_heavy != 0: - out = max(cg_light / cg_heavy, 0) - - return out - - -def _get_weight(w, atK): - if not w: - return 1.0 - elif len(w) <= atK: - return 0.0 - else: - return w[atK] - - -def recall_at_nk(labels, predictions, n=None, k=None, w=None): - """ - Recall at N-K ranking metric - Args: - labels (list): A list of label values - predictions (list): A list of prediction values - n (int): size of the list at which of predictions to compute recall. (Light Ranking Predictions) - The default is None in which case the length of the provided predictions is used as L - k (int): size of the list at which of labels to compute recall. (Heavy Ranking Predictions) - The default is None in which case the length of the provided labels is used as L - w (list): weight vector sorted by labels - """ - assert len(labels) == len(predictions) - - if not any(labels): - out = None - else: - - safe_n = len(predictions) if not n else min(len(predictions), n) - safe_k = len(labels) if not k else min(len(labels), k) - - labels_with_predictions = zip(labels, predictions) - sorted_labels_with_predictions = sorted( - labels_with_predictions, key=lambda x: x[0], reverse=True - ) - - order_sorted_labels_predictions = zip(range(len(labels)), *zip(*sorted_labels_with_predictions)) - - order_with_predictions = [ - (order, pred) for order, label, pred in order_sorted_labels_predictions - ] - order_with_sorted_predictions = sorted(order_with_predictions, key=lambda x: x[1], reverse=True) - - pred_sorted_order_at_n = [order for order, _ in order_with_sorted_predictions][:safe_n] - - intersection_weight = [ - _get_weight(w, order) if order < safe_k else 0 for order in pred_sorted_order_at_n - ] - - intersection_score = sum(intersection_weight) - full_score = sum(w) if w else float(safe_k) - - out = 0.0 - if full_score != 0: - out = intersection_score / full_score - - return out - - -class ExpectedLossGroupedMetricsConfiguration(GroupedMetricsConfiguration): - """ - This is the Expected Loss Grouped metric computation configuration. - """ - - def __init__(self, lightNs=[]): - """ - Args: - lightNs (list): size of the list at which of Initial candidates to compute Expected Loss. (LightRanking) - """ - self.lightNs = lightNs - - @property - def name(self): - return "ExpectedLoss" - - @property - def metrics_dict(self): - metrics_to_compute = {} - for lightN in self.lightNs: - metric_name = "ExpectedLoss_atLight_" + str(lightN) - metrics_to_compute[metric_name] = partial(score_loss_at_n, lightN=lightN) - return metrics_to_compute - - def extract_label(self, prec, drec, drec_label): - return drec_label - - def extract_prediction(self, prec, drec, drec_label): - return extract_prediction_from_prediction_record(prec) - - -class CGRGroupedMetricsConfiguration(GroupedMetricsConfiguration): - """ - This is the Cumulative Gain Ratio (CGR) Grouped metric computation configuration. - CGR at the max length of each session is the default. - CGR at additional positions can be computed by specifying a list of 'n's and 'k's - """ - - def __init__(self, lightNs=[], heavyKs=[]): - """ - Args: - lightNs (list): size of the list at which of Initial candidates to compute CGR. (LightRanking) - heavyK (int): size of the list at which of Refined candidates to compute CGR. (HeavyRanking) - """ - self.lightNs = lightNs - self.heavyKs = heavyKs - - @property - def name(self): - return "cgr" - - @property - def metrics_dict(self): - metrics_to_compute = {} - for lightN in self.lightNs: - for heavyK in self.heavyKs: - metric_name = "cgr_atLight_" + str(lightN) + "_atHeavy_" + str(heavyK) - metrics_to_compute[metric_name] = partial(cgr_at_nk, lightN=lightN, heavyK=heavyK) - return metrics_to_compute - - def extract_label(self, prec, drec, drec_label): - return drec_label - - def extract_prediction(self, prec, drec, drec_label): - return extract_prediction_from_prediction_record(prec) - - -class RecallGroupedMetricsConfiguration(GroupedMetricsConfiguration): - """ - This is the Recall Grouped metric computation configuration. - Recall at the max length of each session is the default. - Recall at additional positions can be computed by specifying a list of 'n's and 'k's - """ - - def __init__(self, n=[], k=[], w=[]): - """ - Args: - n (list): A list of ints. List of prediction rank thresholds (for light) - k (list): A list of ints. List of label rank thresholds (for heavy) - """ - self.predN = n - self.labelK = k - self.weight = w - - @property - def name(self): - return "group_recall" - - @property - def metrics_dict(self): - metrics_to_compute = {"group_recall_unweighted": recall_at_nk} - if not self.weight: - metrics_to_compute["group_recall_weighted"] = partial(recall_at_nk, w=self.weight) - - if self.predN and self.labelK: - for n in self.predN: - for k in self.labelK: - if n >= k: - metrics_to_compute[ - "group_recall_unweighted_at_L" + str(n) + "_at_H" + str(k) - ] = partial(recall_at_nk, n=n, k=k) - if self.weight: - metrics_to_compute[ - "group_recall_weighted_at_L" + str(n) + "_at_H" + str(k) - ] = partial(recall_at_nk, n=n, k=k, w=self.weight) - - if self.labelK and not self.predN: - for k in self.labelK: - metrics_to_compute["group_recall_unweighted_at_full_at_H" + str(k)] = partial( - recall_at_nk, k=k - ) - if self.weight: - metrics_to_compute["group_recall_weighted_at_full_at_H" + str(k)] = partial( - recall_at_nk, k=k, w=self.weight - ) - return metrics_to_compute - - def extract_label(self, prec, drec, drec_label): - return drec_label - - def extract_prediction(self, prec, drec, drec_label): - return extract_prediction_from_prediction_record(prec) diff --git a/pushservice/src/main/python/models/libs/metric_fn_utils.py b/pushservice/src/main/python/models/libs/metric_fn_utils.py deleted file mode 100644 index fc26a1305..000000000 --- a/pushservice/src/main/python/models/libs/metric_fn_utils.py +++ /dev/null @@ -1,294 +0,0 @@ -""" -Utilties for constructing a metric_fn for magic recs. -""" - -from twml.contrib.metrics.metrics import ( - get_dual_binary_tasks_metric_fn, - get_numeric_metric_fn, - get_partial_multi_binary_class_metric_fn, - get_single_binary_task_metric_fn, -) - -from .model_utils import generate_disliked_mask - -import tensorflow.compat.v1 as tf - - -METRIC_BOOK = { - "OONC": ["OONC"], - "OONC_Engagement": ["OONC", "Engagement"], - "Sent": ["Sent"], - "HeavyRankPosition": ["HeavyRankPosition"], - "HeavyRankProbability": ["HeavyRankProbability"], -} - -USER_AGE_FEATURE_NAME = "accountAge" -NEW_USER_AGE_CUTOFF = 0 - - -def remove_padding_and_flatten(tensor, valid_batch_size): - """Remove the padding of the input padded tensor given the valid batch size tensor, - then flatten the output with respect to the first dimension. - Args: - tensor: A tensor of size [META_BATCH_SIZE, BATCH_SIZE, FEATURE_DIM]. - valid_batch_size: A tensor of size [META_BATCH_SIZE], with each element indicating - the effective batch size of the BATCH_SIZE dimension. - - Returns: - A tesnor of size [tf.reduce_sum(valid_batch_size), FEATURE_DIM]. - """ - unpadded_ragged_tensor = tf.RaggedTensor.from_tensor(tensor=tensor, lengths=valid_batch_size) - - return unpadded_ragged_tensor.flat_values - - -def safe_mask(values, mask): - """Mask values if possible. - - Boolean mask inputed values if and only if values is a tensor of the same dimension as mask (or can be broadcasted to that dimension). - - Args: - values (Any or Tensor): Input tensor to mask. Dim 0 should be size N. - mask (boolean tensor): A boolean tensor of size N. - - Returns Values or Values masked. - """ - if values is None: - return values - if not tf.is_tensor(values): - return values - values_shape = values.get_shape() - if not values_shape or len(values_shape) == 0: - return values - if not mask.get_shape().is_compatible_with(values_shape[0]): - return values - return tf.boolean_mask(values, mask) - - -def add_new_user_metrics(metric_fn): - """Will stratify the metric_fn by adding new user metrics. - - Given an input metric_fn, double every metric: One will be the orignal and the other will only include those for new users. - - Args: - metric_fn (python function): Base twml metric_fn. - - Returns a metric_fn with new user metrics included. - """ - - def metric_fn_with_new_users(graph_output, labels, weights): - if USER_AGE_FEATURE_NAME not in graph_output: - raise ValueError( - "In order to get metrics stratified by user age, {name} feature should be added to model graph output. However, only the following output keys were found: {keys}.".format( - name=USER_AGE_FEATURE_NAME, keys=graph_output.keys() - ) - ) - - metric_ops = metric_fn(graph_output, labels, weights) - - is_new = tf.reshape( - tf.math.less_equal( - tf.cast(graph_output[USER_AGE_FEATURE_NAME], tf.int64), - tf.cast(NEW_USER_AGE_CUTOFF, tf.int64), - ), - [-1], - ) - - labels = safe_mask(labels, is_new) - weights = safe_mask(weights, is_new) - graph_output = {key: safe_mask(values, is_new) for key, values in graph_output.items()} - - new_user_metric_ops = metric_fn(graph_output, labels, weights) - new_user_metric_ops = {name + "_new_users": ops for name, ops in new_user_metric_ops.items()} - metric_ops.update(new_user_metric_ops) - return metric_ops - - return metric_fn_with_new_users - - -def get_meta_learn_single_binary_task_metric_fn( - metrics, classnames, top_k=(5, 5, 5), use_top_k=False -): - """Wrapper function to use the metric_fn with meta learning evaluation scheme. - - Args: - metrics: A list of string representing metric names. - classnames: A list of string repsenting class names, In case of multiple binary class models, - the names for each class or label. - top_k: A tuple of int to specify top K metrics. - use_top_k: A boolean value indicating of top K of metrics is used. - - Returns: - A customized metric_fn function. - """ - - def get_eval_metric_ops(graph_output, labels, weights): - """The op func of the eval_metrics. Comparing with normal version, - the difference is we flatten the output, label, and weights. - - Args: - graph_output: A dict of tensors. - labels: A tensor of int32 be the value of either 0 or 1. - weights: A tensor of float32 to indicate the per record weight. - - Returns: - A dict of metric names and values. - """ - metric_op_weighted = get_partial_multi_binary_class_metric_fn( - metrics, predcols=0, classes=classnames - ) - classnames_unweighted = ["unweighted_" + classname for classname in classnames] - metric_op_unweighted = get_partial_multi_binary_class_metric_fn( - metrics, predcols=0, classes=classnames_unweighted - ) - - valid_batch_size = graph_output["valid_batch_size"] - graph_output["output"] = remove_padding_and_flatten(graph_output["output"], valid_batch_size) - labels = remove_padding_and_flatten(labels, valid_batch_size) - weights = remove_padding_and_flatten(weights, valid_batch_size) - - tf.ensure_shape(graph_output["output"], [None, 1]) - tf.ensure_shape(labels, [None, 1]) - tf.ensure_shape(weights, [None, 1]) - - metrics_weighted = metric_op_weighted(graph_output, labels, weights) - metrics_unweighted = metric_op_unweighted(graph_output, labels, None) - metrics_weighted.update(metrics_unweighted) - - if use_top_k: - metric_op_numeric = get_numeric_metric_fn(metrics=None, topK=top_k, predcol=0, labelcol=1) - metrics_numeric = metric_op_numeric(graph_output, labels, weights) - metrics_weighted.update(metrics_numeric) - return metrics_weighted - - return get_eval_metric_ops - - -def get_meta_learn_dual_binary_tasks_metric_fn( - metrics, classnames, top_k=(5, 5, 5), use_top_k=False -): - """Wrapper function to use the metric_fn with meta learning evaluation scheme. - - Args: - metrics: A list of string representing metric names. - classnames: A list of string repsenting class names, In case of multiple binary class models, - the names for each class or label. - top_k: A tuple of int to specify top K metrics. - use_top_k: A boolean value indicating of top K of metrics is used. - - Returns: - A customized metric_fn function. - """ - - def get_eval_metric_ops(graph_output, labels, weights): - """The op func of the eval_metrics. Comparing with normal version, - the difference is we flatten the output, label, and weights. - - Args: - graph_output: A dict of tensors. - labels: A tensor of int32 be the value of either 0 or 1. - weights: A tensor of float32 to indicate the per record weight. - - Returns: - A dict of metric names and values. - """ - metric_op_weighted = get_partial_multi_binary_class_metric_fn( - metrics, predcols=[0, 1], classes=classnames - ) - classnames_unweighted = ["unweighted_" + classname for classname in classnames] - metric_op_unweighted = get_partial_multi_binary_class_metric_fn( - metrics, predcols=[0, 1], classes=classnames_unweighted - ) - - valid_batch_size = graph_output["valid_batch_size"] - graph_output["output"] = remove_padding_and_flatten(graph_output["output"], valid_batch_size) - labels = remove_padding_and_flatten(labels, valid_batch_size) - weights = remove_padding_and_flatten(weights, valid_batch_size) - - tf.ensure_shape(graph_output["output"], [None, 2]) - tf.ensure_shape(labels, [None, 2]) - tf.ensure_shape(weights, [None, 1]) - - metrics_weighted = metric_op_weighted(graph_output, labels, weights) - metrics_unweighted = metric_op_unweighted(graph_output, labels, None) - metrics_weighted.update(metrics_unweighted) - - if use_top_k: - metric_op_numeric = get_numeric_metric_fn(metrics=None, topK=top_k, predcol=2, labelcol=2) - metrics_numeric = metric_op_numeric(graph_output, labels, weights) - metrics_weighted.update(metrics_numeric) - return metrics_weighted - - return get_eval_metric_ops - - -def get_metric_fn(task_name, use_stratify_metrics, use_meta_batch=False): - """Will retrieve the metric_fn for magic recs. - - Args: - task_name (string): Which task is being used for this model. - use_stratify_metrics (boolean): Should we add stratified metrics (new user metrics). - use_meta_batch (boolean): If the output/label/weights are passed in 3D shape instead of - 2D shape. - - Returns: - A metric_fn function to pass in twml Trainer. - """ - if task_name not in METRIC_BOOK: - raise ValueError( - "Task name of {task_name} not recognized. Unable to retrieve metrics.".format( - task_name=task_name - ) - ) - class_names = METRIC_BOOK[task_name] - if use_meta_batch: - get_n_binary_task_metric_fn = ( - get_meta_learn_single_binary_task_metric_fn - if len(class_names) == 1 - else get_meta_learn_dual_binary_tasks_metric_fn - ) - else: - get_n_binary_task_metric_fn = ( - get_single_binary_task_metric_fn if len(class_names) == 1 else get_dual_binary_tasks_metric_fn - ) - - metric_fn = get_n_binary_task_metric_fn(metrics=None, classnames=METRIC_BOOK[task_name]) - - if use_stratify_metrics: - metric_fn = add_new_user_metrics(metric_fn) - - return metric_fn - - -def flip_disliked_labels(metric_fn): - """This function returns an adapted metric_fn which flips the labels of the OONCed evaluation data to 0 if it is disliked. - Args: - metric_fn: A metric_fn function to pass in twml Trainer. - - Returns: - _adapted_metric_fn: A customized metric_fn function with disliked OONC labels flipped. - """ - - def _adapted_metric_fn(graph_output, labels, weights): - """A customized metric_fn function with disliked OONC labels flipped. - - Args: - graph_output: A dict of tensors. - labels: labels of training samples, which is a 2D tensor of shape batch_size x 3: [OONCs, engagements, dislikes] - weights: A tensor of float32 to indicate the per record weight. - - Returns: - A dict of metric names and values. - """ - # We want to multiply the label of the observation by 0 only when it is disliked - disliked_mask = generate_disliked_mask(labels) - - # Extract OONC and engagement labels only. - labels = tf.reshape(labels[:, 0:2], shape=[-1, 2]) - - # Labels will be set to 0 if it is disliked. - adapted_labels = labels * tf.cast(tf.logical_not(disliked_mask), dtype=labels.dtype) - - return metric_fn(graph_output, adapted_labels, weights) - - return _adapted_metric_fn diff --git a/pushservice/src/main/python/models/libs/model_args.py b/pushservice/src/main/python/models/libs/model_args.py deleted file mode 100644 index ae142d818..000000000 --- a/pushservice/src/main/python/models/libs/model_args.py +++ /dev/null @@ -1,231 +0,0 @@ -from twml.trainers import DataRecordTrainer - - -# checkstyle: noqa - - -def get_arg_parser(): - parser = DataRecordTrainer.add_parser_arguments() - - parser.add_argument( - "--input_size_bits", - type=int, - default=18, - help="number of bits allocated to the input size", - ) - parser.add_argument( - "--model_trainer_name", - default="magic_recs_mlp_calibration_MTL_OONC_Engagement", - type=str, - help="specify the model trainer name.", - ) - - parser.add_argument( - "--model_type", - default="deepnorm_gbdt_inputdrop2_rescale", - type=str, - help="specify the model type to use.", - ) - parser.add_argument( - "--feat_config_type", - default="get_feature_config_with_sparse_continuous", - type=str, - help="specify the feature configure function to use.", - ) - - parser.add_argument( - "--directly_export_best", - default=False, - action="store_true", - help="whether to directly_export best_checkpoint", - ) - - parser.add_argument( - "--warm_start_base_dir", - default="none", - type=str, - help="latest ckpt in this folder will be used to ", - ) - - parser.add_argument( - "--feature_list", - default="none", - type=str, - help="Which features to use for training", - ) - parser.add_argument( - "--warm_start_from", default=None, type=str, help="model dir to warm start from" - ) - - parser.add_argument( - "--momentum", default=0.99999, type=float, help="Momentum term for batch normalization" - ) - parser.add_argument( - "--dropout", - default=0.2, - type=float, - help="input_dropout_rate to rescale output by (1 - input_dropout_rate)", - ) - parser.add_argument( - "--out_layer_1_size", default=256, type=int, help="Size of MLP_branch layer 1" - ) - parser.add_argument( - "--out_layer_2_size", default=128, type=int, help="Size of MLP_branch layer 2" - ) - parser.add_argument("--out_layer_3_size", default=64, type=int, help="Size of MLP_branch layer 3") - parser.add_argument( - "--sparse_embedding_size", default=50, type=int, help="Dimensionality of sparse embedding layer" - ) - parser.add_argument( - "--dense_embedding_size", default=128, type=int, help="Dimensionality of dense embedding layer" - ) - - parser.add_argument( - "--use_uam_label", - default=False, - type=str, - help="Whether to use uam_label or not", - ) - - parser.add_argument( - "--task_name", - default="OONC_Engagement", - type=str, - help="specify the task name to use: OONC or OONC_Engagement.", - ) - parser.add_argument( - "--init_weight", - default=0.9, - type=float, - help="Initial OONC Task Weight MTL: OONC+Engagement.", - ) - parser.add_argument( - "--use_engagement_weight", - default=False, - action="store_true", - help="whether to use engagement weight for base model.", - ) - parser.add_argument( - "--mtl_num_extra_layers", - type=int, - default=1, - help="Number of Hidden Layers for each TaskBranch.", - ) - parser.add_argument( - "--mtl_neuron_scale", type=int, default=4, help="Scaling Factor of Neurons in MTL Extra Layers." - ) - parser.add_argument( - "--use_oonc_score", - default=False, - action="store_true", - help="whether to use oonc score only or combined score.", - ) - parser.add_argument( - "--use_stratified_metrics", - default=False, - action="store_true", - help="Use stratified metrics: Break out new-user metrics.", - ) - parser.add_argument( - "--run_group_metrics", - default=False, - action="store_true", - help="Will run evaluation metrics grouped by user.", - ) - parser.add_argument( - "--use_full_scope", - default=False, - action="store_true", - help="Will add extra scope and naming to graph.", - ) - parser.add_argument( - "--trainable_regexes", - default=None, - nargs="*", - help="The union of variables specified by the list of regexes will be considered trainable.", - ) - parser.add_argument( - "--fine_tuning.ckpt_to_initialize_from", - dest="fine_tuning_ckpt_to_initialize_from", - type=str, - default=None, - help="Checkpoint path from which to warm start. Indicates the pre-trained model.", - ) - parser.add_argument( - "--fine_tuning.warm_start_scope_regex", - dest="fine_tuning_warm_start_scope_regex", - type=str, - default=None, - help="All variables matching this will be restored.", - ) - - return parser - - -def get_params(args=None): - parser = get_arg_parser() - if args is None: - return parser.parse_args() - else: - return parser.parse_args(args) - - -def get_arg_parser_light_ranking(): - parser = get_arg_parser() - - parser.add_argument( - "--use_record_weight", - default=False, - action="store_true", - help="whether to use record weight for base model.", - ) - parser.add_argument( - "--min_record_weight", default=0.0, type=float, help="Minimum record weight to use." - ) - parser.add_argument( - "--smooth_weight", default=0.0, type=float, help="Factor to smooth Rank Position Weight." - ) - - parser.add_argument( - "--num_mlp_layers", type=int, default=3, help="Number of Hidden Layers for MLP model." - ) - parser.add_argument( - "--mlp_neuron_scale", type=int, default=4, help="Scaling Factor of Neurons in MLP Layers." - ) - parser.add_argument( - "--run_light_ranking_group_metrics", - default=False, - action="store_true", - help="Will run evaluation metrics grouped by user for Light Ranking.", - ) - parser.add_argument( - "--use_missing_sub_branch", - default=False, - action="store_true", - help="Whether to use missing value sub-branch for Light Ranking.", - ) - parser.add_argument( - "--use_gbdt_features", - default=False, - action="store_true", - help="Whether to use GBDT features for Light Ranking.", - ) - parser.add_argument( - "--run_light_ranking_group_metrics_in_bq", - default=False, - action="store_true", - help="Whether to get_predictions for Light Ranking to compute group metrics in BigQuery.", - ) - parser.add_argument( - "--pred_file_path", - default=None, - type=str, - help="path", - ) - parser.add_argument( - "--pred_file_name", - default=None, - type=str, - help="path", - ) - return parser diff --git a/pushservice/src/main/python/models/libs/model_utils.py b/pushservice/src/main/python/models/libs/model_utils.py deleted file mode 100644 index 1c5306911..000000000 --- a/pushservice/src/main/python/models/libs/model_utils.py +++ /dev/null @@ -1,339 +0,0 @@ -import sys - -import twml - -from .initializer import customized_glorot_uniform - -import tensorflow.compat.v1 as tf -import yaml - - -# checkstyle: noqa - - -def read_config(whitelist_yaml_file): - with tf.gfile.FastGFile(whitelist_yaml_file) as f: - try: - return yaml.safe_load(f) - except yaml.YAMLError as exc: - print(exc) - sys.exit(1) - - -def _sparse_feature_fixup(features, input_size_bits): - """Rebuild a sparse tensor feature so that its dense shape attribute is present. - - Arguments: - features (SparseTensor): Sparse feature tensor of shape ``(B, sparse_feature_dim)``. - input_size_bits (int): Number of columns in ``log2`` scale. Must be positive. - - Returns: - SparseTensor: Rebuilt and non-faulty version of `features`.""" - sparse_feature_dim = tf.constant(2**input_size_bits, dtype=tf.int64) - sparse_shape = tf.stack([features.dense_shape[0], sparse_feature_dim]) - sparse_tf = tf.SparseTensor(features.indices, features.values, sparse_shape) - return sparse_tf - - -def self_atten_dense(input, out_dim, activation=None, use_bias=True, name=None): - def safe_concat(base, suffix): - """Concats variables name components if base is given.""" - if not base: - return base - return f"{base}:{suffix}" - - input_dim = input.shape.as_list()[1] - - sigmoid_out = twml.layers.FullDense( - input_dim, dtype=tf.float32, activation=tf.nn.sigmoid, name=safe_concat(name, "sigmoid_out") - )(input) - atten_input = sigmoid_out * input - mlp_out = twml.layers.FullDense( - out_dim, - dtype=tf.float32, - activation=activation, - use_bias=use_bias, - name=safe_concat(name, "mlp_out"), - )(atten_input) - return mlp_out - - -def get_dense_out(input, out_dim, activation, dense_type): - if dense_type == "full_dense": - out = twml.layers.FullDense(out_dim, dtype=tf.float32, activation=activation)(input) - elif dense_type == "self_atten_dense": - out = self_atten_dense(input, out_dim, activation=activation) - return out - - -def get_input_trans_func(bn_normalized_dense, is_training): - gw_normalized_dense = tf.expand_dims(bn_normalized_dense, -1) - group_num = bn_normalized_dense.shape.as_list()[1] - - gw_normalized_dense = GroupWiseTrans(group_num, 1, 8, name="groupwise_1", activation=tf.tanh)( - gw_normalized_dense - ) - gw_normalized_dense = GroupWiseTrans(group_num, 8, 4, name="groupwise_2", activation=tf.tanh)( - gw_normalized_dense - ) - gw_normalized_dense = GroupWiseTrans(group_num, 4, 1, name="groupwise_3", activation=tf.tanh)( - gw_normalized_dense - ) - - gw_normalized_dense = tf.squeeze(gw_normalized_dense, [-1]) - - bn_gw_normalized_dense = tf.layers.batch_normalization( - gw_normalized_dense, - training=is_training, - renorm_momentum=0.9999, - momentum=0.9999, - renorm=is_training, - trainable=True, - ) - - return bn_gw_normalized_dense - - -def tensor_dropout( - input_tensor, - rate, - is_training, - sparse_tensor=None, -): - """ - Implements dropout layer for both dense and sparse input_tensor - - Arguments: - input_tensor: - B x D dense tensor, or a sparse tensor - rate (float32): - dropout rate - is_training (bool): - training stage or not. - sparse_tensor (bool): - whether the input_tensor is sparse tensor or not. Default to be None, this value has to be passed explicitly. - rescale_sparse_dropout (bool): - Do we need to do rescaling or not. - Returns: - tensor dropped out""" - if sparse_tensor == True: - if is_training: - with tf.variable_scope("sparse_dropout"): - values = input_tensor.values - keep_mask = tf.keras.backend.random_binomial( - tf.shape(values), p=1 - rate, dtype=tf.float32, seed=None - ) - keep_mask.set_shape([None]) - keep_mask = tf.cast(keep_mask, tf.bool) - - keep_indices = tf.boolean_mask(input_tensor.indices, keep_mask, axis=0) - keep_values = tf.boolean_mask(values, keep_mask, axis=0) - - dropped_tensor = tf.SparseTensor(keep_indices, keep_values, input_tensor.dense_shape) - return dropped_tensor - else: - return input_tensor - elif sparse_tensor == False: - return tf.layers.dropout(input_tensor, rate=rate, training=is_training) - - -def adaptive_transformation(bn_normalized_dense, is_training, func_type="default"): - assert func_type in [ - "default", - "tiny", - ], f"fun_type can only be one of default and tiny, but get {func_type}" - - gw_normalized_dense = tf.expand_dims(bn_normalized_dense, -1) - group_num = bn_normalized_dense.shape.as_list()[1] - - if func_type == "default": - gw_normalized_dense = FastGroupWiseTrans( - group_num, 1, 8, name="groupwise_1", activation=tf.tanh, init_multiplier=8 - )(gw_normalized_dense) - - gw_normalized_dense = FastGroupWiseTrans( - group_num, 8, 4, name="groupwise_2", activation=tf.tanh, init_multiplier=8 - )(gw_normalized_dense) - - gw_normalized_dense = FastGroupWiseTrans( - group_num, 4, 1, name="groupwise_3", activation=tf.tanh, init_multiplier=8 - )(gw_normalized_dense) - elif func_type == "tiny": - gw_normalized_dense = FastGroupWiseTrans( - group_num, 1, 2, name="groupwise_1", activation=tf.tanh, init_multiplier=8 - )(gw_normalized_dense) - - gw_normalized_dense = FastGroupWiseTrans( - group_num, 2, 1, name="groupwise_2", activation=tf.tanh, init_multiplier=8 - )(gw_normalized_dense) - - gw_normalized_dense = FastGroupWiseTrans( - group_num, 1, 1, name="groupwise_3", activation=tf.tanh, init_multiplier=8 - )(gw_normalized_dense) - - gw_normalized_dense = tf.squeeze(gw_normalized_dense, [-1]) - bn_gw_normalized_dense = tf.layers.batch_normalization( - gw_normalized_dense, - training=is_training, - renorm_momentum=0.9999, - momentum=0.9999, - renorm=is_training, - trainable=True, - ) - - return bn_gw_normalized_dense - - -class FastGroupWiseTrans(object): - """ - used to apply group-wise fully connected layers to the input. - it applies a tiny, unique MLP to each individual feature.""" - - def __init__(self, group_num, input_dim, out_dim, name, activation=None, init_multiplier=1): - self.group_num = group_num - self.input_dim = input_dim - self.out_dim = out_dim - self.activation = activation - self.init_multiplier = init_multiplier - - self.w = tf.get_variable( - name + "_group_weight", - [1, group_num, input_dim, out_dim], - initializer=customized_glorot_uniform( - fan_in=input_dim * init_multiplier, fan_out=out_dim * init_multiplier - ), - trainable=True, - ) - self.b = tf.get_variable( - name + "_group_bias", - [1, group_num, out_dim], - initializer=tf.constant_initializer(0.0), - trainable=True, - ) - - def __call__(self, input_tensor): - """ - input_tensor: batch_size x group_num x input_dim - output_tensor: batch_size x group_num x out_dim""" - input_tensor_expand = tf.expand_dims(input_tensor, axis=-1) - - output_tensor = tf.add( - tf.reduce_sum(tf.multiply(input_tensor_expand, self.w), axis=-2, keepdims=False), - self.b, - ) - - if self.activation is not None: - output_tensor = self.activation(output_tensor) - return output_tensor - - -class GroupWiseTrans(object): - """ - Used to apply group fully connected layers to the input. - """ - - def __init__(self, group_num, input_dim, out_dim, name, activation=None): - self.group_num = group_num - self.input_dim = input_dim - self.out_dim = out_dim - self.activation = activation - - w_list, b_list = [], [] - for idx in range(out_dim): - this_w = tf.get_variable( - name + f"_group_weight_{idx}", - [1, group_num, input_dim], - initializer=tf.keras.initializers.glorot_uniform(), - trainable=True, - ) - this_b = tf.get_variable( - name + f"_group_bias_{idx}", - [1, group_num, 1], - initializer=tf.constant_initializer(0.0), - trainable=True, - ) - w_list.append(this_w) - b_list.append(this_b) - self.w_list = w_list - self.b_list = b_list - - def __call__(self, input_tensor): - """ - input_tensor: batch_size x group_num x input_dim - output_tensor: batch_size x group_num x out_dim - """ - out_tensor_list = [] - for idx in range(self.out_dim): - this_res = ( - tf.reduce_sum(input_tensor * self.w_list[idx], axis=-1, keepdims=True) + self.b_list[idx] - ) - out_tensor_list.append(this_res) - output_tensor = tf.concat(out_tensor_list, axis=-1) - - if self.activation is not None: - output_tensor = self.activation(output_tensor) - return output_tensor - - -def add_scalar_summary(var, name, name_scope="hist_dense_feature/"): - with tf.name_scope("summaries/"): - with tf.name_scope(name_scope): - tf.summary.scalar(name, var) - - -def add_histogram_summary(var, name, name_scope="hist_dense_feature/"): - with tf.name_scope("summaries/"): - with tf.name_scope(name_scope): - tf.summary.histogram(name, tf.reshape(var, [-1])) - - -def sparse_clip_by_value(sparse_tf, min_val, max_val): - new_vals = tf.clip_by_value(sparse_tf.values, min_val, max_val) - return tf.SparseTensor(sparse_tf.indices, new_vals, sparse_tf.dense_shape) - - -def check_numerics_with_msg(tensor, message="", sparse_tensor=False): - if sparse_tensor: - values = tf.debugging.check_numerics(tensor.values, message=message) - return tf.SparseTensor(tensor.indices, values, tensor.dense_shape) - else: - return tf.debugging.check_numerics(tensor, message=message) - - -def pad_empty_sparse_tensor(tensor): - dummy_tensor = tf.SparseTensor( - indices=[[0, 0]], - values=[0.00001], - dense_shape=tensor.dense_shape, - ) - result = tf.cond( - tf.equal(tf.size(tensor.values), 0), - lambda: dummy_tensor, - lambda: tensor, - ) - return result - - -def filter_nans_and_infs(tensor, sparse_tensor=False): - if sparse_tensor: - sparse_values = tensor.values - filtered_val = tf.where( - tf.logical_or(tf.is_nan(sparse_values), tf.is_inf(sparse_values)), - tf.zeros_like(sparse_values), - sparse_values, - ) - return tf.SparseTensor(tensor.indices, filtered_val, tensor.dense_shape) - else: - return tf.where( - tf.logical_or(tf.is_nan(tensor), tf.is_inf(tensor)), tf.zeros_like(tensor), tensor - ) - - -def generate_disliked_mask(labels): - """Generate a disliked mask where only samples with dislike labels are set to 1 otherwise set to 0. - Args: - labels: labels of training samples, which is a 2D tensor of shape batch_size x 3: [OONCs, engagements, dislikes] - Returns: - 1D tensor of shape batch_size x 1: [dislikes (booleans)] - """ - return tf.equal(tf.reshape(labels[:, 2], shape=[-1, 1]), 1) diff --git a/pushservice/src/main/python/models/libs/warm_start_utils.py b/pushservice/src/main/python/models/libs/warm_start_utils.py deleted file mode 100644 index ca83df585..000000000 --- a/pushservice/src/main/python/models/libs/warm_start_utils.py +++ /dev/null @@ -1,309 +0,0 @@ -from collections import OrderedDict -import json -import os -from os.path import join - -from twitter.magicpony.common import file_access -import twml - -from .model_utils import read_config - -import numpy as np -from scipy import stats -import tensorflow.compat.v1 as tf - - -# checkstyle: noqa - - -def get_model_type_to_tensors_to_change_axis(): - model_type_to_tensors_to_change_axis = { - "magic_recs/model/batch_normalization/beta": ([0], "continuous"), - "magic_recs/model/batch_normalization/gamma": ([0], "continuous"), - "magic_recs/model/batch_normalization/moving_mean": ([0], "continuous"), - "magic_recs/model/batch_normalization/moving_stddev": ([0], "continuous"), - "magic_recs/model/batch_normalization/moving_variance": ([0], "continuous"), - "magic_recs/model/batch_normalization/renorm_mean": ([0], "continuous"), - "magic_recs/model/batch_normalization/renorm_stddev": ([0], "continuous"), - "magic_recs/model/logits/EngagementGivenOONC_logits/clem_net_1/block2_4/channel_wise_dense_4/kernel": ( - [1], - "all", - ), - "magic_recs/model/logits/OONC_logits/clem_net/block2/channel_wise_dense/kernel": ([1], "all"), - } - - return model_type_to_tensors_to_change_axis - - -def mkdirp(dirname): - if not tf.io.gfile.exists(dirname): - tf.io.gfile.makedirs(dirname) - - -def rename_dir(dirname, dst): - file_access.hdfs.mv(dirname, dst) - - -def rmdir(dirname): - if tf.io.gfile.exists(dirname): - if tf.io.gfile.isdir(dirname): - tf.io.gfile.rmtree(dirname) - else: - tf.io.gfile.remove(dirname) - - -def get_var_dict(checkpoint_path): - checkpoint = tf.train.get_checkpoint_state(checkpoint_path) - var_dict = OrderedDict() - with tf.Session() as sess: - all_var_list = tf.train.list_variables(checkpoint_path) - for var_name, _ in all_var_list: - # Load the variable - var = tf.train.load_variable(checkpoint_path, var_name) - var_dict[var_name] = var - return var_dict - - -def get_continunous_mapping_from_feat_list(old_feature_list, new_feature_list): - """ - get var_ind for old_feature and corresponding var_ind for new_feature - """ - new_var_ind, old_var_ind = [], [] - for this_new_id, this_new_name in enumerate(new_feature_list): - if this_new_name in old_feature_list: - this_old_id = old_feature_list.index(this_new_name) - new_var_ind.append(this_new_id) - old_var_ind.append(this_old_id) - return np.asarray(old_var_ind), np.asarray(new_var_ind) - - -def get_continuous_mapping_from_feat_dict(old_feature_dict, new_feature_dict): - """ - get var_ind for old_feature and corresponding var_ind for new_feature - """ - old_cont = old_feature_dict["continuous"] - old_bin = old_feature_dict["binary"] - - new_cont = new_feature_dict["continuous"] - new_bin = new_feature_dict["binary"] - - _dummy_sparse_feat = [f"sparse_feature_{_idx}" for _idx in range(100)] - - cont_old_var_ind, cont_new_var_ind = get_continunous_mapping_from_feat_list(old_cont, new_cont) - - all_old_var_ind, all_new_var_ind = get_continunous_mapping_from_feat_list( - old_cont + old_bin + _dummy_sparse_feat, new_cont + new_bin + _dummy_sparse_feat - ) - - _res = { - "continuous": (cont_old_var_ind, cont_new_var_ind), - "all": (all_old_var_ind, all_new_var_ind), - } - - return _res - - -def warm_start_from_var_dict( - old_ckpt_path, - var_ind_dict, - output_dir, - new_len_var, - var_to_change_dict_fn=get_model_type_to_tensors_to_change_axis, -): - """ - Parameters: - old_ckpt_path (str): path to the old checkpoint path - new_var_ind (array of int): index to overlapping features in new var between old and new feature list. - old_var_ind (array of int): index to overlapping features in old var between old and new feature list. - - output_dir (str): dir that used to write modified checkpoint - new_len_var ({str:int}): number of feature in the new feature list. - var_to_change_dict_fn (dict): A function to get the dictionary of format {var_name: dim_to_change} - """ - old_var_dict = get_var_dict(old_ckpt_path) - - ckpt_file_name = os.path.basename(old_ckpt_path) - mkdirp(output_dir) - output_path = join(output_dir, ckpt_file_name) - - tensors_to_change = var_to_change_dict_fn() - tf.compat.v1.reset_default_graph() - - with tf.Session() as sess: - var_name_shape_list = tf.train.list_variables(old_ckpt_path) - count = 0 - - for var_name, var_shape in var_name_shape_list: - old_var = old_var_dict[var_name] - if var_name in tensors_to_change.keys(): - _info_tuple = tensors_to_change[var_name] - dims_to_remove_from, var_type = _info_tuple - - new_var_ind, old_var_ind = var_ind_dict[var_type] - - this_shape = list(old_var.shape) - for this_dim in dims_to_remove_from: - this_shape[this_dim] = new_len_var[var_type] - - stddev = np.std(old_var) - truncated_norm_generator = stats.truncnorm(-0.5, 0.5, loc=0, scale=stddev) - size = np.prod(this_shape) - new_var = truncated_norm_generator.rvs(size).reshape(this_shape) - new_var = new_var.astype(old_var.dtype) - - new_var = copy_feat_based_on_mapping( - new_var, old_var, dims_to_remove_from, new_var_ind, old_var_ind - ) - count = count + 1 - else: - new_var = old_var - var = tf.Variable(new_var, name=var_name) - assert count == len(tensors_to_change.keys()), "not all variables are exchanged.\n" - saver = tf.train.Saver() - sess.run(tf.global_variables_initializer()) - saver.save(sess, output_path) - return output_path - - -def copy_feat_based_on_mapping(new_array, old_array, dims_to_remove_from, new_var_ind, old_var_ind): - if dims_to_remove_from == [0, 1]: - for this_new_ind, this_old_ind in zip(new_var_ind, old_var_ind): - new_array[this_new_ind, new_var_ind] = old_array[this_old_ind, old_var_ind] - elif dims_to_remove_from == [0]: - new_array[new_var_ind] = old_array[old_var_ind] - elif dims_to_remove_from == [1]: - new_array[:, new_var_ind] = old_array[:, old_var_ind] - else: - raise RuntimeError(f"undefined dims_to_remove_from pattern: ({dims_to_remove_from})") - return new_array - - -def read_file(filename, decode=False): - """ - Reads contents from a file and optionally decodes it. - - Arguments: - filename: - path to file where the contents will be loaded from. - Accepts HDFS and local paths. - decode: - False or 'json'. When decode='json', contents is decoded - with json.loads. When False, contents is returned as is. - """ - graph = tf.Graph() - with graph.as_default(): - read = tf.read_file(filename) - - with tf.Session(graph=graph) as sess: - contents = sess.run(read) - if not isinstance(contents, str): - contents = contents.decode() - - if decode == "json": - contents = json.loads(contents) - - return contents - - -def read_feat_list_from_disk(file_path): - return read_file(file_path, decode="json") - - -def get_feature_list_for_light_ranking(feature_list_path, data_spec_path): - feature_list = read_config(feature_list_path).items() - string_feat_list = [f[0] for f in feature_list if f[1] != "S"] - - feature_config_builder = twml.contrib.feature_config.FeatureConfigBuilder( - data_spec_path=data_spec_path - ) - feature_config_builder = feature_config_builder.extract_feature_group( - feature_regexes=string_feat_list, - group_name="continuous", - default_value=-1, - type_filter=["CONTINUOUS"], - ) - feature_config = feature_config_builder.build() - feature_list = feature_config_builder._feature_group_extraction_configs[0].feature_map[ - "CONTINUOUS" - ] - return feature_list - - -def get_feature_list_for_heavy_ranking(feature_list_path, data_spec_path): - feature_list = read_config(feature_list_path).items() - string_feat_list = [f[0] for f in feature_list if f[1] != "S"] - - feature_config_builder = twml.contrib.feature_config.FeatureConfigBuilder( - data_spec_path=data_spec_path - ) - feature_config_builder = feature_config_builder.extract_feature_group( - feature_regexes=string_feat_list, - group_name="continuous", - default_value=-1, - type_filter=["CONTINUOUS"], - ) - - feature_config_builder = feature_config_builder.extract_feature_group( - feature_regexes=string_feat_list, - group_name="binary", - default_value=False, - type_filter=["BINARY"], - ) - - feature_config_builder = feature_config_builder.build() - - continuous_feature_list = feature_config_builder._feature_group_extraction_configs[0].feature_map[ - "CONTINUOUS" - ] - - binary_feature_list = feature_config_builder._feature_group_extraction_configs[1].feature_map[ - "BINARY" - ] - return {"continuous": continuous_feature_list, "binary": binary_feature_list} - - -def warm_start_checkpoint( - old_best_ckpt_folder, - old_feature_list_path, - feature_allow_list_path, - data_spec_path, - output_ckpt_folder, - *args, -): - """ - Reads old checkpoint and the old feature list, and create a new ckpt warm started from old ckpt using new features . - - Arguments: - old_best_ckpt_folder: - path to the best_checkpoint_folder for old model - old_feature_list_path: - path to the json file that stores the list of continuous features used in old models. - feature_allow_list_path: - yaml file that contain the feature allow list. - data_spec_path: - path to the data_spec file - output_ckpt_folder: - folder that contains the modified ckpt. - - Returns: - path to the modified ckpt.""" - old_ckpt_path = tf.train.latest_checkpoint(old_best_ckpt_folder, latest_filename=None) - - new_feature_dict = get_feature_list(feature_allow_list_path, data_spec_path) - old_feature_dict = read_feat_list_from_disk(old_feature_list_path) - - var_ind_dict = get_continuous_mapping_from_feat_dict(new_feature_dict, old_feature_dict) - - new_len_var = { - "continuous": len(new_feature_dict["continuous"]), - "all": len(new_feature_dict["continuous"] + new_feature_dict["binary"]) + 100, - } - - warm_started_ckpt_path = warm_start_from_var_dict( - old_ckpt_path, - var_ind_dict, - output_dir=output_ckpt_folder, - new_len_var=new_len_var, - ) - - return warm_started_ckpt_path diff --git a/pushservice/src/main/python/models/light_ranking/BUILD b/pushservice/src/main/python/models/light_ranking/BUILD deleted file mode 100644 index e88d7de7c..000000000 --- a/pushservice/src/main/python/models/light_ranking/BUILD +++ /dev/null @@ -1,69 +0,0 @@ -#":mlwf_libs", - -python37_binary( - name = "eval_model", - source = "eval_model.py", - dependencies = [ - ":libs", - "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/light_ranking:eval_model", - ], -) - -python37_binary( - name = "train_model", - source = "deep_norm.py", - dependencies = [ - ":libs", - "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/light_ranking:train_model", - ], -) - -python37_binary( - name = "train_model_local", - source = "deep_norm.py", - dependencies = [ - ":libs", - "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/light_ranking:train_model_local", - "twml", - ], -) - -python37_binary( - name = "eval_model_local", - source = "eval_model.py", - dependencies = [ - ":libs", - "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/light_ranking:eval_model_local", - "twml", - ], -) - -python37_binary( - name = "mlwf_model", - source = "deep_norm.py", - dependencies = [ - ":mlwf_libs", - "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/light_ranking:mlwf_model", - ], -) - -python3_library( - name = "libs", - sources = ["**/*.py"], - tags = ["no-mypy"], - dependencies = [ - "src/python/twitter/deepbird/projects/magic_recs/libs", - "src/python/twitter/deepbird/util/data", - "twml:twml-nodeps", - ], -) - -python3_library( - name = "mlwf_libs", - sources = ["**/*.py"], - tags = ["no-mypy"], - dependencies = [ - "src/python/twitter/deepbird/projects/magic_recs/libs", - "twml", - ], -) diff --git a/pushservice/src/main/python/models/light_ranking/README.md b/pushservice/src/main/python/models/light_ranking/README.md deleted file mode 100644 index 9d7bd2682..000000000 --- a/pushservice/src/main/python/models/light_ranking/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# Notification Light Ranker Model - -## Model Context -There are 4 major components of Twitter notifications recommendation system: 1) candidate generation 2) light ranking 3) heavy ranking & 4) quality control. This notification light ranker model bridges candidate generation and heavy ranking by pre-selecting highly-relevant candidates from the initial huge candidate pool. It’s a light-weight model to reduce system cost during heavy ranking without hurting user experience. - -## Directory Structure -- BUILD: this file defines python library dependencies -- model_pools_mlp.py: this file defines tensorflow model architecture for the notification light ranker model -- deep_norm.py: this file contains 1) how to build the tensorflow graph with specified model architecture, loss function and training configuration. 2) how to set up the overall model training & evaluation pipeline -- eval_model.py: the main python entry file to set up the overall model evaluation pipeline - - - - diff --git a/pushservice/src/main/python/models/light_ranking/__init__.py b/pushservice/src/main/python/models/light_ranking/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/pushservice/src/main/python/models/light_ranking/deep_norm.py b/pushservice/src/main/python/models/light_ranking/deep_norm.py deleted file mode 100644 index bc90deba4..000000000 --- a/pushservice/src/main/python/models/light_ranking/deep_norm.py +++ /dev/null @@ -1,226 +0,0 @@ -from datetime import datetime -from functools import partial -import os - -from twitter.cortex.ml.embeddings.common.helpers import decode_str_or_unicode -import twml -from twml.trainers import DataRecordTrainer - -from ..libs.get_feat_config import get_feature_config_light_ranking, LABELS_LR -from ..libs.graph_utils import get_trainable_variables -from ..libs.group_metrics import ( - run_group_metrics_light_ranking, - run_group_metrics_light_ranking_in_bq, -) -from ..libs.metric_fn_utils import get_metric_fn -from ..libs.model_args import get_arg_parser_light_ranking -from ..libs.model_utils import read_config -from ..libs.warm_start_utils import get_feature_list_for_light_ranking -from .model_pools_mlp import light_ranking_mlp_ngbdt - -import tensorflow.compat.v1 as tf -from tensorflow.compat.v1 import logging - - -# checkstyle: noqa - - -def build_graph( - features, label, mode, params, config=None, run_light_ranking_group_metrics_in_bq=False -): - is_training = mode == tf.estimator.ModeKeys.TRAIN - this_model_func = light_ranking_mlp_ngbdt - model_output = this_model_func(features, is_training, params, label) - - logits = model_output["output"] - graph_output = {} - # -------------------------------------------------------- - # define graph output dict - # -------------------------------------------------------- - if mode == tf.estimator.ModeKeys.PREDICT: - loss = None - output_label = "prediction" - if params.task_name in LABELS_LR: - output = tf.nn.sigmoid(logits) - output = tf.clip_by_value(output, 0, 1) - - if run_light_ranking_group_metrics_in_bq: - graph_output["trace_id"] = features["meta.trace_id"] - graph_output["target"] = features["meta.ranking.weighted_oonc_model_score"] - - else: - raise ValueError("Invalid Task Name !") - - else: - output_label = "output" - weights = tf.cast(features["weights"], dtype=tf.float32, name="RecordWeights") - - if params.task_name in LABELS_LR: - if params.use_record_weight: - weights = tf.clip_by_value( - 1.0 / (1.0 + weights + params.smooth_weight), params.min_record_weight, 1.0 - ) - - loss = tf.reduce_sum( - tf.nn.sigmoid_cross_entropy_with_logits(labels=label, logits=logits) * weights - ) / (tf.reduce_sum(weights)) - else: - loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=label, logits=logits)) - output = tf.nn.sigmoid(logits) - - else: - raise ValueError("Invalid Task Name !") - - train_op = None - if mode == tf.estimator.ModeKeys.TRAIN: - # -------------------------------------------------------- - # get train_op - # -------------------------------------------------------- - optimizer = tf.train.GradientDescentOptimizer(learning_rate=params.learning_rate) - update_ops = set(tf.get_collection(tf.GraphKeys.UPDATE_OPS)) - variables = get_trainable_variables( - all_trainable_variables=tf.trainable_variables(), trainable_regexes=params.trainable_regexes - ) - with tf.control_dependencies(update_ops): - train_op = twml.optimizers.optimize_loss( - loss=loss, - variables=variables, - global_step=tf.train.get_global_step(), - optimizer=optimizer, - learning_rate=params.learning_rate, - learning_rate_decay_fn=twml.learning_rate_decay.get_learning_rate_decay_fn(params), - ) - - graph_output[output_label] = output - graph_output["loss"] = loss - graph_output["train_op"] = train_op - return graph_output - - -def get_params(args=None): - parser = get_arg_parser_light_ranking() - if args is None: - return parser.parse_args() - else: - return parser.parse_args(args) - - -def _main(): - opt = get_params() - logging.info("parse is: ") - logging.info(opt) - - feature_list = read_config(opt.feature_list).items() - feature_config = get_feature_config_light_ranking( - data_spec_path=opt.data_spec, - feature_list_provided=feature_list, - opt=opt, - add_gbdt=opt.use_gbdt_features, - run_light_ranking_group_metrics_in_bq=opt.run_light_ranking_group_metrics_in_bq, - ) - feature_list_path = opt.feature_list - - # -------------------------------------------------------- - # Create Trainer - # -------------------------------------------------------- - trainer = DataRecordTrainer( - name=opt.model_trainer_name, - params=opt, - build_graph_fn=build_graph, - save_dir=opt.save_dir, - run_config=None, - feature_config=feature_config, - metric_fn=get_metric_fn(opt.task_name, use_stratify_metrics=False), - ) - if opt.directly_export_best: - logging.info("Directly exporting the model without training") - else: - # ---------------------------------------------------- - # Model Training & Evaluation - # ---------------------------------------------------- - eval_input_fn = trainer.get_eval_input_fn(repeat=False, shuffle=False) - train_input_fn = trainer.get_train_input_fn(shuffle=True) - - if opt.distributed or opt.num_workers is not None: - learn = trainer.train_and_evaluate - else: - learn = trainer.learn - logging.info("Training...") - start = datetime.now() - - early_stop_metric = "rce_unweighted_" + opt.task_name - learn( - early_stop_minimize=False, - early_stop_metric=early_stop_metric, - early_stop_patience=opt.early_stop_patience, - early_stop_tolerance=opt.early_stop_tolerance, - eval_input_fn=eval_input_fn, - train_input_fn=train_input_fn, - ) - - end = datetime.now() - logging.info("Training time: " + str(end - start)) - - logging.info("Exporting the models...") - - # -------------------------------------------------------- - # Do the model exporting - # -------------------------------------------------------- - start = datetime.now() - if not opt.export_dir: - opt.export_dir = os.path.join(opt.save_dir, "exported_models") - - raw_model_path = twml.contrib.export.export_fn.export_all_models( - trainer=trainer, - export_dir=opt.export_dir, - parse_fn=feature_config.get_parse_fn(), - serving_input_receiver_fn=feature_config.get_serving_input_receiver_fn(), - export_output_fn=twml.export_output_fns.batch_prediction_continuous_output_fn, - ) - export_model_dir = decode_str_or_unicode(raw_model_path) - - logging.info("Model export time: " + str(datetime.now() - start)) - logging.info("The saved model directory is: " + opt.save_dir) - - tf.logging.info("getting default continuous_feature_list") - continuous_feature_list = get_feature_list_for_light_ranking(feature_list_path, opt.data_spec) - continous_feature_list_save_path = os.path.join(opt.save_dir, "continuous_feature_list.json") - twml.util.write_file(continous_feature_list_save_path, continuous_feature_list, encode="json") - tf.logging.info(f"Finish writting files to {continous_feature_list_save_path}") - - if opt.run_light_ranking_group_metrics: - # -------------------------------------------- - # Run Light Ranking Group Metrics - # -------------------------------------------- - run_group_metrics_light_ranking( - trainer=trainer, - data_dir=os.path.join(opt.eval_data_dir, opt.eval_start_datetime), - model_path=export_model_dir, - parse_fn=feature_config.get_parse_fn(), - ) - - if opt.run_light_ranking_group_metrics_in_bq: - # ---------------------------------------------------------------------------------------- - # Get Light/Heavy Ranker Predictions for Light Ranking Group Metrics in BigQuery - # ---------------------------------------------------------------------------------------- - trainer_pred = DataRecordTrainer( - name=opt.model_trainer_name, - params=opt, - build_graph_fn=partial(build_graph, run_light_ranking_group_metrics_in_bq=True), - save_dir=opt.save_dir + "/tmp/", - run_config=None, - feature_config=feature_config, - metric_fn=get_metric_fn(opt.task_name, use_stratify_metrics=False), - ) - checkpoint_folder = os.path.join(opt.save_dir, "best_checkpoint") - checkpoint = tf.train.latest_checkpoint(checkpoint_folder, latest_filename=None) - tf.logging.info("\n\nPrediction from Checkpoint: {:}.\n\n".format(checkpoint)) - run_group_metrics_light_ranking_in_bq( - trainer=trainer_pred, params=opt, checkpoint_path=checkpoint - ) - - tf.logging.info("Done Training & Prediction.") - - -if __name__ == "__main__": - _main() diff --git a/pushservice/src/main/python/models/light_ranking/eval_model.py b/pushservice/src/main/python/models/light_ranking/eval_model.py deleted file mode 100644 index 1726685cf..000000000 --- a/pushservice/src/main/python/models/light_ranking/eval_model.py +++ /dev/null @@ -1,89 +0,0 @@ -from datetime import datetime -from functools import partial -import os - -from ..libs.group_metrics import ( - run_group_metrics_light_ranking, - run_group_metrics_light_ranking_in_bq, -) -from ..libs.metric_fn_utils import get_metric_fn -from ..libs.model_args import get_arg_parser_light_ranking -from ..libs.model_utils import read_config -from .deep_norm import build_graph, DataRecordTrainer, get_config_func, logging - - -# checkstyle: noqa - -if __name__ == "__main__": - parser = get_arg_parser_light_ranking() - parser.add_argument( - "--eval_checkpoint", - default=None, - type=str, - help="Which checkpoint to use for evaluation", - ) - parser.add_argument( - "--saved_model_path", - default=None, - type=str, - help="Path to saved model for evaluation", - ) - parser.add_argument( - "--run_binary_metrics", - default=False, - action="store_true", - help="Whether to compute the basic binary metrics for Light Ranking.", - ) - - opt = parser.parse_args() - logging.info("parse is: ") - logging.info(opt) - - feature_list = read_config(opt.feature_list).items() - feature_config = get_config_func(opt.feat_config_type)( - data_spec_path=opt.data_spec, - feature_list_provided=feature_list, - opt=opt, - add_gbdt=opt.use_gbdt_features, - run_light_ranking_group_metrics_in_bq=opt.run_light_ranking_group_metrics_in_bq, - ) - - # ----------------------------------------------- - # Create Trainer - # ----------------------------------------------- - trainer = DataRecordTrainer( - name=opt.model_trainer_name, - params=opt, - build_graph_fn=partial(build_graph, run_light_ranking_group_metrics_in_bq=True), - save_dir=opt.save_dir, - run_config=None, - feature_config=feature_config, - metric_fn=get_metric_fn(opt.task_name, use_stratify_metrics=False), - ) - - # ----------------------------------------------- - # Model Evaluation - # ----------------------------------------------- - logging.info("Evaluating...") - start = datetime.now() - - if opt.run_binary_metrics: - eval_input_fn = trainer.get_eval_input_fn(repeat=False, shuffle=False) - eval_steps = None if (opt.eval_steps is not None and opt.eval_steps < 0) else opt.eval_steps - trainer.estimator.evaluate(eval_input_fn, steps=eval_steps, checkpoint_path=opt.eval_checkpoint) - - if opt.run_light_ranking_group_metrics_in_bq: - run_group_metrics_light_ranking_in_bq( - trainer=trainer, params=opt, checkpoint_path=opt.eval_checkpoint - ) - - if opt.run_light_ranking_group_metrics: - run_group_metrics_light_ranking( - trainer=trainer, - data_dir=os.path.join(opt.eval_data_dir, opt.eval_start_datetime), - model_path=opt.saved_model_path, - parse_fn=feature_config.get_parse_fn(), - ) - - end = datetime.now() - logging.info("Evaluating time: " + str(end - start)) diff --git a/pushservice/src/main/python/models/light_ranking/model_pools_mlp.py b/pushservice/src/main/python/models/light_ranking/model_pools_mlp.py deleted file mode 100644 index b45c85e47..000000000 --- a/pushservice/src/main/python/models/light_ranking/model_pools_mlp.py +++ /dev/null @@ -1,187 +0,0 @@ -import warnings - -from twml.contrib.layers import ZscoreNormalization - -from ...libs.customized_full_sparse import FullSparse -from ...libs.get_feat_config import FEAT_CONFIG_DEFAULT_VAL as MISSING_VALUE_MARKER -from ...libs.model_utils import ( - _sparse_feature_fixup, - adaptive_transformation, - filter_nans_and_infs, - get_dense_out, - tensor_dropout, -) - -import tensorflow.compat.v1 as tf -# checkstyle: noqa - -def light_ranking_mlp_ngbdt(features, is_training, params, label=None): - return deepnorm_light_ranking( - features, - is_training, - params, - label=label, - decay=params.momentum, - dense_emb_size=params.dense_embedding_size, - base_activation=tf.keras.layers.LeakyReLU(), - input_dropout_rate=params.dropout, - use_gbdt=False, - ) - - -def deepnorm_light_ranking( - features, - is_training, - params, - label=None, - decay=0.99999, - dense_emb_size=128, - base_activation=None, - input_dropout_rate=None, - input_dense_type="self_atten_dense", - emb_dense_type="self_atten_dense", - mlp_dense_type="self_atten_dense", - use_gbdt=False, -): - # -------------------------------------------------------- - # Initial Parameter Checking - # -------------------------------------------------------- - if base_activation is None: - base_activation = tf.keras.layers.LeakyReLU() - - if label is not None: - warnings.warn( - "Label is unused in deepnorm_gbdt. Stop using this argument.", - DeprecationWarning, - ) - - with tf.variable_scope("helper_layers"): - full_sparse_layer = FullSparse( - output_size=params.sparse_embedding_size, - activation=base_activation, - use_sparse_grads=is_training, - use_binary_values=False, - dtype=tf.float32, - ) - input_normalizing_layer = ZscoreNormalization(decay=decay, name="input_normalizing_layer") - - # -------------------------------------------------------- - # Feature Selection & Embedding - # -------------------------------------------------------- - if use_gbdt: - sparse_gbdt_features = _sparse_feature_fixup(features["gbdt_sparse"], params.input_size_bits) - if input_dropout_rate is not None: - sparse_gbdt_features = tensor_dropout( - sparse_gbdt_features, input_dropout_rate, is_training, sparse_tensor=True - ) - - total_embed = full_sparse_layer(sparse_gbdt_features, use_binary_values=True) - - if (input_dropout_rate is not None) and is_training: - total_embed = total_embed / (1 - input_dropout_rate) - - else: - with tf.variable_scope("dense_branch"): - dense_continuous_features = filter_nans_and_infs(features["continuous"]) - - if params.use_missing_sub_branch: - is_missing = tf.equal(dense_continuous_features, MISSING_VALUE_MARKER) - continuous_features_filled = tf.where( - is_missing, - tf.zeros_like(dense_continuous_features), - dense_continuous_features, - ) - normalized_features = input_normalizing_layer( - continuous_features_filled, is_training, tf.math.logical_not(is_missing) - ) - - with tf.variable_scope("missing_sub_branch"): - missing_feature_embed = get_dense_out( - tf.cast(is_missing, tf.float32), - dense_emb_size, - activation=base_activation, - dense_type=input_dense_type, - ) - - else: - continuous_features_filled = dense_continuous_features - normalized_features = input_normalizing_layer(continuous_features_filled, is_training) - - with tf.variable_scope("continuous_sub_branch"): - normalized_features = adaptive_transformation( - normalized_features, is_training, func_type="tiny" - ) - - if input_dropout_rate is not None: - normalized_features = tensor_dropout( - normalized_features, - input_dropout_rate, - is_training, - sparse_tensor=False, - ) - filled_feature_embed = get_dense_out( - normalized_features, - dense_emb_size, - activation=base_activation, - dense_type=input_dense_type, - ) - - if params.use_missing_sub_branch: - dense_embed = tf.concat( - [filled_feature_embed, missing_feature_embed], axis=1, name="merge_dense_emb" - ) - else: - dense_embed = filled_feature_embed - - with tf.variable_scope("sparse_branch"): - sparse_discrete_features = _sparse_feature_fixup( - features["sparse_no_continuous"], params.input_size_bits - ) - if input_dropout_rate is not None: - sparse_discrete_features = tensor_dropout( - sparse_discrete_features, input_dropout_rate, is_training, sparse_tensor=True - ) - - discrete_features_embed = full_sparse_layer(sparse_discrete_features, use_binary_values=True) - - if (input_dropout_rate is not None) and is_training: - discrete_features_embed = discrete_features_embed / (1 - input_dropout_rate) - - total_embed = tf.concat( - [dense_embed, discrete_features_embed], - axis=1, - name="total_embed", - ) - - total_embed = tf.layers.batch_normalization( - total_embed, - training=is_training, - renorm_momentum=decay, - momentum=decay, - renorm=is_training, - trainable=True, - ) - - # -------------------------------------------------------- - # MLP Layers - # -------------------------------------------------------- - with tf.variable_scope("MLP_branch"): - - assert params.num_mlp_layers >= 0 - embed_list = [total_embed] + [None for _ in range(params.num_mlp_layers)] - dense_types = [emb_dense_type] + [mlp_dense_type for _ in range(params.num_mlp_layers - 1)] - - for xl in range(1, params.num_mlp_layers + 1): - neurons = params.mlp_neuron_scale ** (params.num_mlp_layers + 1 - xl) - embed_list[xl] = get_dense_out( - embed_list[xl - 1], neurons, activation=base_activation, dense_type=dense_types[xl - 1] - ) - - if params.task_name in ["Sent", "HeavyRankPosition", "HeavyRankProbability"]: - logits = get_dense_out(embed_list[-1], 1, activation=None, dense_type=mlp_dense_type) - - else: - raise ValueError("Invalid Task Name !") - - output_dict = {"output": logits} - return output_dict diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/BUILD.bazel b/pushservice/src/main/scala/com/twitter/frigate/pushservice/BUILD.bazel deleted file mode 100644 index d53d4e251..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/BUILD.bazel +++ /dev/null @@ -1,337 +0,0 @@ -scala_library( - sources = ["**/*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = [ - "bazel-compatible", - ], - dependencies = [ - "3rdparty/jvm/com/twitter/bijection:scrooge", - "3rdparty/jvm/com/twitter/storehaus:core", - "abdecider", - "abuse/detection/src/main/thrift/com/twitter/abuse/detection/scoring:thrift-scala", - "ann/src/main/scala/com/twitter/ann/common", - "ann/src/main/thrift/com/twitter/ann/common:ann-common-scala", - "audience-rewards/thrift/src/main/thrift:thrift-scala", - "communities/thrift/src/main/thrift/com/twitter/communities:thrift-scala", - "configapi/configapi-core", - "configapi/configapi-decider", - "content-mixer/thrift/src/main/thrift:thrift-scala", - "content-recommender/thrift/src/main/thrift:thrift-scala", - "copyselectionservice/server/src/main/scala/com/twitter/copyselectionservice/algorithms", - "copyselectionservice/thrift/src/main/thrift:copyselectionservice-scala", - "cortex-deepbird/thrift/src/main/thrift:thrift-java", - "cr-mixer/thrift/src/main/thrift:thrift-scala", - "cuad/projects/hashspace/thrift:thrift-scala", - "cuad/projects/tagspace/thrift/src/main/thrift:thrift-scala", - "detopic/thrift/src/main/thrift:thrift-scala", - "discovery-common/src/main/scala/com/twitter/discovery/common/configapi", - "discovery-common/src/main/scala/com/twitter/discovery/common/ddg", - "discovery-common/src/main/scala/com/twitter/discovery/common/environment", - "discovery-common/src/main/scala/com/twitter/discovery/common/fatigue", - "discovery-common/src/main/scala/com/twitter/discovery/common/nackwarmupfilter", - "discovery-common/src/main/scala/com/twitter/discovery/common/server", - "discovery-ds/src/main/thrift/com/twitter/dds/scio/searcher_aggregate_history_srp:searcher_aggregate_history_srp-scala", - "escherbird/src/scala/com/twitter/escherbird/util/metadatastitch", - "escherbird/src/scala/com/twitter/escherbird/util/uttclient", - "escherbird/src/thrift/com/twitter/escherbird/utt:strato-columns-scala", - "eventbus/client", - "eventdetection/event_context/src/main/scala/com/twitter/eventdetection/event_context/util", - "events-recos/events-recos-service/src/main/thrift:events-recos-thrift-scala", - "explore/explore-ranker/thrift/src/main/thrift:thrift-scala", - "featureswitches/featureswitches-core/src/main/scala", - "featureswitches/featureswitches-core/src/main/scala:dynmap", - "featureswitches/featureswitches-core/src/main/scala:recipient", - "featureswitches/featureswitches-core/src/main/scala:useragent", - "featureswitches/featureswitches-core/src/main/scala/com/twitter/featureswitches/v2/builder", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/server", - "finagle-internal/ostrich-stats", - "finagle/finagle-core/src/main", - "finagle/finagle-http/src/main/scala", - "finagle/finagle-memcached/src/main/scala", - "finagle/finagle-stats", - "finagle/finagle-thriftmux", - "finagle/finagle-tunable/src/main/scala", - "finagle/finagle-zipkin-scribe", - "finatra-internal/abdecider", - "finatra-internal/decider", - "finatra-internal/mtls-http/src/main/scala", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/http-client/src/main/scala", - "finatra/http-core/src/main/java/com/twitter/finatra/http", - "finatra/http-core/src/main/scala/com/twitter/finatra/http/response", - "finatra/http-server/src/main/scala/com/twitter/finatra/http", - "finatra/http-server/src/main/scala/com/twitter/finatra/http/filters", - "finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations", - "finatra/inject/inject-app/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-server/src/main/scala", - "finatra/inject/inject-slf4j/src/main/scala/com/twitter/inject", - "finatra/inject/inject-thrift-client/src/main/scala", - "finatra/inject/inject-utils/src/main/scala", - "finatra/utils/src/main/java/com/twitter/finatra/annotations", - "fleets/fleets-proxy/thrift/src/main/thrift:fleet-scala", - "fleets/fleets-proxy/thrift/src/main/thrift/service:baseservice-scala", - "flock-client/src/main/scala", - "flock-client/src/main/thrift:thrift-scala", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "frigate/frigate-common:base", - "frigate/frigate-common:config", - "frigate/frigate-common:debug", - "frigate/frigate-common:entity_graph_client", - "frigate/frigate-common:history", - "frigate/frigate-common:logger", - "frigate/frigate-common:ml-base", - "frigate/frigate-common:ml-feature", - "frigate/frigate-common:ml-prediction", - "frigate/frigate-common:ntab", - "frigate/frigate-common:predicate", - "frigate/frigate-common:rec_types", - "frigate/frigate-common:score_summary", - "frigate/frigate-common:util", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/candidate", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/experiments", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/filter", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/modules/store:semantic_core_stores", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/deviceinfo", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/interests", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/strato", - "frigate/push-mixer/thrift/src/main/thrift:thrift-scala", - "geo/geo-prediction/src/main/thrift:local-viral-tweets-thrift-scala", - "geoduck/service/src/main/scala/com/twitter/geoduck/service/common/clientmodules", - "geoduck/util/country", - "gizmoduck/client/src/main/scala/com/twitter/gizmoduck/testusers/client", - "hermit/hermit-core:model-user_state", - "hermit/hermit-core:predicate", - "hermit/hermit-core:predicate-gizmoduck", - "hermit/hermit-core:predicate-scarecrow", - "hermit/hermit-core:predicate-socialgraph", - "hermit/hermit-core:predicate-tweetypie", - "hermit/hermit-core:store-labeled_push_recs", - "hermit/hermit-core:store-metastore", - "hermit/hermit-core:store-timezone", - "hermit/hermit-core:store-tweetypie", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/constants", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/model", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/gizmoduck", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/scarecrow", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/semantic_core", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/user_htl_session_store", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/user_interest", - "hmli/hss/src/main/thrift/com/twitter/hss:thrift-scala", - "ibis2/service/src/main/scala/com/twitter/ibis2/lib", - "ibis2/service/src/main/thrift/com/twitter/ibis2/service:ibis2-service-scala", - "interests-service/thrift/src/main/thrift:thrift-scala", - "interests_discovery/thrift/src/main/thrift:batch-thrift-scala", - "interests_discovery/thrift/src/main/thrift:service-thrift-scala", - "kujaku/thrift/src/main/thrift:domain-scala", - "live-video-timeline/client/src/main/scala/com/twitter/livevideo/timeline/client/v2", - "live-video-timeline/domain/src/main/scala/com/twitter/livevideo/timeline/domain", - "live-video-timeline/domain/src/main/scala/com/twitter/livevideo/timeline/domain/v2", - "live-video-timeline/thrift/src/main/thrift/com/twitter/livevideo/timeline:thrift-scala", - "live-video/common/src/main/scala/com/twitter/livevideo/common/domain/v2", - "live-video/common/src/main/scala/com/twitter/livevideo/common/ids", - "notifications-platform/inbound-notifications/src/main/thrift/com/twitter/inbound_notifications:exception-scala", - "notifications-platform/inbound-notifications/src/main/thrift/com/twitter/inbound_notifications:thrift-scala", - "notifications-platform/platform-lib/src/main/thrift/com/twitter/notifications/platform:custom-notification-actions-scala", - "notifications-platform/platform-lib/src/main/thrift/com/twitter/notifications/platform:thrift-scala", - "notifications-relevance/src/scala/com/twitter/nrel/heavyranker", - "notifications-relevance/src/scala/com/twitter/nrel/hydration/base", - "notifications-relevance/src/scala/com/twitter/nrel/hydration/frigate", - "notifications-relevance/src/scala/com/twitter/nrel/hydration/push", - "notifications-relevance/src/scala/com/twitter/nrel/lightranker", - "notificationservice/common/src/main/scala/com/twitter/notificationservice/genericfeedbackstore", - "notificationservice/common/src/main/scala/com/twitter/notificationservice/model:alias", - "notificationservice/common/src/main/scala/com/twitter/notificationservice/model/service", - "notificationservice/common/src/test/scala/com/twitter/notificationservice/mocks", - "notificationservice/scribe/src/main/scala/com/twitter/notificationservice/scribe/manhattan:mh_wrapper", - "notificationservice/thrift/src/main/thrift/com/twitter/notificationservice/api:thrift-scala", - "notificationservice/thrift/src/main/thrift/com/twitter/notificationservice/badgecount-api:thrift-scala", - "notificationservice/thrift/src/main/thrift/com/twitter/notificationservice/generic_notifications:thrift-scala", - "notifinfra/ni-lib/src/main/scala/com/twitter/ni/lib/logged_out_transform", - "observability/observability-manhattan-client/src/main/scala", - "onboarding/service/src/main/scala/com/twitter/onboarding/task/service/models/external", - "onboarding/service/thrift/src/main/thrift:thrift-scala", - "people-discovery/api/thrift/src/main/thrift:thrift-scala", - "periscope/api-proxy-thrift/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter", - "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", - "qig-ranker/thrift/src/main/thrift:thrift-scala", - "rux-ds/src/main/thrift/com/twitter/ruxds/jobs/user_past_aggregate:user_past_aggregate-scala", - "rux/common/src/main/scala/com/twitter/rux/common/encode", - "rux/common/thrift/src/main/thrift/rux-context:rux-context-scala", - "rux/common/thrift/src/main/thrift/strato:strato-scala", - "scribelib/marshallers/src/main/scala/com/twitter/scribelib/marshallers", - "scrooge/scrooge-core", - "scrooge/scrooge-serializer/src/main/scala", - "sensitive-ds/src/main/thrift/com/twitter/scio/nsfw_user_segmentation:nsfw_user_segmentation-scala", - "servo/decider/src/main/scala", - "servo/request/src/main/scala", - "servo/util/src/main/scala", - "src/java/com/twitter/ml/api:api-base", - "src/java/com/twitter/ml/prediction/core", - "src/scala/com/twitter/frigate/data_pipeline/common", - "src/scala/com/twitter/frigate/data_pipeline/embedding_cg:embedding_cg-test-user-ids", - "src/scala/com/twitter/frigate/data_pipeline/features_common", - "src/scala/com/twitter/frigate/news_article_recs/news_articles_metadata:thrift-scala", - "src/scala/com/twitter/frontpage/stream/util", - "src/scala/com/twitter/language/normalization", - "src/scala/com/twitter/ml/api/embedding", - "src/scala/com/twitter/ml/api/util:datarecord", - "src/scala/com/twitter/ml/featurestore/catalog/entities/core", - "src/scala/com/twitter/ml/featurestore/catalog/entities/magicrecs", - "src/scala/com/twitter/ml/featurestore/catalog/features/core:aggregate", - "src/scala/com/twitter/ml/featurestore/catalog/features/cuad:aggregate", - "src/scala/com/twitter/ml/featurestore/catalog/features/embeddings", - "src/scala/com/twitter/ml/featurestore/catalog/features/magicrecs:aggregate", - "src/scala/com/twitter/ml/featurestore/catalog/features/topic_signals:aggregate", - "src/scala/com/twitter/ml/featurestore/lib", - "src/scala/com/twitter/ml/featurestore/lib/data", - "src/scala/com/twitter/ml/featurestore/lib/dynamic", - "src/scala/com/twitter/ml/featurestore/lib/entity", - "src/scala/com/twitter/ml/featurestore/lib/online", - "src/scala/com/twitter/recommendation/interests/discovery/core/config", - "src/scala/com/twitter/recommendation/interests/discovery/core/deploy", - "src/scala/com/twitter/recommendation/interests/discovery/core/model", - "src/scala/com/twitter/recommendation/interests/discovery/popgeo/deploy", - "src/scala/com/twitter/simclusters_v2/common", - "src/scala/com/twitter/storehaus_internal/manhattan", - "src/scala/com/twitter/storehaus_internal/manhattan/config", - "src/scala/com/twitter/storehaus_internal/memcache", - "src/scala/com/twitter/storehaus_internal/memcache/config", - "src/scala/com/twitter/storehaus_internal/util", - "src/scala/com/twitter/taxi/common", - "src/scala/com/twitter/taxi/config", - "src/scala/com/twitter/taxi/deploy", - "src/scala/com/twitter/taxi/trending/common", - "src/thrift/com/twitter/ads/adserver:adserver_rpc-scala", - "src/thrift/com/twitter/clientapp/gen:clientapp-scala", - "src/thrift/com/twitter/core_workflows/user_model:user_model-scala", - "src/thrift/com/twitter/escherbird/common:constants-scala", - "src/thrift/com/twitter/escherbird/metadata:megadata-scala", - "src/thrift/com/twitter/escherbird/metadata:metadata-service-scala", - "src/thrift/com/twitter/escherbird/search:search-service-scala", - "src/thrift/com/twitter/expandodo:only-scala", - "src/thrift/com/twitter/frigate:frigate-common-thrift-scala", - "src/thrift/com/twitter/frigate:frigate-ml-thrift-scala", - "src/thrift/com/twitter/frigate:frigate-notification-thrift-scala", - "src/thrift/com/twitter/frigate:frigate-secondary-accounts-thrift-scala", - "src/thrift/com/twitter/frigate:frigate-thrift-scala", - "src/thrift/com/twitter/frigate:frigate-user-media-representation-thrift-scala", - "src/thrift/com/twitter/frigate/data_pipeline:frigate-user-history-thrift-scala", - "src/thrift/com/twitter/frigate/dau_model:frigate-dau-thrift-scala", - "src/thrift/com/twitter/frigate/magic_events:frigate-magic-events-thrift-scala", - "src/thrift/com/twitter/frigate/magic_events/scribe:thrift-scala", - "src/thrift/com/twitter/frigate/pushcap:frigate-pushcap-thrift-scala", - "src/thrift/com/twitter/frigate/pushservice:frigate-pushservice-thrift-scala", - "src/thrift/com/twitter/frigate/scribe:frigate-scribe-thrift-scala", - "src/thrift/com/twitter/frigate/subscribed_search:frigate-subscribed-search-thrift-scala", - "src/thrift/com/twitter/frigate/user_states:frigate-userstates-thrift-scala", - "src/thrift/com/twitter/geoduck:geoduck-scala", - "src/thrift/com/twitter/gizmoduck:thrift-scala", - "src/thrift/com/twitter/gizmoduck:user-thrift-scala", - "src/thrift/com/twitter/hermit:hermit-scala", - "src/thrift/com/twitter/hermit/pop_geo:hermit-pop-geo-scala", - "src/thrift/com/twitter/hermit/stp:hermit-stp-scala", - "src/thrift/com/twitter/ibis:service-scala", - "src/thrift/com/twitter/manhattan:v1-scala", - "src/thrift/com/twitter/manhattan:v2-scala", - "src/thrift/com/twitter/ml/api:data-java", - "src/thrift/com/twitter/ml/api:data-scala", - "src/thrift/com/twitter/ml/featurestore/timelines:ml-features-timelines-scala", - "src/thrift/com/twitter/ml/featurestore/timelines:ml-features-timelines-strato", - "src/thrift/com/twitter/ml/prediction_service:prediction_service-java", - "src/thrift/com/twitter/permissions_storage:thrift-scala", - "src/thrift/com/twitter/pink-floyd/thrift:thrift-scala", - "src/thrift/com/twitter/recos:recos-common-scala", - "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", - "src/thrift/com/twitter/recos/user_user_graph:user_user_graph-scala", - "src/thrift/com/twitter/relevance/feature_store:feature_store-scala", - "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/search/common:features-scala", - "src/thrift/com/twitter/search/query_interaction_graph:query_interaction_graph-scala", - "src/thrift/com/twitter/search/query_interaction_graph/service:qig-service-scala", - "src/thrift/com/twitter/service/metastore/gen:thrift-scala", - "src/thrift/com/twitter/service/scarecrow/gen:scarecrow-scala", - "src/thrift/com/twitter/service/scarecrow/gen:tiered-actions-scala", - "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "src/thrift/com/twitter/spam/rtf:safety-level-scala", - "src/thrift/com/twitter/timelinemixer:thrift-scala", - "src/thrift/com/twitter/timelinemixer/server/internal:thrift-scala", - "src/thrift/com/twitter/timelines/author_features/user_health:thrift-scala", - "src/thrift/com/twitter/timelines/real_graph:real_graph-scala", - "src/thrift/com/twitter/timelinescorer:thrift-scala", - "src/thrift/com/twitter/timelinescorer/server/internal:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala", - "src/thrift/com/twitter/trends/common:common-scala", - "src/thrift/com/twitter/trends/trip_v1:trip-tweets-thrift-scala", - "src/thrift/com/twitter/tweetypie:service-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - "src/thrift/com/twitter/user_session_store:thrift-scala", - "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", - "src/thrift/com/twitter/wtf/interest:interest-thrift-scala", - "src/thrift/com/twitter/wtf/scalding/common:thrift-scala", - "stitch/stitch-core", - "stitch/stitch-gizmoduck", - "stitch/stitch-socialgraph/src/main/scala", - "stitch/stitch-storehaus/src/main/scala", - "stitch/stitch-tweetypie/src/main/scala", - "storage/clients/manhattan/client/src/main/scala", - "strato/config/columns/clients:clients-strato-client", - "strato/config/columns/geo/user:user-strato-client", - "strato/config/columns/globe/curation:curation-strato-client", - "strato/config/columns/interests:interests-strato-client", - "strato/config/columns/ml/featureStore:featureStore-strato-client", - "strato/config/columns/notifications:notifications-strato-client", - "strato/config/columns/notifinfra:notifinfra-strato-client", - "strato/config/columns/periscope:periscope-strato-client", - "strato/config/columns/rux", - "strato/config/columns/rux:rux-strato-client", - "strato/config/columns/rux/open-app:open-app-strato-client", - "strato/config/columns/socialgraph/graphs:graphs-strato-client", - "strato/config/columns/socialgraph/service/soft_users:soft_users-strato-client", - "strato/config/columns/translation/service:service-strato-client", - "strato/config/columns/translation/service/platform:platform-strato-client", - "strato/config/columns/trends/trip:trip-strato-client", - "strato/config/src/thrift/com/twitter/strato/columns/frigate:logged-out-web-notifications-scala", - "strato/config/src/thrift/com/twitter/strato/columns/notifications:thrift-scala", - "strato/src/main/scala/com/twitter/strato/config", - "strato/src/main/scala/com/twitter/strato/response", - "thrift-web-forms", - "timeline-training-service/service/thrift/src/main/thrift:thrift-scala", - "timelines/src/main/scala/com/twitter/timelines/features/app", - "topic-social-proof/server/src/main/thrift:thrift-scala", - "topiclisting/topiclisting-core/src/main/scala/com/twitter/topiclisting", - "topiclisting/topiclisting-utt/src/main/scala/com/twitter/topiclisting/utt", - "trends/common/src/main/thrift/com/twitter/trends/common:thrift-scala", - "tweetypie/src/scala/com/twitter/tweetypie/tweettext", - "twitter-context/src/main/scala", - "twitter-server-internal", - "twitter-server/server/src/main/scala", - "twitter-text/lib/java/src/main/java/com/twitter/twittertext", - "twml/runtime/src/main/scala/com/twitter/deepbird/runtime/prediction_engine:prediction_engine_mkl", - "ubs/common/src/main/thrift/com/twitter/ubs:broadcast-thrift-scala", - "ubs/common/src/main/thrift/com/twitter/ubs:seller_application-thrift-scala", - "user_session_store/src/main/scala/com/twitter/user_session_store/impl/manhattan/readwrite", - "util-internal/scribe", - "util-internal/tunable/src/main/scala/com/twitter/util/tunable", - "util/util-app", - "util/util-hashing/src/main/scala", - "util/util-slf4j-api/src/main/scala", - "util/util-stats/src/main/scala", - "visibility/lib/src/main/scala/com/twitter/visibility/builder", - "visibility/lib/src/main/scala/com/twitter/visibility/interfaces/push_service", - "visibility/lib/src/main/scala/com/twitter/visibility/interfaces/spaces", - "visibility/lib/src/main/scala/com/twitter/visibility/util", - ], - exports = [ - "strato/config/src/thrift/com/twitter/strato/columns/frigate:logged-out-web-notifications-scala", - ], -) diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/PushMixerThriftServerWarmupHandler.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/PushMixerThriftServerWarmupHandler.scala deleted file mode 100644 index b13d3b093..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/PushMixerThriftServerWarmupHandler.scala +++ /dev/null @@ -1,93 +0,0 @@ -package com.twitter.frigate.pushservice - -import com.google.inject.Inject -import com.google.inject.Singleton -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.thrift.ClientId -import com.twitter.finatra.thrift.routing.ThriftWarmup -import com.twitter.util.logging.Logging -import com.twitter.inject.utils.Handler -import com.twitter.frigate.pushservice.{thriftscala => t} -import com.twitter.frigate.thriftscala.NotificationDisplayLocation -import com.twitter.util.Stopwatch -import com.twitter.scrooge.Request -import com.twitter.scrooge.Response -import com.twitter.util.Return -import com.twitter.util.Throw -import com.twitter.util.Try - -/** - * Warms up the refresh request path. - * If service is running as pushservice-send then the warmup does nothing. - * - * When making the warmup refresh requests we - * - Set skipFilters to true to execute as much of the request path as possible - * - Set darkWrite to true to prevent sending a push - */ -@Singleton -class PushMixerThriftServerWarmupHandler @Inject() ( - warmup: ThriftWarmup, - serviceIdentifier: ServiceIdentifier) - extends Handler - with Logging { - - private val clientId = ClientId("thrift-warmup-client") - - def handle(): Unit = { - val refreshServices = Set( - "frigate-pushservice", - "frigate-pushservice-canary", - "frigate-pushservice-canary-control", - "frigate-pushservice-canary-treatment" - ) - val isRefresh = refreshServices.contains(serviceIdentifier.service) - if (isRefresh && !serviceIdentifier.isLocal) refreshWarmup() - } - - def refreshWarmup(): Unit = { - val elapsed = Stopwatch.start() - val testIds = Seq( - 1, - 2, - 3 - ) - try { - clientId.asCurrent { - testIds.foreach { id => - val warmupReq = warmupQuery(id) - info(s"Sending warm-up request to service with query: $warmupReq") - warmup.sendRequest( - method = t.PushService.Refresh, - req = Request(t.PushService.Refresh.Args(warmupReq)))(assertWarmupResponse) - } - } - } catch { - case e: Throwable => - error(e.getMessage, e) - } - info(s"Warm up complete. Time taken: ${elapsed().toString}") - } - - private def warmupQuery(userId: Long): t.RefreshRequest = { - t.RefreshRequest( - userId = userId, - notificationDisplayLocation = NotificationDisplayLocation.PushToMobileDevice, - context = Some( - t.PushContext( - skipFilters = Some(true), - darkWrite = Some(true) - )) - ) - } - - private def assertWarmupResponse( - result: Try[Response[t.PushService.Refresh.SuccessType]] - ): Unit = { - result match { - case Return(_) => // ok - case Throw(exception) => - warn("Error performing warm-up request.") - error(exception.getMessage, exception) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/PushServiceMain.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/PushServiceMain.scala deleted file mode 100644 index c60f6e352..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/PushServiceMain.scala +++ /dev/null @@ -1,193 +0,0 @@ -package com.twitter.frigate.pushservice - -import com.twitter.discovery.common.environment.modules.EnvironmentModule -import com.twitter.finagle.Filter -import com.twitter.finatra.annotations.DarkTrafficFilterType -import com.twitter.finatra.decider.modules.DeciderModule -import com.twitter.finatra.http.HttpServer -import com.twitter.finatra.http.filters.CommonFilters -import com.twitter.finatra.http.routing.HttpRouter -import com.twitter.finatra.mtls.http.{Mtls => HttpMtls} -import com.twitter.finatra.mtls.thriftmux.{Mtls => ThriftMtls} -import com.twitter.finatra.mtls.thriftmux.filters.MtlsServerSessionTrackerFilter -import com.twitter.finatra.thrift.ThriftServer -import com.twitter.finatra.thrift.filters.ExceptionMappingFilter -import com.twitter.finatra.thrift.filters.LoggingMDCFilter -import com.twitter.finatra.thrift.filters.StatsFilter -import com.twitter.finatra.thrift.filters.ThriftMDCFilter -import com.twitter.finatra.thrift.filters.TraceIdMDCFilter -import com.twitter.finatra.thrift.routing.ThriftRouter -import com.twitter.frigate.common.logger.MRLoggerGlobalVariables -import com.twitter.frigate.pushservice.controller.PushServiceController -import com.twitter.frigate.pushservice.module._ -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flags -import com.twitter.inject.thrift.modules.ThriftClientIdModule -import com.twitter.logging.BareFormatter -import com.twitter.logging.Level -import com.twitter.logging.LoggerFactory -import com.twitter.logging.{Logging => JLogging} -import com.twitter.logging.QueueingHandler -import com.twitter.logging.ScribeHandler -import com.twitter.product_mixer.core.module.product_mixer_flags.ProductMixerFlagModule -import com.twitter.product_mixer.core.module.ABDeciderModule -import com.twitter.product_mixer.core.module.FeatureSwitchesModule -import com.twitter.product_mixer.core.module.StratoClientModule - -object PushServiceMain extends PushServiceFinatraServer - -class PushServiceFinatraServer - extends ThriftServer - with ThriftMtls - with HttpServer - with HttpMtls - with JLogging { - - override val name = "PushService" - - override val modules: Seq[TwitterModule] = { - Seq( - ABDeciderModule, - DeciderModule, - FeatureSwitchesModule, - FilterModule, - FlagModule, - EnvironmentModule, - ThriftClientIdModule, - DeployConfigModule, - ProductMixerFlagModule, - StratoClientModule, - PushHandlerModule, - PushTargetUserBuilderModule, - PushServiceDarkTrafficModule, - LoggedOutPushTargetUserBuilderModule, - new ThriftWebFormsModule(this), - ) - } - - override def configureThrift(router: ThriftRouter): Unit = { - router - .filter[ExceptionMappingFilter] - .filter[LoggingMDCFilter] - .filter[TraceIdMDCFilter] - .filter[ThriftMDCFilter] - .filter[MtlsServerSessionTrackerFilter] - .filter[StatsFilter] - .filter[Filter.TypeAgnostic, DarkTrafficFilterType] - .add[PushServiceController] - } - - override def configureHttp(router: HttpRouter): Unit = - router - .filter[CommonFilters] - - override protected def start(): Unit = { - MRLoggerGlobalVariables.setRequiredFlags( - traceLogFlag = injector.instance[Boolean](Flags.named(FlagModule.mrLoggerIsTraceAll.name)), - nthLogFlag = injector.instance[Boolean](Flags.named(FlagModule.mrLoggerNthLog.name)), - nthLogValFlag = injector.instance[Long](Flags.named(FlagModule.mrLoggerNthVal.name)) - ) - } - - override protected def warmup(): Unit = { - handle[PushMixerThriftServerWarmupHandler]() - } - - override protected def configureLoggerFactories(): Unit = { - loggerFactories.foreach { _() } - } - - override def loggerFactories: List[LoggerFactory] = { - val scribeScope = statsReceiver.scope("scribe") - List( - LoggerFactory( - level = Some(levelFlag()), - handlers = handlers - ), - LoggerFactory( - node = "request_scribe", - level = Some(Level.INFO), - useParents = false, - handlers = QueueingHandler( - maxQueueSize = 10000, - handler = ScribeHandler( - category = "frigate_pushservice_log", - formatter = BareFormatter, - statsReceiver = scribeScope.scope("frigate_pushservice_log") - ) - ) :: Nil - ), - LoggerFactory( - node = "notification_scribe", - level = Some(Level.INFO), - useParents = false, - handlers = QueueingHandler( - maxQueueSize = 10000, - handler = ScribeHandler( - category = "frigate_notifier", - formatter = BareFormatter, - statsReceiver = scribeScope.scope("frigate_notifier") - ) - ) :: Nil - ), - LoggerFactory( - node = "push_scribe", - level = Some(Level.INFO), - useParents = false, - handlers = QueueingHandler( - maxQueueSize = 10000, - handler = ScribeHandler( - category = "test_frigate_push", - formatter = BareFormatter, - statsReceiver = scribeScope.scope("test_frigate_push") - ) - ) :: Nil - ), - LoggerFactory( - node = "push_subsample_scribe", - level = Some(Level.INFO), - useParents = false, - handlers = QueueingHandler( - maxQueueSize = 2500, - handler = ScribeHandler( - category = "magicrecs_candidates_subsample_scribe", - maxMessagesPerTransaction = 250, - maxMessagesToBuffer = 2500, - formatter = BareFormatter, - statsReceiver = scribeScope.scope("magicrecs_candidates_subsample_scribe") - ) - ) :: Nil - ), - LoggerFactory( - node = "mr_request_scribe", - level = Some(Level.INFO), - useParents = false, - handlers = QueueingHandler( - maxQueueSize = 2500, - handler = ScribeHandler( - category = "mr_request_scribe", - maxMessagesPerTransaction = 250, - maxMessagesToBuffer = 2500, - formatter = BareFormatter, - statsReceiver = scribeScope.scope("mr_request_scribe") - ) - ) :: Nil - ), - LoggerFactory( - node = "high_quality_candidates_scribe", - level = Some(Level.INFO), - useParents = false, - handlers = QueueingHandler( - maxQueueSize = 2500, - handler = ScribeHandler( - category = "frigate_high_quality_candidates_log", - maxMessagesPerTransaction = 250, - maxMessagesToBuffer = 2500, - formatter = BareFormatter, - statsReceiver = scribeScope.scope("high_quality_candidates_scribe") - ) - ) :: Nil - ), - ) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ContentRecommenderMixerAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ContentRecommenderMixerAdaptor.scala deleted file mode 100644 index 946923fb9..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ContentRecommenderMixerAdaptor.scala +++ /dev/null @@ -1,323 +0,0 @@ -package com.twitter.frigate.pushservice.adaptor - -import com.twitter.contentrecommender.thriftscala.MetricTag -import com.twitter.cr_mixer.thriftscala.CrMixerTweetRequest -import com.twitter.cr_mixer.thriftscala.NotificationsContext -import com.twitter.cr_mixer.thriftscala.Product -import com.twitter.cr_mixer.thriftscala.ProductContext -import com.twitter.cr_mixer.thriftscala.{MetricTag => CrMixerMetricTag} -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.AlgorithmScore -import com.twitter.frigate.common.base.CandidateSource -import com.twitter.frigate.common.base.CandidateSourceEligible -import com.twitter.frigate.common.base.CrMixerCandidate -import com.twitter.frigate.common.base.TopicCandidate -import com.twitter.frigate.common.base.TopicProofTweetCandidate -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.common.predicate.CommonOutNetworkTweetCandidatesSourcePredicates.filterOutInNetworkTweets -import com.twitter.frigate.common.predicate.CommonOutNetworkTweetCandidatesSourcePredicates.filterOutReplyTweet -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.store.CrMixerTweetStore -import com.twitter.frigate.pushservice.store.UttEntityHydrationStore -import com.twitter.frigate.pushservice.util.AdaptorUtils -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.frigate.pushservice.util.TopicsUtil -import com.twitter.frigate.pushservice.util.TweetWithTopicProof -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.hermit.predicate.socialgraph.RelationEdge -import com.twitter.product_mixer.core.thriftscala.ClientContext -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.topiclisting.utt.LocalizedEntity -import com.twitter.tsp.thriftscala.TopicSocialProofRequest -import com.twitter.tsp.thriftscala.TopicSocialProofResponse -import com.twitter.util.Future -import scala.collection.Map - -case class ContentRecommenderMixerAdaptor( - crMixerTweetStore: CrMixerTweetStore, - tweetyPieStore: ReadableStore[Long, TweetyPieResult], - edgeStore: ReadableStore[RelationEdge, Boolean], - topicSocialProofServiceStore: ReadableStore[TopicSocialProofRequest, TopicSocialProofResponse], - uttEntityHydrationStore: UttEntityHydrationStore, - globalStats: StatsReceiver) - extends CandidateSource[Target, RawCandidate] - with CandidateSourceEligible[Target, RawCandidate] { - - override val name: String = this.getClass.getSimpleName - - private[this] val stats = globalStats.scope("ContentRecommenderMixerAdaptor") - private[this] val numOfValidAuthors = stats.stat("num_of_valid_authors") - private[this] val numOutOfMaximumDropped = stats.stat("dropped_due_out_of_maximum") - private[this] val totalInputRecs = stats.counter("input_recs") - private[this] val totalOutputRecs = stats.stat("output_recs") - private[this] val totalRequests = stats.counter("total_requests") - private[this] val nonReplyTweetsCounter = stats.counter("non_reply_tweets") - private[this] val totalOutNetworkRecs = stats.counter("out_network_tweets") - private[this] val totalInNetworkRecs = stats.counter("in_network_tweets") - - /** - * Builds OON raw candidates based on input OON Tweets - */ - def buildOONRawCandidates( - inputTarget: Target, - oonTweets: Seq[TweetyPieResult], - tweetScoreMap: Map[Long, Double], - tweetIdToTagsMap: Map[Long, Seq[CrMixerMetricTag]], - maxNumOfCandidates: Int - ): Option[Seq[RawCandidate]] = { - val cands = oonTweets.flatMap { tweetResult => - val tweetId = tweetResult.tweet.id - generateOONRawCandidate( - inputTarget, - tweetId, - Some(tweetResult), - tweetScoreMap, - tweetIdToTagsMap - ) - } - - val candidates = restrict( - maxNumOfCandidates, - cands, - numOutOfMaximumDropped, - totalOutputRecs - ) - - Some(candidates) - } - - /** - * Builds a single RawCandidate With TopicProofTweetCandidate - */ - def buildTopicTweetRawCandidate( - inputTarget: Target, - tweetWithTopicProof: TweetWithTopicProof, - localizedEntity: LocalizedEntity, - tags: Option[Seq[MetricTag]], - ): RawCandidate with TopicProofTweetCandidate = { - new RawCandidate with TopicProofTweetCandidate { - override def target: Target = inputTarget - override def topicListingSetting: Option[String] = Some( - tweetWithTopicProof.topicListingSetting) - override def tweetId: Long = tweetWithTopicProof.tweetId - override def tweetyPieResult: Option[TweetyPieResult] = Some( - tweetWithTopicProof.tweetyPieResult) - override def semanticCoreEntityId: Option[Long] = Some(tweetWithTopicProof.topicId) - override def localizedUttEntity: Option[LocalizedEntity] = Some(localizedEntity) - override def algorithmCR: Option[String] = tweetWithTopicProof.algorithmCR - override def tagsCR: Option[Seq[MetricTag]] = tags - override def isOutOfNetwork: Boolean = tweetWithTopicProof.isOON - } - } - - /** - * Takes a group of TopicTweets and transforms them into RawCandidates - */ - def buildTopicTweetRawCandidates( - inputTarget: Target, - topicProofCandidates: Seq[TweetWithTopicProof], - tweetIdToTagsMap: Map[Long, Seq[CrMixerMetricTag]], - maxNumberOfCands: Int - ): Future[Option[Seq[RawCandidate]]] = { - val semanticCoreEntityIds = topicProofCandidates - .map(_.topicId) - .toSet - - TopicsUtil - .getLocalizedEntityMap(inputTarget, semanticCoreEntityIds, uttEntityHydrationStore) - .map { localizedEntityMap => - val rawCandidates = topicProofCandidates.collect { - case topicSocialProof: TweetWithTopicProof - if localizedEntityMap.contains(topicSocialProof.topicId) => - // Once we deprecate CR calls, we should replace this code to use the CrMixerMetricTag - val tags = tweetIdToTagsMap.get(topicSocialProof.tweetId).map { - _.flatMap { tag => MetricTag.get(tag.value) } - } - buildTopicTweetRawCandidate( - inputTarget, - topicSocialProof, - localizedEntityMap(topicSocialProof.topicId), - tags - ) - } - - val candResult = restrict( - maxNumberOfCands, - rawCandidates, - numOutOfMaximumDropped, - totalOutputRecs - ) - - Some(candResult) - } - } - - private def generateOONRawCandidate( - inputTarget: Target, - id: Long, - result: Option[TweetyPieResult], - tweetScoreMap: Map[Long, Double], - tweetIdToTagsMap: Map[Long, Seq[CrMixerMetricTag]] - ): Option[RawCandidate with TweetCandidate] = { - val tagsFromCR = tweetIdToTagsMap.get(id).map { _.flatMap { tag => MetricTag.get(tag.value) } } - val candidate = new RawCandidate with CrMixerCandidate with TopicCandidate with AlgorithmScore { - override val tweetId = id - override val target = inputTarget - override val tweetyPieResult = result - override val localizedUttEntity = None - override val semanticCoreEntityId = None - override def commonRecType = - getMediaBasedCRT( - CommonRecommendationType.TwistlyTweet, - CommonRecommendationType.TwistlyPhoto, - CommonRecommendationType.TwistlyVideo) - override def tagsCR = tagsFromCR - override def algorithmScore = tweetScoreMap.get(id) - override def algorithmCR = None - } - Some(candidate) - } - - private def restrict( - maxNumToReturn: Int, - candidates: Seq[RawCandidate], - numOutOfMaximumDropped: Stat, - totalOutputRecs: Stat - ): Seq[RawCandidate] = { - val newCandidates = candidates.take(maxNumToReturn) - val numDropped = candidates.length - newCandidates.length - numOutOfMaximumDropped.add(numDropped) - totalOutputRecs.add(newCandidates.size) - newCandidates - } - - private def buildCrMixerRequest( - target: Target, - countryCode: Option[String], - language: Option[String], - seenTweets: Seq[Long] - ): CrMixerTweetRequest = { - CrMixerTweetRequest( - clientContext = ClientContext( - userId = Some(target.targetId), - countryCode = countryCode, - languageCode = language - ), - product = Product.Notifications, - productContext = Some(ProductContext.NotificationsContext(NotificationsContext())), - excludedTweetIds = Some(seenTweets) - ) - } - - private def selectCandidatesToSendBasedOnSettings( - isRecommendationsEligible: Boolean, - isTopicsEligible: Boolean, - oonRawCandidates: Option[Seq[RawCandidate]], - topicTweetCandidates: Option[Seq[RawCandidate]] - ): Option[Seq[RawCandidate]] = { - if (isRecommendationsEligible && isTopicsEligible) { - Some(topicTweetCandidates.getOrElse(Seq.empty) ++ oonRawCandidates.getOrElse(Seq.empty)) - } else if (isRecommendationsEligible) { - oonRawCandidates - } else if (isTopicsEligible) { - topicTweetCandidates - } else None - } - - override def get(target: Target): Future[Option[Seq[RawCandidate]]] = { - Future - .join( - target.seenTweetIds, - target.countryCode, - target.inferredUserDeviceLanguage, - PushDeviceUtil.isTopicsEligible(target), - PushDeviceUtil.isRecommendationsEligible(target) - ).flatMap { - case (seenTweets, countryCode, language, isTopicsEligible, isRecommendationsEligible) => - val request = buildCrMixerRequest(target, countryCode, language, seenTweets) - crMixerTweetStore.getTweetRecommendations(request).flatMap { - case Some(response) => - totalInputRecs.incr(response.tweets.size) - totalRequests.incr() - AdaptorUtils - .getTweetyPieResults( - response.tweets.map(_.tweetId).toSet, - tweetyPieStore).flatMap { tweetyPieResultMap => - filterOutInNetworkTweets( - target, - filterOutReplyTweet(tweetyPieResultMap.toMap, nonReplyTweetsCounter), - edgeStore, - numOfValidAuthors).flatMap { - outNetworkTweetsWithId: Seq[(Long, TweetyPieResult)] => - totalOutNetworkRecs.incr(outNetworkTweetsWithId.size) - totalInNetworkRecs.incr(response.tweets.size - outNetworkTweetsWithId.size) - val outNetworkTweets: Seq[TweetyPieResult] = outNetworkTweetsWithId.map { - case (_, tweetyPieResult) => tweetyPieResult - } - - val tweetIdToTagsMap = response.tweets.map { tweet => - tweet.tweetId -> tweet.metricTags.getOrElse(Seq.empty) - }.toMap - - val tweetScoreMap = response.tweets.map { tweet => - tweet.tweetId -> tweet.score - }.toMap - - val maxNumOfCandidates = - target.params(PushFeatureSwitchParams.NumberOfMaxCrMixerCandidatesParam) - - val oonRawCandidates = - buildOONRawCandidates( - target, - outNetworkTweets, - tweetScoreMap, - tweetIdToTagsMap, - maxNumOfCandidates) - - TopicsUtil - .getTopicSocialProofs( - target, - outNetworkTweets, - topicSocialProofServiceStore, - edgeStore, - PushFeatureSwitchParams.TopicProofTweetCandidatesTopicScoreThreshold).flatMap { - tweetsWithTopicProof => - buildTopicTweetRawCandidates( - target, - tweetsWithTopicProof, - tweetIdToTagsMap, - maxNumOfCandidates) - }.map { topicTweetCandidates => - selectCandidatesToSendBasedOnSettings( - isRecommendationsEligible, - isTopicsEligible, - oonRawCandidates, - topicTweetCandidates) - } - } - } - case _ => Future.None - } - } - } - - /** - * For a user to be available the following news to happen - */ - override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { - Future - .join( - PushDeviceUtil.isRecommendationsEligible(target), - PushDeviceUtil.isTopicsEligible(target) - ).map { - case (isRecommendationsEligible, isTopicsEligible) => - (isRecommendationsEligible || isTopicsEligible) && - target.params(PushParams.ContentRecommenderMixerAdaptorDecider) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/EarlyBirdFirstDegreeCandidateAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/EarlyBirdFirstDegreeCandidateAdaptor.scala deleted file mode 100644 index ab631841a..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/EarlyBirdFirstDegreeCandidateAdaptor.scala +++ /dev/null @@ -1,293 +0,0 @@ -package com.twitter.frigate.pushservice.adaptor - -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.candidate._ -import com.twitter.frigate.common.predicate.CommonOutNetworkTweetCandidatesSourcePredicates.filterOutReplyTweet -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.hermit.store.tweetypie.UserTweet -import com.twitter.recos.recos_common.thriftscala.SocialProofType -import com.twitter.search.common.features.thriftscala.ThriftSearchResultFeatures -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi.Param -import com.twitter.util.Future -import com.twitter.util.Time -import scala.collection.Map - -case class EarlyBirdFirstDegreeCandidateAdaptor( - earlyBirdFirstDegreeCandidates: CandidateSource[ - EarlybirdCandidateSource.Query, - EarlybirdCandidate - ], - tweetyPieStore: ReadableStore[Long, TweetyPieResult], - tweetyPieStoreNoVF: ReadableStore[Long, TweetyPieResult], - userTweetTweetyPieStore: ReadableStore[UserTweet, TweetyPieResult], - maxResultsParam: Param[Int], - globalStats: StatsReceiver) - extends CandidateSource[Target, RawCandidate] - with CandidateSourceEligible[Target, RawCandidate] { - - type EBCandidate = EarlybirdCandidate with TweetDetails - private val stats = globalStats.scope("EarlyBirdFirstDegreeAdaptor") - private val earlyBirdCandsStat: Stat = stats.stat("early_bird_cands_dist") - private val emptyEarlyBirdCands = stats.counter("empty_early_bird_candidates") - private val seedSetEmpty = stats.counter("empty_seedset") - private val seenTweetsStat = stats.stat("filtered_by_seen_tweets") - private val emptyTweetyPieResult = stats.stat("empty_tweetypie_result") - private val nonReplyTweetsCounter = stats.counter("non_reply_tweets") - private val enableRetweets = stats.counter("enable_retweets") - private val f1withoutSocialContexts = stats.counter("f1_without_social_context") - private val userTweetTweetyPieStoreCounter = stats.counter("user_tweet_tweetypie_store") - - override val name: String = earlyBirdFirstDegreeCandidates.name - - private def getAllSocialContextActions( - socialProofTypes: Seq[(SocialProofType, Seq[Long])] - ): Seq[SocialContextAction] = { - socialProofTypes.flatMap { - case (SocialProofType.Favorite, scIds) => - scIds.map { scId => - SocialContextAction( - scId, - Time.now.inMilliseconds, - socialContextActionType = Some(SocialContextActionType.Favorite) - ) - } - case (SocialProofType.Retweet, scIds) => - scIds.map { scId => - SocialContextAction( - scId, - Time.now.inMilliseconds, - socialContextActionType = Some(SocialContextActionType.Retweet) - ) - } - case (SocialProofType.Reply, scIds) => - scIds.map { scId => - SocialContextAction( - scId, - Time.now.inMilliseconds, - socialContextActionType = Some(SocialContextActionType.Reply) - ) - } - case (SocialProofType.Tweet, scIds) => - scIds.map { scId => - SocialContextAction( - scId, - Time.now.inMilliseconds, - socialContextActionType = Some(SocialContextActionType.Tweet) - ) - } - case _ => Nil - } - } - - private def generateRetweetCandidate( - inputTarget: Target, - candidate: EBCandidate, - scIds: Seq[Long], - socialProofTypes: Seq[(SocialProofType, Seq[Long])] - ): RawCandidate = { - val scActions = scIds.map { scId => SocialContextAction(scId, Time.now.inMilliseconds) } - new RawCandidate with TweetRetweetCandidate with EarlybirdTweetFeatures { - override val socialContextActions = scActions - override val socialContextAllTypeActions = getAllSocialContextActions(socialProofTypes) - override val tweetId = candidate.tweetId - override val target = inputTarget - override val tweetyPieResult = candidate.tweetyPieResult - override val features = candidate.features - } - } - - private def generateF1CandidateWithoutSocialContext( - inputTarget: Target, - candidate: EBCandidate - ): RawCandidate = { - f1withoutSocialContexts.incr() - new RawCandidate with F1FirstDegree with EarlybirdTweetFeatures { - override val tweetId = candidate.tweetId - override val target = inputTarget - override val tweetyPieResult = candidate.tweetyPieResult - override val features = candidate.features - } - } - - private def generateEarlyBirdCandidate( - id: Long, - result: Option[TweetyPieResult], - ebFeatures: Option[ThriftSearchResultFeatures] - ): EBCandidate = { - new EarlybirdCandidate with TweetDetails { - override val tweetyPieResult: Option[TweetyPieResult] = result - override val tweetId: Long = id - override val features: Option[ThriftSearchResultFeatures] = ebFeatures - } - } - - private def filterOutSeenTweets(seenTweetIds: Seq[Long], inputTweetIds: Seq[Long]): Seq[Long] = { - inputTweetIds.filterNot(seenTweetIds.contains) - } - - private def filterInvalidTweets( - tweetIds: Seq[Long], - target: Target - ): Future[Seq[(Long, TweetyPieResult)]] = { - - val resMap = { - if (target.params(PushFeatureSwitchParams.EnableF1FromProtectedTweetAuthors)) { - userTweetTweetyPieStoreCounter.incr() - val keys = tweetIds.map { tweetId => - UserTweet(tweetId, Some(target.targetId)) - } - - userTweetTweetyPieStore - .multiGet(keys.toSet).map { - case (userTweet, resultFut) => - userTweet.tweetId -> resultFut - }.toMap - } else { - (target.params(PushFeatureSwitchParams.EnableVFInTweetypie) match { - case true => tweetyPieStore - case false => tweetyPieStoreNoVF - }).multiGet(tweetIds.toSet) - } - } - Future.collect(resMap).map { tweetyPieResultMap => - val cands = filterOutReplyTweet(tweetyPieResultMap, nonReplyTweetsCounter).collect { - case (id: Long, Some(result)) => - id -> result - } - - emptyTweetyPieResult.add(tweetyPieResultMap.size - cands.size) - cands.toSeq - } - } - - private def getEBRetweetCandidates( - inputTarget: Target, - retweets: Seq[(Long, TweetyPieResult)] - ): Seq[RawCandidate] = { - retweets.flatMap { - case (_, tweetypieResult) => - tweetypieResult.tweet.coreData.flatMap { coreData => - tweetypieResult.sourceTweet.map { sourceTweet => - val tweetId = sourceTweet.id - val scId = coreData.userId - val socialProofTypes = Seq((SocialProofType.Retweet, Seq(scId))) - val candidate = generateEarlyBirdCandidate( - tweetId, - Some(TweetyPieResult(sourceTweet, None, None)), - None - ) - generateRetweetCandidate( - inputTarget, - candidate, - Seq(scId), - socialProofTypes - ) - } - } - } - } - - private def getEBFirstDegreeCands( - tweets: Seq[(Long, TweetyPieResult)], - ebTweetIdMap: Map[Long, Option[ThriftSearchResultFeatures]] - ): Seq[EBCandidate] = { - tweets.map { - case (id, tweetypieResult) => - val features = ebTweetIdMap.getOrElse(id, None) - generateEarlyBirdCandidate(id, Some(tweetypieResult), features) - } - } - - /** - * Returns a combination of raw candidates made of: f1 recs, topic social proof recs, sc recs and retweet candidates - */ - def buildRawCandidates( - inputTarget: Target, - firstDegreeCandidates: Seq[EBCandidate], - retweetCandidates: Seq[RawCandidate] - ): Seq[RawCandidate] = { - val hydratedF1Recs = - firstDegreeCandidates.map(generateF1CandidateWithoutSocialContext(inputTarget, _)) - hydratedF1Recs ++ retweetCandidates - } - - override def get(inputTarget: Target): Future[Option[Seq[RawCandidate]]] = { - inputTarget.seedsWithWeight.flatMap { seedsetOpt => - val seedsetMap = seedsetOpt.getOrElse(Map.empty) - - if (seedsetMap.isEmpty) { - seedSetEmpty.incr() - Future.None - } else { - val maxResultsToReturn = inputTarget.params(maxResultsParam) - val maxTweetAge = inputTarget.params(PushFeatureSwitchParams.F1CandidateMaxTweetAgeParam) - val earlybirdQuery = EarlybirdCandidateSource.Query( - maxNumResultsToReturn = maxResultsToReturn, - seedset = seedsetMap, - maxConsecutiveResultsByTheSameUser = Some(1), - maxTweetAge = maxTweetAge, - disableTimelinesMLModel = false, - searcherId = Some(inputTarget.targetId), - isProtectTweetsEnabled = - inputTarget.params(PushFeatureSwitchParams.EnableF1FromProtectedTweetAuthors), - followedUserIds = Some(seedsetMap.keySet.toSeq) - ) - - Future - .join(inputTarget.seenTweetIds, earlyBirdFirstDegreeCandidates.get(earlybirdQuery)) - .flatMap { - case (seenTweetIds, Some(candidates)) => - earlyBirdCandsStat.add(candidates.size) - - val ebTweetIdMap = candidates.map { cand => cand.tweetId -> cand.features }.toMap - - val ebTweetIds = ebTweetIdMap.keys.toSeq - - val tweetIds = filterOutSeenTweets(seenTweetIds, ebTweetIds) - seenTweetsStat.add(ebTweetIds.size - tweetIds.size) - - filterInvalidTweets(tweetIds, inputTarget) - .map { validTweets => - val (retweets, tweets) = validTweets.partition { - case (_, tweetypieResult) => - tweetypieResult.sourceTweet.isDefined - } - - val firstDegreeCandidates = getEBFirstDegreeCands(tweets, ebTweetIdMap) - - val retweetCandidates = { - if (inputTarget.params(PushParams.EarlyBirdSCBasedCandidatesParam) && - inputTarget.params(PushParams.MRTweetRetweetRecsParam)) { - enableRetweets.incr() - getEBRetweetCandidates(inputTarget, retweets) - } else Nil - } - - Some( - buildRawCandidates( - inputTarget, - firstDegreeCandidates, - retweetCandidates - )) - } - - case _ => - emptyEarlyBirdCands.incr() - Future.None - } - } - } - } - - override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { - PushDeviceUtil.isRecommendationsEligible(target) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ExploreVideoTweetCandidateAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ExploreVideoTweetCandidateAdaptor.scala deleted file mode 100644 index 345fdbd3c..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ExploreVideoTweetCandidateAdaptor.scala +++ /dev/null @@ -1,120 +0,0 @@ -package com.twitter.frigate.pushservice.adaptor - -import com.twitter.explore_ranker.thriftscala.ExploreRankerProductResponse -import com.twitter.explore_ranker.thriftscala.ExploreRankerRequest -import com.twitter.explore_ranker.thriftscala.ExploreRankerResponse -import com.twitter.explore_ranker.thriftscala.ExploreRecommendation -import com.twitter.explore_ranker.thriftscala.ImmersiveRecsResponse -import com.twitter.explore_ranker.thriftscala.ImmersiveRecsResult -import com.twitter.explore_ranker.thriftscala.NotificationsVideoRecs -import com.twitter.explore_ranker.thriftscala.Product -import com.twitter.explore_ranker.thriftscala.ProductContext -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateSource -import com.twitter.frigate.common.base.CandidateSourceEligible -import com.twitter.frigate.common.base.OutOfNetworkTweetCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.util.AdaptorUtils -import com.twitter.frigate.pushservice.util.MediaCRT -import com.twitter.frigate.pushservice.util.PushAdaptorUtil -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.product_mixer.core.thriftscala.ClientContext -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -case class ExploreVideoTweetCandidateAdaptor( - exploreRankerStore: ReadableStore[ExploreRankerRequest, ExploreRankerResponse], - tweetyPieStore: ReadableStore[Long, TweetyPieResult], - globalStats: StatsReceiver) - extends CandidateSource[Target, RawCandidate] - with CandidateSourceEligible[Target, RawCandidate] { - - override def name: String = this.getClass.getSimpleName - private[this] val stats = globalStats.scope("ExploreVideoTweetCandidateAdaptor") - private[this] val totalInputRecs = stats.stat("input_recs") - private[this] val totalRequests = stats.counter("total_requests") - private[this] val totalEmptyResponse = stats.counter("total_empty_response") - - private def buildExploreRankerRequest( - target: Target, - countryCode: Option[String], - language: Option[String], - ): ExploreRankerRequest = { - ExploreRankerRequest( - clientContext = ClientContext( - userId = Some(target.targetId), - countryCode = countryCode, - languageCode = language, - ), - product = Product.NotificationsVideoRecs, - productContext = Some(ProductContext.NotificationsVideoRecs(NotificationsVideoRecs())), - maxResults = Some(target.params(PushFeatureSwitchParams.MaxExploreVideoTweets)) - ) - } - - override def get(target: Target): Future[Option[Seq[RawCandidate]]] = { - Future - .join( - target.countryCode, - target.inferredUserDeviceLanguage - ).flatMap { - case (countryCode, language) => - val request = buildExploreRankerRequest(target, countryCode, language) - exploreRankerStore.get(request).flatMap { - case Some(response) => - val exploreResonseTweetIds = response match { - case ExploreRankerResponse(ExploreRankerProductResponse - .ImmersiveRecsResponse(ImmersiveRecsResponse(immersiveRecsResult))) => - immersiveRecsResult.collect { - case ImmersiveRecsResult(ExploreRecommendation - .ExploreTweetRecommendation(exploreTweetRecommendation)) => - exploreTweetRecommendation.tweetId - } - case _ => - Seq.empty - } - - totalInputRecs.add(exploreResonseTweetIds.size) - totalRequests.incr() - AdaptorUtils - .getTweetyPieResults(exploreResonseTweetIds.toSet, tweetyPieStore).map { - tweetyPieResultMap => - val candidates = tweetyPieResultMap.values.flatten - .map(buildVideoRawCandidates(target, _)) - Some(candidates.toSeq) - } - case _ => - totalEmptyResponse.incr() - Future.None - } - case _ => - Future.None - } - } - - override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { - PushDeviceUtil.isRecommendationsEligible(target).map { userRecommendationsEligible => - userRecommendationsEligible && target.params(PushFeatureSwitchParams.EnableExploreVideoTweets) - } - } - private def buildVideoRawCandidates( - target: Target, - tweetyPieResult: TweetyPieResult - ): RawCandidate with OutOfNetworkTweetCandidate = { - PushAdaptorUtil.generateOutOfNetworkTweetCandidates( - inputTarget = target, - id = tweetyPieResult.tweet.id, - mediaCRT = MediaCRT( - CommonRecommendationType.ExploreVideoTweet, - CommonRecommendationType.ExploreVideoTweet, - CommonRecommendationType.ExploreVideoTweet - ), - result = Some(tweetyPieResult), - localizedEntity = None - ) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/FRSTweetCandidateAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/FRSTweetCandidateAdaptor.scala deleted file mode 100644 index 49610c645..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/FRSTweetCandidateAdaptor.scala +++ /dev/null @@ -1,272 +0,0 @@ -package com.twitter.frigate.pushservice.adaptor - -import com.twitter.cr_mixer.thriftscala.FrsTweetRequest -import com.twitter.cr_mixer.thriftscala.NotificationsContext -import com.twitter.cr_mixer.thriftscala.Product -import com.twitter.cr_mixer.thriftscala.ProductContext -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateSource -import com.twitter.frigate.common.base.CandidateSourceEligible -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.predicate.CommonOutNetworkTweetCandidatesSourcePredicates.filterOutReplyTweet -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.store.CrMixerTweetStore -import com.twitter.frigate.pushservice.store.UttEntityHydrationStore -import com.twitter.frigate.pushservice.util.MediaCRT -import com.twitter.frigate.pushservice.util.PushAdaptorUtil -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.frigate.pushservice.util.TopicsUtil -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.hermit.constants.AlgorithmFeedbackTokens -import com.twitter.hermit.model.Algorithm.Algorithm -import com.twitter.hermit.model.Algorithm.CrowdSearchAccounts -import com.twitter.hermit.model.Algorithm.ForwardEmailBook -import com.twitter.hermit.model.Algorithm.ForwardPhoneBook -import com.twitter.hermit.model.Algorithm.ReverseEmailBookIbis -import com.twitter.hermit.model.Algorithm.ReversePhoneBook -import com.twitter.hermit.store.tweetypie.UserTweet -import com.twitter.product_mixer.core.thriftscala.ClientContext -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.tsp.thriftscala.TopicSocialProofRequest -import com.twitter.tsp.thriftscala.TopicSocialProofResponse -import com.twitter.util.Future - -object FRSAlgorithmFeedbackTokenUtil { - private val crtsByAlgoToken = Map( - getAlgorithmToken(ReverseEmailBookIbis) -> CommonRecommendationType.ReverseAddressbookTweet, - getAlgorithmToken(ReversePhoneBook) -> CommonRecommendationType.ReverseAddressbookTweet, - getAlgorithmToken(ForwardEmailBook) -> CommonRecommendationType.ForwardAddressbookTweet, - getAlgorithmToken(ForwardPhoneBook) -> CommonRecommendationType.ForwardAddressbookTweet, - getAlgorithmToken(CrowdSearchAccounts) -> CommonRecommendationType.CrowdSearchTweet - ) - - def getAlgorithmToken(algorithm: Algorithm): Int = { - AlgorithmFeedbackTokens.AlgorithmToFeedbackTokenMap(algorithm) - } - - def getCRTForAlgoToken(algorithmToken: Int): Option[CommonRecommendationType] = { - crtsByAlgoToken.get(algorithmToken) - } -} - -case class FRSTweetCandidateAdaptor( - crMixerTweetStore: CrMixerTweetStore, - tweetyPieStore: ReadableStore[Long, TweetyPieResult], - tweetyPieStoreNoVF: ReadableStore[Long, TweetyPieResult], - userTweetTweetyPieStore: ReadableStore[UserTweet, TweetyPieResult], - uttEntityHydrationStore: UttEntityHydrationStore, - topicSocialProofServiceStore: ReadableStore[TopicSocialProofRequest, TopicSocialProofResponse], - globalStats: StatsReceiver) - extends CandidateSource[Target, RawCandidate] - with CandidateSourceEligible[Target, RawCandidate] { - - private val stats = globalStats.scope(this.getClass.getSimpleName) - private val crtStats = stats.scope("CandidateDistribution") - private val totalRequests = stats.counter("total_requests") - - // Candidate Distribution stats - private val reverseAddressbookCounter = crtStats.counter("reverse_addressbook") - private val forwardAddressbookCounter = crtStats.counter("forward_addressbook") - private val frsTweetCounter = crtStats.counter("frs_tweet") - private val nonReplyTweetsCounter = stats.counter("non_reply_tweets") - private val crtToCounterMapping: Map[CommonRecommendationType, Counter] = Map( - CommonRecommendationType.ReverseAddressbookTweet -> reverseAddressbookCounter, - CommonRecommendationType.ForwardAddressbookTweet -> forwardAddressbookCounter, - CommonRecommendationType.FrsTweet -> frsTweetCounter - ) - - private val emptyTweetyPieResult = stats.stat("empty_tweetypie_result") - - private[this] val numberReturnedCandidates = stats.stat("returned_candidates_from_earlybird") - private[this] val numberCandidateWithTopic: Counter = stats.counter("num_can_with_topic") - private[this] val numberCandidateWithoutTopic: Counter = stats.counter("num_can_without_topic") - - private val userTweetTweetyPieStoreCounter = stats.counter("user_tweet_tweetypie_store") - - override val name: String = this.getClass.getSimpleName - - private def filterInvalidTweets( - tweetIds: Seq[Long], - target: Target - ): Future[Map[Long, TweetyPieResult]] = { - val resMap = { - if (target.params(PushFeatureSwitchParams.EnableF1FromProtectedTweetAuthors)) { - userTweetTweetyPieStoreCounter.incr() - val keys = tweetIds.map { tweetId => - UserTweet(tweetId, Some(target.targetId)) - } - userTweetTweetyPieStore - .multiGet(keys.toSet).map { - case (userTweet, resultFut) => - userTweet.tweetId -> resultFut - }.toMap - } else { - (if (target.params(PushFeatureSwitchParams.EnableVFInTweetypie)) { - tweetyPieStore - } else { - tweetyPieStoreNoVF - }).multiGet(tweetIds.toSet) - } - } - - Future.collect(resMap).map { tweetyPieResultMap => - // Filter out replies and generate earlybird candidates only for non-empty tweetypie result - val cands = filterOutReplyTweet(tweetyPieResultMap, nonReplyTweetsCounter).collect { - case (id: Long, Some(result)) => - id -> result - } - - emptyTweetyPieResult.add(tweetyPieResultMap.size - cands.size) - cands - } - } - - private def buildRawCandidates( - target: Target, - ebCandidates: Seq[FRSTweetCandidate] - ): Future[Option[Seq[RawCandidate with TweetCandidate]]] = { - - val enableTopic = target.params(PushFeatureSwitchParams.EnableFrsTweetCandidatesTopicAnnotation) - val topicScoreThre = - target.params(PushFeatureSwitchParams.FrsTweetCandidatesTopicScoreThreshold) - - val ebTweets = ebCandidates.map { ebCandidate => - ebCandidate.tweetId -> ebCandidate.tweetyPieResult - }.toMap - - val tweetIdLocalizedEntityMapFut = TopicsUtil.getTweetIdLocalizedEntityMap( - target, - ebTweets, - uttEntityHydrationStore, - topicSocialProofServiceStore, - enableTopic, - topicScoreThre - ) - - Future.join(target.deviceInfo, tweetIdLocalizedEntityMapFut).map { - case (Some(deviceInfo), tweetIdLocalizedEntityMap) => - val candidates = ebCandidates - .map { ebCandidate => - val crt = ebCandidate.commonRecType - crtToCounterMapping.get(crt).foreach(_.incr()) - - val tweetId = ebCandidate.tweetId - val localizedEntityOpt = { - if (tweetIdLocalizedEntityMap - .contains(tweetId) && tweetIdLocalizedEntityMap.contains( - tweetId) && deviceInfo.isTopicsEligible) { - tweetIdLocalizedEntityMap(tweetId) - } else { - None - } - } - - PushAdaptorUtil.generateOutOfNetworkTweetCandidates( - inputTarget = target, - id = ebCandidate.tweetId, - mediaCRT = MediaCRT( - crt, - crt, - crt - ), - result = ebCandidate.tweetyPieResult, - localizedEntity = localizedEntityOpt) - }.filter { candidate => - // If user only has the topic setting enabled, filter out all non-topic cands - deviceInfo.isRecommendationsEligible || (deviceInfo.isTopicsEligible && candidate.semanticCoreEntityId.nonEmpty) - } - - candidates.map { candidate => - if (candidate.semanticCoreEntityId.nonEmpty) { - numberCandidateWithTopic.incr() - } else { - numberCandidateWithoutTopic.incr() - } - } - - numberReturnedCandidates.add(candidates.length) - Some(candidates) - case _ => Some(Seq.empty) - } - } - - def getTweetCandidatesFromCrMixer( - inputTarget: Target, - showAllResultsFromFrs: Boolean, - ): Future[Option[Seq[RawCandidate with TweetCandidate]]] = { - Future - .join( - inputTarget.seenTweetIds, - inputTarget.pushRecItems, - inputTarget.countryCode, - inputTarget.targetLanguage).flatMap { - case (seenTweetIds, pastRecItems, countryCode, language) => - val pastUserRecs = pastRecItems.userIds.toSeq - val request = FrsTweetRequest( - clientContext = ClientContext( - userId = Some(inputTarget.targetId), - countryCode = countryCode, - languageCode = language - ), - product = Product.Notifications, - productContext = Some(ProductContext.NotificationsContext(NotificationsContext())), - excludedUserIds = Some(pastUserRecs), - excludedTweetIds = Some(seenTweetIds) - ) - crMixerTweetStore.getFRSTweetCandidates(request).flatMap { - case Some(response) => - val tweetIds = response.tweets.map(_.tweetId) - val validTweets = filterInvalidTweets(tweetIds, inputTarget) - validTweets.flatMap { tweetypieMap => - val ebCandidates = response.tweets - .map { frsTweet => - val candidateTweetId = frsTweet.tweetId - val resultFromTweetyPie = tweetypieMap.get(candidateTweetId) - new FRSTweetCandidate { - override val tweetId = candidateTweetId - override val features = None - override val tweetyPieResult = resultFromTweetyPie - override val feedbackToken = frsTweet.frsPrimarySource - override val commonRecType: CommonRecommendationType = feedbackToken - .flatMap(token => - FRSAlgorithmFeedbackTokenUtil.getCRTForAlgoToken(token)).getOrElse( - CommonRecommendationType.FrsTweet) - } - }.filter { ebCandidate => - showAllResultsFromFrs || ebCandidate.commonRecType == CommonRecommendationType.ReverseAddressbookTweet - } - - numberReturnedCandidates.add(ebCandidates.length) - buildRawCandidates( - inputTarget, - ebCandidates - ) - } - case _ => Future.None - } - } - } - - override def get(inputTarget: Target): Future[Option[Seq[RawCandidate with TweetCandidate]]] = { - totalRequests.incr() - val enableResultsFromFrs = - inputTarget.params(PushFeatureSwitchParams.EnableResultFromFrsCandidates) - getTweetCandidatesFromCrMixer(inputTarget, enableResultsFromFrs) - } - - override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { - lazy val enableFrsCandidates = target.params(PushFeatureSwitchParams.EnableFrsCandidates) - PushDeviceUtil.isRecommendationsEligible(target).flatMap { isEnabledForRecosSetting => - PushDeviceUtil.isTopicsEligible(target).map { topicSettingEnabled => - val isEnabledForTopics = - topicSettingEnabled && target.params( - PushFeatureSwitchParams.EnableFrsTweetCandidatesTopicSetting) - (isEnabledForRecosSetting || isEnabledForTopics) && enableFrsCandidates - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/GenericCandidateAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/GenericCandidateAdaptor.scala deleted file mode 100644 index 24d0cb64a..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/GenericCandidateAdaptor.scala +++ /dev/null @@ -1,107 +0,0 @@ -package com.twitter.frigate.pushservice.adaptor - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.candidate._ -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -object GenericCandidates { - type Target = - TargetUser - with UserDetails - with TargetDecider - with TargetABDecider - with TweetImpressionHistory - with HTLVisitHistory - with MaxTweetAge - with NewUserDetails - with FrigateHistory - with TargetWithSeedUsers -} - -case class GenericCandidateAdaptor( - genericCandidates: CandidateSource[GenericCandidates.Target, Candidate], - tweetyPieStore: ReadableStore[Long, TweetyPieResult], - tweetyPieStoreNoVF: ReadableStore[Long, TweetyPieResult], - stats: StatsReceiver) - extends CandidateSource[Target, RawCandidate] - with CandidateSourceEligible[Target, RawCandidate] { - - override val name: String = genericCandidates.name - - private def generateTweetFavCandidate( - _target: Target, - _tweetId: Long, - _socialContextActions: Seq[SocialContextAction], - socialContextActionsAllTypes: Seq[SocialContextAction], - _tweetyPieResult: Option[TweetyPieResult] - ): RawCandidate = { - new RawCandidate with TweetFavoriteCandidate { - override val socialContextActions = _socialContextActions - override val socialContextAllTypeActions = - socialContextActionsAllTypes - val tweetId = _tweetId - val target = _target - val tweetyPieResult = _tweetyPieResult - } - } - - private def generateTweetRetweetCandidate( - _target: Target, - _tweetId: Long, - _socialContextActions: Seq[SocialContextAction], - socialContextActionsAllTypes: Seq[SocialContextAction], - _tweetyPieResult: Option[TweetyPieResult] - ): RawCandidate = { - new RawCandidate with TweetRetweetCandidate { - override val socialContextActions = _socialContextActions - override val socialContextAllTypeActions = socialContextActionsAllTypes - val tweetId = _tweetId - val target = _target - val tweetyPieResult = _tweetyPieResult - } - } - - override def get(inputTarget: Target): Future[Option[Seq[RawCandidate]]] = { - genericCandidates.get(inputTarget).map { candidatesOpt => - candidatesOpt - .map { candidates => - val candidatesSeq = - candidates.collect { - case tweetRetweet: TweetRetweetCandidate - if inputTarget.params(PushParams.MRTweetRetweetRecsParam) => - generateTweetRetweetCandidate( - inputTarget, - tweetRetweet.tweetId, - tweetRetweet.socialContextActions, - tweetRetweet.socialContextAllTypeActions, - tweetRetweet.tweetyPieResult) - case tweetFavorite: TweetFavoriteCandidate - if inputTarget.params(PushParams.MRTweetFavRecsParam) => - generateTweetFavCandidate( - inputTarget, - tweetFavorite.tweetId, - tweetFavorite.socialContextActions, - tweetFavorite.socialContextAllTypeActions, - tweetFavorite.tweetyPieResult) - } - candidatesSeq.foreach { candidate => - stats.counter(s"${candidate.commonRecType}_count").incr() - } - candidatesSeq - } - } - } - - override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { - PushDeviceUtil.isRecommendationsEligible(target).map { isAvailable => - isAvailable && target.params(PushParams.GenericCandidateAdaptorDecider) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/HighQualityTweetsAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/HighQualityTweetsAdaptor.scala deleted file mode 100644 index 37d11535f..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/HighQualityTweetsAdaptor.scala +++ /dev/null @@ -1,280 +0,0 @@ -package com.twitter.frigate.pushservice.adaptor - -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateSource -import com.twitter.frigate.common.base.CandidateSourceEligible -import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.HighQualityCandidateGroupEnum -import com.twitter.frigate.pushservice.params.HighQualityCandidateGroupEnum._ -import com.twitter.frigate.pushservice.params.PushConstants.targetUserAgeFeatureName -import com.twitter.frigate.pushservice.params.PushConstants.targetUserPreferredLanguage -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} -import com.twitter.frigate.pushservice.predicate.TargetPredicates -import com.twitter.frigate.pushservice.util.MediaCRT -import com.twitter.frigate.pushservice.util.PushAdaptorUtil -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.frigate.pushservice.util.TopicsUtil -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.interests.thriftscala.InterestId.SemanticCore -import com.twitter.interests.thriftscala.UserInterests -import com.twitter.language.normalization.UserDisplayLanguage -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweet -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweets -import com.twitter.util.Future - -object HighQualityTweetsHelper { - def getFollowedTopics( - target: Target, - interestsWithLookupContextStore: ReadableStore[ - InterestsLookupRequestWithContext, - UserInterests - ], - followedTopicsStats: Stat - ): Future[Seq[Long]] = { - TopicsUtil - .getTopicsFollowedByUser(target, interestsWithLookupContextStore, followedTopicsStats).map { - userInterestsOpt => - val userInterests = userInterestsOpt.getOrElse(Seq.empty) - val extractedTopicIds = userInterests.flatMap { - _.interestId match { - case SemanticCore(semanticCore) => Some(semanticCore.id) - case _ => None - } - } - extractedTopicIds - } - } - - def getTripQueries( - target: Target, - enabledGroups: Set[HighQualityCandidateGroupEnum.Value], - interestsWithLookupContextStore: ReadableStore[ - InterestsLookupRequestWithContext, - UserInterests - ], - sourceIds: Seq[String], - stat: Stat - ): Future[Set[TripDomain]] = { - - val followedTopicIdsSetFut: Future[Set[Long]] = if (enabledGroups.contains(Topic)) { - getFollowedTopics(target, interestsWithLookupContextStore, stat).map(topicIds => - topicIds.toSet) - } else { - Future.value(Set.empty) - } - - Future - .join(target.featureMap, target.inferredUserDeviceLanguage, followedTopicIdsSetFut).map { - case ( - featureMap, - deviceLanguageOpt, - followedTopicIds - ) => - val ageBucketOpt = if (enabledGroups.contains(AgeBucket)) { - featureMap.categoricalFeatures.get(targetUserAgeFeatureName) - } else { - None - } - - val languageOptions: Set[Option[String]] = if (enabledGroups.contains(Language)) { - val userPreferredLanguages = featureMap.sparseBinaryFeatures - .getOrElse(targetUserPreferredLanguage, Set.empty[String]) - if (userPreferredLanguages.nonEmpty) { - userPreferredLanguages.map(lang => Some(UserDisplayLanguage.toTweetLanguage(lang))) - } else { - Set(deviceLanguageOpt.map(UserDisplayLanguage.toTweetLanguage)) - } - } else Set(None) - - val followedTopicOptions: Set[Option[Long]] = if (followedTopicIds.nonEmpty) { - followedTopicIds.map(topic => Some(topic)) - } else Set(None) - - val tripQueries = followedTopicOptions.flatMap { topicOption => - languageOptions.flatMap { languageOption => - sourceIds.map { sourceId => - TripDomain( - sourceId = sourceId, - language = languageOption, - placeId = None, - topicId = topicOption, - gender = None, - ageBucket = ageBucketOpt - ) - } - } - } - - tripQueries - } - } -} - -case class HighQualityTweetsAdaptor( - tripTweetCandidateStore: ReadableStore[TripDomain, TripTweets], - interestsWithLookupContextStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests], - tweetyPieStore: ReadableStore[Long, TweetyPieResult], - tweetyPieStoreNoVF: ReadableStore[Long, TweetyPieResult], - globalStats: StatsReceiver) - extends CandidateSource[Target, RawCandidate] - with CandidateSourceEligible[Target, RawCandidate] { - - override def name: String = this.getClass.getSimpleName - - private val stats = globalStats.scope("HighQualityCandidateAdaptor") - private val followedTopicsStats = stats.stat("followed_topics") - private val missingResponseCounter = stats.counter("missing_respond_counter") - private val crtFatigueCounter = stats.counter("fatigue_by_crt") - private val fallbackRequestsCounter = stats.counter("fallback_requests") - - override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { - PushDeviceUtil.isRecommendationsEligible(target).map { - _ && target.params(FS.HighQualityCandidatesEnableCandidateSource) - } - } - - private val highQualityCandidateFrequencyPredicate = { - TargetPredicates - .pushRecTypeFatiguePredicate( - CommonRecommendationType.TripHqTweet, - FS.HighQualityTweetsPushInterval, - FS.MaxHighQualityTweetsPushGivenInterval, - stats - ) - } - - private def getTripCandidatesStrato( - target: Target - ): Future[Map[Long, Set[TripDomain]]] = { - val tripQueriesF: Future[Set[TripDomain]] = HighQualityTweetsHelper.getTripQueries( - target = target, - enabledGroups = target.params(FS.HighQualityCandidatesEnableGroups).toSet, - interestsWithLookupContextStore = interestsWithLookupContextStore, - sourceIds = target.params(FS.TripTweetCandidateSourceIds), - stat = followedTopicsStats - ) - - lazy val fallbackTripQueriesFut: Future[Set[TripDomain]] = - if (target.params(FS.HighQualityCandidatesEnableFallback)) - HighQualityTweetsHelper.getTripQueries( - target = target, - enabledGroups = target.params(FS.HighQualityCandidatesFallbackEnabledGroups).toSet, - interestsWithLookupContextStore = interestsWithLookupContextStore, - sourceIds = target.params(FS.HighQualityCandidatesFallbackSourceIds), - stat = followedTopicsStats - ) - else Future.value(Set.empty) - - val initialTweetsFut: Future[Map[TripDomain, Seq[TripTweet]]] = tripQueriesF.flatMap { - tripQueries => getTripTweetsByDomains(tripQueries) - } - - val tweetsByDomainFut: Future[Map[TripDomain, Seq[TripTweet]]] = - if (target.params(FS.HighQualityCandidatesEnableFallback)) { - initialTweetsFut.flatMap { candidates => - val minCandidatesForFallback: Int = - target.params(FS.HighQualityCandidatesMinNumOfCandidatesToFallback) - val validCandidates = candidates.filter(_._2.size >= minCandidatesForFallback) - - if (validCandidates.nonEmpty) { - Future.value(validCandidates) - } else { - fallbackTripQueriesFut.flatMap { fallbackTripDomains => - fallbackRequestsCounter.incr(fallbackTripDomains.size) - getTripTweetsByDomains(fallbackTripDomains) - } - } - } - } else { - initialTweetsFut - } - - val numOfCandidates: Int = target.params(FS.HighQualityCandidatesNumberOfCandidates) - tweetsByDomainFut.map(tweetsByDomain => reformatDomainTweetMap(tweetsByDomain, numOfCandidates)) - } - - private def getTripTweetsByDomains( - tripQueries: Set[TripDomain] - ): Future[Map[TripDomain, Seq[TripTweet]]] = { - Future.collect(tripTweetCandidateStore.multiGet(tripQueries)).map { response => - response - .filter(p => p._2.exists(_.tweets.nonEmpty)) - .mapValues(_.map(_.tweets).getOrElse(Seq.empty)) - } - } - - private def reformatDomainTweetMap( - tweetsByDomain: Map[TripDomain, Seq[TripTweet]], - numOfCandidates: Int - ): Map[Long, Set[TripDomain]] = tweetsByDomain - .flatMap { - case (tripDomain, tripTweets) => - tripTweets - .sortBy(_.score)(Ordering[Double].reverse) - .take(numOfCandidates) - .map { tweet => (tweet.tweetId, tripDomain) } - }.groupBy(_._1).mapValues(_.map(_._2).toSet) - - private def buildRawCandidate( - target: Target, - tweetyPieResult: TweetyPieResult, - tripDomain: Option[scala.collection.Set[TripDomain]] - ): RawCandidate = { - PushAdaptorUtil.generateOutOfNetworkTweetCandidates( - inputTarget = target, - id = tweetyPieResult.tweet.id, - mediaCRT = MediaCRT( - CommonRecommendationType.TripHqTweet, - CommonRecommendationType.TripHqTweet, - CommonRecommendationType.TripHqTweet - ), - result = Some(tweetyPieResult), - tripTweetDomain = tripDomain - ) - } - - private def getTweetyPieResults( - target: Target, - tweetToTripDomain: Map[Long, Set[TripDomain]] - ): Future[Map[Long, Option[TweetyPieResult]]] = { - Future.collect((if (target.params(FS.EnableVFInTweetypie)) { - tweetyPieStore - } else { - tweetyPieStoreNoVF - }).multiGet(tweetToTripDomain.keySet)) - } - - override def get(target: Target): Future[Option[Seq[RawCandidate]]] = { - for { - tweetsToTripDomainMap <- getTripCandidatesStrato(target) - tweetyPieResults <- getTweetyPieResults(target, tweetsToTripDomainMap) - } yield { - val candidates = tweetyPieResults.flatMap { - case (tweetId, tweetyPieResultOpt) => - tweetyPieResultOpt.map(buildRawCandidate(target, _, tweetsToTripDomainMap.get(tweetId))) - } - if (candidates.nonEmpty) { - highQualityCandidateFrequencyPredicate(Seq(target)) - .map(_.head) - .map { isTargetFatigueEligible => - if (isTargetFatigueEligible) Some(candidates) - else { - crtFatigueCounter.incr() - None - } - } - - Some(candidates.toSeq) - } else { - missingResponseCounter.incr() - None - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ListsToRecommendCandidateAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ListsToRecommendCandidateAdaptor.scala deleted file mode 100644 index 59744b375..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ListsToRecommendCandidateAdaptor.scala +++ /dev/null @@ -1,152 +0,0 @@ -package com.twitter.frigate.pushservice.adaptor - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateSource -import com.twitter.frigate.common.base.CandidateSourceEligible -import com.twitter.frigate.common.base.ListPushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.predicate.TargetPredicates -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.geoduck.service.thriftscala.LocationResponse -import com.twitter.interests_discovery.thriftscala.DisplayLocation -import com.twitter.interests_discovery.thriftscala.NonPersonalizedRecommendedLists -import com.twitter.interests_discovery.thriftscala.RecommendedListsRequest -import com.twitter.interests_discovery.thriftscala.RecommendedListsResponse -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -case class ListsToRecommendCandidateAdaptor( - listRecommendationsStore: ReadableStore[String, NonPersonalizedRecommendedLists], - geoDuckV2Store: ReadableStore[Long, LocationResponse], - idsStore: ReadableStore[RecommendedListsRequest, RecommendedListsResponse], - globalStats: StatsReceiver) - extends CandidateSource[Target, RawCandidate] - with CandidateSourceEligible[Target, RawCandidate] { - - override val name: String = this.getClass.getSimpleName - - private[this] val stats = globalStats.scope(name) - private[this] val noLocationCodeCounter = stats.counter("no_location_code") - private[this] val noCandidatesCounter = stats.counter("no_candidates_for_geo") - private[this] val disablePopGeoListsCounter = stats.counter("disable_pop_geo_lists") - private[this] val disableIDSListsCounter = stats.counter("disable_ids_lists") - - private def getListCandidate( - targetUser: Target, - _listId: Long - ): RawCandidate with ListPushCandidate = { - new RawCandidate with ListPushCandidate { - override val listId: Long = _listId - - override val commonRecType: CommonRecommendationType = CommonRecommendationType.List - - override val target: Target = targetUser - } - } - - private def getListsRecommendedFromHistory( - target: Target - ): Future[Seq[Long]] = { - target.history.map { history => - history.sortedHistory.flatMap { - case (_, notif) if notif.commonRecommendationType == List => - notif.listNotification.map(_.listId) - case _ => None - } - } - } - - private def getIDSListRecs( - target: Target, - historicalListIds: Seq[Long] - ): Future[Seq[Long]] = { - val request = RecommendedListsRequest( - target.targetId, - DisplayLocation.ListDiscoveryPage, - Some(historicalListIds) - ) - if (target.params(PushFeatureSwitchParams.EnableIDSListRecommendations)) { - idsStore.get(request).map { - case Some(response) => - response.channels.map(_.id) - case _ => Nil - } - } else { - disableIDSListsCounter.incr() - Future.Nil - } - } - - private def getPopGeoLists( - target: Target, - historicalListIds: Seq[Long] - ): Future[Seq[Long]] = { - if (target.params(PushFeatureSwitchParams.EnablePopGeoListRecommendations)) { - geoDuckV2Store.get(target.targetId).flatMap { - case Some(locationResponse) if locationResponse.geohash.isDefined => - val geoHashLength = - target.params(PushFeatureSwitchParams.ListRecommendationsGeoHashLength) - val geoHash = locationResponse.geohash.get.take(geoHashLength) - listRecommendationsStore - .get(s"geohash_$geoHash") - .map { - case Some(recommendedLists) => - recommendedLists.recommendedListsByAlgo.flatMap { topLists => - topLists.lists.collect { - case list if !historicalListIds.contains(list.listId) => list.listId - } - } - case _ => Nil - } - case _ => - noLocationCodeCounter.incr() - Future.Nil - } - } else { - disablePopGeoListsCounter.incr() - Future.Nil - } - } - - override def get(target: Target): Future[Option[Seq[RawCandidate]]] = { - getListsRecommendedFromHistory(target).flatMap { historicalListIds => - Future - .join( - getPopGeoLists(target, historicalListIds), - getIDSListRecs(target, historicalListIds) - ) - .map { - case (popGeoListsIds, idsListIds) => - val candidates = (idsListIds ++ popGeoListsIds).map(getListCandidate(target, _)) - Some(candidates) - case _ => - noCandidatesCounter.incr() - None - } - } - } - - private val pushCapFatiguePredicate = TargetPredicates.pushRecTypeFatiguePredicate( - CommonRecommendationType.List, - PushFeatureSwitchParams.ListRecommendationsPushInterval, - PushFeatureSwitchParams.MaxListRecommendationsPushGivenInterval, - stats, - ) - override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { - - val isNotFatigued = pushCapFatiguePredicate.apply(Seq(target)).map(_.head) - - Future - .join( - PushDeviceUtil.isRecommendationsEligible(target), - isNotFatigued - ).map { - case (userRecommendationsEligible, isUnderCAP) => - userRecommendationsEligible && isUnderCAP && target.params( - PushFeatureSwitchParams.EnableListRecommendations) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/LoggedOutPushCandidateSourceGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/LoggedOutPushCandidateSourceGenerator.scala deleted file mode 100644 index e5ac0b516..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/LoggedOutPushCandidateSourceGenerator.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.frigate.pushservice.adaptor - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateSource -import com.twitter.frigate.common.base.CandidateSourceEligible -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.geoduck.service.thriftscala.LocationResponse -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweets -import com.twitter.content_mixer.thriftscala.ContentMixerRequest -import com.twitter.content_mixer.thriftscala.ContentMixerResponse -import com.twitter.geoduck.common.thriftscala.Location -import com.twitter.hermit.pop_geo.thriftscala.PopTweetsInPlace -import com.twitter.recommendation.interests.discovery.core.model.InterestDomain - -class LoggedOutPushCandidateSourceGenerator( - tripTweetCandidateStore: ReadableStore[TripDomain, TripTweets], - geoDuckV2Store: ReadableStore[Long, LocationResponse], - safeCachedTweetyPieStoreV2: ReadableStore[Long, TweetyPieResult], - cachedTweetyPieStoreV2NoVF: ReadableStore[Long, TweetyPieResult], - cachedTweetyPieStoreV2: ReadableStore[Long, TweetyPieResult], - contentMixerStore: ReadableStore[ContentMixerRequest, ContentMixerResponse], - softUserLocationStore: ReadableStore[Long, Location], - topTweetsByGeoStore: ReadableStore[InterestDomain[String], Map[String, List[(Long, Double)]]], - topTweetsByGeoV2VersionedStore: ReadableStore[String, PopTweetsInPlace], -)( - implicit val globalStats: StatsReceiver) { - val sources: Seq[CandidateSource[Target, RawCandidate] with CandidateSourceEligible[ - Target, - RawCandidate - ]] = { - Seq( - TripGeoCandidatesAdaptor( - tripTweetCandidateStore, - contentMixerStore, - safeCachedTweetyPieStoreV2, - cachedTweetyPieStoreV2NoVF, - globalStats - ), - TopTweetsByGeoAdaptor( - geoDuckV2Store, - softUserLocationStore, - topTweetsByGeoStore, - topTweetsByGeoV2VersionedStore, - cachedTweetyPieStoreV2, - cachedTweetyPieStoreV2NoVF, - globalStats - ) - ) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/OnboardingPushCandidateAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/OnboardingPushCandidateAdaptor.scala deleted file mode 100644 index 98568e9dc..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/OnboardingPushCandidateAdaptor.scala +++ /dev/null @@ -1,101 +0,0 @@ -package com.twitter.frigate.pushservice.adaptor - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateSource -import com.twitter.frigate.common.base.CandidateSourceEligible -import com.twitter.frigate.common.base.DiscoverTwitterCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} -import com.twitter.frigate.pushservice.predicate.DiscoverTwitterPredicate -import com.twitter.frigate.pushservice.predicate.TargetPredicates -import com.twitter.frigate.pushservice.util.PushAppPermissionUtil -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.frigate.thriftscala.{CommonRecommendationType => CRT} -import com.twitter.util.Future - -class OnboardingPushCandidateAdaptor( - globalStats: StatsReceiver) - extends CandidateSource[Target, RawCandidate] - with CandidateSourceEligible[Target, RawCandidate] { - - override val name: String = this.getClass.getSimpleName - - private[this] val stats = globalStats.scope(name) - private[this] val requestNum = stats.counter("request_num") - private[this] val addressBookCandNum = stats.counter("address_book_cand_num") - private[this] val completeOnboardingCandNum = stats.counter("complete_onboarding_cand_num") - - private def generateOnboardingPushRawCandidate( - _target: Target, - _commonRecType: CRT - ): RawCandidate = { - new RawCandidate with DiscoverTwitterCandidate { - override val target = _target - override val commonRecType = _commonRecType - } - } - - private def getEligibleCandsForTarget( - target: Target - ): Future[Option[Seq[RawCandidate]]] = { - val addressBookFatigue = - TargetPredicates - .pushRecTypeFatiguePredicate( - CRT.AddressBookUploadPush, - FS.FatigueForOnboardingPushes, - FS.MaxOnboardingPushInInterval, - stats)(Seq(target)).map(_.head) - val completeOnboardingFatigue = - TargetPredicates - .pushRecTypeFatiguePredicate( - CRT.CompleteOnboardingPush, - FS.FatigueForOnboardingPushes, - FS.MaxOnboardingPushInInterval, - stats)(Seq(target)).map(_.head) - - Future - .join( - target.appPermissions, - addressBookFatigue, - completeOnboardingFatigue - ).map { - case (appPermissionOpt, addressBookPredicate, completeOnboardingPredicate) => - val addressBookUploaded = - PushAppPermissionUtil.hasTargetUploadedAddressBook(appPermissionOpt) - val abUploadCandidate = - if (!addressBookUploaded && addressBookPredicate && target.params( - FS.EnableAddressBookPush)) { - addressBookCandNum.incr() - Some(generateOnboardingPushRawCandidate(target, CRT.AddressBookUploadPush)) - } else if (!addressBookUploaded && (completeOnboardingPredicate || - target.params(FS.DisableOnboardingPushFatigue)) && target.params( - FS.EnableCompleteOnboardingPush)) { - completeOnboardingCandNum.incr() - Some(generateOnboardingPushRawCandidate(target, CRT.CompleteOnboardingPush)) - } else None - - val allCandidates = - Seq(abUploadCandidate).filter(_.isDefined).flatten - if (allCandidates.nonEmpty) Some(allCandidates) else None - } - } - - override def get(inputTarget: Target): Future[Option[Seq[RawCandidate]]] = { - requestNum.incr() - val minDurationForMRElapsed = - DiscoverTwitterPredicate - .minDurationElapsedSinceLastMrPushPredicate( - name, - FS.MrMinDurationSincePushForOnboardingPushes, - stats)(Seq(inputTarget)).map(_.head) - minDurationForMRElapsed.flatMap { minDurationElapsed => - if (minDurationElapsed) getEligibleCandsForTarget(inputTarget) else Future.None - } - } - - override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { - PushDeviceUtil - .isRecommendationsEligible(target).map(_ && target.params(FS.EnableOnboardingPushes)) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/PushCandidateSourceGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/PushCandidateSourceGenerator.scala deleted file mode 100644 index ea2dcd008..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/PushCandidateSourceGenerator.scala +++ /dev/null @@ -1,162 +0,0 @@ -package com.twitter.frigate.pushservice.adaptor - -import com.twitter.content_mixer.thriftscala.ContentMixerRequest -import com.twitter.content_mixer.thriftscala.ContentMixerResponse -import com.twitter.explore_ranker.thriftscala.ExploreRankerRequest -import com.twitter.explore_ranker.thriftscala.ExploreRankerResponse -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.candidate._ -import com.twitter.frigate.common.store.RecentTweetsQuery -import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.store._ -import com.twitter.geoduck.common.thriftscala.Location -import com.twitter.geoduck.service.thriftscala.LocationResponse -import com.twitter.hermit.pop_geo.thriftscala.PopTweetsInPlace -import com.twitter.hermit.predicate.socialgraph.RelationEdge -import com.twitter.hermit.store.tweetypie.UserTweet -import com.twitter.interests.thriftscala.UserInterests -import com.twitter.interests_discovery.thriftscala.NonPersonalizedRecommendedLists -import com.twitter.interests_discovery.thriftscala.RecommendedListsRequest -import com.twitter.interests_discovery.thriftscala.RecommendedListsResponse -import com.twitter.recommendation.interests.discovery.core.model.InterestDomain -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweets -import com.twitter.tsp.thriftscala.TopicSocialProofRequest -import com.twitter.tsp.thriftscala.TopicSocialProofResponse - -/** - * PushCandidateSourceGenerator generates candidate source list for a given Target user - */ -class PushCandidateSourceGenerator( - earlybirdCandidates: CandidateSource[EarlybirdCandidateSource.Query, EarlybirdCandidate], - userTweetEntityGraphCandidates: CandidateSource[UserTweetEntityGraphCandidates.Target, Candidate], - cachedTweetyPieStoreV2: ReadableStore[Long, TweetyPieResult], - safeCachedTweetyPieStoreV2: ReadableStore[Long, TweetyPieResult], - userTweetTweetyPieStore: ReadableStore[UserTweet, TweetyPieResult], - safeUserTweetTweetyPieStore: ReadableStore[UserTweet, TweetyPieResult], - cachedTweetyPieStoreV2NoVF: ReadableStore[Long, TweetyPieResult], - edgeStore: ReadableStore[RelationEdge, Boolean], - interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests], - uttEntityHydrationStore: UttEntityHydrationStore, - geoDuckV2Store: ReadableStore[Long, LocationResponse], - topTweetsByGeoStore: ReadableStore[InterestDomain[String], Map[String, List[(Long, Double)]]], - topTweetsByGeoV2VersionedStore: ReadableStore[String, PopTweetsInPlace], - tweetImpressionsStore: TweetImpressionsStore, - recommendedTrendsCandidateSource: RecommendedTrendsCandidateSource, - recentTweetsByAuthorStore: ReadableStore[RecentTweetsQuery, Seq[Seq[Long]]], - topicSocialProofServiceStore: ReadableStore[TopicSocialProofRequest, TopicSocialProofResponse], - crMixerStore: CrMixerTweetStore, - contentMixerStore: ReadableStore[ContentMixerRequest, ContentMixerResponse], - exploreRankerStore: ReadableStore[ExploreRankerRequest, ExploreRankerResponse], - softUserLocationStore: ReadableStore[Long, Location], - tripTweetCandidateStore: ReadableStore[TripDomain, TripTweets], - listRecsStore: ReadableStore[String, NonPersonalizedRecommendedLists], - idsStore: ReadableStore[RecommendedListsRequest, RecommendedListsResponse] -)( - implicit val globalStats: StatsReceiver) { - - private val earlyBirdFirstDegreeCandidateAdaptor = EarlyBirdFirstDegreeCandidateAdaptor( - earlybirdCandidates, - cachedTweetyPieStoreV2, - cachedTweetyPieStoreV2NoVF, - userTweetTweetyPieStore, - PushFeatureSwitchParams.NumberOfMaxEarlybirdInNetworkCandidatesParam, - globalStats - ) - - private val frsTweetCandidateAdaptor = FRSTweetCandidateAdaptor( - crMixerStore, - cachedTweetyPieStoreV2, - cachedTweetyPieStoreV2NoVF, - userTweetTweetyPieStore, - uttEntityHydrationStore, - topicSocialProofServiceStore, - globalStats - ) - - private val contentRecommenderMixerAdaptor = ContentRecommenderMixerAdaptor( - crMixerStore, - safeCachedTweetyPieStoreV2, - edgeStore, - topicSocialProofServiceStore, - uttEntityHydrationStore, - globalStats - ) - - private val tripGeoCandidatesAdaptor = TripGeoCandidatesAdaptor( - tripTweetCandidateStore, - contentMixerStore, - safeCachedTweetyPieStoreV2, - cachedTweetyPieStoreV2NoVF, - globalStats - ) - - val sources: Seq[ - CandidateSource[Target, RawCandidate] with CandidateSourceEligible[ - Target, - RawCandidate - ] - ] = { - Seq( - earlyBirdFirstDegreeCandidateAdaptor, - GenericCandidateAdaptor( - userTweetEntityGraphCandidates, - cachedTweetyPieStoreV2, - cachedTweetyPieStoreV2NoVF, - globalStats.scope("UserTweetEntityGraphCandidates") - ), - new OnboardingPushCandidateAdaptor(globalStats), - TopTweetsByGeoAdaptor( - geoDuckV2Store, - softUserLocationStore, - topTweetsByGeoStore, - topTweetsByGeoV2VersionedStore, - cachedTweetyPieStoreV2, - cachedTweetyPieStoreV2NoVF, - globalStats - ), - frsTweetCandidateAdaptor, - TopTweetImpressionsCandidateAdaptor( - recentTweetsByAuthorStore, - cachedTweetyPieStoreV2, - cachedTweetyPieStoreV2NoVF, - tweetImpressionsStore, - globalStats - ), - TrendsCandidatesAdaptor( - softUserLocationStore, - recommendedTrendsCandidateSource, - safeCachedTweetyPieStoreV2, - cachedTweetyPieStoreV2NoVF, - safeUserTweetTweetyPieStore, - globalStats - ), - contentRecommenderMixerAdaptor, - tripGeoCandidatesAdaptor, - HighQualityTweetsAdaptor( - tripTweetCandidateStore, - interestsLookupStore, - cachedTweetyPieStoreV2, - cachedTweetyPieStoreV2NoVF, - globalStats - ), - ExploreVideoTweetCandidateAdaptor( - exploreRankerStore, - cachedTweetyPieStoreV2, - globalStats - ), - ListsToRecommendCandidateAdaptor( - listRecsStore, - geoDuckV2Store, - idsStore, - globalStats - ) - ) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TopTweetImpressionsCandidateAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TopTweetImpressionsCandidateAdaptor.scala deleted file mode 100644 index 25ab31e85..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TopTweetImpressionsCandidateAdaptor.scala +++ /dev/null @@ -1,326 +0,0 @@ -package com.twitter.frigate.pushservice.adaptor - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateSource -import com.twitter.frigate.common.base.CandidateSourceEligible -import com.twitter.frigate.common.base.TopTweetImpressionsCandidate -import com.twitter.frigate.common.store.RecentTweetsQuery -import com.twitter.frigate.common.util.SnowflakeUtils -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} -import com.twitter.frigate.pushservice.store.TweetImpressionsStore -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.FutureOps -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -case class TweetImpressionsCandidate( - tweetId: Long, - tweetyPieResultOpt: Option[TweetyPieResult], - impressionsCountOpt: Option[Long]) - -case class TopTweetImpressionsCandidateAdaptor( - recentTweetsFromTflockStore: ReadableStore[RecentTweetsQuery, Seq[Seq[Long]]], - tweetyPieStore: ReadableStore[Long, TweetyPieResult], - tweetyPieStoreNoVF: ReadableStore[Long, TweetyPieResult], - tweetImpressionsStore: TweetImpressionsStore, - globalStats: StatsReceiver) - extends CandidateSource[Target, RawCandidate] - with CandidateSourceEligible[Target, RawCandidate] { - - private val stats = globalStats.scope("TopTweetImpressionsAdaptor") - private val tweetImpressionsCandsStat = stats.stat("top_tweet_impressions_cands_dist") - - private val eligibleUsersCounter = stats.counter("eligible_users") - private val noneligibleUsersCounter = stats.counter("noneligible_users") - private val meetsMinTweetsRequiredCounter = stats.counter("meets_min_tweets_required") - private val belowMinTweetsRequiredCounter = stats.counter("below_min_tweets_required") - private val aboveMaxInboundFavoritesCounter = stats.counter("above_max_inbound_favorites") - private val meetsImpressionsRequiredCounter = stats.counter("meets_impressions_required") - private val belowImpressionsRequiredCounter = stats.counter("below_impressions_required") - private val meetsFavoritesThresholdCounter = stats.counter("meets_favorites_threshold") - private val aboveFavoritesThresholdCounter = stats.counter("above_favorites_threshold") - private val emptyImpressionsMapCounter = stats.counter("empty_impressions_map") - - private val tflockResultsStat = stats.stat("tflock", "results") - private val emptyTflockResult = stats.counter("tflock", "empty_result") - private val nonEmptyTflockResult = stats.counter("tflock", "non_empty_result") - - private val originalTweetsStat = stats.stat("tweets", "original_tweets") - private val retweetsStat = stats.stat("tweets", "retweets") - private val allRetweetsOnlyCounter = stats.counter("tweets", "all_retweets_only") - private val allOriginalTweetsOnlyCounter = stats.counter("tweets", "all_original_tweets_only") - - private val emptyTweetypieMap = stats.counter("", "empty_tweetypie_map") - private val emptyTweetyPieResult = stats.stat("", "empty_tweetypie_result") - private val allEmptyTweetypieResults = stats.counter("", "all_empty_tweetypie_results") - - private val eligibleUsersAfterImpressionsFilter = - stats.counter("eligible_users_after_impressions_filter") - private val eligibleUsersAfterFavoritesFilter = - stats.counter("eligible_users_after_favorites_filter") - private val eligibleUsersWithEligibleTweets = - stats.counter("eligible_users_with_eligible_tweets") - - private val eligibleTweetCands = stats.stat("eligible_tweet_cands") - private val getCandsRequestCounter = - stats.counter("top_tweet_impressions_get_request") - - override val name: String = this.getClass.getSimpleName - - override def get(inputTarget: Target): Future[Option[Seq[RawCandidate]]] = { - getCandsRequestCounter.incr() - val eligibleCandidatesFut = getTweetImpressionsCandidates(inputTarget) - eligibleCandidatesFut.map { eligibleCandidates => - if (eligibleCandidates.nonEmpty) { - eligibleUsersWithEligibleTweets.incr() - eligibleTweetCands.add(eligibleCandidates.size) - val candidate = getMostImpressionsTweet(eligibleCandidates) - Some( - Seq( - generateTopTweetImpressionsCandidate( - inputTarget, - candidate.tweetId, - candidate.tweetyPieResultOpt, - candidate.impressionsCountOpt.getOrElse(0L)))) - } else None - } - } - - private def getTweetImpressionsCandidates( - inputTarget: Target - ): Future[Seq[TweetImpressionsCandidate]] = { - val originalTweets = getRecentOriginalTweetsForUser(inputTarget) - originalTweets.flatMap { tweetyPieResultsMap => - val numDaysSearchForOriginalTweets = - inputTarget.params(FS.TopTweetImpressionsOriginalTweetsNumDaysSearch) - val moreRecentTweetIds = - getMoreRecentTweetIds(tweetyPieResultsMap.keySet.toSeq, numDaysSearchForOriginalTweets) - val isEligible = isEligibleUser(inputTarget, tweetyPieResultsMap, moreRecentTweetIds) - if (isEligible) filterByEligibility(inputTarget, tweetyPieResultsMap, moreRecentTweetIds) - else Future.Nil - } - } - - private def getRecentOriginalTweetsForUser( - targetUser: Target - ): Future[Map[Long, TweetyPieResult]] = { - val tweetyPieResultsMapFut = getTflockStoreResults(targetUser).flatMap { recentTweetIds => - FutureOps.mapCollect((targetUser.params(FS.EnableVFInTweetypie) match { - case true => tweetyPieStore - case false => tweetyPieStoreNoVF - }).multiGet(recentTweetIds.toSet)) - } - tweetyPieResultsMapFut.map { tweetyPieResultsMap => - if (tweetyPieResultsMap.isEmpty) { - emptyTweetypieMap.incr() - Map.empty - } else removeRetweets(tweetyPieResultsMap) - } - } - - private def getTflockStoreResults(targetUser: Target): Future[Seq[Long]] = { - val maxResults = targetUser.params(FS.TopTweetImpressionsRecentTweetsByAuthorStoreMaxResults) - val maxAge = targetUser.params(FS.TopTweetImpressionsTotalFavoritesLimitNumDaysSearch) - val recentTweetsQuery = - RecentTweetsQuery( - userIds = Seq(targetUser.targetId), - maxResults = maxResults, - maxAge = maxAge.days - ) - recentTweetsFromTflockStore - .get(recentTweetsQuery).map { - case Some(tweetIdsAll) => - val tweetIds = tweetIdsAll.headOption.getOrElse(Seq.empty) - val numTweets = tweetIds.size - if (numTweets > 0) { - tflockResultsStat.add(numTweets) - nonEmptyTflockResult.incr() - } else emptyTflockResult.incr() - tweetIds - case _ => Nil - } - } - - private def removeRetweets( - tweetyPieResultsMap: Map[Long, Option[TweetyPieResult]] - ): Map[Long, TweetyPieResult] = { - val nonEmptyTweetyPieResults: Map[Long, TweetyPieResult] = tweetyPieResultsMap.collect { - case (key, Some(value)) => (key, value) - } - emptyTweetyPieResult.add(tweetyPieResultsMap.size - nonEmptyTweetyPieResults.size) - - if (nonEmptyTweetyPieResults.nonEmpty) { - val originalTweets = nonEmptyTweetyPieResults.filter { - case (_, tweetyPieResult) => - tweetyPieResult.sourceTweet.isEmpty - } - val numOriginalTweets = originalTweets.size - val numRetweets = nonEmptyTweetyPieResults.size - originalTweets.size - originalTweetsStat.add(numOriginalTweets) - retweetsStat.add(numRetweets) - if (numRetweets == 0) allOriginalTweetsOnlyCounter.incr() - if (numOriginalTweets == 0) allRetweetsOnlyCounter.incr() - originalTweets - } else { - allEmptyTweetypieResults.incr() - Map.empty - } - } - - private def getMoreRecentTweetIds( - tweetIds: Seq[Long], - numDays: Int - ): Seq[Long] = { - tweetIds.filter { tweetId => - SnowflakeUtils.isRecent(tweetId, numDays.days) - } - } - - private def isEligibleUser( - inputTarget: Target, - tweetyPieResults: Map[Long, TweetyPieResult], - recentTweetIds: Seq[Long] - ): Boolean = { - val minNumTweets = inputTarget.params(FS.TopTweetImpressionsMinNumOriginalTweets) - lazy val totalFavoritesLimit = - inputTarget.params(FS.TopTweetImpressionsTotalInboundFavoritesLimit) - if (recentTweetIds.size >= minNumTweets) { - meetsMinTweetsRequiredCounter.incr() - val isUnderLimit = isUnderTotalInboundFavoritesLimit(tweetyPieResults, totalFavoritesLimit) - if (isUnderLimit) eligibleUsersCounter.incr() - else { - aboveMaxInboundFavoritesCounter.incr() - noneligibleUsersCounter.incr() - } - isUnderLimit - } else { - belowMinTweetsRequiredCounter.incr() - noneligibleUsersCounter.incr() - false - } - } - - private def getFavoriteCounts( - tweetyPieResult: TweetyPieResult - ): Long = tweetyPieResult.tweet.counts.flatMap(_.favoriteCount).getOrElse(0L) - - private def isUnderTotalInboundFavoritesLimit( - tweetyPieResults: Map[Long, TweetyPieResult], - totalFavoritesLimit: Long - ): Boolean = { - val favoritesIterator = tweetyPieResults.valuesIterator.map(getFavoriteCounts) - val totalInboundFavorites = favoritesIterator.sum - totalInboundFavorites <= totalFavoritesLimit - } - - def filterByEligibility( - inputTarget: Target, - tweetyPieResults: Map[Long, TweetyPieResult], - tweetIds: Seq[Long] - ): Future[Seq[TweetImpressionsCandidate]] = { - lazy val minNumImpressions: Long = inputTarget.params(FS.TopTweetImpressionsMinRequired) - lazy val maxNumLikes: Long = inputTarget.params(FS.TopTweetImpressionsMaxFavoritesPerTweet) - for { - filteredImpressionsMap <- getFilteredImpressionsMap(tweetIds, minNumImpressions) - tweetIdsFilteredByFavorites <- - getTweetIdsFilteredByFavorites(filteredImpressionsMap.keySet, tweetyPieResults, maxNumLikes) - } yield { - if (filteredImpressionsMap.nonEmpty) eligibleUsersAfterImpressionsFilter.incr() - if (tweetIdsFilteredByFavorites.nonEmpty) eligibleUsersAfterFavoritesFilter.incr() - - val candidates = tweetIdsFilteredByFavorites.map { tweetId => - TweetImpressionsCandidate( - tweetId, - tweetyPieResults.get(tweetId), - filteredImpressionsMap.get(tweetId)) - } - tweetImpressionsCandsStat.add(candidates.length) - candidates - } - } - - private def getFilteredImpressionsMap( - tweetIds: Seq[Long], - minNumImpressions: Long - ): Future[Map[Long, Long]] = { - getImpressionsCounts(tweetIds).map { impressionsMap => - if (impressionsMap.isEmpty) emptyImpressionsMapCounter.incr() - impressionsMap.filter { - case (_, numImpressions) => - val isValid = numImpressions >= minNumImpressions - if (isValid) { - meetsImpressionsRequiredCounter.incr() - } else { - belowImpressionsRequiredCounter.incr() - } - isValid - } - } - } - - private def getTweetIdsFilteredByFavorites( - filteredTweetIds: Set[Long], - tweetyPieResults: Map[Long, TweetyPieResult], - maxNumLikes: Long - ): Future[Seq[Long]] = { - val filteredByFavoritesTweetIds = filteredTweetIds.filter { tweetId => - val tweetyPieResultOpt = tweetyPieResults.get(tweetId) - val isValid = tweetyPieResultOpt.exists { tweetyPieResult => - getFavoriteCounts(tweetyPieResult) <= maxNumLikes - } - if (isValid) meetsFavoritesThresholdCounter.incr() - else aboveFavoritesThresholdCounter.incr() - isValid - } - Future(filteredByFavoritesTweetIds.toSeq) - } - - private def getMostImpressionsTweet( - filteredResults: Seq[TweetImpressionsCandidate] - ): TweetImpressionsCandidate = { - val maxImpressions: Long = filteredResults.map { - _.impressionsCountOpt.getOrElse(0L) - }.max - - val mostImpressionsCandidates: Seq[TweetImpressionsCandidate] = - filteredResults.filter(_.impressionsCountOpt.getOrElse(0L) == maxImpressions) - - mostImpressionsCandidates.maxBy(_.tweetId) - } - - private def getImpressionsCounts( - tweetIds: Seq[Long] - ): Future[Map[Long, Long]] = { - val impressionCountMap = tweetIds.map { tweetId => - tweetId -> tweetImpressionsStore - .getCounts(tweetId).map(_.getOrElse(0L)) - }.toMap - Future.collect(impressionCountMap) - } - - private def generateTopTweetImpressionsCandidate( - inputTarget: Target, - _tweetId: Long, - result: Option[TweetyPieResult], - _impressionsCount: Long - ): RawCandidate = { - new RawCandidate with TopTweetImpressionsCandidate { - override val target: Target = inputTarget - override val tweetId: Long = _tweetId - override val tweetyPieResult: Option[TweetyPieResult] = result - override val impressionsCount: Long = _impressionsCount - } - } - - override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { - val enabledTopTweetImpressionsNotification = - target.params(FS.EnableTopTweetImpressionsNotification) - - PushDeviceUtil - .isRecommendationsEligible(target).map(_ && enabledTopTweetImpressionsNotification) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TopTweetsByGeoAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TopTweetsByGeoAdaptor.scala deleted file mode 100644 index 3228760fd..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TopTweetsByGeoAdaptor.scala +++ /dev/null @@ -1,413 +0,0 @@ -package com.twitter.frigate.pushservice.adaptor - -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateSource -import com.twitter.frigate.common.base.CandidateSourceEligible -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.common.predicate.CommonOutNetworkTweetCandidatesSourcePredicates.filterOutReplyTweet -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.model.PushTypes -import com.twitter.frigate.pushservice.params.PopGeoTweetVersion -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.params.TopTweetsForGeoCombination -import com.twitter.frigate.pushservice.params.TopTweetsForGeoRankingFunction -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} -import com.twitter.frigate.pushservice.predicate.DiscoverTwitterPredicate -import com.twitter.frigate.pushservice.predicate.TargetPredicates -import com.twitter.frigate.pushservice.util.MediaCRT -import com.twitter.frigate.pushservice.util.PushAdaptorUtil -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.geoduck.common.thriftscala.{Location => GeoLocation} -import com.twitter.geoduck.service.thriftscala.LocationResponse -import com.twitter.gizmoduck.thriftscala.UserType -import com.twitter.hermit.pop_geo.thriftscala.PopTweetsInPlace -import com.twitter.recommendation.interests.discovery.core.model.InterestDomain -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.FutureOps -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future -import com.twitter.util.Time -import scala.collection.Map - -case class PlaceTweetScore(place: String, tweetId: Long, score: Double) { - def toTweetScore: (Long, Double) = (tweetId, score) -} -case class TopTweetsByGeoAdaptor( - geoduckStoreV2: ReadableStore[Long, LocationResponse], - softUserGeoLocationStore: ReadableStore[Long, GeoLocation], - topTweetsByGeoStore: ReadableStore[InterestDomain[String], Map[String, List[(Long, Double)]]], - topTweetsByGeoStoreV2: ReadableStore[String, PopTweetsInPlace], - tweetyPieStore: ReadableStore[Long, TweetyPieResult], - tweetyPieStoreNoVF: ReadableStore[Long, TweetyPieResult], - globalStats: StatsReceiver) - extends CandidateSource[Target, RawCandidate] - with CandidateSourceEligible[Target, RawCandidate] { - - override def name: String = this.getClass.getSimpleName - - private[this] val stats = globalStats.scope("TopTweetsByGeoAdaptor") - private[this] val noGeohashUserCounter: Counter = stats.counter("users_with_no_geohash_counter") - private[this] val incomingRequestCounter: Counter = stats.counter("incoming_request_counter") - private[this] val incomingLoggedOutRequestCounter: Counter = - stats.counter("incoming_logged_out_request_counter") - private[this] val loggedOutRawCandidatesCounter = - stats.counter("logged_out_raw_candidates_counter") - private[this] val emptyLoggedOutRawCandidatesCounter = - stats.counter("logged_out_empty_raw_candidates") - private[this] val outputTopTweetsByGeoCounter: Stat = - stats.stat("output_top_tweets_by_geo_counter") - private[this] val loggedOutPopByGeoV2CandidatesCounter: Counter = - stats.counter("logged_out_pop_by_geo_candidates") - private[this] val dormantUsersSince14DaysCounter: Counter = - stats.counter("dormant_user_since_14_days_counter") - private[this] val dormantUsersSince30DaysCounter: Counter = - stats.counter("dormant_user_since_30_days_counter") - private[this] val nonDormantUsersSince14DaysCounter: Counter = - stats.counter("non_dormant_user_since_14_days_counter") - private[this] val topTweetsByGeoTake100Counter: Counter = - stats.counter("top_tweets_by_geo_take_100_counter") - private[this] val combinationRequestsCounter = - stats.scope("combination_method_request_counter") - private[this] val popGeoTweetVersionCounter = - stats.scope("popgeo_tweet_version_counter") - private[this] val nonReplyTweetsCounter = stats.counter("non_reply_tweets") - - val MaxGeoHashSize = 4 - - private def constructKeys( - geohash: Option[String], - accountCountryCode: Option[String], - keyLengths: Seq[Int], - version: PopGeoTweetVersion.Value - ): Set[String] = { - val geohashKeys = geohash match { - case Some(hash) => keyLengths.map { version + "_geohash_" + hash.take(_) } - case _ => Seq.empty - } - - val accountCountryCodeKeys = - accountCountryCode.toSeq.map(version + "_country_" + _.toUpperCase) - (geohashKeys ++ accountCountryCodeKeys).toSet - } - - def convertToPlaceTweetScore( - popTweetsInPlace: Seq[PopTweetsInPlace] - ): Seq[PlaceTweetScore] = { - popTweetsInPlace.flatMap { - case p => - p.popTweets.map { - case popTweet => PlaceTweetScore(p.place, popTweet.tweetId, popTweet.score) - } - } - } - - def sortGeoHashTweets( - placeTweetScores: Seq[PlaceTweetScore], - rankingFunction: TopTweetsForGeoRankingFunction.Value - ): Seq[PlaceTweetScore] = { - rankingFunction match { - case TopTweetsForGeoRankingFunction.Score => - placeTweetScores.sortBy(_.score)(Ordering[Double].reverse) - case TopTweetsForGeoRankingFunction.GeohashLengthAndThenScore => - placeTweetScores - .sortBy(row => (row.place.length, row.score))(Ordering[(Int, Double)].reverse) - } - } - - def getResultsForLambdaStore( - inputTarget: Target, - geohash: Option[String], - store: ReadableStore[String, PopTweetsInPlace], - topk: Int, - version: PopGeoTweetVersion.Value - ): Future[Seq[(Long, Double)]] = { - inputTarget.accountCountryCode.flatMap { countryCode => - val keys = { - if (inputTarget.params(FS.EnableCountryCodeBackoffTopTweetsByGeo)) - constructKeys(geohash, countryCode, inputTarget.params(FS.GeoHashLengthList), version) - else - constructKeys(geohash, None, inputTarget.params(FS.GeoHashLengthList), version) - } - FutureOps - .mapCollect(store.multiGet(keys)).map { - case geohashTweetMap => - val popTweets = - geohashTweetMap.values.flatten.toSeq - val results = sortGeoHashTweets( - convertToPlaceTweetScore(popTweets), - inputTarget.params(FS.RankingFunctionForTopTweetsByGeo)) - .map(_.toTweetScore).take(topk) - results - } - } - } - - def getPopGeoTweetsForLoggedOutUsers( - inputTarget: Target, - store: ReadableStore[String, PopTweetsInPlace] - ): Future[Seq[(Long, Double)]] = { - inputTarget.countryCode.flatMap { countryCode => - val keys = constructKeys(None, countryCode, Seq(4), PopGeoTweetVersion.Prod) - FutureOps.mapCollect(store.multiGet(keys)).map { - case tweetMap => - val tweets = tweetMap.values.flatten.toSeq - loggedOutPopByGeoV2CandidatesCounter.incr(tweets.size) - val popTweets = sortGeoHashTweets( - convertToPlaceTweetScore(tweets), - TopTweetsForGeoRankingFunction.Score).map(_.toTweetScore) - popTweets - } - } - } - - def getRankedTweets( - inputTarget: Target, - geohash: Option[String] - ): Future[Seq[(Long, Double)]] = { - val MaxTopTweetsByGeoCandidatesToTake = - inputTarget.params(FS.MaxTopTweetsByGeoCandidatesToTake) - val scoringFn: String = inputTarget.params(FS.ScoringFuncForTopTweetsByGeo) - val combinationMethod = inputTarget.params(FS.TopTweetsByGeoCombinationParam) - val popGeoTweetVersion = inputTarget.params(FS.PopGeoTweetVersionParam) - - inputTarget.isHeavyUserState.map { isHeavyUser => - stats - .scope(combinationMethod.toString).scope(popGeoTweetVersion.toString).scope( - "IsHeavyUser_" + isHeavyUser.toString).counter().incr() - } - combinationRequestsCounter.scope(combinationMethod.toString).counter().incr() - popGeoTweetVersionCounter.scope(popGeoTweetVersion.toString).counter().incr() - lazy val geoStoreResults = if (geohash.isDefined) { - val hash = geohash.get.take(MaxGeoHashSize) - topTweetsByGeoStore - .get( - InterestDomain[String](hash) - ) - .map { - case Some(scoringFnToTweetsMapOpt) => - val tweetsWithScore = scoringFnToTweetsMapOpt - .getOrElse(scoringFn, List.empty) - val sortedResults = sortGeoHashTweets( - tweetsWithScore.map { - case (tweetId, score) => PlaceTweetScore(hash, tweetId, score) - }, - TopTweetsForGeoRankingFunction.Score - ).map(_.toTweetScore).take( - MaxTopTweetsByGeoCandidatesToTake - ) - sortedResults - case _ => Seq.empty - } - } else Future.value(Seq.empty) - lazy val versionPopGeoTweetResults = - getResultsForLambdaStore( - inputTarget, - geohash, - topTweetsByGeoStoreV2, - MaxTopTweetsByGeoCandidatesToTake, - popGeoTweetVersion - ) - combinationMethod match { - case TopTweetsForGeoCombination.Default => geoStoreResults - case TopTweetsForGeoCombination.AccountsTweetFavAsBackfill => - Future.join(geoStoreResults, versionPopGeoTweetResults).map { - case (geoStoreTweets, versionPopGeoTweets) => - (geoStoreTweets ++ versionPopGeoTweets).take(MaxTopTweetsByGeoCandidatesToTake) - } - case TopTweetsForGeoCombination.AccountsTweetFavIntermixed => - Future.join(geoStoreResults, versionPopGeoTweetResults).map { - case (geoStoreTweets, versionPopGeoTweets) => - CandidateSource.interleaveSeqs(Seq(geoStoreTweets, versionPopGeoTweets)) - } - } - } - - override def get(inputTarget: Target): Future[Option[Seq[RawCandidate]]] = { - if (inputTarget.isLoggedOutUser) { - incomingLoggedOutRequestCounter.incr() - val rankedTweets = getPopGeoTweetsForLoggedOutUsers(inputTarget, topTweetsByGeoStoreV2) - val rawCandidates = { - rankedTweets.map { rt => - FutureOps - .mapCollect( - tweetyPieStore - .multiGet(rt.map { case (tweetId, _) => tweetId }.toSet)) - .map { tweetyPieResultMap => - val results = buildTopTweetsByGeoRawCandidates( - inputTarget, - None, - tweetyPieResultMap - ) - if (results.isEmpty) { - emptyLoggedOutRawCandidatesCounter.incr() - } - loggedOutRawCandidatesCounter.incr(results.size) - Some(results) - } - }.flatten - } - rawCandidates - } else { - incomingRequestCounter.incr() - getGeoHashForUsers(inputTarget).flatMap { geohash => - if (geohash.isEmpty) noGeohashUserCounter.incr() - getRankedTweets(inputTarget, geohash).map { rt => - if (rt.size == 100) { - topTweetsByGeoTake100Counter.incr(1) - } - FutureOps - .mapCollect((inputTarget.params(FS.EnableVFInTweetypie) match { - case true => tweetyPieStore - case false => tweetyPieStoreNoVF - }).multiGet(rt.map { case (tweetId, _) => tweetId }.toSet)) - .map { tweetyPieResultMap => - Some( - buildTopTweetsByGeoRawCandidates( - inputTarget, - None, - filterOutReplyTweet( - tweetyPieResultMap, - nonReplyTweetsCounter - ) - ) - ) - } - }.flatten - } - } - } - - private def getGeoHashForUsers( - inputTarget: Target - ): Future[Option[String]] = { - - inputTarget.targetUser.flatMap { - case Some(user) => - user.userType match { - case UserType.Soft => - softUserGeoLocationStore - .get(inputTarget.targetId) - .map(_.flatMap(_.geohash.flatMap(_.stringGeohash))) - - case _ => - geoduckStoreV2.get(inputTarget.targetId).map(_.flatMap(_.geohash)) - } - - case None => Future.None - } - } - - private def buildTopTweetsByGeoRawCandidates( - target: PushTypes.Target, - locationName: Option[String], - topTweets: Map[Long, Option[TweetyPieResult]] - ): Seq[RawCandidate with TweetCandidate] = { - val candidates = topTweets.map { tweetIdTweetyPieResultMap => - PushAdaptorUtil.generateOutOfNetworkTweetCandidates( - inputTarget = target, - id = tweetIdTweetyPieResultMap._1, - mediaCRT = MediaCRT( - CommonRecommendationType.GeoPopTweet, - CommonRecommendationType.GeoPopTweet, - CommonRecommendationType.GeoPopTweet - ), - result = tweetIdTweetyPieResultMap._2, - localizedEntity = None - ) - }.toSeq - outputTopTweetsByGeoCounter.add(candidates.length) - candidates - } - - private val topTweetsByGeoFrequencyPredicate = { - TargetPredicates - .pushRecTypeFatiguePredicate( - CommonRecommendationType.GeoPopTweet, - FS.TopTweetsByGeoPushInterval, - FS.MaxTopTweetsByGeoPushGivenInterval, - stats - ) - } - - def getAvailabilityForDormantUser(target: Target): Future[Boolean] = { - lazy val isDormantUserNotFatigued = topTweetsByGeoFrequencyPredicate(Seq(target)).map(_.head) - lazy val enableTopTweetsByGeoForDormantUsers = - target.params(FS.EnableTopTweetsByGeoCandidatesForDormantUsers) - - target.lastHTLVisitTimestamp.flatMap { - case Some(lastHTLTimestamp) => - val minTimeSinceLastLogin = - target.params(FS.MinimumTimeSinceLastLoginForGeoPopTweetPush).ago - val timeSinceInactive = target.params(FS.TimeSinceLastLoginForGeoPopTweetPush).ago - val lastActiveTimestamp = Time.fromMilliseconds(lastHTLTimestamp) - if (lastActiveTimestamp > minTimeSinceLastLogin) { - nonDormantUsersSince14DaysCounter.incr() - Future.False - } else { - dormantUsersSince14DaysCounter.incr() - isDormantUserNotFatigued.map { isUserNotFatigued => - lastActiveTimestamp < timeSinceInactive && - enableTopTweetsByGeoForDormantUsers && - isUserNotFatigued - } - } - case _ => - dormantUsersSince30DaysCounter.incr() - isDormantUserNotFatigued.map { isUserNotFatigued => - enableTopTweetsByGeoForDormantUsers && isUserNotFatigued - } - } - } - - def getAvailabilityForPlaybookSetUp(target: Target): Future[Boolean] = { - lazy val enableTopTweetsByGeoForNewUsers = target.params(FS.EnableTopTweetsByGeoCandidates) - val isTargetEligibleForMrFatigueCheck = target.isAccountAtleastNDaysOld( - target.params(FS.MrMinDurationSincePushForTopTweetsByGeoPushes)) - val isMrFatigueCheckEnabled = - target.params(FS.EnableMrMinDurationSinceMrPushFatigue) - val applyPredicateForTopTweetsByGeo = - if (isMrFatigueCheckEnabled) { - if (isTargetEligibleForMrFatigueCheck) { - DiscoverTwitterPredicate - .minDurationElapsedSinceLastMrPushPredicate( - name, - FS.MrMinDurationSincePushForTopTweetsByGeoPushes, - stats - ).andThen( - topTweetsByGeoFrequencyPredicate - )(Seq(target)).map(_.head) - } else { - Future.False - } - } else { - topTweetsByGeoFrequencyPredicate(Seq(target)).map(_.head) - } - applyPredicateForTopTweetsByGeo.map { predicateResult => - predicateResult && enableTopTweetsByGeoForNewUsers - } - } - - override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { - if (target.isLoggedOutUser) { - Future.True - } else { - PushDeviceUtil - .isRecommendationsEligible(target).map( - _ && target.params(PushParams.PopGeoCandidatesDecider)).flatMap { isAvailable => - if (isAvailable) { - Future - .join(getAvailabilityForDormantUser(target), getAvailabilityForPlaybookSetUp(target)) - .map { - case (isAvailableForDormantUser, isAvailableForPlaybook) => - isAvailableForDormantUser || isAvailableForPlaybook - case _ => false - } - } else Future.False - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TrendsCandidatesAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TrendsCandidatesAdaptor.scala deleted file mode 100644 index 4e7ec3314..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TrendsCandidatesAdaptor.scala +++ /dev/null @@ -1,215 +0,0 @@ -package com.twitter.frigate.pushservice.adaptor - -import com.twitter.events.recos.thriftscala.DisplayLocation -import com.twitter.events.recos.thriftscala.TrendsContext -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateSource -import com.twitter.frigate.common.base.CandidateSourceEligible -import com.twitter.frigate.common.base.TrendTweetCandidate -import com.twitter.frigate.common.base.TrendsCandidate -import com.twitter.frigate.common.candidate.RecommendedTrendsCandidateSource -import com.twitter.frigate.common.candidate.RecommendedTrendsCandidateSource.Query -import com.twitter.frigate.common.predicate.CommonOutNetworkTweetCandidatesSourcePredicates.filterOutReplyTweet -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.adaptor.TrendsCandidatesAdaptor._ -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.predicate.TargetPredicates -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.geoduck.common.thriftscala.Location -import com.twitter.gizmoduck.thriftscala.UserType -import com.twitter.hermit.store.tweetypie.UserTweet -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future -import scala.collection.Map - -object TrendsCandidatesAdaptor { - type TweetId = Long - type EventId = Long -} - -case class TrendsCandidatesAdaptor( - softUserGeoLocationStore: ReadableStore[Long, Location], - recommendedTrendsCandidateSource: RecommendedTrendsCandidateSource, - tweetyPieStore: ReadableStore[Long, TweetyPieResult], - tweetyPieStoreNoVF: ReadableStore[Long, TweetyPieResult], - safeUserTweetTweetyPieStore: ReadableStore[UserTweet, TweetyPieResult], - statsReceiver: StatsReceiver) - extends CandidateSource[Target, RawCandidate] - with CandidateSourceEligible[Target, RawCandidate] { - override val name = this.getClass.getSimpleName - - private val trendAdaptorStats = statsReceiver.scope("TrendsCandidatesAdaptor") - private val trendTweetCandidateNumber = trendAdaptorStats.counter("trend_tweet_candidate") - private val nonReplyTweetsCounter = trendAdaptorStats.counter("non_reply_tweets") - - private def getQuery(target: Target): Future[Query] = { - def getUserCountryCode(target: Target): Future[Option[String]] = { - target.targetUser.flatMap { - case Some(user) if user.userType == UserType.Soft => - softUserGeoLocationStore - .get(user.id) - .map(_.flatMap(_.simpleRgcResult.flatMap(_.countryCodeAlpha2))) - - case _ => target.accountCountryCode - } - } - - for { - countryCode <- getUserCountryCode(target) - inferredLanguage <- target.inferredUserDeviceLanguage - } yield { - Query( - userId = target.targetId, - displayLocation = DisplayLocation.MagicRecs, - languageCode = inferredLanguage, - countryCode = countryCode, - maxResults = target.params(PushFeatureSwitchParams.MaxRecommendedTrendsToQuery) - ) - } - } - - /** - * Query candidates only if sent at most [[PushFeatureSwitchParams.MaxTrendTweetNotificationsInDuration]] - * trend tweet notifications in [[PushFeatureSwitchParams.TrendTweetNotificationsFatigueDuration]] - */ - val trendTweetFatiguePredicate = TargetPredicates.pushRecTypeFatiguePredicate( - CommonRecommendationType.TrendTweet, - PushFeatureSwitchParams.TrendTweetNotificationsFatigueDuration, - PushFeatureSwitchParams.MaxTrendTweetNotificationsInDuration, - trendAdaptorStats - ) - - private val recommendedTrendsWithTweetsCandidateSource: CandidateSource[ - Target, - RawCandidate with TrendsCandidate - ] = recommendedTrendsCandidateSource - .convert[Target, TrendsCandidate]( - getQuery, - recommendedTrendsCandidateSource.identityCandidateMapper - ) - .batchMapValues[Target, RawCandidate with TrendsCandidate]( - trendsCandidatesToTweetCandidates(_, _, getTweetyPieResults)) - - private def getTweetyPieResults( - tweetIds: Seq[TweetId], - target: Target - ): Future[Map[TweetId, TweetyPieResult]] = { - if (target.params(PushFeatureSwitchParams.EnableSafeUserTweetTweetypieStore)) { - Future - .collect( - safeUserTweetTweetyPieStore.multiGet( - tweetIds.toSet.map(UserTweet(_, Some(target.targetId))))).map { - _.collect { - case (userTweet, Some(tweetyPieResult)) => userTweet.tweetId -> tweetyPieResult - } - } - } else { - Future - .collect((target.params(PushFeatureSwitchParams.EnableVFInTweetypie) match { - case true => tweetyPieStore - case false => tweetyPieStoreNoVF - }).multiGet(tweetIds.toSet)).map { tweetyPieResultMap => - filterOutReplyTweet(tweetyPieResultMap, nonReplyTweetsCounter).collect { - case (tweetId, Some(tweetyPieResult)) => tweetId -> tweetyPieResult - } - } - } - } - - /** - * - * @param _target: [[Target]] object representing notificaion recipient user - * @param trendsCandidates: Sequence of [[TrendsCandidate]] returned from ERS - * @return: Seq of trends candidates expanded to associated tweets. - */ - private def trendsCandidatesToTweetCandidates( - _target: Target, - trendsCandidates: Seq[TrendsCandidate], - getTweetyPieResults: (Seq[TweetId], Target) => Future[Map[TweetId, TweetyPieResult]] - ): Future[Seq[RawCandidate with TrendsCandidate]] = { - - def generateTrendTweetCandidates( - trendCandidate: TrendsCandidate, - tweetyPieResults: Map[TweetId, TweetyPieResult] - ) = { - val tweetIds = trendCandidate.context.curatedRepresentativeTweets.getOrElse(Seq.empty) ++ - trendCandidate.context.algoRepresentativeTweets.getOrElse(Seq.empty) - - tweetIds.flatMap { tweetId => - tweetyPieResults.get(tweetId).map { _tweetyPieResult => - new RawCandidate with TrendTweetCandidate { - override val trendId: String = trendCandidate.trendId - override val trendName: String = trendCandidate.trendName - override val landingUrl: String = trendCandidate.landingUrl - override val timeBoundedLandingUrl: Option[String] = - trendCandidate.timeBoundedLandingUrl - override val context: TrendsContext = trendCandidate.context - override val tweetyPieResult: Option[TweetyPieResult] = Some(_tweetyPieResult) - override val tweetId: TweetId = _tweetyPieResult.tweet.id - override val target: Target = _target - } - } - } - } - - // collect all tweet ids associated with all trends - val allTweetIds = trendsCandidates.flatMap { trendsCandidate => - val context = trendsCandidate.context - context.curatedRepresentativeTweets.getOrElse(Seq.empty) ++ - context.algoRepresentativeTweets.getOrElse(Seq.empty) - } - - getTweetyPieResults(allTweetIds, _target) - .map { tweetIdToTweetyPieResult => - val trendTweetCandidates = trendsCandidates.flatMap { trendCandidate => - val allTrendTweetCandidates = generateTrendTweetCandidates( - trendCandidate, - tweetIdToTweetyPieResult - ) - - val (tweetCandidatesFromCuratedTrends, tweetCandidatesFromNonCuratedTrends) = - allTrendTweetCandidates.partition(_.isCuratedTrend) - - tweetCandidatesFromCuratedTrends.filter( - _.target.params(PushFeatureSwitchParams.EnableCuratedTrendTweets)) ++ - tweetCandidatesFromNonCuratedTrends.filter( - _.target.params(PushFeatureSwitchParams.EnableNonCuratedTrendTweets)) - } - - trendTweetCandidateNumber.incr(trendTweetCandidates.size) - trendTweetCandidates - } - } - - /** - * - * @param target: [[Target]] user - * @return: true if customer is eligible to receive trend tweet notifications - * - */ - override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { - PushDeviceUtil - .isRecommendationsEligible(target) - .map(target.params(PushParams.TrendsCandidateDecider) && _) - } - - override def get(target: Target): Future[Option[Seq[RawCandidate with TrendsCandidate]]] = { - recommendedTrendsWithTweetsCandidateSource - .get(target) - .flatMap { - case Some(candidates) if candidates.nonEmpty => - trendTweetFatiguePredicate(Seq(target)) - .map(_.head) - .map { isTargetFatigueEligible => - if (isTargetFatigueEligible) Some(candidates) - else None - } - - case _ => Future.None - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TripGeoCandidatesAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TripGeoCandidatesAdaptor.scala deleted file mode 100644 index 2bdef162c..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TripGeoCandidatesAdaptor.scala +++ /dev/null @@ -1,188 +0,0 @@ -package com.twitter.frigate.pushservice.adaptor - -import com.twitter.content_mixer.thriftscala.ContentMixerProductResponse -import com.twitter.content_mixer.thriftscala.ContentMixerRequest -import com.twitter.content_mixer.thriftscala.ContentMixerResponse -import com.twitter.content_mixer.thriftscala.NotificationsTripTweetsProductContext -import com.twitter.content_mixer.thriftscala.Product -import com.twitter.content_mixer.thriftscala.ProductContext -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateSource -import com.twitter.frigate.common.base.CandidateSourceEligible -import com.twitter.frigate.common.predicate.CommonOutNetworkTweetCandidatesSourcePredicates.filterOutReplyTweet -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.util.MediaCRT -import com.twitter.frigate.pushservice.util.PushAdaptorUtil -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.geoduck.util.country.CountryInfo -import com.twitter.product_mixer.core.thriftscala.ClientContext -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweets -import com.twitter.util.Future - -case class TripGeoCandidatesAdaptor( - tripTweetCandidateStore: ReadableStore[TripDomain, TripTweets], - contentMixerStore: ReadableStore[ContentMixerRequest, ContentMixerResponse], - tweetyPieStore: ReadableStore[Long, TweetyPieResult], - tweetyPieStoreNoVF: ReadableStore[Long, TweetyPieResult], - statsReceiver: StatsReceiver) - extends CandidateSource[Target, RawCandidate] - with CandidateSourceEligible[Target, RawCandidate] { - - override def name: String = this.getClass.getSimpleName - - private val stats = statsReceiver.scope(name.stripSuffix("$")) - - private val contentMixerRequests = stats.counter("getTripCandidatesContentMixerRequests") - private val loggedOutTripTweetIds = stats.counter("logged_out_trip_tweet_ids_count") - private val loggedOutRawCandidates = stats.counter("logged_out_raw_candidates_count") - private val rawCandidates = stats.counter("raw_candidates_count") - private val loggedOutEmptyplaceId = stats.counter("logged_out_empty_place_id_count") - private val loggedOutPlaceId = stats.counter("logged_out_place_id_count") - private val nonReplyTweetsCounter = stats.counter("non_reply_tweets") - - override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { - if (target.isLoggedOutUser) { - Future.True - } else { - for { - isRecommendationsSettingEnabled <- PushDeviceUtil.isRecommendationsEligible(target) - inferredLanguage <- target.inferredUserDeviceLanguage - } yield { - isRecommendationsSettingEnabled && - inferredLanguage.nonEmpty && - target.params(PushParams.TripGeoTweetCandidatesDecider) - } - } - - } - - private def buildRawCandidate(target: Target, tweetyPieResult: TweetyPieResult): RawCandidate = { - PushAdaptorUtil.generateOutOfNetworkTweetCandidates( - inputTarget = target, - id = tweetyPieResult.tweet.id, - mediaCRT = MediaCRT( - CommonRecommendationType.TripGeoTweet, - CommonRecommendationType.TripGeoTweet, - CommonRecommendationType.TripGeoTweet - ), - result = Some(tweetyPieResult), - localizedEntity = None - ) - } - - override def get(target: Target): Future[Option[Seq[RawCandidate]]] = { - if (target.isLoggedOutUser) { - for { - tripTweetIds <- getTripCandidatesForLoggedOutTarget(target) - tweetyPieResults <- Future.collect(tweetyPieStoreNoVF.multiGet(tripTweetIds)) - } yield { - val candidates = tweetyPieResults.values.flatten.map(buildRawCandidate(target, _)) - if (candidates.nonEmpty) { - loggedOutRawCandidates.incr(candidates.size) - Some(candidates.toSeq) - } else None - } - } else { - for { - tripTweetIds <- getTripCandidatesContentMixer(target) - tweetyPieResults <- - Future.collect((target.params(PushFeatureSwitchParams.EnableVFInTweetypie) match { - case true => tweetyPieStore - case false => tweetyPieStoreNoVF - }).multiGet(tripTweetIds)) - } yield { - val nonReplyTweets = filterOutReplyTweet(tweetyPieResults, nonReplyTweetsCounter) - val candidates = nonReplyTweets.values.flatten.map(buildRawCandidate(target, _)) - if (candidates.nonEmpty && target.params( - PushFeatureSwitchParams.TripTweetCandidateReturnEnable)) { - rawCandidates.incr(candidates.size) - Some(candidates.toSeq) - } else None - } - } - } - - private def getTripCandidatesContentMixer( - target: Target - ): Future[Set[Long]] = { - contentMixerRequests.incr() - Future - .join( - target.inferredUserDeviceLanguage, - target.deviceInfo - ) - .flatMap { - case (languageOpt, deviceInfoOpt) => - contentMixerStore - .get( - ContentMixerRequest( - clientContext = ClientContext( - userId = Some(target.targetId), - languageCode = languageOpt, - userAgent = deviceInfoOpt.flatMap(_.guessedPrimaryDeviceUserAgent.map(_.toString)) - ), - product = Product.NotificationsTripTweets, - productContext = Some( - ProductContext.NotificationsTripTweetsProductContext( - NotificationsTripTweetsProductContext() - )), - cursor = None, - maxResults = - Some(target.params(PushFeatureSwitchParams.TripTweetMaxTotalCandidates)) - ) - ).map { - _.map { rawResponse => - val tripResponse = - rawResponse.contentMixerProductResponse - .asInstanceOf[ - ContentMixerProductResponse.NotificationsTripTweetsProductResponse] - .notificationsTripTweetsProductResponse - - tripResponse.results.map(_.tweetResult.tweetId).toSet - }.getOrElse(Set.empty) - } - } - } - - private def getTripCandidatesForLoggedOutTarget( - target: Target - ): Future[Set[Long]] = { - Future.join(target.targetLanguage, target.countryCode).flatMap { - case (Some(lang), Some(country)) => - val placeId = CountryInfo.lookupByCode(country).map(_.placeIdLong) - if (placeId.nonEmpty) { - loggedOutPlaceId.incr() - } else { - loggedOutEmptyplaceId.incr() - } - val tripSource = "TOP_GEO_V3_LR" - val tripQuery = TripDomain( - sourceId = tripSource, - language = Some(lang), - placeId = placeId, - topicId = None - ) - val response = tripTweetCandidateStore.get(tripQuery) - val tripTweetIds = - response.map { res => - if (res.isDefined) { - res.get.tweets - .sortBy(_.score)(Ordering[Double].reverse).map(_.tweetId).toSet - } else { - Set.empty[Long] - } - } - tripTweetIds.map { ids => loggedOutTripTweetIds.incr(ids.size) } - tripTweetIds - - case (_, _) => Future.value(Set.empty) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/Config.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/Config.scala deleted file mode 100644 index 3a0e1dc70..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/Config.scala +++ /dev/null @@ -1,461 +0,0 @@ -package com.twitter.frigate.pushservice.config - -import com.twitter.abdecider.LoggingABDecider -import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringRequest -import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringResponse -import com.twitter.audience_rewards.thriftscala.HasSuperFollowingRelationshipRequest -import com.twitter.channels.common.thriftscala.ApiList -import com.twitter.datatools.entityservice.entities.sports.thriftscala._ -import com.twitter.decider.Decider -import com.twitter.discovery.common.configapi.ConfigParamsBuilder -import com.twitter.escherbird.common.thriftscala.QualifiedId -import com.twitter.escherbird.metadata.thriftscala.EntityMegadata -import com.twitter.eventbus.client.EventBusPublisher -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.candidate._ -import com.twitter.frigate.common.history._ -import com.twitter.frigate.common.ml.base._ -import com.twitter.frigate.common.ml.feature._ -import com.twitter.frigate.common.store._ -import com.twitter.frigate.common.store.deviceinfo.DeviceInfo -import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext -import com.twitter.frigate.common.store.interests.UserId -import com.twitter.frigate.common.util._ -import com.twitter.frigate.data_pipeline.features_common._ -import com.twitter.frigate.data_pipeline.thriftscala.UserHistoryKey -import com.twitter.frigate.data_pipeline.thriftscala.UserHistoryValue -import com.twitter.frigate.dau_model.thriftscala.DauProbability -import com.twitter.frigate.magic_events.thriftscala.FanoutEvent -import com.twitter.frigate.pushcap.thriftscala.PushcapUserHistory -import com.twitter.frigate.pushservice.ml._ -import com.twitter.frigate.pushservice.params.DeciderKey -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushFeatureSwitches -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.send_handler.SendHandlerPushCandidateHydrator -import com.twitter.frigate.pushservice.refresh_handler.PushCandidateHydrator -import com.twitter.frigate.pushservice.store._ -import com.twitter.frigate.pushservice.store.{Ibis2Store => PushIbis2Store} -import com.twitter.frigate.pushservice.take.NotificationServiceRequest -import com.twitter.frigate.pushservice.thriftscala.PushRequestScribe -import com.twitter.frigate.scribe.thriftscala.NotificationScribe -import com.twitter.frigate.thriftscala._ -import com.twitter.frigate.user_states.thriftscala.MRUserHmmState -import com.twitter.geoduck.common.thriftscala.{Location => GeoLocation} -import com.twitter.geoduck.service.thriftscala.LocationResponse -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.hermit.pop_geo.thriftscala.PopTweetsInPlace -import com.twitter.hermit.predicate.socialgraph.RelationEdge -import com.twitter.hermit.predicate.tweetypie.Perspective -import com.twitter.hermit.predicate.tweetypie.UserTweet -import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery -import com.twitter.hermit.store.tweetypie.{UserTweet => TweetyPieUserTweet} -import com.twitter.hermit.stp.thriftscala.STPResult -import com.twitter.hss.api.thriftscala.UserHealthSignalResponse -import com.twitter.interests.thriftscala.InterestId -import com.twitter.interests.thriftscala.{UserInterests => Interests} -import com.twitter.interests_discovery.thriftscala.NonPersonalizedRecommendedLists -import com.twitter.interests_discovery.thriftscala.RecommendedListsRequest -import com.twitter.interests_discovery.thriftscala.RecommendedListsResponse -import com.twitter.livevideo.timeline.domain.v2.{Event => LiveEvent} -import com.twitter.ml.api.thriftscala.{DataRecord => ThriftDataRecord} -import com.twitter.ml.featurestore.lib.dynamic.DynamicFeatureStoreClient -import com.twitter.notificationservice.genericfeedbackstore.FeedbackPromptValue -import com.twitter.notificationservice.genericfeedbackstore.GenericFeedbackStore -import com.twitter.notificationservice.scribe.manhattan.GenericNotificationsFeedbackRequest -import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationResponse -import com.twitter.nrel.heavyranker.CandidateFeatureHydrator -import com.twitter.nrel.heavyranker.{FeatureHydrator => MRFeatureHydrator} -import com.twitter.nrel.heavyranker.{TargetFeatureHydrator => RelevanceTargetFeatureHydrator} -import com.twitter.onboarding.task.service.thriftscala.FatigueFlowEnrollment -import com.twitter.permissions_storage.thriftscala.AppPermission -import com.twitter.recommendation.interests.discovery.core.model.InterestDomain -import com.twitter.recos.user_tweet_entity_graph.thriftscala.RecommendTweetEntityRequest -import com.twitter.recos.user_tweet_entity_graph.thriftscala.RecommendTweetEntityResponse -import com.twitter.recos.user_user_graph.thriftscala.RecommendUserRequest -import com.twitter.recos.user_user_graph.thriftscala.RecommendUserResponse -import com.twitter.rux.common.strato.thriftscala.UserTargetingProperty -import com.twitter.scio.nsfw_user_segmentation.thriftscala.NSFWProducer -import com.twitter.scio.nsfw_user_segmentation.thriftscala.NSFWUserSegmentation -import com.twitter.search.common.features.thriftscala.ThriftSearchResultFeatures -import com.twitter.search.earlybird.thriftscala.EarlybirdRequest -import com.twitter.search.earlybird.thriftscala.ThriftSearchResult -import com.twitter.service.gen.scarecrow.thriftscala.Event -import com.twitter.service.gen.scarecrow.thriftscala.TieredActionResult -import com.twitter.service.metastore.gen.thriftscala.Location -import com.twitter.service.metastore.gen.thriftscala.UserLanguages -import com.twitter.servo.decider.DeciderGateBuilder -import com.twitter.simclusters_v2.thriftscala.SimClustersInferredEntities -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.columns.frigate.logged_out_web_notifications.thriftscala.LOWebNotificationMetadata -import com.twitter.strato.columns.notifications.thriftscala.SourceDestUserRequest -import com.twitter.strato.client.{UserId => StratoUserId} -import com.twitter.timelines.configapi -import com.twitter.timelines.configapi.CompositeConfig -import com.twitter.timelinescorer.thriftscala.v1.ScoredTweet -import com.twitter.topiclisting.TopicListing -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweets -import com.twitter.tsp.thriftscala.TopicSocialProofRequest -import com.twitter.tsp.thriftscala.TopicSocialProofResponse -import com.twitter.ubs.thriftscala.SellerTrack -import com.twitter.ubs.thriftscala.AudioSpace -import com.twitter.ubs.thriftscala.Participants -import com.twitter.ubs.thriftscala.SellerApplicationState -import com.twitter.user_session_store.thriftscala.UserSession -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.wtf.scalding.common.thriftscala.UserFeatures - -trait Config { - self => - - def isServiceLocal: Boolean - - def localConfigRepoPath: String - - def inMemCacheOff: Boolean - - def historyStore: PushServiceHistoryStore - - def emailHistoryStore: PushServiceHistoryStore - - def strongTiesStore: ReadableStore[Long, STPResult] - - def safeUserStore: ReadableStore[Long, User] - - def deviceInfoStore: ReadableStore[Long, DeviceInfo] - - def edgeStore: ReadableStore[RelationEdge, Boolean] - - def socialGraphServiceProcessStore: ReadableStore[RelationEdge, Boolean] - - def userUtcOffsetStore: ReadableStore[Long, Duration] - - def cachedTweetyPieStoreV2: ReadableStore[Long, TweetyPieResult] - - def safeCachedTweetyPieStoreV2: ReadableStore[Long, TweetyPieResult] - - def userTweetTweetyPieStore: ReadableStore[TweetyPieUserTweet, TweetyPieResult] - - def safeUserTweetTweetyPieStore: ReadableStore[TweetyPieUserTweet, TweetyPieResult] - - def cachedTweetyPieStoreV2NoVF: ReadableStore[Long, TweetyPieResult] - - def tweetContentFeatureCacheStore: ReadableStore[Long, ThriftDataRecord] - - def scarecrowCheckEventStore: ReadableStore[Event, TieredActionResult] - - def userTweetPerspectiveStore: ReadableStore[UserTweet, Perspective] - - def userCountryStore: ReadableStore[Long, Location] - - def pushInfoStore: ReadableStore[Long, UserForPushTargeting] - - def loggedOutPushInfoStore: ReadableStore[Long, LOWebNotificationMetadata] - - def tweetImpressionStore: ReadableStore[Long, Seq[Long]] - - def audioSpaceStore: ReadableStore[String, AudioSpace] - - def basketballGameScoreStore: ReadableStore[QualifiedId, BasketballGameLiveUpdate] - - def baseballGameScoreStore: ReadableStore[QualifiedId, BaseballGameLiveUpdate] - - def cricketMatchScoreStore: ReadableStore[QualifiedId, CricketMatchLiveUpdate] - - def soccerMatchScoreStore: ReadableStore[QualifiedId, SoccerMatchLiveUpdate] - - def nflGameScoreStore: ReadableStore[QualifiedId, NflFootballGameLiveUpdate] - - def topicSocialProofServiceStore: ReadableStore[TopicSocialProofRequest, TopicSocialProofResponse] - - def spaceDeviceFollowStore: ReadableStore[SourceDestUserRequest, Boolean] - - def audioSpaceParticipantsStore: ReadableStore[String, Participants] - - def notificationServiceSender: ReadableStore[ - NotificationServiceRequest, - CreateGenericNotificationResponse - ] - - def ocfFatigueStore: ReadableStore[OCFHistoryStoreKey, FatigueFlowEnrollment] - - def dauProbabilityStore: ReadableStore[Long, DauProbability] - - def hydratedLabeledPushRecsStore: ReadableStore[UserHistoryKey, UserHistoryValue] - - def userHTLLastVisitStore: ReadableStore[Long, Seq[Long]] - - def userLanguagesStore: ReadableStore[Long, UserLanguages] - - def topTweetsByGeoStore: ReadableStore[InterestDomain[String], Map[String, List[ - (Long, Double) - ]]] - - def topTweetsByGeoV2VersionedStore: ReadableStore[String, PopTweetsInPlace] - - lazy val pushRecItemStore: ReadableStore[PushRecItemsKey, RecItems] = PushRecItemStore( - hydratedLabeledPushRecsStore - ) - - lazy val labeledPushRecsVerifyingStore: ReadableStore[ - LabeledPushRecsVerifyingStoreKey, - LabeledPushRecsVerifyingStoreResponse - ] = - LabeledPushRecsVerifyingStore( - hydratedLabeledPushRecsStore, - historyStore - ) - - lazy val labeledPushRecsDecideredStore: ReadableStore[LabeledPushRecsStoreKey, UserHistoryValue] = - LabeledPushRecsDecideredStore( - labeledPushRecsVerifyingStore, - useHydratedLabeledSendsForFeaturesDeciderKey, - verifyHydratedLabeledSendsForFeaturesDeciderKey - ) - - def onlineUserHistoryStore: ReadableStore[OnlineUserHistoryKey, UserHistoryValue] - - def nsfwConsumerStore: ReadableStore[Long, NSFWUserSegmentation] - - def nsfwProducerStore: ReadableStore[Long, NSFWProducer] - - def popGeoLists: ReadableStore[String, NonPersonalizedRecommendedLists] - - def listAPIStore: ReadableStore[Long, ApiList] - - def openedPushByHourAggregatedStore: ReadableStore[Long, Map[Int, Int]] - - def userHealthSignalStore: ReadableStore[Long, UserHealthSignalResponse] - - def reactivatedUserInfoStore: ReadableStore[Long, String] - - def weightedOpenOrNtabClickModelScorer: PushMLModelScorer - - def optoutModelScorer: PushMLModelScorer - - def filteringModelScorer: PushMLModelScorer - - def recentFollowsStore: ReadableStore[Long, Seq[Long]] - - def geoDuckV2Store: ReadableStore[UserId, LocationResponse] - - def realGraphScoresTop500InStore: ReadableStore[Long, Map[Long, Double]] - - def tweetEntityGraphStore: ReadableStore[ - RecommendTweetEntityRequest, - RecommendTweetEntityResponse - ] - - def userUserGraphStore: ReadableStore[RecommendUserRequest, RecommendUserResponse] - - def userFeaturesStore: ReadableStore[Long, UserFeatures] - - def userTargetingPropertyStore: ReadableStore[Long, UserTargetingProperty] - - def timelinesUserSessionStore: ReadableStore[Long, UserSession] - - def optOutUserInterestsStore: ReadableStore[UserId, Seq[InterestId]] - - def ntabCaretFeedbackStore: ReadableStore[GenericNotificationsFeedbackRequest, Seq[ - CaretFeedbackDetails - ]] - - def genericFeedbackStore: ReadableStore[FeedbackRequest, Seq[ - FeedbackPromptValue - ]] - - def genericNotificationFeedbackStore: GenericFeedbackStore - - def semanticCoreMegadataStore: ReadableStore[ - SemanticEntityForQuery, - EntityMegadata - ] - - def tweetHealthScoreStore: ReadableStore[TweetScoringRequest, TweetScoringResponse] - - def earlybirdFeatureStore: ReadableStore[Long, ThriftSearchResultFeatures] - - def earlybirdFeatureBuilder: FeatureBuilder[Long] - - // Feature builders - - def tweetAuthorLocationFeatureBuilder: FeatureBuilder[Location] - - def tweetAuthorLocationFeatureBuilderById: FeatureBuilder[Long] - - def socialContextActionsFeatureBuilder: FeatureBuilder[SocialContextActions] - - def tweetContentFeatureBuilder: FeatureBuilder[Long] - - def tweetAuthorRecentRealGraphFeatureBuilder: FeatureBuilder[RealGraphEdge] - - def socialContextRecentRealGraphFeatureBuilder: FeatureBuilder[Set[RealGraphEdge]] - - def tweetSocialProofFeatureBuilder: FeatureBuilder[TweetSocialProofKey] - - def targetUserFullRealGraphFeatureBuilder: FeatureBuilder[TargetFullRealGraphFeatureKey] - - def postProcessingFeatureBuilder: PostProcessingFeatureBuilder - - def mrOfflineUserCandidateSparseAggregatesFeatureBuilder: FeatureBuilder[ - OfflineSparseAggregateKey - ] - - def mrOfflineUserAggregatesFeatureBuilder: FeatureBuilder[Long] - - def mrOfflineUserCandidateAggregatesFeatureBuilder: FeatureBuilder[OfflineAggregateKey] - - def tweetAnnotationsFeatureBuilder: FeatureBuilder[Long] - - def targetUserMediaRepresentationFeatureBuilder: FeatureBuilder[Long] - - def targetLevelFeatureBuilder: FeatureBuilder[MrRequestContextForFeatureStore] - - def candidateLevelFeatureBuilder: FeatureBuilder[EntityRequestContextForFeatureStore] - - def targetFeatureHydrator: RelevanceTargetFeatureHydrator - - def useHydratedLabeledSendsForFeaturesDeciderKey: String = - DeciderKey.useHydratedLabeledSendsForFeaturesDeciderKey.toString - - def verifyHydratedLabeledSendsForFeaturesDeciderKey: String = - DeciderKey.verifyHydratedLabeledSendsForFeaturesDeciderKey.toString - - def lexServiceStore: ReadableStore[EventRequest, LiveEvent] - - def userMediaRepresentationStore: ReadableStore[Long, UserMediaRepresentation] - - def producerMediaRepresentationStore: ReadableStore[Long, UserMediaRepresentation] - - def mrUserStatePredictionStore: ReadableStore[Long, MRUserHmmState] - - def pushcapDynamicPredictionStore: ReadableStore[Long, PushcapUserHistory] - - def earlybirdCandidateSource: EarlybirdCandidateSource - - def earlybirdSearchStore: ReadableStore[EarlybirdRequest, Seq[ThriftSearchResult]] - - def earlybirdSearchDest: String - - def pushserviceThriftClientId: ClientId - - def simClusterToEntityStore: ReadableStore[Int, SimClustersInferredEntities] - - def fanoutMetadataStore: ReadableStore[(Long, Long), FanoutEvent] - - /** - * PostRanking Feature Store Client - */ - def postRankingFeatureStoreClient: DynamicFeatureStoreClient[MrRequestContextForFeatureStore] - - /** - * ReadableStore to fetch [[UserInterests]] from INTS service - */ - def interestsWithLookupContextStore: ReadableStore[InterestsLookupRequestWithContext, Interests] - - /** - * - * @return: [[TopicListing]] object to fetch paused topics and scope from productId - */ - def topicListing: TopicListing - - /** - * - * @return: [[UttEntityHydrationStore]] object - */ - def uttEntityHydrationStore: UttEntityHydrationStore - - def appPermissionStore: ReadableStore[(Long, (String, String)), AppPermission] - - lazy val userTweetEntityGraphCandidates: UserTweetEntityGraphCandidates = - UserTweetEntityGraphCandidates( - cachedTweetyPieStoreV2, - tweetEntityGraphStore, - PushParams.UTEGTweetCandidateSourceParam, - PushFeatureSwitchParams.NumberOfMaxUTEGCandidatesQueriedParam, - PushParams.AllowOneSocialProofForTweetInUTEGParam, - PushParams.OutNetworkTweetsOnlyForUTEGParam, - PushFeatureSwitchParams.MaxTweetAgeParam - )(statsReceiver) - - def pushSendEventBusPublisher: EventBusPublisher[NotificationScribe] - - // miscs. - - def isProd: Boolean - - implicit def statsReceiver: StatsReceiver - - def decider: Decider - - def abDecider: LoggingABDecider - - def casLock: CasLock - - def pushIbisV2Store: PushIbis2Store - - // scribe - def notificationScribe(data: NotificationScribe): Unit - - def requestScribe(data: PushRequestScribe): Unit - - def init(): Future[Unit] = Future.Done - - def configParamsBuilder: ConfigParamsBuilder - - def candidateFeatureHydrator: CandidateFeatureHydrator - - def featureHydrator: MRFeatureHydrator - - def candidateHydrator: PushCandidateHydrator - - def sendHandlerCandidateHydrator: SendHandlerPushCandidateHydrator - - lazy val overridesConfig: configapi.Config = { - val pushFeatureSwitchConfigs: configapi.Config = PushFeatureSwitches( - deciderGateBuilder = new DeciderGateBuilder(decider), - statsReceiver = statsReceiver - ).config - - new CompositeConfig(Seq(pushFeatureSwitchConfigs)) - } - - def realTimeClientEventStore: RealTimeClientEventStore - - def inlineActionHistoryStore: ReadableStore[Long, Seq[(Long, String)]] - - def softUserGeoLocationStore: ReadableStore[Long, GeoLocation] - - def tweetTranslationStore: ReadableStore[TweetTranslationStore.Key, TweetTranslationStore.Value] - - def tripTweetCandidateStore: ReadableStore[TripDomain, TripTweets] - - def softUserFollowingStore: ReadableStore[User, Seq[Long]] - - def superFollowEligibilityUserStore: ReadableStore[Long, Boolean] - - def superFollowCreatorTweetCountStore: ReadableStore[StratoUserId, Int] - - def hasSuperFollowingRelationshipStore: ReadableStore[ - HasSuperFollowingRelationshipRequest, - Boolean - ] - - def superFollowApplicationStatusStore: ReadableStore[(Long, SellerTrack), SellerApplicationState] - - def recentHistoryCacheClient: RecentHistoryCacheClient - - def openAppUserStore: ReadableStore[Long, Boolean] - - def loggedOutHistoryStore: PushServiceHistoryStore - - def idsStore: ReadableStore[RecommendedListsRequest, RecommendedListsResponse] - - def htlScoreStore(userId: Long): ReadableStore[Long, ScoredTweet] -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/DeployConfig.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/DeployConfig.scala deleted file mode 100644 index 8d6e95a67..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/DeployConfig.scala +++ /dev/null @@ -1,2150 +0,0 @@ -package com.twitter.frigate.pushservice.config - -import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringRequest -import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringResponse -import com.twitter.audience_rewards.thriftscala.HasSuperFollowingRelationshipRequest -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.bijection.scrooge.CompactScalaCodec -import com.twitter.channels.common.thriftscala.ApiList -import com.twitter.channels.common.thriftscala.ApiListDisplayLocation -import com.twitter.channels.common.thriftscala.ApiListView -import com.twitter.content_mixer.thriftscala.ContentMixer -import com.twitter.conversions.DurationOps._ -import com.twitter.cortex.deepbird.thriftjava.DeepbirdPredictionService -import com.twitter.cr_mixer.thriftscala.CrMixer -import com.twitter.datatools.entityservice.entities.sports.thriftscala.BaseballGameLiveUpdate -import com.twitter.datatools.entityservice.entities.sports.thriftscala.BasketballGameLiveUpdate -import com.twitter.datatools.entityservice.entities.sports.thriftscala.CricketMatchLiveUpdate -import com.twitter.datatools.entityservice.entities.sports.thriftscala.NflFootballGameLiveUpdate -import com.twitter.datatools.entityservice.entities.sports.thriftscala.SoccerMatchLiveUpdate -import com.twitter.discovery.common.configapi.ConfigParamsBuilder -import com.twitter.discovery.common.configapi.FeatureContextBuilder -import com.twitter.discovery.common.environment.{Environment => NotifEnvironment} -import com.twitter.escherbird.common.thriftscala.Domains -import com.twitter.escherbird.common.thriftscala.QualifiedId -import com.twitter.escherbird.metadata.thriftscala.EntityMegadata -import com.twitter.escherbird.metadata.thriftscala.MetadataService -import com.twitter.escherbird.util.metadatastitch.MetadataStitchClient -import com.twitter.escherbird.util.uttclient -import com.twitter.escherbird.util.uttclient.CacheConfigV2 -import com.twitter.escherbird.util.uttclient.CachedUttClientV2 -import com.twitter.escherbird.utt.strato.thriftscala.Environment -import com.twitter.eventbus.client.EventBusPublisherBuilder -import com.twitter.events.recos.thriftscala.EventsRecosService -import com.twitter.explore_ranker.thriftscala.ExploreRanker -import com.twitter.featureswitches.v2.FeatureSwitches -import com.twitter.finagle.Memcached -import com.twitter.finagle.ThriftMux -import com.twitter.finagle.client.BackupRequestFilter -import com.twitter.finagle.client.ClientRegistry -import com.twitter.finagle.loadbalancer.Balancers -import com.twitter.finagle.memcached.Client -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.mtls.client.MtlsStackClient._ -import com.twitter.finagle.mux.transport.OpportunisticTls -import com.twitter.finagle.service.Retries -import com.twitter.finagle.service.RetryPolicy -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.finagle.thrift.RichClientParam -import com.twitter.finagle.util.DefaultTimer -import com.twitter.flockdb.client._ -import com.twitter.flockdb.client.thriftscala.FlockDB -import com.twitter.frigate.common.base.RandomRanker -import com.twitter.frigate.common.candidate._ -import com.twitter.frigate.common.config.RateLimiterGenerator -import com.twitter.frigate.common.entity_graph_client.RecommendedTweetEntitiesStore -import com.twitter.frigate.common.filter.DynamicRequestMeterFilter -import com.twitter.frigate.common.history._ -import com.twitter.frigate.common.ml.feature._ -import com.twitter.frigate.common.store._ -import com.twitter.frigate.common.store.deviceinfo.DeviceInfoStore -import com.twitter.frigate.common.store.deviceinfo.MobileSdkStore -import com.twitter.frigate.common.store.interests._ -import com.twitter.frigate.common.store.strato.StratoFetchableStore -import com.twitter.frigate.common.store.strato.StratoScannableStore -import com.twitter.frigate.common.util.Finagle.readOnlyThriftService -import com.twitter.frigate.common.util._ -import com.twitter.frigate.data_pipeline.features_common.FeatureStoreUtil -import com.twitter.frigate.data_pipeline.features_common._ -import com.twitter.frigate.data_pipeline.thriftscala.UserHistoryKey -import com.twitter.frigate.data_pipeline.thriftscala.UserHistoryValue -import com.twitter.frigate.dau_model.thriftscala.DauProbability -import com.twitter.frigate.magic_events.thriftscala.FanoutEvent -import com.twitter.frigate.pushcap.thriftscala.PushcapUserHistory -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.adaptor.LoggedOutPushCandidateSourceGenerator -import com.twitter.frigate.pushservice.adaptor.PushCandidateSourceGenerator -import com.twitter.frigate.pushservice.config.mlconfig.DeepbirdV2ModelConfig -import com.twitter.frigate.pushservice.ml._ -import com.twitter.frigate.pushservice.params._ -import com.twitter.frigate.pushservice.rank.LoggedOutRanker -import com.twitter.frigate.pushservice.rank.RFPHLightRanker -import com.twitter.frigate.pushservice.rank.RFPHRanker -import com.twitter.frigate.pushservice.rank.SubscriptionCreatorRanker -import com.twitter.frigate.pushservice.refresh_handler._ -import com.twitter.frigate.pushservice.refresh_handler.cross.CandidateCopyExpansion -import com.twitter.frigate.pushservice.send_handler.SendHandlerPushCandidateHydrator -import com.twitter.frigate.pushservice.store._ -import com.twitter.frigate.pushservice.take.CandidateNotifier -import com.twitter.frigate.pushservice.take.NotificationSender -import com.twitter.frigate.pushservice.take.NotificationServiceRequest -import com.twitter.frigate.pushservice.take.NotificationServiceSender -import com.twitter.frigate.pushservice.take.NtabOnlyChannelSelector -import com.twitter.frigate.pushservice.take.history.EventBusWriter -import com.twitter.frigate.pushservice.take.history.HistoryWriter -import com.twitter.frigate.pushservice.take.sender.Ibis2Sender -import com.twitter.frigate.pushservice.take.sender.NtabSender -import com.twitter.frigate.pushservice.take.LoggedOutRefreshForPushNotifier -import com.twitter.frigate.pushservice.util.RFPHTakeStepUtil -import com.twitter.frigate.pushservice.util.SendHandlerPredicateUtil -import com.twitter.frigate.scribe.thriftscala.NotificationScribe -import com.twitter.frigate.thriftscala._ -import com.twitter.frigate.user_states.thriftscala.MRUserHmmState -import com.twitter.geoduck.backend.hydration.thriftscala.Hydration -import com.twitter.geoduck.common.thriftscala.PlaceQueryFields -import com.twitter.geoduck.common.thriftscala.PlaceType -import com.twitter.geoduck.common.thriftscala.{Location => GeoLocation} -import com.twitter.geoduck.service.common.clientmodules.GeoduckUserLocate -import com.twitter.geoduck.service.common.clientmodules.GeoduckUserLocateModule -import com.twitter.geoduck.service.thriftscala.LocationResponse -import com.twitter.geoduck.thriftscala.LocationService -import com.twitter.gizmoduck.context.thriftscala.ReadConfig -import com.twitter.gizmoduck.context.thriftscala.TestUserConfig -import com.twitter.gizmoduck.testusers.client.TestUserClientBuilder -import com.twitter.gizmoduck.thriftscala.LookupContext -import com.twitter.gizmoduck.thriftscala.QueryFields -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.gizmoduck.thriftscala.UserService -import com.twitter.hermit.pop_geo.thriftscala.PopTweetsInPlace -import com.twitter.hermit.predicate.socialgraph.SocialGraphPredicate -import com.twitter.hermit.predicate.tweetypie.PerspectiveReadableStore -import com.twitter.hermit.store._ -import com.twitter.hermit.store.common._ -import com.twitter.hermit.store.gizmoduck.GizmoduckUserStore -import com.twitter.hermit.store.metastore.UserCountryStore -import com.twitter.hermit.store.metastore.UserLanguagesStore -import com.twitter.hermit.store.scarecrow.ScarecrowCheckEventStore -import com.twitter.hermit.store.semantic_core.MetaDataReadableStore -import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery -import com.twitter.hermit.store.timezone.GizmoduckUserUtcOffsetStore -import com.twitter.hermit.store.timezone.UtcOffsetStore -import com.twitter.hermit.store.tweetypie.TweetyPieStore -import com.twitter.hermit.store.tweetypie.UserTweet -import com.twitter.hermit.store.user_htl_session_store.UserHTLLastVisitReadableStore -import com.twitter.hermit.stp.thriftscala.STPResult -import com.twitter.hss.api.thriftscala.UserHealthSignal -import com.twitter.hss.api.thriftscala.UserHealthSignal._ -import com.twitter.hss.api.thriftscala.UserHealthSignalResponse -import com.twitter.interests.thriftscala.InterestId -import com.twitter.interests.thriftscala.InterestsThriftService -import com.twitter.interests.thriftscala.{UserInterests => Interests} -import com.twitter.interests_discovery.thriftscala.InterestsDiscoveryService -import com.twitter.interests_discovery.thriftscala.NonPersonalizedRecommendedLists -import com.twitter.interests_discovery.thriftscala.RecommendedListsRequest -import com.twitter.interests_discovery.thriftscala.RecommendedListsResponse -import com.twitter.kujaku.domain.thriftscala.MachineTranslationResponse -import com.twitter.livevideo.timeline.client.v2.LiveVideoTimelineClient -import com.twitter.livevideo.timeline.domain.v2.{Event => LiveEvent} -import com.twitter.livevideo.timeline.thrift.thriftscala.TimelineService -import com.twitter.logging.Logger -import com.twitter.ml.api.thriftscala.{DataRecord => ThriftDataRecord} -import com.twitter.ml.featurestore.catalog.entities.core.{Author => TweetAuthorEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{User => TargetUserEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{UserAuthor => UserAuthorEntity} -import com.twitter.ml.featurestore.catalog.entities.magicrecs.{SocialContext => SocialContextEntity} -import com.twitter.ml.featurestore.catalog.entities.magicrecs.{UserSocialContext => TargetUserSocialContextEntity} -import com.twitter.ml.featurestore.timelines.thriftscala.TimelineScorerScoreView -import com.twitter.notificationservice.api.thriftscala.DeleteCurrentTimelineForUserRequest -import com.twitter.notificationservice.genericfeedbackstore.FeedbackPromptValue -import com.twitter.notificationservice.genericfeedbackstore.GenericFeedbackStore -import com.twitter.notificationservice.genericfeedbackstore.GenericFeedbackStoreBuilder -import com.twitter.notificationservice.scribe.manhattan.FeedbackSignalManhattanClient -import com.twitter.notificationservice.scribe.manhattan.GenericNotificationsFeedbackRequest -import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationRequest -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationResponse -import com.twitter.notificationservice.thriftscala.DeleteGenericNotificationRequest -import com.twitter.notificationservice.thriftscala.GenericNotificationOverrideKey -import com.twitter.notificationservice.thriftscala.NotificationService$FinagleClient -import com.twitter.nrel.heavyranker.CandidateFeatureHydrator -import com.twitter.nrel.heavyranker.FeatureHydrator -import com.twitter.nrel.heavyranker.{PushPredictionServiceStore => RelevancePushPredictionServiceStore} -import com.twitter.nrel.heavyranker.{TargetFeatureHydrator => RelevanceTargetFeatureHydrator} -import com.twitter.nrel.lightranker.MagicRecsServeDataRecordLightRanker -import com.twitter.nrel.lightranker.{Config => LightRankerConfig} -import com.twitter.onboarding.task.service.thriftscala.FatigueFlowEnrollment -import com.twitter.periscope.api.thriftscala.AudioSpacesLookupContext -import com.twitter.permissions_storage.thriftscala.AppPermission -import com.twitter.recommendation.interests.discovery.core.config.{DeployConfig => InterestDeployConfig} -import com.twitter.recommendation.interests.discovery.popgeo.deploy.PopGeoInterestProvider -import com.twitter.recos.user_tweet_entity_graph.thriftscala.UserTweetEntityGraph -import com.twitter.recos.user_user_graph.thriftscala.UserUserGraph -import com.twitter.rux.common.strato.thriftscala.UserTargetingProperty -import com.twitter.scio.nsfw_user_segmentation.thriftscala.NSFWProducer -import com.twitter.scio.nsfw_user_segmentation.thriftscala.NSFWUserSegmentation -import com.twitter.search.earlybird.thriftscala.EarlybirdService -import com.twitter.service.gen.scarecrow.thriftscala.ScarecrowService -import com.twitter.service.metastore.gen.thriftscala.Location -import com.twitter.simclusters_v2.thriftscala.SimClustersInferredEntities -import com.twitter.socialgraph.thriftscala.SocialGraphService -import com.twitter.spam.rtf.thriftscala.SafetyLevel -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storage.client.manhattan.kv.Guarantee -import com.twitter.storage.client.manhattan.kv.ManhattanKVClient -import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams -import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpoint -import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpointBuilder -import com.twitter.storehaus.ReadableStore -import com.twitter.storehaus_internal.manhattan.Apollo -import com.twitter.storehaus_internal.manhattan.Athena -import com.twitter.storehaus_internal.manhattan.Dataset -import com.twitter.storehaus_internal.manhattan.ManhattanStore -import com.twitter.storehaus_internal.manhattan.Nash -import com.twitter.storehaus_internal.manhattan.Omega -import com.twitter.storehaus_internal.memcache.MemcacheStore -import com.twitter.storehaus_internal.util.ClientName -import com.twitter.storehaus_internal.util.ZkEndPoint -import com.twitter.strato.catalog.Scan.Slice -import com.twitter.strato.client.Strato -import com.twitter.strato.client.UserId -import com.twitter.strato.columns.frigate.logged_out_web_notifications.thriftscala.LOWebNotificationMetadata -import com.twitter.strato.columns.notifications.thriftscala.SourceDestUserRequest -import com.twitter.strato.generated.client.geo.user.FrequentSoftUserLocationClientColumn -import com.twitter.strato.generated.client.ml.featureStore.TimelineScorerTweetScoresV1ClientColumn -import com.twitter.strato.generated.client.notifications.space_device_follow_impl.SpaceDeviceFollowingClientColumn -import com.twitter.strato.generated.client.periscope.CoreOnAudioSpaceClientColumn -import com.twitter.strato.generated.client.periscope.ParticipantsOnAudioSpaceClientColumn -import com.twitter.strato.generated.client.rux.TargetingPropertyOnUserClientColumn -import com.twitter.strato.generated.client.socialgraph.graphs.creatorSubscriptionTimeline.{CountEdgesBySourceClientColumn => CreatorSubscriptionNumTweetsColumn} -import com.twitter.strato.generated.client.translation.service.IsTweetTranslatableClientColumn -import com.twitter.strato.generated.client.translation.service.platform.MachineTranslateTweetClientColumn -import com.twitter.strato.generated.client.trends.trip.TripTweetsAirflowProdClientColumn -import com.twitter.strato.thrift.ScroogeConvImplicits._ -import com.twitter.taxi.common.AppId -import com.twitter.taxi.deploy.Cluster -import com.twitter.taxi.deploy.Env -import com.twitter.topiclisting.TopicListing -import com.twitter.topiclisting.TopicListingBuilder -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweets -import com.twitter.tsp.thriftscala.TopicSocialProofRequest -import com.twitter.tsp.thriftscala.TopicSocialProofResponse -import com.twitter.tweetypie.thriftscala.GetTweetOptions -import com.twitter.tweetypie.thriftscala.Tweet.VisibleTextRangeField -import com.twitter.tweetypie.thriftscala.TweetService -import com.twitter.ubs.thriftscala.AudioSpace -import com.twitter.ubs.thriftscala.Participants -import com.twitter.ubs.thriftscala.SellerApplicationState -import com.twitter.user_session_store.thriftscala.UserSession -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.util.Timer -import com.twitter.util.tunable.TunableMap -import com.twitter.wtf.scalding.common.thriftscala.UserFeatures -import org.apache.thrift.protocol.TCompactProtocol -import com.twitter.timelinescorer.thriftscala.v1.ScoredTweet -import com.twitter.ubs.thriftscala.SellerTrack -import com.twitter.wtf.candidate.thriftscala.CandidateSeq - -trait DeployConfig extends Config { - // Any finagle clients should not be defined as lazy. If defined lazy, - // ClientRegistry.expAllRegisteredClientsResolved() call in init will not ensure that the clients - // are active before thrift endpoint is active. We want the clients to be active, because zookeeper - // resolution triggered by first request(s) might result in the request(s) failing. - - def serviceIdentifier: ServiceIdentifier - - def tunableMap: TunableMap - - def featureSwitches: FeatureSwitches - - override val isProd: Boolean = - serviceIdentifier.environment == PushConstants.ServiceProdEnvironmentName - - def shardParams: ShardParams - - def log: Logger - - implicit def statsReceiver: StatsReceiver - - implicit val timer: Timer = DefaultTimer - - def notifierThriftClientId: ClientId - - def loggedOutNotifierThriftClientId: ClientId - - def pushserviceThriftClientId: ClientId - - def deepbirdv2PredictionServiceDest: String - - def featureStoreUtil: FeatureStoreUtil - - def targetLevelFeaturesConfig: PushFeaturesConfig - - private val manhattanClientMtlsParams = ManhattanKVClientMtlsParams( - serviceIdentifier = serviceIdentifier, - opportunisticTls = OpportunisticTls.Required - ) - - // Commonly used clients - val gizmoduckClient = { - - val client = ThriftMux.client - .withMutualTls(serviceIdentifier) - .withClientId(pushserviceThriftClientId) - .build[UserService.MethodPerEndpoint]( - dest = "/s/gizmoduck/gizmoduck" - ) - - /** - * RequestContext test user config to allow reading test user accounts on pushservice for load - * testing - */ - val GizmoduckTestUserConfig = TestUserConfig( - clientId = Some(pushserviceThriftClientId.name), - readConfig = Some(ReadConfig(includeTestUsers = true)) - ) - - TestUserClientBuilder[UserService.MethodPerEndpoint] - .withClient(client) - .withConfig(GizmoduckTestUserConfig) - .build() - } - - val sgsClient = { - val service = readOnlyThriftService( - "", - "/s/socialgraph/socialgraph", - statsReceiver, - pushserviceThriftClientId, - mTLSServiceIdentifier = Some(serviceIdentifier) - ) - new SocialGraphService.FinagledClient(service) - } - - val tweetyPieClient = { - val service = readOnlyThriftService( - "", - "/s/tweetypie/tweetypie", - statsReceiver, - notifierThriftClientId, - mTLSServiceIdentifier = Some(serviceIdentifier) - ) - new TweetService.FinagledClient(service) - } - - lazy val geoduckHydrationClient: Hydration.MethodPerEndpoint = { - val servicePerEndpoint = ThriftMux.client - .withLabel("geoduck_hydration") - .withClientId(pushserviceThriftClientId) - .withMutualTls(serviceIdentifier) - .methodBuilder("/s/geo/hydration") - .withTimeoutPerRequest(10.seconds) - .withTimeoutTotal(10.seconds) - .idempotent(maxExtraLoad = 0.0) - .servicePerEndpoint[Hydration.ServicePerEndpoint] - Hydration.MethodPerEndpoint(servicePerEndpoint) - } - - lazy val geoduckLocationClient: LocationService.MethodPerEndpoint = { - val servicePerEndpoint = ThriftMux.client - .withLabel("geoduck_location") - .withClientId(pushserviceThriftClientId) - .withMutualTls(serviceIdentifier) - .methodBuilder("/s/geo/geoduck_locationservice") - .withTimeoutPerRequest(10.seconds) - .withTimeoutTotal(10.seconds) - .idempotent(maxExtraLoad = 0.0) - .servicePerEndpoint[LocationService.ServicePerEndpoint] - LocationService.MethodPerEndpoint(servicePerEndpoint) - } - - override val geoDuckV2Store: ReadableStore[Long, LocationResponse] = { - val geoduckLocate: GeoduckUserLocate = GeoduckUserLocateModule.providesGeoduckUserLocate( - locationServiceClient = geoduckLocationClient, - hydrationClient = geoduckHydrationClient, - unscopedStatsReceiver = statsReceiver - ) - - val store: ReadableStore[Long, LocationResponse] = ReadableStore - .convert[GeoduckRequest, Long, LocationResponse, LocationResponse]( - GeoduckStoreV2(geoduckLocate))({ userId: Long => - GeoduckRequest( - userId, - placeTypes = Set( - PlaceType.City, - PlaceType.Metro, - PlaceType.Country, - PlaceType.ZipCode, - PlaceType.Admin0, - PlaceType.Admin1), - placeFields = Set(PlaceQueryFields.PlaceNames), - includeCountryCode = true - ) - })({ locationResponse: LocationResponse => Future.value(locationResponse) }) - - val _cacheName = "geoduckv2_in_memory_cache" - ObservedCachedReadableStore.from( - store, - ttl = 20.seconds, - maxKeys = 1000, - cacheName = _cacheName, - windowSize = 10000L - )(statsReceiver.scope(_cacheName)) - } - - private val deepbirdServiceBase = ThriftMux.client - .withClientId(pushserviceThriftClientId) - .withMutualTls(serviceIdentifier) - .withLoadBalancer(Balancers.p2c()) - .newService(deepbirdv2PredictionServiceDest, "DeepbirdV2PredictionService") - val deepbirdPredictionServiceClient = new DeepbirdPredictionService.ServiceToClient( - Finagle - .retryReadFilter( - tries = 3, - statsReceiver = statsReceiver.scope("DeepbirdV2PredictionService")) - .andThen(Finagle.timeoutFilter(timeout = 10.seconds)) - .andThen(deepbirdServiceBase), - RichClientParam(serviceName = "DeepbirdV2PredictionService", clientStats = statsReceiver) - ) - - val manhattanStarbuckAppId = "frigate_pushservice_starbuck" - val metastoreLocationAppId = "frigate_notifier_metastore_location" - val manhattanMetastoreAppId = "frigate_pushservice_penguin" - - def pushServiceMHCacheDest: String - def pushServiceCoreSvcsCacheDest: String - def poptartImpressionsCacheDest: String = "/srv#/prod/local/cache/poptart_impressions" - def entityGraphCacheDest: String - - val pushServiceCacheClient: Client = MemcacheStore.memcachedClient( - name = ClientName("memcache-pushservice"), - dest = ZkEndPoint(pushServiceMHCacheDest), - statsReceiver = statsReceiver, - timeout = 2.seconds, - serviceIdentifier = serviceIdentifier - ) - - val pushServiceCoreSvcsCacheClient: Client = - MemcacheStore.memcachedClient( - name = ClientName("memcache-pushservice-core-svcs"), - dest = ZkEndPoint(pushServiceCoreSvcsCacheDest), - statsReceiver = statsReceiver, - serviceIdentifier = serviceIdentifier, - timeout = 2.seconds, - ) - - val poptartImpressionsCacheClient: Client = - MemcacheStore.memcachedClient( - name = ClientName("memcache-pushservice-poptart-impressions"), - dest = ZkEndPoint(poptartImpressionsCacheDest), - statsReceiver = statsReceiver, - serviceIdentifier = serviceIdentifier, - timeout = 2.seconds - ) - - val entityGraphCacheClient: Client = MemcacheStore.memcachedClient( - name = ClientName("memcache-pushservice-entity-graph"), - dest = ZkEndPoint(entityGraphCacheDest), - statsReceiver = statsReceiver, - serviceIdentifier = serviceIdentifier, - timeout = 2.seconds - ) - - val stratoClient = { - val pushserviceThriftClient = ThriftMux.client.withClientId(pushserviceThriftClientId) - val baseBuilder = Strato - .Client(pushserviceThriftClient) - .withMutualTls(serviceIdentifier) - val finalBuilder = if (isServiceLocal) { - baseBuilder.withRequestTimeout(Duration.fromSeconds(15)) - } else { - baseBuilder.withRequestTimeout(Duration.fromSeconds(3)) - } - finalBuilder.build() - } - - val interestThriftServiceClient = ThriftMux.client - .withClientId(pushserviceThriftClientId) - .withMutualTls(serviceIdentifier) - .withRequestTimeout(3.seconds) - .configured(Retries.Policy(RetryPolicy.tries(1))) - .configured(BackupRequestFilter.Configured(maxExtraLoad = 0.0, sendInterrupts = false)) - .withStatsReceiver(statsReceiver) - .build[InterestsThriftService.MethodPerEndpoint]( - dest = "/s/interests-thrift-service/interests-thrift-service", - label = "interests-lookup" - ) - - def memcacheCASDest: String - - override val casLock: CasLock = { - val magicrecsCasMemcacheClient = Memcached.client - .withMutualTls(serviceIdentifier) - .withLabel("mr-cas-memcache-client") - .withRequestTimeout(3.seconds) - .withStatsReceiver(statsReceiver) - .configured(Retries.Policy(RetryPolicy.tries(3))) - .newTwemcacheClient(memcacheCASDest) - .withStrings - - MemcacheCasLock(magicrecsCasMemcacheClient) - } - - override val pushInfoStore: ReadableStore[Long, UserForPushTargeting] = { - StratoFetchableStore.withUnitView[Long, UserForPushTargeting]( - stratoClient, - "frigate/magicrecs/pushRecsTargeting.User") - } - - override val loggedOutPushInfoStore: ReadableStore[Long, LOWebNotificationMetadata] = { - StratoFetchableStore.withUnitView[Long, LOWebNotificationMetadata]( - stratoClient, - "frigate/magicrecs/web/loggedOutWebUserStoreMh" - ) - } - - // Setting up model stores - override val dauProbabilityStore: ReadableStore[Long, DauProbability] = { - StratoFetchableStore - .withUnitView[Long, DauProbability](stratoClient, "frigate/magicrecs/dauProbability.User") - } - - override val nsfwConsumerStore = { - StratoFetchableStore.withUnitView[Long, NSFWUserSegmentation]( - stratoClient, - "frigate/nsfw-user-segmentation/nsfwUserSegmentation.User") - } - - override val nsfwProducerStore = { - StratoFetchableStore.withUnitView[Long, NSFWProducer]( - stratoClient, - "frigate/nsfw-user-segmentation/nsfwProducer.User" - ) - } - - override val idsStore: ReadableStore[RecommendedListsRequest, RecommendedListsResponse] = { - val service = Finagle.readOnlyThriftService( - name = "interests-discovery-service", - dest = "/s/interests_discovery/interests_discovery", - statsReceiver, - pushserviceThriftClientId, - requestTimeout = 4.seconds, - tries = 2, - mTLSServiceIdentifier = Some(serviceIdentifier) - ) - val client = new InterestsDiscoveryService.FinagledClient( - service = service, - RichClientParam(serviceName = "interests-discovery-service") - ) - - InterestDiscoveryStore(client) - } - - override val popGeoLists = { - StratoFetchableStore.withUnitView[String, NonPersonalizedRecommendedLists]( - stratoClient, - column = "recommendations/interests_discovery/recommendations_mh/OrganicPopgeoLists" - ) - } - - override val listAPIStore = { - val fetcher = stratoClient - .fetcher[Long, ApiListView, ApiList]("channels/hydration/apiList.List") - StratoFetchableStore.withView[Long, ApiListView, ApiList]( - fetcher, - ApiListView(ApiListDisplayLocation.Recommendations) - ) - } - - override val reactivatedUserInfoStore = { - val stratoFetchableStore = StratoFetchableStore - .withUnitView[Long, String](stratoClient, "ml/featureStore/recentReactivationTime.User") - - ObservedReadableStore( - stratoFetchableStore - )(statsReceiver.scope("RecentReactivationTime")) - } - - override val openedPushByHourAggregatedStore: ReadableStore[Long, Map[Int, Int]] = { - StratoFetchableStore - .withUnitView[Long, Map[Int, Int]]( - stratoClient, - "frigate/magicrecs/opendPushByHourAggregated.User") - } - - private val lexClient: LiveVideoTimelineClient = { - val lexService = - new TimelineService.FinagledClient( - readOnlyThriftService( - name = "lex", - dest = lexServiceDest, - statsReceiver = statsReceiver.scope("lex-service"), - thriftClientId = pushserviceThriftClientId, - requestTimeout = 5.seconds, - mTLSServiceIdentifier = Some(serviceIdentifier) - ), - clientParam = RichClientParam(serviceName = "lex") - ) - new LiveVideoTimelineClient(lexService) - } - - override val lexServiceStore = { - ObservedCachedReadableStore.from[EventRequest, LiveEvent]( - buildStore(LexServiceStore(lexClient), "lexServiceStore"), - ttl = 1.hour, - maxKeys = 1000, - cacheName = "lexServiceStore_cache", - windowSize = 10000L - )(statsReceiver.scope("lexServiceStore_cache")) - } - - val inferredEntitiesFromInterestedInKeyedByClusterColumn = - "recommendations/simclusters_v2/inferred_entities/inferredEntitiesFromInterestedInKeyedByCluster" - override val simClusterToEntityStore: ReadableStore[Int, SimClustersInferredEntities] = { - val store = StratoFetchableStore - .withUnitView[Int, SimClustersInferredEntities]( - stratoClient, - inferredEntitiesFromInterestedInKeyedByClusterColumn) - ObservedCachedReadableStore.from[Int, SimClustersInferredEntities]( - buildStore(store, "simcluster_entity_store_cache"), - ttl = 6.hours, - maxKeys = 1000, - cacheName = "simcluster_entity_store_cache", - windowSize = 10000L - )(statsReceiver.scope("simcluster_entity_store_cache")) - } - - def fanoutMetadataColumn: String - - override val fanoutMetadataStore: ReadableStore[(Long, Long), FanoutEvent] = { - val store = StratoFetchableStore - .withUnitView[(Long, Long), FanoutEvent](stratoClient, fanoutMetadataColumn) - ObservedCachedReadableStore.from[(Long, Long), FanoutEvent]( - buildStore(store, "fanoutMetadataStore"), - ttl = 10.minutes, - maxKeys = 1000, - cacheName = "fanoutMetadataStore_cache", - windowSize = 10000L - )(statsReceiver.scope("fanoutMetadataStore_cache")) - } - - /** - * PostRanking Feature Store Client - */ - override def postRankingFeatureStoreClient = { - val clientStats = statsReceiver.scope("post_ranking_feature_store_client") - val clientConfig = - FeatureStoreClientBuilder.getClientConfig(PostRankingFeaturesConfig(), featureStoreUtil) - - FeatureStoreClientBuilder.getDynamicFeatureStoreClient(clientConfig, clientStats) - } - - /** - * Interests lookup store - */ - override val interestsWithLookupContextStore = { - ObservedCachedReadableStore.from[InterestsLookupRequestWithContext, Interests]( - buildStore( - new InterestsWithLookupContextStore(interestThriftServiceClient, statsReceiver), - "InterestsWithLookupContextStore" - ), - ttl = 1.minute, - maxKeys = 1000, - cacheName = "interestsWithLookupContextStore_cache", - windowSize = 10000L - ) - } - - /** - * OptOutInterestsStore - */ - override lazy val optOutUserInterestsStore: ReadableStore[Long, Seq[InterestId]] = { - buildStore( - InterestsOptOutwithLookUpContextStore(interestThriftServiceClient), - "InterestsOptOutStore" - ) - } - - override val topicListing: TopicListing = - if (isServiceLocal) { - new TopicListingBuilder(statsReceiver.scope("topiclisting"), Some(localConfigRepoPath)).build - } else { - new TopicListingBuilder(statsReceiver.scope("topiclisting"), None).build - } - - val cachedUttClient = { - val DefaultUttCacheConfig = CacheConfigV2(capacity = 100) - val uttClientCacheConfigs = uttclient.UttClientCacheConfigsV2( - DefaultUttCacheConfig, - DefaultUttCacheConfig, - DefaultUttCacheConfig, - DefaultUttCacheConfig - ) - new CachedUttClientV2(stratoClient, Environment.Prod, uttClientCacheConfigs, statsReceiver) - } - - override val uttEntityHydrationStore = - new UttEntityHydrationStore(cachedUttClient, statsReceiver, log) - - private lazy val dbv2PredictionServiceScoreStore: RelevancePushPredictionServiceStore = - DeepbirdV2ModelConfig.buildPredictionServiceScoreStore( - deepbirdPredictionServiceClient, - "deepbirdv2_magicrecs" - ) - - // Customized model to PredictionServiceStoreMap - // It is used to specify the predictionServiceStore for the models not in the default dbv2PredictionServiceScoreStore - private lazy val modelToPredictionServiceStoreMap: Map[ - WeightedOpenOrNtabClickModel.ModelNameType, - RelevancePushPredictionServiceStore - ] = Map() - - override lazy val weightedOpenOrNtabClickModelScorer = new PushMLModelScorer( - PushMLModel.WeightedOpenOrNtabClickProbability, - modelToPredictionServiceStoreMap, - dbv2PredictionServiceScoreStore, - statsReceiver.scope("weighted_oonc_scoring") - ) - - override lazy val optoutModelScorer = new PushMLModelScorer( - PushMLModel.OptoutProbability, - Map.empty, - dbv2PredictionServiceScoreStore, - statsReceiver.scope("optout_scoring") - ) - - override lazy val filteringModelScorer = new PushMLModelScorer( - PushMLModel.FilteringProbability, - Map.empty, - dbv2PredictionServiceScoreStore, - statsReceiver.scope("filtering_scoring") - ) - - private val queryFields: Set[QueryFields] = Set( - QueryFields.Profile, - QueryFields.Account, - QueryFields.Roles, - QueryFields.Discoverability, - QueryFields.Safety, - QueryFields.Takedowns, - QueryFields.Labels, - QueryFields.Counts, - QueryFields.ExtendedProfile - ) - - // Setting up safeUserStore - override val safeUserStore = - // in-memory cache - ObservedCachedReadableStore.from[Long, User]( - ObservedReadableStore( - GizmoduckUserStore.safeStore( - client = gizmoduckClient, - queryFields = queryFields, - safetyLevel = SafetyLevel.FilterNone, - statsReceiver = statsReceiver - ) - )(statsReceiver.scope("SafeUserStore")), - ttl = 1.minute, - maxKeys = 5e4.toInt, - cacheName = "safeUserStore_cache", - windowSize = 10000L - )(statsReceiver.scope("safeUserStore_cache")) - - val mobileSdkStore = MobileSdkStore( - "frigate_mobile_sdk_version_apollo", - "mobile_sdk_versions_scalding", - manhattanClientMtlsParams, - Apollo - ) - - val deviceUserStore = ObservedReadableStore( - GizmoduckUserStore( - client = gizmoduckClient, - queryFields = Set(QueryFields.Devices), - context = LookupContext(includeSoftUsers = true), - statsReceiver = statsReceiver - ) - )(statsReceiver.scope("devicesUserStore")) - - override val deviceInfoStore = DeviceInfoStore( - ObservedMemcachedReadableStore.fromCacheClient( - backingStore = ObservedReadableStore( - mobileSdkStore - )(statsReceiver.scope("uncachedMobileSdkVersionsStore")), - cacheClient = pushServiceCacheClient, - ttl = 12.hours - )( - valueInjection = BinaryScalaCodec(SdkVersionValue), - statsReceiver = statsReceiver.scope("MobileSdkVersionsStore"), - keyToString = { - case SdkVersionKey(Some(userId), Some(clientId)) => - s"DeviceInfoStore/$userId/$clientId" - case SdkVersionKey(Some(userId), None) => s"DeviceInfoStore/$userId/_" - case SdkVersionKey(None, Some(clientId)) => - s"DeviceInfoStore/_/$clientId" - case SdkVersionKey(None, None) => s"DeviceInfoStore/_" - } - ), - deviceUserStore - ) - - // Setting up edgeStore - override val edgeStore = SocialGraphPredicate.buildEdgeStore(sgsClient) - - override val socialGraphServiceProcessStore = SocialGraphServiceProcessStore(edgeStore) - - def userTweetEntityGraphDest: String - def userUserGraphDest: String - def lexServiceDest: String - - // Setting up the history store - def frigateHistoryCacheDest: String - - val notificationHistoryStore: NotificationHistoryStore = { - - val manhattanStackBasedClient = ThriftMux.client - .withClientId(notifierThriftClientId) - .withOpportunisticTls(OpportunisticTls.Required) - .withMutualTls( - serviceIdentifier - ) - - val manhattanHistoryMethodBuilder = manhattanStackBasedClient - .withLabel("manhattan_history_v2") - .withRequestTimeout(10.seconds) - .withStatsReceiver(statsReceiver) - .methodBuilder(Omega.wilyName) - .withMaxRetries(3) - - NotificationHistoryStore.build( - "frigate_notifier", - "frigate_notifications_v2", - manhattanHistoryMethodBuilder, - maxRetryCount = 3 - ) - } - - val emailNotificationHistoryStore: ReadOnlyHistoryStore = { - val client = ManhattanKVClient( - appId = "frigate_email_history", - dest = "/s/manhattan/omega.native-thrift", - mtlsParams = ManhattanKVClientMtlsParams( - serviceIdentifier = serviceIdentifier, - opportunisticTls = OpportunisticTls.Required - ) - ) - val endpoint = ManhattanKVEndpointBuilder(client) - .defaultGuarantee(Guarantee.SoftDcReadMyWrites) - .statsReceiver(statsReceiver) - .build() - - ReadOnlyHistoryStore(ManhattanKVHistoryStore(endpoint, dataset = "frigate_email_history"))( - statsReceiver) - } - - val manhattanKVLoggedOutHistoryStoreEndpoint: ManhattanKVEndpoint = { - val mhClient = ManhattanKVClient( - "frigate_notification_logged_out_history", - Nash.wilyName, - manhattanClientMtlsParams) - ManhattanKVEndpointBuilder(mhClient) - .defaultGuarantee(Guarantee.SoftDcReadMyWrites) - .defaultMaxTimeout(5.seconds) - .maxRetryCount(3) - .statsReceiver(statsReceiver) - .build() - } - - val manhattanKVNtabHistoryStoreEndpoint: ManhattanKVEndpoint = { - val mhClient = ManhattanKVClient("frigate_ntab", Omega.wilyName, manhattanClientMtlsParams) - ManhattanKVEndpointBuilder(mhClient) - .defaultGuarantee(Guarantee.SoftDcReadMyWrites) - .defaultMaxTimeout(5.seconds) - .maxRetryCount(3) - .statsReceiver(statsReceiver) - .build() - } - - val nTabHistoryStore: ReadableWritableStore[(Long, String), GenericNotificationOverrideKey] = { - ObservedReadableWritableStore( - NTabHistoryStore(manhattanKVNtabHistoryStoreEndpoint, "frigate_ntab_generic_notif_history") - )(statsReceiver.scope("NTabHistoryStore")) - } - - override lazy val ocfFatigueStore: ReadableStore[OCFHistoryStoreKey, FatigueFlowEnrollment] = - new OCFPromptHistoryStore( - manhattanAppId = "frigate_pushservice_ocf_fatigue_store", - dataset = "fatigue_v1", - manhattanClientMtlsParams - ) - - def historyStore: PushServiceHistoryStore - - def emailHistoryStore: PushServiceHistoryStore - - def loggedOutHistoryStore: PushServiceHistoryStore - - override val hydratedLabeledPushRecsStore: ReadableStore[UserHistoryKey, UserHistoryValue] = { - val labeledHistoryMemcacheClient = { - MemcacheStore.memcachedClient( - name = ClientName("history-memcache"), - dest = ZkEndPoint(frigateHistoryCacheDest), - statsReceiver = statsReceiver, - timeout = 2.seconds, - serviceIdentifier = serviceIdentifier - ) - } - - implicit val keyCodec = CompactScalaCodec(UserHistoryKey) - implicit val valueCodec = CompactScalaCodec(UserHistoryValue) - val dataset: Dataset[UserHistoryKey, UserHistoryValue] = - Dataset( - "", - "frigate_data_pipeline_pushservice", - "labeled_push_recs_aggregated_hydrated", - Athena - ) - ObservedMemcachedReadableStore.fromCacheClient( - backingStore = ObservedReadableStore(buildManhattanStore(dataset))( - statsReceiver.scope("UncachedHydratedLabeledPushRecsStore") - ), - cacheClient = labeledHistoryMemcacheClient, - ttl = 6.hours - )( - valueInjection = valueCodec, - statsReceiver = statsReceiver.scope("HydratedLabeledPushRecsStore"), - keyToString = { - case UserHistoryKey.UserId(userId) => s"HLPRS/$userId" - case unknownKey => - throw new IllegalArgumentException(s"Unknown userHistoryStore cache key $unknownKey") - } - ) - } - - override val realTimeClientEventStore: RealTimeClientEventStore = { - val client = ManhattanKVClient( - "frigate_eventstream", - "/s/manhattan/omega.native-thrift", - manhattanClientMtlsParams - ) - val endpoint = - ManhattanKVEndpointBuilder(client) - .defaultGuarantee(Guarantee.SoftDcReadMyWrites) - .defaultMaxTimeout(3.seconds) - .statsReceiver(statsReceiver) - .build() - - ManhattanRealTimeClientEventStore(endpoint, "realtime_client_events", statsReceiver, None) - } - - override val onlineUserHistoryStore: ReadableStore[OnlineUserHistoryKey, UserHistoryValue] = { - OnlineUserHistoryStore(realTimeClientEventStore) - } - - override val userMediaRepresentationStore = UserMediaRepresentationStore( - "user_media_representation", - "user_media_representation_dataset", - manhattanClientMtlsParams - ) - - override val producerMediaRepresentationStore = ObservedMemcachedReadableStore.fromCacheClient( - backingStore = UserMediaRepresentationStore( - "user_media_representation", - "producer_media_representation_dataset", - manhattanClientMtlsParams - )(statsReceiver.scope("UncachedProducerMediaRepStore")), - cacheClient = pushServiceCacheClient, - ttl = 4.hours - )( - valueInjection = BinaryScalaCodec(UserMediaRepresentation), - keyToString = { k: Long => s"ProducerMediaRepStore/$k" }, - statsReceiver.scope("ProducerMediaRepStore") - ) - - override val mrUserStatePredictionStore = { - StratoFetchableStore.withUnitView[Long, MRUserHmmState]( - stratoClient, - "frigate/magicrecs/mrUserStatePrediction.User") - } - - override val userHTLLastVisitStore = - UserHTLLastVisitReadableStore( - "pushservice_htl_user_session", - "tls_user_session_store", - statsReceiver.scope("userHTLLastVisitStore"), - manhattanClientMtlsParams - ) - - val crMixerClient: CrMixer.MethodPerEndpoint = new CrMixer.FinagledClient( - readOnlyThriftService( - "cr-mixer", - "/s/cr-mixer/cr-mixer-plus", - statsReceiver, - pushserviceThriftClientId, - requestTimeout = 5.seconds, - mTLSServiceIdentifier = Some(serviceIdentifier) - ), - clientParam = RichClientParam(serviceName = "cr-mixer") - ) - - val crMixerStore = CrMixerTweetStore(crMixerClient)(statsReceiver.scope("CrMixerTweetStore")) - - val contentMixerClient: ContentMixer.MethodPerEndpoint = new ContentMixer.FinagledClient( - readOnlyThriftService( - "content-mixer", - "/s/corgi-shared/content-mixer", - statsReceiver, - pushserviceThriftClientId, - requestTimeout = 5.seconds, - mTLSServiceIdentifier = Some(serviceIdentifier) - ), - clientParam = RichClientParam(serviceName = "content-mixer") - ) - - val exploreRankerClient: ExploreRanker.MethodPerEndpoint = - new ExploreRanker.FinagledClient( - readOnlyThriftService( - "explore-ranker", - "/s/explore-ranker/explore-ranker", - statsReceiver, - pushserviceThriftClientId, - requestTimeout = 5.seconds, - mTLSServiceIdentifier = Some(serviceIdentifier) - ), - clientParam = RichClientParam(serviceName = "explore-ranker") - ) - - val contentMixerStore = { - ObservedReadableStore(ContentMixerStore(contentMixerClient))( - statsReceiver.scope("ContentMixerStore")) - } - - val exploreRankerStore = { - ObservedReadableStore(ExploreRankerStore(exploreRankerClient))( - statsReceiver.scope("ExploreRankerStore") - ) - } - - val gizmoduckUtcOffsetStore = ObservedReadableStore( - GizmoduckUserUtcOffsetStore.fromUserStore(safeUserStore) - )(statsReceiver.scope("GizmoUserUtcOffsetStore")) - - override val userUtcOffsetStore = - UtcOffsetStore - .makeMemcachedUtcOffsetStore( - gizmoduckUtcOffsetStore, - pushServiceCoreSvcsCacheClient, - ReadableStore.empty, - manhattanStarbuckAppId, - manhattanClientMtlsParams - )(statsReceiver) - .mapValues(Duration.fromSeconds) - - override val cachedTweetyPieStoreV2 = { - val getTweetOptions = Some( - GetTweetOptions( - safetyLevel = Some(SafetyLevel.MagicRecsV2), - includeRetweetCount = true, - includeReplyCount = true, - includeFavoriteCount = true, - includeQuotedTweet = true, - additionalFieldIds = Seq(VisibleTextRangeField.id) - ) - ) - buildCachedTweetyPieStore(getTweetOptions, "tp_v2") - } - - override val cachedTweetyPieStoreV2NoVF = { - val getTweetOptions = Some( - GetTweetOptions( - safetyLevel = Some(SafetyLevel.FilterDefault), - includeRetweetCount = true, - includeReplyCount = true, - includeFavoriteCount = true, - includeQuotedTweet = true, - additionalFieldIds = Seq(VisibleTextRangeField.id), - ) - ) - buildCachedTweetyPieStore(getTweetOptions, "tp_v2_noVF") - } - - override val safeCachedTweetyPieStoreV2 = { - val getTweetOptions = Some( - GetTweetOptions( - safetyLevel = Some(SafetyLevel.MagicRecsAggressiveV2), - includeRetweetCount = true, - includeReplyCount = true, - includeFavoriteCount = true, - includeQuotedTweet = true, - additionalFieldIds = Seq(VisibleTextRangeField.id) - ) - ) - buildCachedTweetyPieStore(getTweetOptions, "sftp_v2") - } - - override val userTweetTweetyPieStore: ReadableStore[UserTweet, TweetyPieResult] = { - val getTweetOptions = Some( - GetTweetOptions( - safetyLevel = Some(SafetyLevel.MagicRecsV2), - includeRetweetCount = true, - includeReplyCount = true, - includeFavoriteCount = true, - includeQuotedTweet = true, - additionalFieldIds = Seq(VisibleTextRangeField.id) - ) - ) - TweetyPieStore.buildUserTweetStore( - client = tweetyPieClient, - options = getTweetOptions - ) - } - - override val safeUserTweetTweetyPieStore: ReadableStore[UserTweet, TweetyPieResult] = { - val getTweetOptions = Some( - GetTweetOptions( - safetyLevel = Some(SafetyLevel.MagicRecsAggressiveV2), - includeRetweetCount = true, - includeReplyCount = true, - includeFavoriteCount = true, - includeQuotedTweet = true, - additionalFieldIds = Seq(VisibleTextRangeField.id) - ) - ) - TweetyPieStore.buildUserTweetStore( - client = tweetyPieClient, - options = getTweetOptions - ) - } - - override val tweetContentFeatureCacheStore: ReadableStore[Long, ThriftDataRecord] = { - ObservedMemcachedReadableStore.fromCacheClient( - backingStore = TweetContentFeatureReadableStore(stratoClient), - cacheClient = poptartImpressionsCacheClient, - ttl = 12.hours - )( - valueInjection = BinaryScalaCodec(ThriftDataRecord), - statsReceiver = statsReceiver.scope("TweetContentFeaturesCacheStore"), - keyToString = { k: Long => s"tcf/$k" } - ) - } - - lazy val tweetTranslationStore: ReadableStore[ - TweetTranslationStore.Key, - TweetTranslationStore.Value - ] = { - val isTweetTranslatableStore = - StratoFetchableStore - .withUnitView[IsTweetTranslatableClientColumn.Key, Boolean]( - fetcher = new IsTweetTranslatableClientColumn(stratoClient).fetcher - ) - - val translateTweetStore = - StratoFetchableStore - .withUnitView[MachineTranslateTweetClientColumn.Key, MachineTranslationResponse]( - fetcher = new MachineTranslateTweetClientColumn(stratoClient).fetcher - ) - - ObservedReadableStore( - TweetTranslationStore(translateTweetStore, isTweetTranslatableStore, statsReceiver) - )(statsReceiver.scope("tweetTranslationStore")) - } - - val scarecrowClient = new ScarecrowService.FinagledClient( - readOnlyThriftService( - "", - "/s/abuse/scarecrow", - statsReceiver, - notifierThriftClientId, - requestTimeout = 5.second, - mTLSServiceIdentifier = Some(serviceIdentifier) - ), - clientParam = RichClientParam(serviceName = "") - ) - - // Setting up scarecrow store - override val scarecrowCheckEventStore = { - ScarecrowCheckEventStore(scarecrowClient) - } - - // setting up the perspective store - override val userTweetPerspectiveStore = { - val service = new DynamicRequestMeterFilter( - tunableMap(PushServiceTunableKeys.TweetPerspectiveStoreQpsLimit), - RateLimiterGenerator.asTuple(_, shardParams.numShards, 40), - PushQPSLimitConstants.PerspectiveStoreQPS)(timer) - .andThen( - readOnlyThriftService( - "tweetypie_perspective_service", - "/s/tweetypie/tweetypie", - statsReceiver, - notifierThriftClientId, - mTLSServiceIdentifier = Some(serviceIdentifier) - ) - ) - - val client = new TweetService.FinagledClient( - service, - clientParam = RichClientParam(serviceName = "tweetypie_perspective_client")) - ObservedReadableStore( - PerspectiveReadableStore(client) - )(statsReceiver.scope("TweetPerspectiveStore")) - } - - //user country code store, used in RecsWithheldContentPredicate - wrapped by memcache based cache - override val userCountryStore = - ObservedMemcachedReadableStore.fromCacheClient( - backingStore = ObservedReadableStore( - UserCountryStore(metastoreLocationAppId, manhattanClientMtlsParams) - )(statsReceiver.scope("userCountryStore")), - cacheClient = pushServiceCacheClient, - ttl = 12.hours - )( - valueInjection = BinaryScalaCodec(Location), - statsReceiver = statsReceiver.scope("UserCountryStore"), - keyToString = { k: Long => s"UserCountryStore/$k" } - ) - - override val audioSpaceParticipantsStore: ReadableStore[String, Participants] = { - val store = StratoFetchableStore - .DefaultStratoFetchableStore( - fetcher = new ParticipantsOnAudioSpaceClientColumn(stratoClient).fetcher - ).composeKeyMapping[String](broadcastId => - (broadcastId, AudioSpacesLookupContext(forUserId = None))) - - ObservedCachedReadableStore - .from( - store = buildStore(store, "AudioSpaceParticipantsStore"), - ttl = 20.seconds, - maxKeys = 200, - cacheName = "AudioSpaceParticipantsStore", - windowSize = 200 - ) - - } - - override val topicSocialProofServiceStore: ReadableStore[ - TopicSocialProofRequest, - TopicSocialProofResponse - ] = { - StratoFetchableStore.withUnitView[TopicSocialProofRequest, TopicSocialProofResponse]( - stratoClient, - "topic-signals/tsp/topic-social-proof") - } - - override val spaceDeviceFollowStore: ReadableStore[SourceDestUserRequest, Boolean] = { - StratoFetchableStore.withUnitView( - fetcher = new SpaceDeviceFollowingClientColumn(stratoClient).fetcher - ) - } - - override val audioSpaceStore: ReadableStore[String, AudioSpace] = { - val store = StratoFetchableStore - .DefaultStratoFetchableStore( - fetcher = new CoreOnAudioSpaceClientColumn(stratoClient).fetcher - ).composeKeyMapping[String] { broadcastId => - (broadcastId, AudioSpacesLookupContext(forUserId = None)) - } - - ObservedCachedReadableStore - .from( - store = buildStore(store, "AudioSpaceVisibilityStore"), - ttl = 1.minute, - maxKeys = 5000, - cacheName = "AudioSpaceVisibilityStore", - windowSize = 10000L) - } - - override val userLanguagesStore = UserLanguagesStore( - manhattanMetastoreAppId, - manhattanClientMtlsParams, - statsReceiver.scope("user_languages_store") - ) - - val tflockClient: TFlockClient = new TFlockClient( - new FlockDB.FinagledClient( - readOnlyThriftService( - "tflockClient", - "/s/tflock/tflock", - statsReceiver, - pushserviceThriftClientId, - mTLSServiceIdentifier = Some(serviceIdentifier) - ), - serviceName = "tflock", - stats = statsReceiver - ), - defaultPageSize = 1000 - ) - - val rawFlockClient = ThriftMux.client - .withClientId(pushserviceThriftClientId) - .withMutualTls(serviceIdentifier) - .build[FlockDB.MethodPerEndpoint]("/s/flock/flock") - - val flockClient: FlockClient = new FlockClient( - rawFlockClient, - defaultPageSize = 100 - ) - - override val recentFollowsStore: FlockFollowStore = { - val dStats = statsReceiver.scope("FlockRecentFollowsStore") - FlockFollowStore(flockClient, dStats) - } - - def notificationServiceClient: NotificationService$FinagleClient - - def notificationServiceSend( - target: Target, - request: CreateGenericNotificationRequest - ): Future[CreateGenericNotificationResponse] - - def notificationServiceDelete( - request: DeleteGenericNotificationRequest - ): Future[Unit] - - def notificationServiceDeleteTimeline( - request: DeleteCurrentTimelineForUserRequest - ): Future[Unit] - - override val notificationServiceSender: ReadableStore[ - NotificationServiceRequest, - CreateGenericNotificationResponse - ] = { - new NotificationServiceSender( - notificationServiceSend, - PushParams.EnableWritesToNotificationServiceParam, - PushParams.EnableWritesToNotificationServiceForAllEmployeesParam, - PushParams.EnableWritesToNotificationServiceForEveryoneParam - ) - } - - val eventRecosServiceClient = { - val dest = "/s/events-recos/events-recos-service" - new EventsRecosService.FinagledClient( - readOnlyThriftService( - "EventRecosService", - dest, - statsReceiver, - pushserviceThriftClientId, - mTLSServiceIdentifier = Some(serviceIdentifier) - ), - clientParam = RichClientParam(serviceName = "EventRecosService") - ) - } - - lazy val recommendedTrendsCandidateSource = RecommendedTrendsCandidateSource( - TrendsRecommendationStore(eventRecosServiceClient, statsReceiver)) - - override val softUserGeoLocationStore: ReadableStore[Long, GeoLocation] = - StratoFetchableStore.withUnitView[Long, GeoLocation](fetcher = - new FrequentSoftUserLocationClientColumn(stratoClient).fetcher) - - lazy val candidateSourceGenerator = new PushCandidateSourceGenerator( - earlybirdCandidateSource, - userTweetEntityGraphCandidates, - cachedTweetyPieStoreV2, - safeCachedTweetyPieStoreV2, - userTweetTweetyPieStore, - safeUserTweetTweetyPieStore, - cachedTweetyPieStoreV2NoVF, - edgeStore, - interestsWithLookupContextStore, - uttEntityHydrationStore, - geoDuckV2Store, - topTweetsByGeoStore, - topTweetsByGeoV2VersionedStore, - ruxTweetImpressionsStore, - recommendedTrendsCandidateSource, - recentTweetsByAuthorsStore, - topicSocialProofServiceStore, - crMixerStore, - contentMixerStore, - exploreRankerStore, - softUserGeoLocationStore, - tripTweetCandidateStore, - popGeoLists, - idsStore - ) - - lazy val loCandidateSourceGenerator = new LoggedOutPushCandidateSourceGenerator( - tripTweetCandidateStore, - geoDuckV2Store, - safeCachedTweetyPieStoreV2, - cachedTweetyPieStoreV2NoVF, - cachedTweetyPieStoreV2, - contentMixerStore, - softUserGeoLocationStore, - topTweetsByGeoStore, - topTweetsByGeoV2VersionedStore - ) - - lazy val rfphStatsRecorder = new RFPHStatsRecorder() - - lazy val rfphRestrictStep = new RFPHRestrictStep() - - lazy val rfphTakeStepUtil = new RFPHTakeStepUtil()(statsReceiver) - - lazy val rfphPrerankFilter = new RFPHPrerankFilter()(statsReceiver) - - lazy val rfphLightRanker = new RFPHLightRanker(lightRanker, statsReceiver) - - lazy val sendHandlerPredicateUtil = new SendHandlerPredicateUtil()(statsReceiver) - - lazy val ntabSender = - new NtabSender( - notificationServiceSender, - nTabHistoryStore, - notificationServiceDelete, - notificationServiceDeleteTimeline - ) - - lazy val ibis2Sender = new Ibis2Sender(pushIbisV2Store, tweetTranslationStore, statsReceiver) - - lazy val historyWriter = new HistoryWriter(historyStore, statsReceiver) - - lazy val loggedOutHistoryWriter = new HistoryWriter(loggedOutHistoryStore, statsReceiver) - - lazy val eventBusWriter = new EventBusWriter(pushSendEventBusPublisher, statsReceiver) - - lazy val ntabOnlyChannelSelector = new NtabOnlyChannelSelector - - lazy val notificationSender = - new NotificationSender( - ibis2Sender, - ntabSender, - statsReceiver, - notificationScribe - ) - - lazy val candidateNotifier = - new CandidateNotifier( - notificationSender, - casLock = casLock, - historyWriter = historyWriter, - eventBusWriter = eventBusWriter, - ntabOnlyChannelSelector = ntabOnlyChannelSelector - )(statsReceiver) - - lazy val loggedOutCandidateNotifier = new CandidateNotifier( - notificationSender, - casLock = casLock, - historyWriter = loggedOutHistoryWriter, - eventBusWriter = null, - ntabOnlyChannelSelector = ntabOnlyChannelSelector - )(statsReceiver) - - lazy val rfphNotifier = - new RefreshForPushNotifier(rfphStatsRecorder, candidateNotifier)(statsReceiver) - - lazy val loRfphNotifier = - new LoggedOutRefreshForPushNotifier(rfphStatsRecorder, loggedOutCandidateNotifier)( - statsReceiver) - - lazy val rfphRanker = { - val randomRanker = RandomRanker[Target, PushCandidate]() - val subscriptionCreatorRanker = - new SubscriptionCreatorRanker(superFollowEligibilityUserStore, statsReceiver) - new RFPHRanker( - randomRanker, - weightedOpenOrNtabClickModelScorer, - subscriptionCreatorRanker, - userHealthSignalStore, - producerMediaRepresentationStore, - statsReceiver - ) - } - - lazy val rfphFeatureHydrator = new RFPHFeatureHydrator(featureHydrator) - lazy val loggedOutRFPHRanker = new LoggedOutRanker(cachedTweetyPieStoreV2, statsReceiver) - - override val userFeaturesStore: ReadableStore[Long, UserFeatures] = { - implicit val valueCodec = new BinaryScalaCodec(UserFeatures) - val dataset: Dataset[Long, UserFeatures] = - Dataset( - "", - "user_features_pushservice_apollo", - "recommendations_user_features_apollo", - Apollo) - - ObservedMemcachedReadableStore.fromCacheClient( - backingStore = ObservedReadableStore(buildManhattanStore(dataset))( - statsReceiver.scope("UncachedUserFeaturesStore") - ), - cacheClient = pushServiceCacheClient, - ttl = 24.hours - )( - valueInjection = valueCodec, - statsReceiver = statsReceiver.scope("UserFeaturesStore"), - keyToString = { k: Long => s"ufts/$k" } - ) - } - - override def htlScoreStore(userId: Long): ReadableStore[Long, ScoredTweet] = { - val fetcher = new TimelineScorerTweetScoresV1ClientColumn(stratoClient).fetcher - val htlStore = buildStore( - StratoFetchableStore.withView[Long, TimelineScorerScoreView, ScoredTweet]( - fetcher, - TimelineScorerScoreView(Some(userId)) - ), - "htlScoreStore" - ) - htlStore - } - - override val userTargetingPropertyStore: ReadableStore[Long, UserTargetingProperty] = { - val name = "userTargetingPropertyStore" - val store = StratoFetchableStore - .withUnitView(new TargetingPropertyOnUserClientColumn(stratoClient).fetcher) - buildStore(store, name) - } - - override val timelinesUserSessionStore: ReadableStore[Long, UserSession] = { - implicit val valueCodec = new CompactScalaCodec(UserSession) - val dataset: Dataset[Long, UserSession] = Dataset[Long, UserSession]( - "", - "frigate_realgraph", - "real_graph_user_features", - Apollo - ) - - ObservedMemcachedReadableStore.fromCacheClient( - backingStore = ObservedReadableStore(buildManhattanStore(dataset))( - statsReceiver.scope("UncachedTimelinesUserSessionStore") - ), - cacheClient = pushServiceCacheClient, - ttl = 6.hours - )( - valueInjection = valueCodec, - statsReceiver = statsReceiver.scope("timelinesUserSessionStore"), - keyToString = { k: Long => s"tluss/$k" } - ) - } - - lazy val recentTweetsFromTflockStore: ReadableStore[Long, Seq[Long]] = - ObservedReadableStore( - RecentTweetsByAuthorsStore.usingRecentTweetsConfig( - tflockClient, - RecentTweetsConfig(maxResults = 1, maxAge = 3.days) - ) - )(statsReceiver.scope("RecentTweetsFromTflockStore")) - - lazy val recentTweetsByAuthorsStore: ReadableStore[RecentTweetsQuery, Seq[Seq[Long]]] = - ObservedReadableStore( - RecentTweetsByAuthorsStore(tflockClient) - )(statsReceiver.scope("RecentTweetsByAuthorsStore")) - - val jobConfig = PopGeoInterestProvider - .getPopularTweetsJobConfig( - InterestDeployConfig( - AppId("PopularTweetsByInterestProd"), - Cluster.ATLA, - Env.Prod, - serviceIdentifier, - manhattanClientMtlsParams - )) - .withManhattanAppId("frigate_pop_by_geo_tweets") - - override val topTweetsByGeoStore = TopTweetsStore.withMemCache( - jobConfig, - pushServiceCacheClient, - 10.seconds - )(statsReceiver) - - override val topTweetsByGeoV2VersionedStore: ReadableStore[String, PopTweetsInPlace] = { - StratoFetchableStore.withUnitView[String, PopTweetsInPlace]( - stratoClient, - "recommendations/popgeo/popGeoTweetsVersioned") - } - - override lazy val pushcapDynamicPredictionStore: ReadableStore[Long, PushcapUserHistory] = { - StratoFetchableStore.withUnitView[Long, PushcapUserHistory]( - stratoClient, - "frigate/magicrecs/pushcapDynamicPrediction.User") - } - - override val tweetAuthorLocationFeatureBuilder = - UserLocationFeatureBuilder(Some("TweetAuthor")) - .withStats() - - override val tweetAuthorLocationFeatureBuilderById = - UserLocationFeatureBuilderById( - userCountryStore, - tweetAuthorLocationFeatureBuilder - ).withStats() - - override val socialContextActionsFeatureBuilder = - SocialContextActionsFeatureBuilder().withStats() - - override val tweetContentFeatureBuilder = - TweetContentFeatureBuilder(tweetContentFeatureCacheStore).withStats() - - override val tweetAuthorRecentRealGraphFeatureBuilder = - RecentRealGraphFeatureBuilder( - stratoClient, - UserAuthorEntity, - TargetUserEntity, - TweetAuthorEntity, - TweetAuthorRecentRealGraphFeatures(statsReceiver.scope("TweetAuthorRecentRealGraphFeatures")) - ).withStats() - - override val socialContextRecentRealGraphFeatureBuilder = - SocialContextRecentRealGraphFeatureBuilder( - RecentRealGraphFeatureBuilder( - stratoClient, - TargetUserSocialContextEntity, - TargetUserEntity, - SocialContextEntity, - SocialContextRecentRealGraphFeatures( - statsReceiver.scope("SocialContextRecentRealGraphFeatures")) - )(statsReceiver - .scope("SocialContextRecentRealGraphFeatureBuilder").scope("RecentRealGraphFeatureBuilder")) - ).withStats() - - override val tweetSocialProofFeatureBuilder = - TweetSocialProofFeatureBuilder(Some("TargetUser")).withStats() - - override val targetUserFullRealGraphFeatureBuilder = - TargetFullRealGraphFeatureBuilder(Some("TargetUser")).withStats() - - override val postProcessingFeatureBuilder: PostProcessingFeatureBuilder = - PostProcessingFeatureBuilder() - - override val mrOfflineUserCandidateSparseAggregatesFeatureBuilder = - MrOfflineUserCandidateSparseAggregatesFeatureBuilder(stratoClient, featureStoreUtil).withStats() - - override val mrOfflineUserAggregatesFeatureBuilder = - MrOfflineUserAggregatesFeatureBuilder(stratoClient, featureStoreUtil).withStats() - - override val mrOfflineUserCandidateAggregatesFeatureBuilder = - MrOfflineUserCandidateAggregatesFeatureBuilder(stratoClient, featureStoreUtil).withStats() - - override val tweetAnnotationsFeatureBuilder = - TweetAnnotationsFeatureBuilder(stratoClient).withStats() - - override val targetUserMediaRepresentationFeatureBuilder = - UserMediaRepresentationFeatureBuilder(userMediaRepresentationStore).withStats() - - override val targetLevelFeatureBuilder = - TargetLevelFeatureBuilder(featureStoreUtil, targetLevelFeaturesConfig).withStats() - - override val candidateLevelFeatureBuilder = - CandidateLevelFeatureBuilder(featureStoreUtil).withStats() - - override lazy val targetFeatureHydrator = RelevanceTargetFeatureHydrator( - targetUserFullRealGraphFeatureBuilder, - postProcessingFeatureBuilder, - targetUserMediaRepresentationFeatureBuilder, - targetLevelFeatureBuilder - ) - - override lazy val featureHydrator = - FeatureHydrator(targetFeatureHydrator, candidateFeatureHydrator) - - val pushServiceLightRankerConfig: LightRankerConfig = new LightRankerConfig( - pushserviceThriftClientId, - serviceIdentifier, - statsReceiver.scope("lightRanker"), - deepbirdv2PredictionServiceDest, - "DeepbirdV2PredictionService" - ) - val lightRanker: MagicRecsServeDataRecordLightRanker = - pushServiceLightRankerConfig.lightRanker - - override val tweetImpressionStore: ReadableStore[Long, Seq[Long]] = { - val name = "htl_impression_store" - val store = buildStore( - HtlTweetImpressionStore.createStoreWithTweetIds( - requestTimeout = 6.seconds, - label = "htl_tweet_impressions", - serviceIdentifier = serviceIdentifier, - statsReceiver = statsReceiver - ), - name - ) - val numTweetsReturned = - statsReceiver.scope(name).stat("num_tweets_returned_per_user") - new TransformedReadableStore(store)((userId: Long, tweetIds: Seq[Long]) => { - numTweetsReturned.add(tweetIds.size) - Future.value(Some(tweetIds)) - }) - } - - val ruxTweetImpressionsStore = new TweetImpressionsStore(stratoClient) - - override val strongTiesStore: ReadableStore[Long, STPResult] = { - implicit val valueCodec = new BinaryScalaCodec(STPResult) - val strongTieScoringDataset: Dataset[Long, STPResult] = - Dataset("", "frigate_stp", "stp_result_rerank", Athena) - buildManhattanStore(strongTieScoringDataset) - } - - override lazy val earlybirdFeatureStore = ObservedReadableStore( - EarlybirdFeatureStore( - clientId = pushserviceThriftClientId.name, - earlybirdSearchStore = earlybirdSearchStore - ) - )(statsReceiver.scope("EarlybirdFeatureStore")) - - override lazy val earlybirdFeatureBuilder = EarlybirdFeatureBuilder(earlybirdFeatureStore) - - override lazy val earlybirdSearchStore = { - val earlybirdClientName: String = "earlybird" - val earlybirdSearchStoreName: String = "EarlybirdSearchStore" - - val earlybirdClient = new EarlybirdService.FinagledClient( - readOnlyThriftService( - earlybirdClientName, - earlybirdSearchDest, - statsReceiver, - pushserviceThriftClientId, - tries = 1, - requestTimeout = 3.seconds, - mTLSServiceIdentifier = Some(serviceIdentifier) - ), - clientParam = RichClientParam(protocolFactory = new TCompactProtocol.Factory) - ) - - ObservedReadableStore( - EarlybirdSearchStore(earlybirdClient)(statsReceiver.scope(earlybirdSearchStoreName)) - )(statsReceiver.scope(earlybirdSearchStoreName)) - } - - override lazy val earlybirdCandidateSource: EarlybirdCandidateSource = EarlybirdCandidateSource( - clientId = pushserviceThriftClientId.name, - earlybirdSearchStore = earlybirdSearchStore - ) - - override val realGraphScoresTop500InStore: RealGraphScoresTop500InStore = { - val stratoRealGraphInStore = - StratoFetchableStore - .withUnitView[Long, CandidateSeq]( - stratoClient, - "frigate/magicrecs/fanoutCoi500pRealGraphV2") - - RealGraphScoresTop500InStore( - ObservedMemcachedReadableStore.fromCacheClient( - backingStore = stratoRealGraphInStore, - cacheClient = entityGraphCacheClient, - ttl = 24.hours - )( - valueInjection = BinaryScalaCodec(CandidateSeq), - statsReceiver = statsReceiver.scope("CachedRealGraphScoresTop500InStore"), - keyToString = { k: Long => s"500p_test/$k" } - ) - ) - } - - override val tweetEntityGraphStore = { - val tweetEntityGraphClient = new UserTweetEntityGraph.FinagledClient( - Finagle.readOnlyThriftService( - "user_tweet_entity_graph", - userTweetEntityGraphDest, - statsReceiver, - pushserviceThriftClientId, - requestTimeout = 5.seconds, - mTLSServiceIdentifier = Some(serviceIdentifier) - ) - ) - ObservedReadableStore( - RecommendedTweetEntitiesStore( - tweetEntityGraphClient, - statsReceiver.scope("RecommendedTweetEntitiesStore") - ) - )(statsReceiver.scope("RecommendedTweetEntitiesStore")) - } - - override val userUserGraphStore = { - val userUserGraphClient = new UserUserGraph.FinagledClient( - Finagle.readOnlyThriftService( - "user_user_graph", - userUserGraphDest, - statsReceiver, - pushserviceThriftClientId, - requestTimeout = 5.seconds, - mTLSServiceIdentifier = Some(serviceIdentifier) - ), - clientParam = RichClientParam(serviceName = "user_user_graph") - ) - ObservedReadableStore( - UserUserGraphStore(userUserGraphClient, statsReceiver.scope("UserUserGraphStore")) - )(statsReceiver.scope("UserUserGraphStore")) - } - - override val ntabCaretFeedbackStore: ReadableStore[GenericNotificationsFeedbackRequest, Seq[ - CaretFeedbackDetails - ]] = { - val client = ManhattanKVClient( - "pushservice_ntab_caret_feedback_omega", - Omega.wilyName, - manhattanClientMtlsParams - ) - val endpoint = ManhattanKVEndpointBuilder(client) - .defaultGuarantee(Guarantee.SoftDcReadMyWrites) - .defaultMaxTimeout(3.seconds) - .maxRetryCount(2) - .statsReceiver(statsReceiver) - .build() - - val feedbackSignalManhattanClient = - FeedbackSignalManhattanClient(endpoint, statsReceiver.scope("FeedbackSignalManhattanClient")) - NtabCaretFeedbackStore(feedbackSignalManhattanClient) - } - - override val genericFeedbackStore: ReadableStore[FeedbackRequest, Seq[ - FeedbackPromptValue - ]] = { - FeedbackStore( - GenericFeedbackStoreBuilder.build( - manhattanKVClientAppId = "frigate_pushservice_ntabfeedback_prompt", - environment = NotifEnvironment.apply(serviceIdentifier.environment), - svcIdentifier = serviceIdentifier, - statsReceiver = statsReceiver - )) - } - - override val genericNotificationFeedbackStore: GenericFeedbackStore = { - - GenericFeedbackStoreBuilder.build( - manhattanKVClientAppId = "frigate_pushservice_ntabfeedback_prompt", - environment = NotifEnvironment.apply(serviceIdentifier.environment), - svcIdentifier = serviceIdentifier, - statsReceiver = statsReceiver - ) - } - - override val earlybirdSearchDest = "/s/earlybird-root-superroot/root-superroot" - - // low latency as compared to default `semanticCoreMetadataClient` - private val lowLatencySemanticCoreMetadataClient: MetadataService.MethodPerEndpoint = - new MetadataService.FinagledClient( - Finagle.readOnlyThriftService( - name = "semantic_core_metadata_service", - dest = "/s/escherbird/metadataservice", - statsReceiver = statsReceiver, - thriftClientId = pushserviceThriftClientId, - tries = 2, // total number of tries. number of retries = tries - 1 - requestTimeout = 2.seconds, - mTLSServiceIdentifier = Some(serviceIdentifier) - ) - ) - - private val semanticCoreMetadataStitchClient = new MetadataStitchClient( - lowLatencySemanticCoreMetadataClient - ) - - override val semanticCoreMegadataStore: ReadableStore[SemanticEntityForQuery, EntityMegadata] = { - val name = "semantic_core_megadata_store_cached" - val store = MetaDataReadableStore.getMegadataReadableStore( - metadataStitchClient = semanticCoreMetadataStitchClient, - typedMetadataDomains = Some(Set(Domains.EventsEntityService)) - ) - ObservedCachedReadableStore - .from( - store = ObservedReadableStore(store)( - statsReceiver - .scope("store") - .scope("semantic_core_megadata_store") - ), - ttl = 1.hour, - maxKeys = 1000, - cacheName = "semantic_core_megadata_cache", - windowSize = 10000L - )(statsReceiver.scope("store", name)) - } - - override val basketballGameScoreStore: ReadableStore[QualifiedId, BasketballGameLiveUpdate] = { - StratoFetchableStore.withUnitView[QualifiedId, BasketballGameLiveUpdate]( - stratoClient, - "semanticCore/basketballGameScore.Entity") - } - - override val baseballGameScoreStore: ReadableStore[QualifiedId, BaseballGameLiveUpdate] = { - StratoFetchableStore.withUnitView[QualifiedId, BaseballGameLiveUpdate]( - stratoClient, - "semanticCore/baseballGameScore.Entity") - } - - override val cricketMatchScoreStore: ReadableStore[QualifiedId, CricketMatchLiveUpdate] = { - StratoFetchableStore.withUnitView[QualifiedId, CricketMatchLiveUpdate]( - stratoClient, - "semanticCore/cricketMatchScore.Entity") - } - - override val soccerMatchScoreStore: ReadableStore[QualifiedId, SoccerMatchLiveUpdate] = { - ObservedCachedReadableStore - .from( - store = StratoFetchableStore.withUnitView[QualifiedId, SoccerMatchLiveUpdate]( - stratoClient, - "semanticCore/soccerMatchScore.Entity"), - ttl = 10.seconds, - maxKeys = 100, - cacheName = "SoccerMatchCachedStore", - windowSize = 100L - )(statsReceiver.scope("SoccerMatchCachedStore")) - - } - - override val nflGameScoreStore: ReadableStore[QualifiedId, NflFootballGameLiveUpdate] = { - ObservedCachedReadableStore - .from( - store = StratoFetchableStore.withUnitView[QualifiedId, NflFootballGameLiveUpdate]( - stratoClient, - "semanticCore/nflFootballGameScore.Entity"), - ttl = 10.seconds, - maxKeys = 100, - cacheName = "NFLMatchCachedStore", - windowSize = 100L - )(statsReceiver.scope("NFLMatchCachedStore")) - - } - - override val userHealthSignalStore: ReadableStore[Long, UserHealthSignalResponse] = { - val userHealthSignalFetcher = - stratoClient.fetcher[Long, Seq[UserHealthSignal], UserHealthSignalResponse]( - "hss/user_signals/api/healthSignals.User" - ) - - val store = buildStore( - StratoFetchableStore.withView[Long, Seq[UserHealthSignal], UserHealthSignalResponse]( - userHealthSignalFetcher, - Seq( - AgathaRecentAbuseStrikeDouble, - AgathaCalibratedNsfwDouble, - AgathaCseDouble, - NsfwTextUserScoreDouble, - NsfwConsumerScoreDouble)), - "UserHealthSignalFetcher" - ) - if (!inMemCacheOff) { - ObservedCachedReadableStore - .from( - store = ObservedReadableStore(store)( - statsReceiver.scope("store").scope("user_health_model_score_store")), - ttl = 12.hours, - maxKeys = 16777215, - cacheName = "user_health_model_score_store_cache", - windowSize = 10000L - )(statsReceiver.scope("store", "user_health_model_score_store_cached")) - } else { - store - } - } - - override val tweetHealthScoreStore: ReadableStore[TweetScoringRequest, TweetScoringResponse] = { - val tweetHealthScoreFetcher = - stratoClient.fetcher[TweetScoringRequest, Unit, TweetScoringResponse]( - "abuse/detection/tweetHealthModelScore" - ) - - val store = buildStore( - StratoFetchableStore.withUnitView(tweetHealthScoreFetcher), - "TweetHealthScoreFetcher" - ) - - ObservedCachedReadableStore - .from( - store = ObservedReadableStore(store)( - statsReceiver.scope("store").scope("tweet_health_model_score_store")), - ttl = 30.minutes, - maxKeys = 1000, - cacheName = "tweet_health_model_score_store_cache", - windowSize = 10000L - )(statsReceiver.scope("store", "tweet_health_model_score_store_cached")) - } - - override val appPermissionStore: ReadableStore[(Long, (String, String)), AppPermission] = { - val store = StratoFetchableStore - .withUnitView[(Long, (String, String)), AppPermission]( - stratoClient, - "clients/permissionsState") - ObservedCachedReadableStore.from[(Long, (String, String)), AppPermission]( - buildStore(store, "mr_app_permission_store"), - ttl = 30.minutes, - maxKeys = 1000, - cacheName = "mr_app_permission_store_cache", - windowSize = 10000L - )(statsReceiver.scope("mr_app_permission_store_cached")) - } - - def pushSendEventStreamName: String - - override val pushSendEventBusPublisher = EventBusPublisherBuilder() - .clientId("frigate_pushservice") - .streamName(pushSendEventStreamName) - .thriftStruct(NotificationScribe) - .statsReceiver(statsReceiver.scope("push_send_eventbus")) - .build() - - override lazy val candidateFeatureHydrator: CandidateFeatureHydrator = - CandidateFeatureHydrator( - socialContextActionsFeatureBuilder = Some(socialContextActionsFeatureBuilder), - tweetSocialProofFeatureBuilder = Some(tweetSocialProofFeatureBuilder), - earlybirdFeatureBuilder = Some(earlybirdFeatureBuilder), - tweetContentFeatureBuilder = Some(tweetContentFeatureBuilder), - tweetAuthorRecentRealGraphFeatureBuilder = Some(tweetAuthorRecentRealGraphFeatureBuilder), - socialContextRecentRealGraphFeatureBuilder = Some(socialContextRecentRealGraphFeatureBuilder), - tweetAnnotationsFeatureBuilder = Some(tweetAnnotationsFeatureBuilder), - mrOfflineUserCandidateSparseAggregatesFeatureBuilder = - Some(mrOfflineUserCandidateSparseAggregatesFeatureBuilder), - candidateLevelFeatureBuilder = Some(candidateLevelFeatureBuilder) - )(statsReceiver.scope("push_feature_hydrator")) - - private val candidateCopyCross = - new CandidateCopyExpansion(statsReceiver.scope("refresh_handler/cross")) - - override lazy val candidateHydrator: PushCandidateHydrator = - PushCandidateHydrator( - this.socialGraphServiceProcessStore, - safeUserStore, - listAPIStore, - candidateCopyCross)( - statsReceiver.scope("push_candidate_hydrator"), - weightedOpenOrNtabClickModelScorer) - - override lazy val sendHandlerCandidateHydrator: SendHandlerPushCandidateHydrator = - SendHandlerPushCandidateHydrator( - lexServiceStore, - fanoutMetadataStore, - semanticCoreMegadataStore, - safeUserStore, - simClusterToEntityStore, - audioSpaceStore, - interestsWithLookupContextStore, - uttEntityHydrationStore, - superFollowCreatorTweetCountStore - )( - statsReceiver.scope("push_candidate_hydrator"), - weightedOpenOrNtabClickModelScorer - ) - - def mrRequestScriberNode: String - def loggedOutMrRequestScriberNode: String - - override lazy val configParamsBuilder: ConfigParamsBuilder = ConfigParamsBuilder( - config = overridesConfig, - featureContextBuilder = FeatureContextBuilder(featureSwitches), - statsReceiver = statsReceiver - ) - - def buildStore[K, V](store: ReadableStore[K, V], name: String): ReadableStore[K, V] = { - ObservedReadableStore(store)(statsReceiver.scope("store").scope(name)) - } - - def buildManhattanStore[K, V](dataset: Dataset[K, V]): ReadableStore[K, V] = { - val manhattanKVClientParams = ManhattanKVClientMtlsParams( - serviceIdentifier = serviceIdentifier, - opportunisticTls = OpportunisticTls.Required - ) - ManhattanStore - .fromDatasetWithMtls[K, V]( - dataset, - mtlsParams = manhattanKVClientParams, - statsReceiver = statsReceiver.scope(dataset.datasetName)) - } - - def buildCachedTweetyPieStore( - getTweetOptions: Option[GetTweetOptions], - keyPrefix: String - ): ReadableStore[Long, TweetyPieResult] = { - def discardAdditionalMediaInfo(tweetypieResult: TweetyPieResult) = { - val updatedMedia = tweetypieResult.tweet.media.map { mediaSeq => - mediaSeq.map { media => media.copy(additionalMetadata = None, sizes = Nil.toSet) } - } - val updatedTweet = tweetypieResult.tweet.copy(media = updatedMedia) - tweetypieResult.copy(tweet = updatedTweet) - } - - val tweetypieStoreWithoutAdditionalMediaInfo = TweetyPieStore( - tweetyPieClient, - getTweetOptions, - transformTweetypieResult = discardAdditionalMediaInfo - )(statsReceiver.scope("tweetypie_without_additional_media_info")) - - ObservedMemcachedReadableStore.fromCacheClient( - backingStore = tweetypieStoreWithoutAdditionalMediaInfo, - cacheClient = pushServiceCoreSvcsCacheClient, - ttl = 12.hours - )( - valueInjection = TweetyPieResultInjection, - statsReceiver = statsReceiver.scope("TweetyPieStore"), - keyToString = { k: Long => s"$keyPrefix/$k" } - ) - } - - override def init(): Future[Unit] = - ClientRegistry.expAllRegisteredClientsResolved().map { clients => - log.info("Done resolving clients: " + clients.mkString("[", ", ", "]")) - } - - val InlineActionsMhColumn = - "frigate/magicrecs/inlineActionsMh" - - override val inlineActionHistoryStore: ReadableStore[Long, Seq[(Long, String)]] = - StratoScannableStore - .withUnitView[(Long, Slice[Long]), (Long, Long), String](stratoClient, InlineActionsMhColumn) - .composeKeyMapping[Long] { userId => - (userId, Slice[Long](from = None, to = None, limit = None)) - }.mapValues { response => - response.map { - case (key, value) => (key._2, value) - } - } - - override val tripTweetCandidateStore: ReadableStore[TripDomain, TripTweets] = { - StratoFetchableStore - .withUnitView[TripDomain, TripTweets]( - new TripTweetsAirflowProdClientColumn(stratoClient).fetcher) - } - - override val softUserFollowingStore: ReadableStore[User, Seq[Long]] = new SoftUserFollowingStore( - stratoClient) - - override val superFollowEligibilityUserStore: ReadableStore[Long, Boolean] = { - StratoFetchableStore.withUnitView[Long, Boolean]( - stratoClient, - "audiencerewards/audienceRewardsService/getSuperFollowEligibility.User") - } - - override val superFollowCreatorTweetCountStore: ReadableStore[UserId, Int] = { - ObservedCachedReadableStore - .from( - store = StratoFetchableStore - .withUnitView[UserId, Int](new CreatorSubscriptionNumTweetsColumn(stratoClient).fetcher), - ttl = 5.minutes, - maxKeys = 1000, - cacheName = "SuperFollowCreatorTweetCountStore", - windowSize = 10000L - )(statsReceiver.scope("SuperFollowCreatorTweetCountStore")) - - } - - override val hasSuperFollowingRelationshipStore: ReadableStore[ - HasSuperFollowingRelationshipRequest, - Boolean - ] = { - StratoFetchableStore.withUnitView[HasSuperFollowingRelationshipRequest, Boolean]( - stratoClient, - "audiencerewards/superFollows/hasSuperFollowingRelationshipV2") - } - - override val superFollowApplicationStatusStore: ReadableStore[ - (Long, SellerTrack), - SellerApplicationState - ] = { - StratoFetchableStore.withUnitView[(Long, SellerTrack), SellerApplicationState]( - stratoClient, - "periscope/eligibility/applicationStatus") - } - - def historyStoreMemcacheDest: String - - override lazy val recentHistoryCacheClient = { - RecentHistoryCacheClient.build(historyStoreMemcacheDest, serviceIdentifier, statsReceiver) - } - - override val openAppUserStore: ReadableStore[Long, Boolean] = { - buildStore(OpenAppUserStore(stratoClient), "OpenAppUserStore") - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/ExperimentsWithStats.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/ExperimentsWithStats.scala deleted file mode 100644 index 923a785ee..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/ExperimentsWithStats.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.frigate.pushservice.config - -import com.twitter.frigate.common.util.Experiments - -object ExperimentsWithStats { - - /** - * Add an experiment here to collect detailed pushservice stats. - * - * ! Important ! - * Keep this set small and remove experiments when you don't need the stats anymore. - */ - final val PushExperiments: Set[String] = Set( - Experiments.MRAndroidInlineActionHoldback.exptName, - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/ProdConfig.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/ProdConfig.scala deleted file mode 100644 index 7edc8d46d..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/ProdConfig.scala +++ /dev/null @@ -1,230 +0,0 @@ -package com.twitter.frigate.pushservice.config - -import com.twitter.abdecider.LoggingABDecider -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.bijection.Base64String -import com.twitter.bijection.Injection -import com.twitter.conversions.DurationOps._ -import com.twitter.decider.Decider -import com.twitter.featureswitches.v2.FeatureSwitches -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.finagle.thrift.RichClientParam -import com.twitter.finagle.util.DefaultTimer -import com.twitter.frigate.common.config.RateLimiterGenerator -import com.twitter.frigate.common.filter.DynamicRequestMeterFilter -import com.twitter.frigate.common.history.ManhattanHistoryStore -import com.twitter.frigate.common.history.InvalidatingAfterWritesPushServiceHistoryStore -import com.twitter.frigate.common.history.ManhattanKVHistoryStore -import com.twitter.frigate.common.history.PushServiceHistoryStore -import com.twitter.frigate.common.history.SimplePushServiceHistoryStore -import com.twitter.frigate.common.util._ -import com.twitter.frigate.data_pipeline.features_common.FeatureStoreUtil -import com.twitter.frigate.data_pipeline.features_common.TargetLevelFeaturesConfig -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.DeciderKey -import com.twitter.frigate.pushservice.params.PushQPSLimitConstants -import com.twitter.frigate.pushservice.params.PushServiceTunableKeys -import com.twitter.frigate.pushservice.params.ShardParams -import com.twitter.frigate.pushservice.store.PushIbis2Store -import com.twitter.frigate.pushservice.thriftscala.PushRequestScribe -import com.twitter.frigate.scribe.thriftscala.NotificationScribe -import com.twitter.ibis2.service.thriftscala.Ibis2Service -import com.twitter.logging.Logger -import com.twitter.notificationservice.api.thriftscala.DeleteCurrentTimelineForUserRequest -import com.twitter.notificationservice.api.thriftscala.NotificationApi -import com.twitter.notificationservice.api.thriftscala.NotificationApi$FinagleClient -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationRequest -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationResponse -import com.twitter.notificationservice.thriftscala.DeleteGenericNotificationRequest -import com.twitter.notificationservice.thriftscala.NotificationService -import com.twitter.notificationservice.thriftscala.NotificationService$FinagleClient -import com.twitter.servo.decider.DeciderGateBuilder -import com.twitter.util.tunable.TunableMap -import com.twitter.util.Future -import com.twitter.util.Timer - -case class ProdConfig( - override val isServiceLocal: Boolean, - override val localConfigRepoPath: String, - override val inMemCacheOff: Boolean, - override val decider: Decider, - override val abDecider: LoggingABDecider, - override val featureSwitches: FeatureSwitches, - override val shardParams: ShardParams, - override val serviceIdentifier: ServiceIdentifier, - override val tunableMap: TunableMap, -)( - implicit val statsReceiver: StatsReceiver) - extends { - // Due to trait initialization logic in Scala, any abstract members declared in Config or - // DeployConfig should be declared in this block. Otherwise the abstract member might initialize to - // null if invoked before object creation finishing. - - val log = Logger("ProdConfig") - - // Deciders - val isPushserviceCanaryDeepbirdv2CanaryClusterEnabled = decider - .feature(DeciderKey.enablePushserviceDeepbirdv2CanaryClusterDeciderKey.toString).isAvailable - - // Client ids - val notifierThriftClientId = ClientId("frigate-notifier.prod") - val loggedOutNotifierThriftClientId = ClientId("frigate-logged-out-notifier.prod") - val pushserviceThriftClientId: ClientId = ClientId("frigate-pushservice.prod") - - // Dests - val frigateHistoryCacheDest = "/s/cache/frigate_history" - val memcacheCASDest = "/s/cache/magic_recs_cas:twemcaches" - val historyStoreMemcacheDest = - "/srv#/prod/local/cache/magic_recs_history:twemcaches" - - val deepbirdv2PredictionServiceDest = - if (serviceIdentifier.service.equals("frigate-pushservice-canary") && - isPushserviceCanaryDeepbirdv2CanaryClusterEnabled) - "/s/frigate/deepbirdv2-magicrecs-canary" - else "/s/frigate/deepbirdv2-magicrecs" - - override val fanoutMetadataColumn = "frigate/magicfanout/prod/mh/fanoutMetadata" - - override val timer: Timer = DefaultTimer - override val featureStoreUtil = FeatureStoreUtil.withParams(Some(serviceIdentifier)) - override val targetLevelFeaturesConfig = TargetLevelFeaturesConfig() - val pushServiceMHCacheDest = "/s/cache/pushservice_mh" - - val pushServiceCoreSvcsCacheDest = "/srv#/prod/local/cache/pushservice_core_svcs" - - val userTweetEntityGraphDest = "/s/cassowary/user_tweet_entity_graph" - val userUserGraphDest = "/s/cassowary/user_user_graph" - val lexServiceDest = "/s/live-video/timeline-thrift" - val entityGraphCacheDest = "/s/cache/pushservice_entity_graph" - - override val pushIbisV2Store = { - val service = Finagle.readOnlyThriftService( - "ibis-v2-service", - "/s/ibis2/ibis2", - statsReceiver, - notifierThriftClientId, - requestTimeout = 3.seconds, - tries = 3, - mTLSServiceIdentifier = Some(serviceIdentifier) - ) - - // according to ibis team, it is safe to retry on timeout, write & channel closed exceptions. - val pushIbisClient = new Ibis2Service.FinagledClient( - new DynamicRequestMeterFilter( - tunableMap(PushServiceTunableKeys.IbisQpsLimitTunableKey), - RateLimiterGenerator.asTuple(_, shardParams.numShards, 20), - PushQPSLimitConstants.IbisOrNTabQPSForRFPH - )(timer).andThen(service), - RichClientParam(serviceName = "ibis-v2-service") - ) - - PushIbis2Store(pushIbisClient) - } - - val notificationServiceClient: NotificationService$FinagleClient = { - val service = Finagle.readWriteThriftService( - "notificationservice", - "/s/notificationservice/notificationservice", - statsReceiver, - pushserviceThriftClientId, - requestTimeout = 10.seconds, - mTLSServiceIdentifier = Some(serviceIdentifier) - ) - - new NotificationService.FinagledClient( - new DynamicRequestMeterFilter( - tunableMap(PushServiceTunableKeys.NtabQpsLimitTunableKey), - RateLimiterGenerator.asTuple(_, shardParams.numShards, 20), - PushQPSLimitConstants.IbisOrNTabQPSForRFPH)(timer).andThen(service), - RichClientParam(serviceName = "notificationservice") - ) - } - - val notificationServiceApiClient: NotificationApi$FinagleClient = { - val service = Finagle.readWriteThriftService( - "notificationservice-api", - "/s/notificationservice/notificationservice-api:thrift", - statsReceiver, - pushserviceThriftClientId, - requestTimeout = 10.seconds, - mTLSServiceIdentifier = Some(serviceIdentifier) - ) - - new NotificationApi.FinagledClient( - new DynamicRequestMeterFilter( - tunableMap(PushServiceTunableKeys.NtabQpsLimitTunableKey), - RateLimiterGenerator.asTuple(_, shardParams.numShards, 20), - PushQPSLimitConstants.IbisOrNTabQPSForRFPH)(timer).andThen(service), - RichClientParam(serviceName = "notificationservice-api") - ) - } - - val mrRequestScriberNode = "mr_request_scribe" - val loggedOutMrRequestScriberNode = "lo_mr_request_scribe" - - override val pushSendEventStreamName = "frigate_pushservice_send_event_prod" -} with DeployConfig { - // Scribe - private val notificationScribeLog = Logger("notification_scribe") - private val notificationScribeInjection: Injection[NotificationScribe, String] = BinaryScalaCodec( - NotificationScribe - ) andThen Injection.connect[Array[Byte], Base64String, String] - - override def notificationScribe(data: NotificationScribe): Unit = { - val logEntry: String = notificationScribeInjection(data) - notificationScribeLog.info(logEntry) - } - - // History Store - Invalidates cached history after writes - override val historyStore = new InvalidatingAfterWritesPushServiceHistoryStore( - ManhattanHistoryStore(notificationHistoryStore, statsReceiver), - recentHistoryCacheClient, - new DeciderGateBuilder(decider) - .idGate(DeciderKey.enableInvalidatingCachedHistoryStoreAfterWrites) - ) - - override val emailHistoryStore: PushServiceHistoryStore = { - statsReceiver.scope("frigate_email_history").counter("request").incr() - new SimplePushServiceHistoryStore(emailNotificationHistoryStore) - } - - override val loggedOutHistoryStore = - new InvalidatingAfterWritesPushServiceHistoryStore( - ManhattanKVHistoryStore( - manhattanKVLoggedOutHistoryStoreEndpoint, - "frigate_notification_logged_out_history"), - recentHistoryCacheClient, - new DeciderGateBuilder(decider) - .idGate(DeciderKey.enableInvalidatingCachedLoggedOutHistoryStoreAfterWrites) - ) - - private val requestScribeLog = Logger("request_scribe") - private val requestScribeInjection: Injection[PushRequestScribe, String] = BinaryScalaCodec( - PushRequestScribe - ) andThen Injection.connect[Array[Byte], Base64String, String] - - override def requestScribe(data: PushRequestScribe): Unit = { - val logEntry: String = requestScribeInjection(data) - requestScribeLog.info(logEntry) - } - - // generic notification server - override def notificationServiceSend( - target: Target, - request: CreateGenericNotificationRequest - ): Future[CreateGenericNotificationResponse] = - notificationServiceClient.createGenericNotification(request) - - // generic notification server - override def notificationServiceDelete( - request: DeleteGenericNotificationRequest - ): Future[Unit] = notificationServiceClient.deleteGenericNotification(request) - - // NTab-api - override def notificationServiceDeleteTimeline( - request: DeleteCurrentTimelineForUserRequest - ): Future[Unit] = notificationServiceApiClient.deleteCurrentTimelineForUser(request) - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/StagingConfig.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/StagingConfig.scala deleted file mode 100644 index c93ca0ea8..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/StagingConfig.scala +++ /dev/null @@ -1,193 +0,0 @@ -package com.twitter.frigate.pushservice.config - -import com.twitter.abdecider.LoggingABDecider -import com.twitter.conversions.DurationOps._ -import com.twitter.decider.Decider -import com.twitter.featureswitches.v2.FeatureSwitches -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.finagle.thrift.RichClientParam -import com.twitter.finagle.util.DefaultTimer -import com.twitter.frigate.common.config.RateLimiterGenerator -import com.twitter.frigate.common.filter.DynamicRequestMeterFilter -import com.twitter.frigate.common.history.InvalidatingAfterWritesPushServiceHistoryStore -import com.twitter.frigate.common.history.ManhattanHistoryStore -import com.twitter.frigate.common.history.ManhattanKVHistoryStore -import com.twitter.frigate.common.history.ReadOnlyHistoryStore -import com.twitter.frigate.common.history.PushServiceHistoryStore -import com.twitter.frigate.common.history.SimplePushServiceHistoryStore -import com.twitter.frigate.common.util.Finagle -import com.twitter.frigate.data_pipeline.features_common.FeatureStoreUtil -import com.twitter.frigate.data_pipeline.features_common.TargetLevelFeaturesConfig -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.DeciderKey -import com.twitter.frigate.pushservice.params.PushQPSLimitConstants -import com.twitter.frigate.pushservice.params.PushServiceTunableKeys -import com.twitter.frigate.pushservice.params.ShardParams -import com.twitter.frigate.pushservice.store._ -import com.twitter.frigate.pushservice.thriftscala.PushRequestScribe -import com.twitter.frigate.scribe.thriftscala.NotificationScribe -import com.twitter.ibis2.service.thriftscala.Ibis2Service -import com.twitter.logging.Logger -import com.twitter.notificationservice.api.thriftscala.DeleteCurrentTimelineForUserRequest -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationRequest -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationResponse -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationResponseType -import com.twitter.notificationservice.thriftscala.DeleteGenericNotificationRequest -import com.twitter.notificationservice.thriftscala.NotificationService -import com.twitter.notificationservice.thriftscala.NotificationService$FinagleClient -import com.twitter.servo.decider.DeciderGateBuilder -import com.twitter.util.tunable.TunableMap -import com.twitter.util.Future -import com.twitter.util.Timer - -case class StagingConfig( - override val isServiceLocal: Boolean, - override val localConfigRepoPath: String, - override val inMemCacheOff: Boolean, - override val decider: Decider, - override val abDecider: LoggingABDecider, - override val featureSwitches: FeatureSwitches, - override val shardParams: ShardParams, - override val serviceIdentifier: ServiceIdentifier, - override val tunableMap: TunableMap, -)( - implicit val statsReceiver: StatsReceiver) - extends { - // Due to trait initialization logic in Scala, any abstract members declared in Config or - // DeployConfig should be declared in this block. Otherwise the abstract member might initialize to - // null if invoked before object creation finishing. - - val log = Logger("StagingConfig") - - // Client ids - val notifierThriftClientId = ClientId("frigate-notifier.dev") - val loggedOutNotifierThriftClientId = ClientId("frigate-logged-out-notifier.dev") - val pushserviceThriftClientId: ClientId = ClientId("frigate-pushservice.staging") - - override val fanoutMetadataColumn = "frigate/magicfanout/staging/mh/fanoutMetadata" - - // dest - val frigateHistoryCacheDest = "/srv#/test/local/cache/twemcache_frigate_history" - val memcacheCASDest = "/srv#/test/local/cache/twemcache_magic_recs_cas_dev:twemcaches" - val pushServiceMHCacheDest = "/srv#/test/local/cache/twemcache_pushservice_test" - val entityGraphCacheDest = "/srv#/test/local/cache/twemcache_pushservice_test" - val pushServiceCoreSvcsCacheDest = "/srv#/test/local/cache/twemcache_pushservice_core_svcs_test" - val historyStoreMemcacheDest = "/srv#/test/local/cache/twemcache_eventstream_test:twemcaches" - val userTweetEntityGraphDest = "/cluster/local/cassowary/staging/user_tweet_entity_graph" - val userUserGraphDest = "/cluster/local/cassowary/staging/user_user_graph" - val lexServiceDest = "/srv#/staging/local/live-video/timeline-thrift" - val deepbirdv2PredictionServiceDest = "/cluster/local/frigate/staging/deepbirdv2-magicrecs" - - override val featureStoreUtil = FeatureStoreUtil.withParams(Some(serviceIdentifier)) - override val targetLevelFeaturesConfig = TargetLevelFeaturesConfig() - val mrRequestScriberNode = "validation_mr_request_scribe" - val loggedOutMrRequestScriberNode = "lo_mr_request_scribe" - - override val timer: Timer = DefaultTimer - - override val pushSendEventStreamName = "frigate_pushservice_send_event_staging" - - override val pushIbisV2Store = { - val service = Finagle.readWriteThriftService( - "ibis-v2-service", - "/s/ibis2/ibis2", - statsReceiver, - notifierThriftClientId, - requestTimeout = 6.seconds, - mTLSServiceIdentifier = Some(serviceIdentifier) - ) - - val pushIbisClient = new Ibis2Service.FinagledClient( - new DynamicRequestMeterFilter( - tunableMap(PushServiceTunableKeys.IbisQpsLimitTunableKey), - RateLimiterGenerator.asTuple(_, shardParams.numShards, 20), - PushQPSLimitConstants.IbisOrNTabQPSForRFPH - )(timer).andThen(service), - RichClientParam(serviceName = "ibis-v2-service") - ) - - StagingIbis2Store(PushIbis2Store(pushIbisClient)) - } - - val notificationServiceClient: NotificationService$FinagleClient = { - val service = Finagle.readWriteThriftService( - "notificationservice", - "/s/notificationservice/notificationservice", - statsReceiver, - pushserviceThriftClientId, - requestTimeout = 10.seconds, - mTLSServiceIdentifier = Some(serviceIdentifier) - ) - - new NotificationService.FinagledClient( - new DynamicRequestMeterFilter( - tunableMap(PushServiceTunableKeys.NtabQpsLimitTunableKey), - RateLimiterGenerator.asTuple(_, shardParams.numShards, 20), - PushQPSLimitConstants.IbisOrNTabQPSForRFPH)(timer).andThen(service), - RichClientParam(serviceName = "notificationservice") - ) - } -} with DeployConfig { - - // Scribe - private val notificationScribeLog = Logger("StagingNotificationScribe") - - override def notificationScribe(data: NotificationScribe): Unit = { - notificationScribeLog.info(data.toString) - } - private val requestScribeLog = Logger("StagingRequestScribe") - - override def requestScribe(data: PushRequestScribe): Unit = { - requestScribeLog.info(data.toString) - } - - // history store - override val historyStore = new InvalidatingAfterWritesPushServiceHistoryStore( - ReadOnlyHistoryStore( - ManhattanHistoryStore(notificationHistoryStore, statsReceiver) - ), - recentHistoryCacheClient, - new DeciderGateBuilder(decider) - .idGate(DeciderKey.enableInvalidatingCachedHistoryStoreAfterWrites) - ) - - override val emailHistoryStore: PushServiceHistoryStore = new SimplePushServiceHistoryStore( - emailNotificationHistoryStore) - - // history store - override val loggedOutHistoryStore = - new InvalidatingAfterWritesPushServiceHistoryStore( - ReadOnlyHistoryStore( - ManhattanKVHistoryStore( - manhattanKVLoggedOutHistoryStoreEndpoint, - "frigate_notification_logged_out_history")), - recentHistoryCacheClient, - new DeciderGateBuilder(decider) - .idGate(DeciderKey.enableInvalidatingCachedLoggedOutHistoryStoreAfterWrites) - ) - - override def notificationServiceSend( - target: Target, - request: CreateGenericNotificationRequest - ): Future[CreateGenericNotificationResponse] = - target.isTeamMember.flatMap { isTeamMember => - if (isTeamMember) { - notificationServiceClient.createGenericNotification(request) - } else { - log.info(s"Mock creating generic notification $request for user: ${target.targetId}") - Future.value( - CreateGenericNotificationResponse(CreateGenericNotificationResponseType.Success) - ) - } - } - - override def notificationServiceDelete( - request: DeleteGenericNotificationRequest - ): Future[Unit] = Future.Unit - - override def notificationServiceDeleteTimeline( - request: DeleteCurrentTimelineForUserRequest - ): Future[Unit] = Future.Unit -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/mlconfig/DeepbirdV2ModelConfig.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/mlconfig/DeepbirdV2ModelConfig.scala deleted file mode 100644 index c45ccc72e..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/mlconfig/DeepbirdV2ModelConfig.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.frigate.pushservice.config.mlconfig - -import com.twitter.cortex.deepbird.thriftjava.DeepbirdPredictionService -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.ml.prediction.DeepbirdPredictionEngineServiceStore -import com.twitter.nrel.heavyranker.PushDBv2PredictionServiceStore - -object DeepbirdV2ModelConfig { - def buildPredictionServiceScoreStore( - predictionServiceClient: DeepbirdPredictionService.ServiceToClient, - serviceName: String - )( - implicit statsReceiver: StatsReceiver - ): PushDBv2PredictionServiceStore = { - - val stats = statsReceiver.scope(serviceName) - val serviceStats = statsReceiver.scope("dbv2PredictionServiceStore") - - new PushDBv2PredictionServiceStore( - DeepbirdPredictionEngineServiceStore(predictionServiceClient, batchSize = Some(32))(stats) - )(serviceStats) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/controller/PushServiceController.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/controller/PushServiceController.scala deleted file mode 100644 index d271e5a57..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/controller/PushServiceController.scala +++ /dev/null @@ -1,114 +0,0 @@ -package com.twitter.frigate.pushservice.controller - -import com.google.inject.Inject -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.finatra.thrift.Controller -import com.twitter.frigate.pushservice.exception.DisplayLocationNotSupportedException -import com.twitter.frigate.pushservice.refresh_handler.RefreshForPushHandler -import com.twitter.frigate.pushservice.send_handler.SendHandler -import com.twitter.frigate.pushservice.refresh_handler.LoggedOutRefreshForPushHandler -import com.twitter.frigate.pushservice.thriftscala.PushService.Loggedout -import com.twitter.frigate.pushservice.thriftscala.PushService.Refresh -import com.twitter.frigate.pushservice.thriftscala.PushService.Send -import com.twitter.frigate.pushservice.{thriftscala => t} -import com.twitter.frigate.thriftscala.NotificationDisplayLocation -import com.twitter.util.logging.Logging -import com.twitter.util.Future - -class PushServiceController @Inject() ( - sendHandler: SendHandler, - refreshForPushHandler: RefreshForPushHandler, - loggedOutRefreshForPushHandler: LoggedOutRefreshForPushHandler, - statsReceiver: StatsReceiver) - extends Controller(t.PushService) - with Logging { - - private val stats: StatsReceiver = statsReceiver.scope(s"${this.getClass.getSimpleName}") - private val failureCount = stats.counter("failures") - private val failureStatsScope = stats.scope("failures") - private val uncaughtErrorCount = failureStatsScope.counter("uncaught") - private val uncaughtErrorScope = failureStatsScope.scope("uncaught") - private val clientIdScope = stats.scope("client_id") - - handle(t.PushService.Send) { request: Send.Args => - send(request) - } - - handle(t.PushService.Refresh) { args: Refresh.Args => - refresh(args) - } - - handle(t.PushService.Loggedout) { request: Loggedout.Args => - loggedOutRefresh(request) - } - - private def loggedOutRefresh( - request: t.PushService.Loggedout.Args - ): Future[t.PushService.Loggedout.SuccessType] = { - val fut = request.request.notificationDisplayLocation match { - case NotificationDisplayLocation.PushToMobileDevice => - loggedOutRefreshForPushHandler.refreshAndSend(request.request) - case _ => - Future.exception( - new DisplayLocationNotSupportedException( - "Specified notification display location is not supported")) - } - fut.onFailure { ex => - logger.error( - s"Failure in push service for logged out refresh request: $request - ${ex.getMessage} - ${ex.getStackTrace - .mkString(", \n\t")}", - ex) - failureCount.incr() - uncaughtErrorCount.incr() - uncaughtErrorScope.counter(ex.getClass.getCanonicalName).incr() - } - } - - private def refresh( - request: t.PushService.Refresh.Args - ): Future[t.PushService.Refresh.SuccessType] = { - - val fut = request.request.notificationDisplayLocation match { - case NotificationDisplayLocation.PushToMobileDevice => - val clientId: String = - ClientId.current - .flatMap { cid => Option(cid.name) } - .getOrElse("none") - clientIdScope.counter(clientId).incr() - refreshForPushHandler.refreshAndSend(request.request) - case _ => - Future.exception( - new DisplayLocationNotSupportedException( - "Specified notification display location is not supported")) - } - fut.onFailure { ex => - logger.error( - s"Failure in push service for refresh request: $request - ${ex.getMessage} - ${ex.getStackTrace - .mkString(", \n\t")}", - ex - ) - - failureCount.incr() - uncaughtErrorCount.incr() - uncaughtErrorScope.counter(ex.getClass.getCanonicalName).incr() - } - - } - - private def send( - request: t.PushService.Send.Args - ): Future[t.PushService.Send.SuccessType] = { - sendHandler(request.request).onFailure { ex => - logger.error( - s"Failure in push service for send request: $request - ${ex.getMessage} - ${ex.getStackTrace - .mkString(", \n\t")}", - ex - ) - - failureCount.incr() - uncaughtErrorCount.incr() - uncaughtErrorScope.counter(ex.getClass.getCanonicalName).incr() - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/DisplayLocationNotSupportedException.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/DisplayLocationNotSupportedException.scala deleted file mode 100644 index 08399c934..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/DisplayLocationNotSupportedException.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.frigate.pushservice.exception - -import scala.util.control.NoStackTrace - -/** - * Throw exception if DisplayLocation is not supported - * - * @param message Exception message - */ -class DisplayLocationNotSupportedException(private val message: String) - extends Exception(message) - with NoStackTrace diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/InvalidSportDomainException.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/InvalidSportDomainException.scala deleted file mode 100644 index 8f0d2b988..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/InvalidSportDomainException.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.frigate.pushservice.exception - -import scala.util.control.NoStackTrace - -/** - * Throw exception if the sport domain is not supported by MagicFanoutSports - * - * @param message Exception message - */ -class InvalidSportDomainException(private val message: String) - extends Exception(message) - with NoStackTrace diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/TweetNTabRequestHydratorException.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/TweetNTabRequestHydratorException.scala deleted file mode 100644 index 069e65d79..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/TweetNTabRequestHydratorException.scala +++ /dev/null @@ -1,7 +0,0 @@ -package com.twitter.frigate.pushservice.exception - -import scala.util.control.NoStackTrace - -class TweetNTabRequestHydratorException(private val message: String) - extends Exception(message) - with NoStackTrace diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/UnsupportedCrtException.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/UnsupportedCrtException.scala deleted file mode 100644 index 5ed6c1c28..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/UnsupportedCrtException.scala +++ /dev/null @@ -1,11 +0,0 @@ -package com.twitter.frigate.pushservice.exception - -import scala.util.control.NoStackTrace - -/** - * Exception for CRT not expected in the scope - * @param message Exception message to log the UnsupportedCrt - */ -class UnsupportedCrtException(private val message: String) - extends Exception(message) - with NoStackTrace diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/UttEntityNotFoundException.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/UttEntityNotFoundException.scala deleted file mode 100644 index 3ac069dac..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/UttEntityNotFoundException.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.frigate.pushservice.exception - -import scala.util.control.NoStackTrace - -/** - * Throw exception if UttEntity is not found where it might be a required data field - * - * @param message Exception message - */ -class UttEntityNotFoundException(private val message: String) - extends Exception(message) - with NoStackTrace diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/HealthFeatureGetter.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/HealthFeatureGetter.scala deleted file mode 100644 index addf5b438..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/HealthFeatureGetter.scala +++ /dev/null @@ -1,220 +0,0 @@ -package com.twitter.frigate.pushservice.ml - -import com.twitter.abuse.detection.scoring.thriftscala.{Model => TweetHealthModel} -import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringRequest -import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringResponse -import com.twitter.frigate.common.base.FeatureMap -import com.twitter.frigate.common.base.TweetAuthor -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushConstants -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.predicate.HealthPredicates.userHealthSignalValueToDouble -import com.twitter.frigate.pushservice.util.CandidateHydrationUtil -import com.twitter.frigate.pushservice.util.CandidateUtil -import com.twitter.frigate.pushservice.util.MediaAnnotationsUtil -import com.twitter.frigate.thriftscala.UserMediaRepresentation -import com.twitter.hss.api.thriftscala.SignalValue -import com.twitter.hss.api.thriftscala.UserHealthSignal -import com.twitter.hss.api.thriftscala.UserHealthSignal.AgathaCalibratedNsfwDouble -import com.twitter.hss.api.thriftscala.UserHealthSignal.NsfwTextUserScoreDouble -import com.twitter.hss.api.thriftscala.UserHealthSignalResponse -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future -import com.twitter.util.Time - -object HealthFeatureGetter { - - def getFeatures( - pushCandidate: PushCandidate, - producerMediaRepresentationStore: ReadableStore[Long, UserMediaRepresentation], - userHealthScoreStore: ReadableStore[Long, UserHealthSignalResponse], - tweetHealthScoreStoreOpt: Option[ReadableStore[TweetScoringRequest, TweetScoringResponse]] = - None - ): Future[FeatureMap] = { - - pushCandidate match { - case cand: PushCandidate with TweetCandidate with TweetAuthor with TweetAuthorDetails => - val pMediaNsfwRequest = - TweetScoringRequest(cand.tweetId, TweetHealthModel.ExperimentalHealthModelScore4) - val pTweetTextNsfwRequest = - TweetScoringRequest(cand.tweetId, TweetHealthModel.ExperimentalHealthModelScore1) - - cand.authorId match { - case Some(authorId) => - Future - .join( - userHealthScoreStore.get(authorId), - producerMediaRepresentationStore.get(authorId), - tweetHealthScoreStoreOpt.map(_.get(pMediaNsfwRequest)).getOrElse(Future.None), - tweetHealthScoreStoreOpt.map(_.get(pTweetTextNsfwRequest)).getOrElse(Future.None), - cand.tweetAuthor - ).map { - case ( - healthSignalsResponseOpt, - producerMuOpt, - pMediaNsfwOpt, - pTweetTextNsfwOpt, - tweetAuthorOpt) => - val healthSignalScoreMap = healthSignalsResponseOpt - .map(_.signalValues).getOrElse(Map.empty[UserHealthSignal, SignalValue]) - val agathaNSFWScore = userHealthSignalValueToDouble( - healthSignalScoreMap - .getOrElse(AgathaCalibratedNsfwDouble, SignalValue.DoubleValue(0.5))) - val userTextNSFWScore = userHealthSignalValueToDouble( - healthSignalScoreMap - .getOrElse(NsfwTextUserScoreDouble, SignalValue.DoubleValue(0.15))) - val pMediaNsfwScore = pMediaNsfwOpt.map(_.score).getOrElse(0.0) - val pTweetTextNsfwScore = pTweetTextNsfwOpt.map(_.score).getOrElse(0.0) - - val mediaRepresentationMap = - producerMuOpt.map(_.mediaRepresentation).getOrElse(Map.empty[String, Double]) - val sumScore: Double = mediaRepresentationMap.values.sum - val nudityRate = - if (sumScore > 0) - mediaRepresentationMap.getOrElse( - MediaAnnotationsUtil.nudityCategoryId, - 0.0) / sumScore - else 0.0 - val beautyRate = - if (sumScore > 0) - mediaRepresentationMap.getOrElse( - MediaAnnotationsUtil.beautyCategoryId, - 0.0) / sumScore - else 0.0 - val singlePersonRate = - if (sumScore > 0) - mediaRepresentationMap.getOrElse( - MediaAnnotationsUtil.singlePersonCategoryId, - 0.0) / sumScore - else 0.0 - val dislikeCt = cand.numericFeatures.getOrElse( - "tweet.magic_recs_tweet_real_time_aggregates_v2.pair.v2.magicrecs.realtime.is_ntab_disliked.any_feature.Duration.Top.count", - 0.0) - val sentCt = cand.numericFeatures.getOrElse( - "tweet.magic_recs_tweet_real_time_aggregates_v2.pair.v2.magicrecs.realtime.is_sent.any_feature.Duration.Top.count", - 0.0) - val dislikeRate = if (sentCt > 0) dislikeCt / sentCt else 0.0 - - val authorDislikeCt = cand.numericFeatures.getOrElse( - "tweet_author_aggregate.pair.label.ntab.isDisliked.any_feature.28.days.count", - 0.0) - val authorReportCt = cand.numericFeatures.getOrElse( - "tweet_author_aggregate.pair.label.reportTweetDone.any_feature.28.days.count", - 0.0) - val authorSentCt = cand.numericFeatures - .getOrElse( - "tweet_author_aggregate.pair.any_label.any_feature.28.days.count", - 0.0) - val authorDislikeRate = - if (authorSentCt > 0) authorDislikeCt / authorSentCt else 0.0 - val authorReportRate = - if (authorSentCt > 0) authorReportCt / authorSentCt else 0.0 - - val (isNsfwAccount, authorAccountAge) = tweetAuthorOpt match { - case Some(tweetAuthor) => - ( - CandidateHydrationUtil.isNsfwAccount( - tweetAuthor, - cand.target.params(PushFeatureSwitchParams.NsfwTokensParam)), - (Time.now - Time.fromMilliseconds(tweetAuthor.createdAtMsec)).inHours - ) - case _ => (false, 0) - } - - val tweetSemanticCoreIds = cand.sparseBinaryFeatures - .getOrElse(PushConstants.TweetSemanticCoreIdFeature, Set.empty[String]) - - val continuousFeatures = Map[String, Double]( - "agathaNsfwScore" -> agathaNSFWScore, - "textNsfwScore" -> userTextNSFWScore, - "pMediaNsfwScore" -> pMediaNsfwScore, - "pTweetTextNsfwScore" -> pTweetTextNsfwScore, - "nudityRate" -> nudityRate, - "beautyRate" -> beautyRate, - "singlePersonRate" -> singlePersonRate, - "numSources" -> CandidateUtil.getTagsCRCount(cand), - "favCount" -> cand.numericFeatures - .getOrElse("tweet.core.tweet_counts.favorite_count", 0.0), - "activeFollowers" -> cand.numericFeatures - .getOrElse("RecTweetAuthor.User.ActiveFollowers", 0.0), - "favorsRcvd28Days" -> cand.numericFeatures - .getOrElse("RecTweetAuthor.User.FavorsRcvd28Days", 0.0), - "tweets28Days" -> cand.numericFeatures - .getOrElse("RecTweetAuthor.User.Tweets28Days", 0.0), - "dislikeCount" -> dislikeCt, - "dislikeRate" -> dislikeRate, - "sentCount" -> sentCt, - "authorDislikeCount" -> authorDislikeCt, - "authorDislikeRate" -> authorDislikeRate, - "authorReportCount" -> authorReportCt, - "authorReportRate" -> authorReportRate, - "authorSentCount" -> authorSentCt, - "authorAgeInHour" -> authorAccountAge.toDouble - ) - - val booleanFeatures = Map[String, Boolean]( - "isSimclusterBased" -> RecTypes.simclusterBasedTweets - .contains(cand.commonRecType), - "isTopicTweet" -> RecTypes.isTopicTweetType(cand.commonRecType), - "isHashSpace" -> RecTypes.tagspaceTypes.contains(cand.commonRecType), - "isFRS" -> RecTypes.frsTypes.contains(cand.commonRecType), - "isModelingBased" -> RecTypes.mrModelingBasedTypes.contains(cand.commonRecType), - "isGeoPop" -> RecTypes.GeoPopTweetTypes.contains(cand.commonRecType), - "hasPhoto" -> cand.booleanFeatures - .getOrElse("RecTweet.TweetyPieResult.HasPhoto", false), - "hasVideo" -> cand.booleanFeatures - .getOrElse("RecTweet.TweetyPieResult.HasVideo", false), - "hasUrl" -> cand.booleanFeatures - .getOrElse("RecTweet.TweetyPieResult.HasUrl", false), - "isMrTwistly" -> CandidateUtil.isMrTwistlyCandidate(cand), - "abuseStrikeTop2Percent" -> tweetSemanticCoreIds.contains( - PushConstants.AbuseStrike_Top2Percent_Id), - "abuseStrikeTop1Percent" -> tweetSemanticCoreIds.contains( - PushConstants.AbuseStrike_Top1Percent_Id), - "abuseStrikeTop05Percent" -> tweetSemanticCoreIds.contains( - PushConstants.AbuseStrike_Top05Percent_Id), - "abuseStrikeTop025Percent" -> tweetSemanticCoreIds.contains( - PushConstants.AbuseStrike_Top025Percent_Id), - "allSpamReportsPerFavTop1Percent" -> tweetSemanticCoreIds.contains( - PushConstants.AllSpamReportsPerFav_Top1Percent_Id), - "reportsPerFavTop1Percent" -> tweetSemanticCoreIds.contains( - PushConstants.ReportsPerFav_Top1Percent_Id), - "reportsPerFavTop2Percent" -> tweetSemanticCoreIds.contains( - PushConstants.ReportsPerFav_Top2Percent_Id), - "isNudity" -> tweetSemanticCoreIds.contains( - PushConstants.MediaUnderstanding_Nudity_Id), - "beautyStyleFashion" -> tweetSemanticCoreIds.contains( - PushConstants.MediaUnderstanding_Beauty_Id), - "singlePerson" -> tweetSemanticCoreIds.contains( - PushConstants.MediaUnderstanding_SinglePerson_Id), - "pornList" -> tweetSemanticCoreIds.contains(PushConstants.PornList_Id), - "pornographyAndNsfwContent" -> tweetSemanticCoreIds.contains( - PushConstants.PornographyAndNsfwContent_Id), - "sexLife" -> tweetSemanticCoreIds.contains(PushConstants.SexLife_Id), - "sexLifeOrSexualOrientation" -> tweetSemanticCoreIds.contains( - PushConstants.SexLifeOrSexualOrientation_Id), - "profanity" -> tweetSemanticCoreIds.contains(PushConstants.ProfanityFilter_Id), - "isVerified" -> cand.booleanFeatures - .getOrElse("RecTweetAuthor.User.IsVerified", false), - "hasNsfwToken" -> isNsfwAccount - ) - - val stringFeatures = Map[String, String]( - "tweetLanguage" -> cand.categoricalFeatures - .getOrElse("tweet.core.tweet_text.language", "") - ) - - FeatureMap( - booleanFeatures = booleanFeatures, - numericFeatures = continuousFeatures, - categoricalFeatures = stringFeatures) - } - case _ => Future.value(FeatureMap()) - } - case _ => Future.value(FeatureMap()) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/HydrationContextBuilder.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/HydrationContextBuilder.scala deleted file mode 100644 index 023adb81e..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/HydrationContextBuilder.scala +++ /dev/null @@ -1,179 +0,0 @@ -package com.twitter.frigate.pushservice.ml - -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.ml.feature.TweetSocialProofKey -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.predicate.quality_model_predicate.PDauCohortUtil -import com.twitter.nrel.hydration.base.FeatureInput -import com.twitter.nrel.hydration.push.HydrationContext -import com.twitter.nrel.hydration.frigate.{FeatureInputs => FI} -import com.twitter.util.Future - -object HydrationContextBuilder { - - private def getRecUserInputs( - pushCandidate: PushCandidate - ): Set[FI.RecUser] = { - pushCandidate match { - case userCandidate: UserCandidate => - Set(FI.RecUser(userCandidate.userId)) - case _ => Set.empty - } - } - - private def getRecTweetInputs( - pushCandidate: PushCandidate - ): Set[FI.RecTweet] = - pushCandidate match { - case tweetCandidateWithAuthor: TweetCandidate with TweetAuthor with TweetAuthorDetails => - val authorIdOpt = tweetCandidateWithAuthor.authorId - Set(FI.RecTweet(tweetCandidateWithAuthor.tweetId, authorIdOpt)) - case _ => Set.empty - } - - private def getMediaInputs( - pushCandidate: PushCandidate - ): Set[FI.Media] = - pushCandidate match { - case tweetCandidateWithMedia: TweetCandidate with TweetDetails => - tweetCandidateWithMedia.mediaKeys - .map { mk => - Set(FI.Media(mk)) - }.getOrElse(Set.empty) - case _ => Set.empty - } - - private def getEventInputs( - pushCandidate: PushCandidate - ): Set[FI.Event] = pushCandidate match { - case mrEventCandidate: EventCandidate => - Set(FI.Event(mrEventCandidate.eventId)) - case mfEventCandidate: MagicFanoutEventCandidate => - Set(FI.Event(mfEventCandidate.eventId)) - case _ => Set.empty - } - - private def getTopicInputs( - pushCandidate: PushCandidate - ): Set[FI.Topic] = - pushCandidate match { - case mrTopicCandidate: TopicCandidate => - mrTopicCandidate.semanticCoreEntityId match { - case Some(topicId) => Set(FI.Topic(topicId)) - case _ => Set.empty - } - case _ => Set.empty - } - - private def getTweetSocialProofKey( - pushCandidate: PushCandidate - ): Future[Set[FI.SocialProofKey]] = { - pushCandidate match { - case candidate: TweetCandidate with SocialContextActions => - val target = pushCandidate.target - target.seedsWithWeight.map { seedsWithWeightOpt => - Set( - FI.SocialProofKey( - TweetSocialProofKey( - seedsWithWeightOpt.getOrElse(Map.empty), - candidate.socialContextAllTypeActions - )) - ) - } - case _ => Future.value(Set.empty) - } - } - - private def getSocialContextInputs( - pushCandidate: PushCandidate - ): Future[Set[FeatureInput]] = - pushCandidate match { - case candidateWithSC: Candidate with SocialContextActions => - val tweetSocialProofKeyFut = getTweetSocialProofKey(pushCandidate) - tweetSocialProofKeyFut.map { tweetSocialProofKeyOpt => - val socialContextUsers = FI.SocialContextUsers(candidateWithSC.socialContextUserIds.toSet) - val socialContextActions = - FI.SocialContextActions(candidateWithSC.socialContextAllTypeActions) - val socialProofKeyOpt = tweetSocialProofKeyOpt - Set(Set(socialContextUsers), Set(socialContextActions), socialProofKeyOpt).flatten - } - case _ => Future.value(Set.empty) - } - - private def getPushStringGroupInputs( - pushCandidate: PushCandidate - ): Set[FI.PushStringGroup] = - Set( - FI.PushStringGroup( - pushCandidate.getPushCopy.flatMap(_.pushStringGroup).map(_.toString).getOrElse("") - )) - - private def getCRTInputs( - pushCandidate: PushCandidate - ): Set[FI.CommonRecommendationType] = - Set(FI.CommonRecommendationType(pushCandidate.commonRecType)) - - private def getFrigateNotification( - pushCandidate: PushCandidate - ): Set[FI.CandidateFrigateNotification] = - Set(FI.CandidateFrigateNotification(pushCandidate.frigateNotification)) - - private def getCopyId( - pushCandidate: PushCandidate - ): Set[FI.CopyId] = - Set(FI.CopyId(pushCandidate.pushCopyId, pushCandidate.ntabCopyId)) - - def build(candidate: PushCandidate): Future[HydrationContext] = { - val socialContextInputsFut = getSocialContextInputs(candidate) - socialContextInputsFut.map { socialContextInputs => - val featureInputs: Set[FeatureInput] = - socialContextInputs ++ - getRecUserInputs(candidate) ++ - getRecTweetInputs(candidate) ++ - getEventInputs(candidate) ++ - getTopicInputs(candidate) ++ - getCRTInputs(candidate) ++ - getPushStringGroupInputs(candidate) ++ - getMediaInputs(candidate) ++ - getFrigateNotification(candidate) ++ - getCopyId(candidate) - - HydrationContext( - candidate.target.targetId, - featureInputs - ) - } - } - - def build(target: Target): Future[HydrationContext] = { - val realGraphFeaturesFut = target.realGraphFeatures - for { - realGraphFeaturesOpt <- realGraphFeaturesFut - dauProb <- PDauCohortUtil.getDauProb(target) - mrUserStateOpt <- target.targetMrUserState - historyInputOpt <- - if (target.params(PushFeatureSwitchParams.EnableHydratingOnlineMRHistoryFeatures)) { - target.onlineLabeledPushRecs.map { mrHistoryValueOpt => - mrHistoryValueOpt.map(FI.MrHistory) - } - } else Future.None - } yield { - val realGraphFeaturesInputOpt = realGraphFeaturesOpt.map { realGraphFeatures => - FI.TargetRealGraphFeatures(realGraphFeatures) - } - val dauProbInput = FI.DauProb(dauProb) - val mrUserStateInput = FI.MrUserState(mrUserStateOpt.map(_.name).getOrElse("unknown")) - HydrationContext( - target.targetId, - Seq( - realGraphFeaturesInputOpt, - historyInputOpt, - Some(dauProbInput), - Some(mrUserStateInput) - ).flatten.toSet - ) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/PushMLModelScorer.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/PushMLModelScorer.scala deleted file mode 100644 index fd702cc3c..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/PushMLModelScorer.scala +++ /dev/null @@ -1,188 +0,0 @@ -package com.twitter.frigate.pushservice.ml - -import com.twitter.cortex.deepbird.thriftjava.ModelSelector -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.common.base.FeatureMap -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushMLModel -import com.twitter.frigate.pushservice.params.PushModelName -import com.twitter.frigate.pushservice.params.WeightedOpenOrNtabClickModel -import com.twitter.nrel.heavyranker.PushCandidateHydrationContextWithModel -import com.twitter.nrel.heavyranker.PushPredictionServiceStore -import com.twitter.nrel.heavyranker.TargetFeatureMapWithModel -import com.twitter.timelines.configapi.FSParam -import com.twitter.util.Future - -/** - * PushMLModelScorer scores the Candidates and populates their ML scores - * - * @param pushMLModel Enum to specify which model to use for scoring the Candidates - * @param modelToPredictionServiceStoreMap Supports all other prediction services. Specifies model ID -> dbv2 ReadableStore - * @param defaultDBv2PredictionServiceStore: Supports models that are not specified in the previous maps (which will be directly configured in the config repo) - * @param scoringStats StatsReceiver for scoping stats - */ -class PushMLModelScorer( - pushMLModel: PushMLModel.Value, - modelToPredictionServiceStoreMap: Map[ - WeightedOpenOrNtabClickModel.ModelNameType, - PushPredictionServiceStore - ], - defaultDBv2PredictionServiceStore: PushPredictionServiceStore, - scoringStats: StatsReceiver) { - - val queriesOutsideTheModelMaps: StatsReceiver = - scoringStats.scope("queries_outside_the_model_maps") - val totalQueriesOutsideTheModelMaps: Counter = - queriesOutsideTheModelMaps.counter("total") - - private def scoreByBatchPredictionForModelFromMultiModelService( - predictionServiceStore: PushPredictionServiceStore, - modelVersion: WeightedOpenOrNtabClickModel.ModelNameType, - candidatesDetails: Seq[CandidateDetails[PushCandidate]], - useCommonFeatures: Boolean, - overridePushMLModel: PushMLModel.Value - ): Seq[CandidateDetails[PushCandidate]] = { - val modelName = - PushModelName(overridePushMLModel, modelVersion).toString - val modelSelector = new ModelSelector() - modelSelector.setId(modelName) - - val candidateHydrationWithFeaturesMap = candidatesDetails.map { candidatesDetail => - ( - candidatesDetail.candidate.candidateHydrationContext, - candidatesDetail.candidate.candidateFeatureMap()) - } - if (candidatesDetails.nonEmpty) { - val candidatesWithScore = predictionServiceStore.getBatchPredictionsForModel( - candidatesDetails.head.candidate.target.targetHydrationContext, - candidatesDetails.head.candidate.target.featureMap, - candidateHydrationWithFeaturesMap, - Some(modelSelector), - useCommonFeatures - ) - candidatesDetails.zip(candidatesWithScore).foreach { - case (candidateDetail, (_, scoreOptFut)) => - candidateDetail.candidate.populateQualityModelScore( - overridePushMLModel, - modelVersion, - scoreOptFut - ) - } - } - - candidatesDetails - } - - private def scoreByBatchPrediction( - modelVersion: WeightedOpenOrNtabClickModel.ModelNameType, - candidatesDetails: Seq[CandidateDetails[PushCandidate]], - useCommonFeaturesForDBv2Service: Boolean, - overridePushMLModel: PushMLModel.Value - ): Seq[CandidateDetails[PushCandidate]] = { - if (modelToPredictionServiceStoreMap.contains(modelVersion)) { - scoreByBatchPredictionForModelFromMultiModelService( - modelToPredictionServiceStoreMap(modelVersion), - modelVersion, - candidatesDetails, - useCommonFeaturesForDBv2Service, - overridePushMLModel - ) - } else { - totalQueriesOutsideTheModelMaps.incr() - queriesOutsideTheModelMaps.counter(modelVersion).incr() - scoreByBatchPredictionForModelFromMultiModelService( - defaultDBv2PredictionServiceStore, - modelVersion, - candidatesDetails, - useCommonFeaturesForDBv2Service, - overridePushMLModel - ) - } - } - - def scoreByBatchPredictionForModelVersion( - target: Target, - candidatesDetails: Seq[CandidateDetails[PushCandidate]], - modelVersionParam: FSParam[WeightedOpenOrNtabClickModel.ModelNameType], - useCommonFeaturesForDBv2Service: Boolean = true, - overridePushMLModelOpt: Option[PushMLModel.Value] = None - ): Seq[CandidateDetails[PushCandidate]] = { - scoreByBatchPrediction( - target.params(modelVersionParam), - candidatesDetails, - useCommonFeaturesForDBv2Service, - overridePushMLModelOpt.getOrElse(pushMLModel) - ) - } - - def singlePredicationForModelVersion( - modelVersion: String, - candidate: PushCandidate, - overridePushMLModelOpt: Option[PushMLModel.Value] = None - ): Future[Option[Double]] = { - val modelSelector = new ModelSelector() - modelSelector.setId( - PushModelName(overridePushMLModelOpt.getOrElse(pushMLModel), modelVersion).toString - ) - if (modelToPredictionServiceStoreMap.contains(modelVersion)) { - modelToPredictionServiceStoreMap(modelVersion).get( - PushCandidateHydrationContextWithModel( - candidate.target.targetHydrationContext, - candidate.target.featureMap, - candidate.candidateHydrationContext, - candidate.candidateFeatureMap(), - Some(modelSelector) - ) - ) - } else { - totalQueriesOutsideTheModelMaps.incr() - queriesOutsideTheModelMaps.counter(modelVersion).incr() - defaultDBv2PredictionServiceStore.get( - PushCandidateHydrationContextWithModel( - candidate.target.targetHydrationContext, - candidate.target.featureMap, - candidate.candidateHydrationContext, - candidate.candidateFeatureMap(), - Some(modelSelector) - ) - ) - } - } - - def singlePredictionForTargetLevel( - modelVersion: String, - targetId: Long, - featureMap: Future[FeatureMap] - ): Future[Option[Double]] = { - val modelSelector = new ModelSelector() - modelSelector.setId( - PushModelName(pushMLModel, modelVersion).toString - ) - defaultDBv2PredictionServiceStore.getForTargetLevel( - TargetFeatureMapWithModel(targetId, featureMap, Some(modelSelector)) - ) - } - - def getScoreHistogramCounters( - stats: StatsReceiver, - scopeName: String, - histogramBinSize: Double - ): IndexedSeq[Counter] = { - val histogramScopedStatsReceiver = stats.scope(scopeName) - val numBins = math.ceil(1.0 / histogramBinSize).toInt - - (0 to numBins) map { k => - if (k == 0) - histogramScopedStatsReceiver.counter("candidates_with_scores_zero") - else { - val counterName = "candidates_with_scores_from_%s_to_%s".format( - "%.2f".format(histogramBinSize * (k - 1)).replace(".", ""), - "%.2f".format(math.min(1.0, histogramBinSize * k)).replace(".", "")) - histogramScopedStatsReceiver.counter(counterName) - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/DiscoverTwitter.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/DiscoverTwitter.scala deleted file mode 100644 index dc350a740..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/DiscoverTwitter.scala +++ /dev/null @@ -1,89 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.DiscoverTwitterCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.DiscoverTwitterPushIbis2Hydrator -import com.twitter.frigate.pushservice.model.ntab.DiscoverTwitterNtabRequestHydrator -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate -import com.twitter.frigate.pushservice.take.predicates.BasicRFPHPredicates -import com.twitter.frigate.pushservice.take.predicates.OutOfNetworkTweetPredicates -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.hermit.predicate.NamedPredicate - -class DiscoverTwitterPushCandidate( - candidate: RawCandidate with DiscoverTwitterCandidate, - copyIds: CopyIds, -)( - implicit val statsScoped: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with DiscoverTwitterCandidate - with DiscoverTwitterPushIbis2Hydrator - with DiscoverTwitterNtabRequestHydrator { - - override val pushCopyId: Option[Int] = copyIds.pushCopyId - - override val ntabCopyId: Option[Int] = copyIds.ntabCopyId - - override val copyAggregationId: Option[String] = copyIds.aggregationId - - override val target: Target = candidate.target - - override lazy val commonRecType: CommonRecommendationType = candidate.commonRecType - - override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - - override val statsReceiver: StatsReceiver = - statsScoped.scope("DiscoverTwitterPushCandidate") -} - -case class AddressBookPushCandidatePredicates(config: Config) - extends BasicRFPHPredicates[DiscoverTwitterPushCandidate] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override val predicates: List[ - NamedPredicate[DiscoverTwitterPushCandidate] - ] = - List( - PredicatesForCandidate.paramPredicate( - PushFeatureSwitchParams.EnableAddressBookPush - ) - ) -} - -case class CompleteOnboardingPushCandidatePredicates(config: Config) - extends BasicRFPHPredicates[DiscoverTwitterPushCandidate] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override val predicates: List[ - NamedPredicate[DiscoverTwitterPushCandidate] - ] = - List( - PredicatesForCandidate.paramPredicate( - PushFeatureSwitchParams.EnableCompleteOnboardingPush - ) - ) -} - -case class PopGeoTweetCandidatePredicates(override val config: Config) - extends OutOfNetworkTweetPredicates[OutOfNetworkTweetPushCandidate] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override def postCandidateSpecificPredicates: List[ - NamedPredicate[OutOfNetworkTweetPushCandidate] - ] = List( - PredicatesForCandidate.htlFatiguePredicate( - PushFeatureSwitchParams.NewUserPlaybookAllowedLastLoginHours - ) - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/F1FirstdegreeTweet.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/F1FirstdegreeTweet.scala deleted file mode 100644 index a4e8bec68..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/F1FirstdegreeTweet.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.F1FirstDegree -import com.twitter.frigate.common.base.SocialContextAction -import com.twitter.frigate.common.base.SocialGraphServiceRelationshipMap -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes._ -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.F1FirstDegreeTweetIbis2HydratorForCandidate -import com.twitter.frigate.pushservice.model.ntab.F1FirstDegreeTweetNTabRequestHydrator -import com.twitter.frigate.pushservice.take.predicates.BasicTweetPredicatesForRFPHWithoutSGSPredicates -import com.twitter.frigate.pushservice.util.CandidateHydrationUtil.TweetWithSocialContextTraits -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.hermit.predicate.socialgraph.RelationEdge -import com.twitter.stitch.tweetypie.TweetyPie -import com.twitter.util.Future - -class F1TweetPushCandidate( - candidate: RawCandidate with TweetWithSocialContextTraits, - author: Future[Option[User]], - socialGraphServiceResultMap: Map[RelationEdge, Boolean], - copyIds: CopyIds -)( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with F1FirstDegree - with TweetAuthorDetails - with SocialGraphServiceRelationshipMap - with F1FirstDegreeTweetNTabRequestHydrator - with F1FirstDegreeTweetIbis2HydratorForCandidate { - override val socialContextActions: Seq[SocialContextAction] = - candidate.socialContextActions - override val socialContextAllTypeActions: Seq[SocialContextAction] = - candidate.socialContextActions - override val statsReceiver: StatsReceiver = stats - override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - override val tweetId: Long = candidate.tweetId - override lazy val tweetyPieResult: Option[TweetyPie.TweetyPieResult] = - candidate.tweetyPieResult - override lazy val tweetAuthor: Future[Option[User]] = author - override val target: PushTypes.Target = candidate.target - override lazy val commonRecType: CommonRecommendationType = - candidate.commonRecType - override val pushCopyId: Option[Int] = copyIds.pushCopyId - override val ntabCopyId: Option[Int] = copyIds.ntabCopyId - override val copyAggregationId: Option[String] = copyIds.aggregationId - - override val relationshipMap: Map[RelationEdge, Boolean] = socialGraphServiceResultMap -} - -case class F1TweetCandidatePredicates(override val config: Config) - extends BasicTweetPredicatesForRFPHWithoutSGSPredicates[F1TweetPushCandidate] { - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ListRecommendationPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ListRecommendationPushCandidate.scala deleted file mode 100644 index 412dfbf00..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ListRecommendationPushCandidate.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.channels.common.thriftscala.ApiList -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.ListPushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.ListIbis2Hydrator -import com.twitter.frigate.pushservice.model.ntab.ListCandidateNTabRequestHydrator -import com.twitter.frigate.pushservice.predicate.ListPredicates -import com.twitter.frigate.pushservice.take.predicates.BasicRFPHPredicates -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -class ListRecommendationPushCandidate( - val apiListStore: ReadableStore[Long, ApiList], - candidate: RawCandidate with ListPushCandidate, - copyIds: CopyIds -)( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with ListPushCandidate - with ListIbis2Hydrator - with ListCandidateNTabRequestHydrator { - - override val commonRecType: CommonRecommendationType = candidate.commonRecType - - override val pushCopyId: Option[Int] = copyIds.pushCopyId - - override val ntabCopyId: Option[Int] = copyIds.ntabCopyId - - override val copyAggregationId: Option[String] = copyIds.aggregationId - - override val statsReceiver: StatsReceiver = stats - - override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - - override val target: PushTypes.Target = candidate.target - - override val listId: Long = candidate.listId - - lazy val apiList: Future[Option[ApiList]] = apiListStore.get(listId) - - lazy val listName: Future[Option[String]] = apiList.map { apiListOpt => - apiListOpt.map(_.name) - } - - lazy val listOwnerId: Future[Option[Long]] = apiList.map { apiListOpt => - apiListOpt.map(_.ownerId) - } - -} - -case class ListRecommendationPredicates(config: Config) - extends BasicRFPHPredicates[ListRecommendationPushCandidate] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override val predicates: List[NamedPredicate[ListRecommendationPushCandidate]] = List( - ListPredicates.listNameExistsPredicate(), - ListPredicates.listAuthorExistsPredicate(), - ListPredicates.listAuthorAcceptableToTargetUser(config.edgeStore), - ListPredicates.listAcceptablePredicate(), - ListPredicates.listSubscriberCountPredicate() - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutCreatorEventPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutCreatorEventPushCandidate.scala deleted file mode 100644 index 65633259c..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutCreatorEventPushCandidate.scala +++ /dev/null @@ -1,136 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.HydratedMagicFanoutCreatorEventCandidate -import com.twitter.frigate.common.base.MagicFanoutCreatorEventCandidate -import com.twitter.frigate.magic_events.thriftscala.CreatorFanoutType -import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.MagicFanoutCreatorEventIbis2Hydrator -import com.twitter.frigate.pushservice.model.ntab.MagicFanoutCreatorEventNtabRequestHydrator -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate -import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutPredicatesForCandidate -import com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue.MagicFanoutNtabCaretFatiguePredicate -import com.twitter.frigate.pushservice.take.predicates.BasicSendHandlerPredicates -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.client.UserId -import com.twitter.util.Future -import scala.util.control.NoStackTrace - -class MagicFanoutCreatorEventPushCandidateHydratorException(private val message: String) - extends Exception(message) - with NoStackTrace - -class MagicFanoutCreatorEventPushCandidate( - candidate: RawCandidate with MagicFanoutCreatorEventCandidate, - creatorUser: Option[User], - copyIds: CopyIds, - creatorTweetCountStore: ReadableStore[UserId, Int] -)( - implicit val statsScoped: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with MagicFanoutCreatorEventIbis2Hydrator - with MagicFanoutCreatorEventNtabRequestHydrator - with MagicFanoutCreatorEventCandidate - with HydratedMagicFanoutCreatorEventCandidate { - override def creatorId: Long = candidate.creatorId - - override def hydratedCreator: Option[User] = creatorUser - - override lazy val numberOfTweetsFut: Future[Option[Int]] = - creatorTweetCountStore.get(UserId(creatorId)) - - lazy val userProfile = hydratedCreator - .flatMap(_.profile).getOrElse( - throw new MagicFanoutCreatorEventPushCandidateHydratorException( - s"Unable to get user profile to generate tapThrough for userId: $creatorId")) - - override val frigateNotification: FrigateNotification = candidate.frigateNotification - - override def subscriberId: Option[Long] = candidate.subscriberId - - override def creatorFanoutType: CreatorFanoutType = candidate.creatorFanoutType - - override def target: PushTypes.Target = candidate.target - - override def pushId: Long = candidate.pushId - - override def candidateMagicEventsReasons: Seq[MagicEventsReason] = - candidate.candidateMagicEventsReasons - - override def statsReceiver: StatsReceiver = statsScoped - - override def pushCopyId: Option[Int] = copyIds.pushCopyId - - override def ntabCopyId: Option[Int] = copyIds.ntabCopyId - - override def copyAggregationId: Option[String] = copyIds.aggregationId - - override def commonRecType: CommonRecommendationType = candidate.commonRecType - - override def weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - -} - -case class MagicFanouCreatorSubscriptionEventPushPredicates(config: Config) - extends BasicSendHandlerPredicates[MagicFanoutCreatorEventPushCandidate] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override val preCandidateSpecificPredicates: List[ - NamedPredicate[MagicFanoutCreatorEventPushCandidate] - ] = - List( - PredicatesForCandidate.paramPredicate( - PushFeatureSwitchParams.EnableCreatorSubscriptionPush - ), - PredicatesForCandidate.isDeviceEligibleForCreatorPush, - MagicFanoutPredicatesForCandidate.creatorPushTargetIsNotCreator(), - MagicFanoutPredicatesForCandidate.duplicateCreatorPredicate, - MagicFanoutPredicatesForCandidate.magicFanoutCreatorPushFatiguePredicate(), - ) - - override val postCandidateSpecificPredicates: List[ - NamedPredicate[MagicFanoutCreatorEventPushCandidate] - ] = - List( - MagicFanoutNtabCaretFatiguePredicate(), - MagicFanoutPredicatesForCandidate.isSuperFollowingCreator()(config, statsReceiver).flip - ) -} - -case class MagicFanoutNewCreatorEventPushPredicates(config: Config) - extends BasicSendHandlerPredicates[MagicFanoutCreatorEventPushCandidate] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override val preCandidateSpecificPredicates: List[ - NamedPredicate[MagicFanoutCreatorEventPushCandidate] - ] = - List( - PredicatesForCandidate.paramPredicate( - PushFeatureSwitchParams.EnableNewCreatorPush - ), - PredicatesForCandidate.isDeviceEligibleForCreatorPush, - MagicFanoutPredicatesForCandidate.duplicateCreatorPredicate, - MagicFanoutPredicatesForCandidate.magicFanoutCreatorPushFatiguePredicate, - ) - - override val postCandidateSpecificPredicates: List[ - NamedPredicate[MagicFanoutCreatorEventPushCandidate] - ] = - List( - MagicFanoutNtabCaretFatiguePredicate(), - MagicFanoutPredicatesForCandidate.isSuperFollowingCreator()(config, statsReceiver).flip - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutEventPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutEventPushCandidate.scala deleted file mode 100644 index e0a5f5386..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutEventPushCandidate.scala +++ /dev/null @@ -1,303 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.escherbird.metadata.thriftscala.EntityMegadata -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.MagicFanoutEventCandidate -import com.twitter.frigate.common.base.RecommendationType -import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext -import com.twitter.frigate.common.util.HighPriorityLocaleUtil -import com.twitter.frigate.magic_events.thriftscala.FanoutEvent -import com.twitter.frigate.magic_events.thriftscala.FanoutMetadata -import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason -import com.twitter.frigate.magic_events.thriftscala.NewsForYouMetadata -import com.twitter.frigate.magic_events.thriftscala.ReasonSource -import com.twitter.frigate.magic_events.thriftscala.TargetID -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.Ibis2HydratorForCandidate -import com.twitter.frigate.pushservice.model.ntab.EventNTabRequestHydrator -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutPredicatesUtil -import com.twitter.frigate.pushservice.store.EventRequest -import com.twitter.frigate.pushservice.store.UttEntityHydrationStore -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.frigate.pushservice.util.TopicsUtil -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.frigate.thriftscala.MagicFanoutEventNotificationDetails -import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery -import com.twitter.interests.thriftscala.InterestId.SemanticCore -import com.twitter.interests.thriftscala.UserInterests -import com.twitter.livevideo.common.ids.CountryId -import com.twitter.livevideo.common.ids.UserId -import com.twitter.livevideo.timeline.domain.v2.Event -import com.twitter.livevideo.timeline.domain.v2.HydrationOptions -import com.twitter.livevideo.timeline.domain.v2.LookupContext -import com.twitter.simclusters_v2.thriftscala.SimClustersInferredEntities -import com.twitter.storehaus.ReadableStore -import com.twitter.topiclisting.utt.LocalizedEntity -import com.twitter.util.Future - -abstract class MagicFanoutEventPushCandidate( - candidate: RawCandidate with MagicFanoutEventCandidate with RecommendationType, - copyIds: CopyIds, - override val fanoutEvent: Option[FanoutEvent], - override val semanticEntityResults: Map[SemanticEntityForQuery, Option[EntityMegadata]], - simClusterToEntities: Map[Int, Option[SimClustersInferredEntities]], - lexServiceStore: ReadableStore[EventRequest, Event], - interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests], - uttEntityHydrationStore: UttEntityHydrationStore -)( - implicit statsScoped: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with MagicFanoutEventHydratedCandidate - with MagicFanoutEventCandidate - with EventNTabRequestHydrator - with RecommendationType - with Ibis2HydratorForCandidate { - - override lazy val eventFut: Future[Option[Event]] = { - eventRequestFut.flatMap { - case Some(eventRequest) => lexServiceStore.get(eventRequest) - case _ => Future.None - } - } - - override val frigateNotification: FrigateNotification = candidate.frigateNotification - - override val pushId: Long = candidate.pushId - - override val candidateMagicEventsReasons: Seq[MagicEventsReason] = - candidate.candidateMagicEventsReasons - - override val eventId: Long = candidate.eventId - - override val momentId: Option[Long] = candidate.momentId - - override val target: Target = candidate.target - - override val eventLanguage: Option[String] = candidate.eventLanguage - - override val details: Option[MagicFanoutEventNotificationDetails] = candidate.details - - override lazy val stats: StatsReceiver = statsScoped.scope("MagicFanoutEventPushCandidate") - - override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - - override val pushCopyId: Option[Int] = copyIds.pushCopyId - - override val ntabCopyId: Option[Int] = copyIds.ntabCopyId - - override val copyAggregationId: Option[String] = copyIds.aggregationId - - override val statsReceiver: StatsReceiver = statsScoped.scope("MagicFanoutEventPushCandidate") - - override val effectiveMagicEventsReasons: Option[Seq[MagicEventsReason]] = Some( - candidateMagicEventsReasons) - - lazy val newsForYouMetadata: Option[NewsForYouMetadata] = - fanoutEvent.flatMap { event => - { - event.fanoutMetadata.collect { - case FanoutMetadata.NewsForYouMetadata(nfyMetadata) => nfyMetadata - } - } - } - - val reverseIndexedTopicIds = candidate.candidateMagicEventsReasons - .filter(_.source.contains(ReasonSource.UttTopicFollowGraph)) - .map(_.reason).collect { - case TargetID.SemanticCoreID(semanticCoreID) => semanticCoreID.entityId - }.toSet - - val ergSemanticCoreIds = candidate.candidateMagicEventsReasons - .filter(_.source.contains(ReasonSource.ErgShortTermInterestSemanticCore)).map( - _.reason).collect { - case TargetID.SemanticCoreID(semanticCoreID) => semanticCoreID.entityId - }.toSet - - override lazy val ergLocalizedEntities = TopicsUtil - .getLocalizedEntityMap(target, ergSemanticCoreIds, uttEntityHydrationStore) - .map { localizedEntityMap => - ergSemanticCoreIds.collect { - case topicId if localizedEntityMap.contains(topicId) => localizedEntityMap(topicId) - } - } - - val eventSemanticCoreEntityIds: Seq[Long] = { - val entityIds = for { - event <- fanoutEvent - targets <- event.targets - } yield { - targets.flatMap { - _.whitelist.map { - _.collect { - case TargetID.SemanticCoreID(semanticCoreID) => semanticCoreID.entityId - } - } - } - } - - entityIds.map(_.flatten).getOrElse(Seq.empty) - } - - val eventSemanticCoreDomainIds: Seq[Long] = { - val domainIds = for { - event <- fanoutEvent - targets <- event.targets - } yield { - targets.flatMap { - _.whitelist.map { - _.collect { - case TargetID.SemanticCoreID(semanticCoreID) => semanticCoreID.domainId - } - } - } - } - - domainIds.map(_.flatten).getOrElse(Seq.empty) - } - - override lazy val followedTopicLocalizedEntities: Future[Set[LocalizedEntity]] = { - - val isNewSignupTargetingReason = candidateMagicEventsReasons.size == 1 && - candidateMagicEventsReasons.headOption.exists(_.source.contains(ReasonSource.NewSignup)) - - val shouldFetchTopicFollows = reverseIndexedTopicIds.nonEmpty || isNewSignupTargetingReason - - val topicFollows = if (shouldFetchTopicFollows) { - TopicsUtil - .getTopicsFollowedByUser( - candidate.target, - interestsLookupStore, - stats.stat("followed_topics") - ).map { _.getOrElse(Seq.empty) }.map { - _.flatMap { - _.interestId match { - case SemanticCore(semanticCore) => Some(semanticCore.id) - case _ => None - } - } - } - } else Future.Nil - - topicFollows.flatMap { followedTopicIds => - val topicIds = if (isNewSignupTargetingReason) { - // if new signup is the only targeting reason then we check the event targeting reason - // against realtime topic follows. - eventSemanticCoreEntityIds.toSet.intersect(followedTopicIds.toSet) - } else { - // check against the fanout reason of topics - followedTopicIds.toSet.intersect(reverseIndexedTopicIds) - } - - TopicsUtil - .getLocalizedEntityMap(target, topicIds, uttEntityHydrationStore) - .map { localizedEntityMap => - topicIds.collect { - case topicId if localizedEntityMap.contains(topicId) => localizedEntityMap(topicId) - } - } - } - } - - lazy val simClusterToEntityMapping: Map[Int, Seq[Long]] = - simClusterToEntities.flatMap { - case (clusterId, Some(inferredEntities)) => - statsReceiver.counter("with_cluster_to_entity_mapping").incr() - Some( - ( - clusterId, - inferredEntities.entities - .map(_.entityId))) - case _ => - statsReceiver.counter("without_cluster_to_entity_mapping").incr() - None - } - - lazy val annotatedAndInferredSemanticCoreEntities: Seq[Long] = - (simClusterToEntityMapping, eventFanoutReasonEntities) match { - case (entityMapping, eventFanoutReasons) => - entityMapping.values.flatten.toSeq ++ - eventFanoutReasons.semanticCoreIds.map(_.entityId) - } - - lazy val shouldHydrateSquareImage = target.deviceInfo.map { deviceInfo => - (PushDeviceUtil.isPrimaryDeviceIOS(deviceInfo) && - target.params(PushFeatureSwitchParams.EnableEventSquareMediaIosMagicFanoutNewsEvent)) || - (PushDeviceUtil.isPrimaryDeviceAndroid(deviceInfo) && - target.params(PushFeatureSwitchParams.EnableEventSquareMediaAndroid)) - } - - lazy val shouldHydratePrimaryImage: Future[Boolean] = target.deviceInfo.map { deviceInfo => - (PushDeviceUtil.isPrimaryDeviceAndroid(deviceInfo) && - target.params(PushFeatureSwitchParams.EnableEventPrimaryMediaAndroid)) - } - - lazy val eventRequestFut: Future[Option[EventRequest]] = - Future - .join( - target.inferredUserDeviceLanguage, - target.accountCountryCode, - shouldHydrateSquareImage, - shouldHydratePrimaryImage).map { - case ( - inferredUserDeviceLanguage, - accountCountryCode, - shouldHydrateSquareImage, - shouldHydratePrimaryImage) => - if (shouldHydrateSquareImage || shouldHydratePrimaryImage) { - Some( - EventRequest( - eventId, - lookupContext = LookupContext( - hydrationOptions = HydrationOptions( - includeSquareImage = shouldHydrateSquareImage, - includePrimaryImage = shouldHydratePrimaryImage - ), - language = inferredUserDeviceLanguage, - countryCode = accountCountryCode, - userId = Some(UserId(target.targetId)) - ) - )) - } else { - Some( - EventRequest( - eventId, - lookupContext = LookupContext( - language = inferredUserDeviceLanguage, - countryCode = accountCountryCode - ) - )) - } - case _ => None - } - - lazy val isHighPriorityEvent: Future[Boolean] = target.accountCountryCode.map { countryCodeOpt => - val isHighPriorityPushOpt = for { - countryCode <- countryCodeOpt - nfyMetadata <- newsForYouMetadata - eventContext <- nfyMetadata.eventContextScribe - } yield { - val highPriorityLocales = HighPriorityLocaleUtil.getHighPriorityLocales( - eventContext = eventContext, - defaultLocalesOpt = nfyMetadata.locales) - val highPriorityGeos = HighPriorityLocaleUtil.getHighPriorityGeos( - eventContext = eventContext, - defaultGeoPlaceIdsOpt = nfyMetadata.placeIds) - val isHighPriorityLocalePush = - highPriorityLocales.flatMap(_.country).map(CountryId(_)).contains(CountryId(countryCode)) - val isHighPriorityGeoPush = MagicFanoutPredicatesUtil - .geoPlaceIdsFromReasons(candidateMagicEventsReasons) - .intersect(highPriorityGeos.toSet) - .nonEmpty - stats.scope("is_high_priority_locale_push").counter(s"$isHighPriorityLocalePush").incr() - stats.scope("is_high_priority_geo_push").counter(s"$isHighPriorityGeoPush").incr() - isHighPriorityLocalePush || isHighPriorityGeoPush - } - isHighPriorityPushOpt.getOrElse(false) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutHydratedCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutHydratedCandidate.scala deleted file mode 100644 index 36196120b..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutHydratedCandidate.scala +++ /dev/null @@ -1,147 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.escherbird.common.thriftscala.QualifiedId -import com.twitter.escherbird.metadata.thriftscala.BasicMetadata -import com.twitter.escherbird.metadata.thriftscala.EntityIndexFields -import com.twitter.escherbird.metadata.thriftscala.EntityMegadata -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.MagicFanoutCandidate -import com.twitter.frigate.common.base.MagicFanoutEventCandidate -import com.twitter.frigate.common.base.RichEventFutCandidate -import com.twitter.frigate.magic_events.thriftscala -import com.twitter.frigate.magic_events.thriftscala.AnnotationAlg -import com.twitter.frigate.magic_events.thriftscala.FanoutEvent -import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason -import com.twitter.frigate.magic_events.thriftscala.SemanticCoreID -import com.twitter.frigate.magic_events.thriftscala.SimClusterID -import com.twitter.frigate.magic_events.thriftscala.TargetID -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery -import com.twitter.livevideo.timeline.domain.v2.Event -import com.twitter.topiclisting.utt.LocalizedEntity -import com.twitter.util.Future - -case class FanoutReasonEntities( - userIds: Set[Long], - placeIds: Set[Long], - semanticCoreIds: Set[SemanticCoreID], - simclusterIds: Set[SimClusterID]) { - val qualifiedIds: Set[QualifiedId] = - semanticCoreIds.map(e => QualifiedId(e.domainId, e.entityId)) -} - -object FanoutReasonEntities { - val empty = FanoutReasonEntities( - userIds = Set.empty, - placeIds = Set.empty, - semanticCoreIds = Set.empty, - simclusterIds = Set.empty - ) - - def from(reasons: Seq[TargetID]): FanoutReasonEntities = { - val userIds: Set[Long] = reasons.collect { - case TargetID.UserID(userId) => userId.id - }.toSet - val placeIds: Set[Long] = reasons.collect { - case TargetID.PlaceID(placeId) => placeId.id - }.toSet - val semanticCoreIds: Set[SemanticCoreID] = reasons.collect { - case TargetID.SemanticCoreID(semanticCoreID) => semanticCoreID - }.toSet - val simclusterIds: Set[SimClusterID] = reasons.collect { - case TargetID.SimClusterID(simClusterID) => simClusterID - }.toSet - - FanoutReasonEntities( - userIds = userIds, - placeIds, - semanticCoreIds = semanticCoreIds, - simclusterIds = simclusterIds - ) - } -} - -trait MagicFanoutHydratedCandidate extends PushCandidate with MagicFanoutCandidate { - lazy val fanoutReasonEntities: FanoutReasonEntities = - FanoutReasonEntities.from(candidateMagicEventsReasons.map(_.reason)) -} - -trait MagicFanoutEventHydratedCandidate - extends MagicFanoutHydratedCandidate - with MagicFanoutEventCandidate - with RichEventFutCandidate { - - def target: PushTypes.Target - - def stats: StatsReceiver - - def fanoutEvent: Option[FanoutEvent] - - def eventFut: Future[Option[Event]] - - def semanticEntityResults: Map[SemanticEntityForQuery, Option[EntityMegadata]] - - def effectiveMagicEventsReasons: Option[Seq[MagicEventsReason]] - - def followedTopicLocalizedEntities: Future[Set[LocalizedEntity]] - - def ergLocalizedEntities: Future[Set[LocalizedEntity]] - - lazy val entityAnnotationAlg: Map[TargetID, Set[AnnotationAlg]] = - fanoutEvent - .flatMap { metadata => - metadata.eventAnnotationInfo.map { eventAnnotationInfo => - eventAnnotationInfo.map { - case (target, annotationInfoSet) => target -> annotationInfoSet.map(_.alg).toSet - }.toMap - } - }.getOrElse(Map.empty) - - lazy val eventSource: Option[String] = fanoutEvent.map { metadata => - val source = metadata.eventSource.getOrElse("undefined") - stats.scope("eventSource").counter(source).incr() - source - } - - lazy val semanticCoreEntityTags: Map[(Long, Long), Set[String]] = - semanticEntityResults.flatMap { - case (semanticEntityForQuery, entityMegadataOpt: Option[EntityMegadata]) => - for { - entityMegadata <- entityMegadataOpt - basicMetadata: BasicMetadata <- entityMegadata.basicMetadata - indexableFields: EntityIndexFields <- basicMetadata.indexableFields - tags <- indexableFields.tags - } yield { - ((semanticEntityForQuery.domainId, semanticEntityForQuery.entityId), tags.toSet) - } - } - - lazy val owningTwitterUserIds: Seq[Long] = semanticEntityResults.values.flatten - .flatMap { - _.basicMetadata.flatMap(_.twitter.flatMap(_.owningTwitterUserIds)) - }.flatten - .toSeq - .distinct - - lazy val eventFanoutReasonEntities: FanoutReasonEntities = - fanoutEvent match { - case Some(fanout) => - fanout.targets - .map { targets: Seq[thriftscala.Target] => - FanoutReasonEntities.from(targets.flatMap(_.whitelist).flatten) - }.getOrElse(FanoutReasonEntities.empty) - case _ => FanoutReasonEntities.empty - } - - override lazy val eventResultFut: Future[Event] = eventFut.map { - case Some(eventResult) => eventResult - case _ => - throw new IllegalArgumentException("event is None for MagicFanoutEventHydratedCandidate") - } - override val rankScore: Option[Double] = None - override val predictionScore: Option[Double] = None -} - -case class MagicFanoutEventHydratedInfo( - fanoutEvent: Option[FanoutEvent], - semanticEntityResults: Map[SemanticEntityForQuery, Option[EntityMegadata]]) diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutNewsEvent.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutNewsEvent.scala deleted file mode 100644 index 61ead1f22..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutNewsEvent.scala +++ /dev/null @@ -1,99 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.escherbird.metadata.thriftscala.EntityMegadata -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.MagicFanoutNewsEventCandidate -import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext -import com.twitter.frigate.magic_events.thriftscala.FanoutEvent -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.MagicFanoutNewsEventIbis2Hydrator -import com.twitter.frigate.pushservice.model.ntab.MagicFanoutNewsEventNTabRequestHydrator -import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate -import com.twitter.frigate.pushservice.predicate.event.EventPredicatesForCandidate -import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutPredicatesForCandidate -import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutTargetingPredicateWrappersForCandidate -import com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue.MagicFanoutNtabCaretFatiguePredicate -import com.twitter.frigate.pushservice.store.EventRequest -import com.twitter.frigate.pushservice.store.UttEntityHydrationStore -import com.twitter.frigate.pushservice.take.predicates.BasicSendHandlerPredicates -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery -import com.twitter.interests.thriftscala.UserInterests -import com.twitter.livevideo.timeline.domain.v2.Event -import com.twitter.simclusters_v2.thriftscala.SimClustersInferredEntities -import com.twitter.storehaus.ReadableStore - -class MagicFanoutNewsEventPushCandidate( - candidate: RawCandidate with MagicFanoutNewsEventCandidate, - copyIds: CopyIds, - override val fanoutEvent: Option[FanoutEvent], - override val semanticEntityResults: Map[SemanticEntityForQuery, Option[EntityMegadata]], - simClusterToEntities: Map[Int, Option[SimClustersInferredEntities]], - lexServiceStore: ReadableStore[EventRequest, Event], - interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests], - uttEntityHydrationStore: UttEntityHydrationStore -)( - implicit statsScoped: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends MagicFanoutEventPushCandidate( - candidate, - copyIds, - fanoutEvent, - semanticEntityResults, - simClusterToEntities, - lexServiceStore, - interestsLookupStore, - uttEntityHydrationStore - )(statsScoped, pushModelScorer) - with MagicFanoutNewsEventCandidate - with MagicFanoutNewsEventIbis2Hydrator - with MagicFanoutNewsEventNTabRequestHydrator { - - override lazy val stats: StatsReceiver = statsScoped.scope("MagicFanoutNewsEventPushCandidate") - override val statsReceiver: StatsReceiver = statsScoped.scope("MagicFanoutNewsEventPushCandidate") -} - -case class MagicFanoutNewsEventCandidatePredicates(config: Config) - extends BasicSendHandlerPredicates[MagicFanoutNewsEventPushCandidate] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override val preCandidateSpecificPredicates: List[ - NamedPredicate[MagicFanoutNewsEventPushCandidate] - ] = - List( - EventPredicatesForCandidate.accountCountryPredicateWithAllowlist, - PredicatesForCandidate.isDeviceEligibleForNewsOrSports, - MagicFanoutPredicatesForCandidate.inferredUserDeviceLanguagePredicate, - PredicatesForCandidate.secondaryDormantAccountPredicate(statsReceiver), - MagicFanoutPredicatesForCandidate.highPriorityNewsEventExceptedPredicate( - MagicFanoutTargetingPredicateWrappersForCandidate - .magicFanoutTargetingPredicate(statsReceiver, config) - )(config), - MagicFanoutPredicatesForCandidate.geoOptOutPredicate(config.safeUserStore), - EventPredicatesForCandidate.isNotDuplicateWithEventId, - MagicFanoutPredicatesForCandidate.highPriorityNewsEventExceptedPredicate( - MagicFanoutPredicatesForCandidate.newsNotificationFatigue() - )(config), - MagicFanoutPredicatesForCandidate.highPriorityNewsEventExceptedPredicate( - MagicFanoutNtabCaretFatiguePredicate() - )(config), - MagicFanoutPredicatesForCandidate.escherbirdMagicfanoutEventParam()(statsReceiver), - MagicFanoutPredicatesForCandidate.hasCustomTargetingForNewsEventsParam( - statsReceiver - ) - ) - - override val postCandidateSpecificPredicates: List[ - NamedPredicate[MagicFanoutNewsEventPushCandidate] - ] = - List( - MagicFanoutPredicatesForCandidate.magicFanoutNoOptoutInterestPredicate, - MagicFanoutPredicatesForCandidate.geoTargetingHoldback(), - MagicFanoutPredicatesForCandidate.userGeneratedEventsPredicate, - EventPredicatesForCandidate.hasTitle, - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutProductLaunchPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutProductLaunchPushCandidate.scala deleted file mode 100644 index 4dc569e2b..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutProductLaunchPushCandidate.scala +++ /dev/null @@ -1,95 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.MagicFanoutProductLaunchCandidate -import com.twitter.frigate.common.util.{FeatureSwitchParams => FS} -import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason -import com.twitter.frigate.magic_events.thriftscala.ProductType -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutPredicatesUtil -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.MagicFanoutProductLaunchIbis2Hydrator -import com.twitter.frigate.pushservice.model.ntab.MagicFanoutProductLaunchNtabRequestHydrator -import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate -import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutPredicatesForCandidate -import com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue.MagicFanoutNtabCaretFatiguePredicate -import com.twitter.frigate.pushservice.take.predicates.BasicSendHandlerPredicates -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.hermit.predicate.NamedPredicate - -class MagicFanoutProductLaunchPushCandidate( - candidate: RawCandidate with MagicFanoutProductLaunchCandidate, - copyIds: CopyIds -)( - implicit val statsScoped: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with MagicFanoutProductLaunchCandidate - with MagicFanoutProductLaunchIbis2Hydrator - with MagicFanoutProductLaunchNtabRequestHydrator { - - override val frigateNotification: FrigateNotification = candidate.frigateNotification - - override val pushCopyId: Option[Int] = copyIds.pushCopyId - - override val ntabCopyId: Option[Int] = copyIds.ntabCopyId - - override val pushId: Long = candidate.pushId - - override val productLaunchType: ProductType = candidate.productLaunchType - - override val candidateMagicEventsReasons: Seq[MagicEventsReason] = - candidate.candidateMagicEventsReasons - - override val copyAggregationId: Option[String] = copyIds.aggregationId - - override val target: Target = candidate.target - - override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - - override val statsReceiver: StatsReceiver = - statsScoped.scope("MagicFanoutProductLaunchPushCandidate") -} - -case class MagicFanoutProductLaunchPushCandidatePredicates(config: Config) - extends BasicSendHandlerPredicates[MagicFanoutProductLaunchPushCandidate] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override val preCandidateSpecificPredicates: List[ - NamedPredicate[MagicFanoutProductLaunchPushCandidate] - ] = - List( - PredicatesForCandidate.isDeviceEligibleForCreatorPush, - PredicatesForCandidate.exceptedPredicate( - "excepted_is_target_blue_verified", - MagicFanoutPredicatesUtil.shouldSkipBlueVerifiedCheckForCandidate, - PredicatesForCandidate.isTargetBlueVerified.flip - ), // no need to send if target is already Blue Verified - PredicatesForCandidate.exceptedPredicate( - "excepted_is_target_legacy_verified", - MagicFanoutPredicatesUtil.shouldSkipLegacyVerifiedCheckForCandidate, - PredicatesForCandidate.isTargetLegacyVerified.flip - ), // no need to send if target is already Legacy Verified - PredicatesForCandidate.exceptedPredicate( - "excepted_is_target_super_follow_creator", - MagicFanoutPredicatesUtil.shouldSkipSuperFollowCreatorCheckForCandidate, - PredicatesForCandidate.isTargetSuperFollowCreator.flip - ), // no need to send if target is already Super Follow Creator - PredicatesForCandidate.paramPredicate( - FS.EnableMagicFanoutProductLaunch - ), - MagicFanoutPredicatesForCandidate.magicFanoutProductLaunchFatigue(), - ) - - override val postCandidateSpecificPredicates: List[ - NamedPredicate[MagicFanoutProductLaunchPushCandidate] - ] = - List( - MagicFanoutNtabCaretFatiguePredicate(), - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutSportsPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutSportsPushCandidate.scala deleted file mode 100644 index 84535e4c2..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutSportsPushCandidate.scala +++ /dev/null @@ -1,119 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.escherbird.metadata.thriftscala.EntityMegadata -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.BaseGameScore -import com.twitter.frigate.common.base.MagicFanoutSportsEventCandidate -import com.twitter.frigate.common.base.MagicFanoutSportsScoreInformation -import com.twitter.frigate.common.base.TeamInfo -import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext -import com.twitter.frigate.magic_events.thriftscala.FanoutEvent -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.MagicFanoutSportsEventIbis2Hydrator -import com.twitter.frigate.pushservice.model.ntab.MagicFanoutSportsEventNTabRequestHydrator -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate -import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutPredicatesForCandidate -import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutTargetingPredicateWrappersForCandidate -import com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue.MagicFanoutNtabCaretFatiguePredicate -import com.twitter.frigate.pushservice.store.EventRequest -import com.twitter.frigate.pushservice.store.UttEntityHydrationStore -import com.twitter.frigate.pushservice.take.predicates.BasicSendHandlerPredicates -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery -import com.twitter.interests.thriftscala.UserInterests -import com.twitter.livevideo.timeline.domain.v2.Event -import com.twitter.livevideo.timeline.domain.v2.HydrationOptions -import com.twitter.livevideo.timeline.domain.v2.LookupContext -import com.twitter.simclusters_v2.thriftscala.SimClustersInferredEntities -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -class MagicFanoutSportsPushCandidate( - candidate: RawCandidate - with MagicFanoutSportsEventCandidate - with MagicFanoutSportsScoreInformation, - copyIds: CopyIds, - override val fanoutEvent: Option[FanoutEvent], - override val semanticEntityResults: Map[SemanticEntityForQuery, Option[EntityMegadata]], - simClusterToEntities: Map[Int, Option[SimClustersInferredEntities]], - lexServiceStore: ReadableStore[EventRequest, Event], - interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests], - uttEntityHydrationStore: UttEntityHydrationStore -)( - implicit statsScoped: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends MagicFanoutEventPushCandidate( - candidate, - copyIds, - fanoutEvent, - semanticEntityResults, - simClusterToEntities, - lexServiceStore, - interestsLookupStore, - uttEntityHydrationStore)(statsScoped, pushModelScorer) - with MagicFanoutSportsEventCandidate - with MagicFanoutSportsScoreInformation - with MagicFanoutSportsEventNTabRequestHydrator - with MagicFanoutSportsEventIbis2Hydrator { - - override val isScoreUpdate: Boolean = candidate.isScoreUpdate - override val gameScores: Future[Option[BaseGameScore]] = candidate.gameScores - override val homeTeamInfo: Future[Option[TeamInfo]] = candidate.homeTeamInfo - override val awayTeamInfo: Future[Option[TeamInfo]] = candidate.awayTeamInfo - - override lazy val stats: StatsReceiver = statsScoped.scope("MagicFanoutSportsPushCandidate") - override val statsReceiver: StatsReceiver = statsScoped.scope("MagicFanoutSportsPushCandidate") - - override lazy val eventRequestFut: Future[Option[EventRequest]] = { - Future.join(target.inferredUserDeviceLanguage, target.accountCountryCode).map { - case (inferredUserDeviceLanguage, accountCountryCode) => - Some( - EventRequest( - eventId, - lookupContext = LookupContext( - hydrationOptions = HydrationOptions( - includeSquareImage = true, - includePrimaryImage = true - ), - language = inferredUserDeviceLanguage, - countryCode = accountCountryCode - ) - )) - } - } -} - -case class MagicFanoutSportsEventCandidatePredicates(config: Config) - extends BasicSendHandlerPredicates[MagicFanoutSportsPushCandidate] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override val preCandidateSpecificPredicates: List[ - NamedPredicate[MagicFanoutSportsPushCandidate] - ] = - List( - PredicatesForCandidate.paramPredicate(PushFeatureSwitchParams.EnableScoreFanoutNotification) - ) - - override val postCandidateSpecificPredicates: List[ - NamedPredicate[MagicFanoutSportsPushCandidate] - ] = - List( - PredicatesForCandidate.isDeviceEligibleForNewsOrSports, - MagicFanoutPredicatesForCandidate.inferredUserDeviceLanguagePredicate, - MagicFanoutPredicatesForCandidate.highPriorityEventExceptedPredicate( - MagicFanoutTargetingPredicateWrappersForCandidate - .magicFanoutTargetingPredicate(statsReceiver, config) - )(config), - PredicatesForCandidate.secondaryDormantAccountPredicate( - statsReceiver - ), - MagicFanoutPredicatesForCandidate.highPriorityEventExceptedPredicate( - MagicFanoutNtabCaretFatiguePredicate() - )(config), - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/OutOfNetworkTweetPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/OutOfNetworkTweetPushCandidate.scala deleted file mode 100644 index 0b8c533ea..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/OutOfNetworkTweetPushCandidate.scala +++ /dev/null @@ -1,68 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.contentrecommender.thriftscala.MetricTag -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.OutOfNetworkTweetCandidate -import com.twitter.frigate.common.base.TopicCandidate -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.OutOfNetworkTweetIbis2HydratorForCandidate -import com.twitter.frigate.pushservice.model.ntab.OutOfNetworkTweetNTabRequestHydrator -import com.twitter.frigate.pushservice.predicate.HealthPredicates -import com.twitter.frigate.pushservice.take.predicates.OutOfNetworkTweetPredicates -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.stitch.tweetypie.TweetyPie -import com.twitter.topiclisting.utt.LocalizedEntity -import com.twitter.util.Future - -class OutOfNetworkTweetPushCandidate( - candidate: RawCandidate with OutOfNetworkTweetCandidate with TopicCandidate, - author: Future[Option[User]], - copyIds: CopyIds -)( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with OutOfNetworkTweetCandidate - with TopicCandidate - with TweetAuthorDetails - with OutOfNetworkTweetNTabRequestHydrator - with OutOfNetworkTweetIbis2HydratorForCandidate { - override val statsReceiver: StatsReceiver = stats - override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - override val tweetId: Long = candidate.tweetId - override lazy val tweetyPieResult: Option[TweetyPie.TweetyPieResult] = - candidate.tweetyPieResult - override lazy val tweetAuthor: Future[Option[User]] = author - override val target: PushTypes.Target = candidate.target - override lazy val commonRecType: CommonRecommendationType = - candidate.commonRecType - override val pushCopyId: Option[Int] = copyIds.pushCopyId - override val ntabCopyId: Option[Int] = copyIds.ntabCopyId - override val copyAggregationId: Option[String] = copyIds.aggregationId - override lazy val semanticCoreEntityId: Option[Long] = candidate.semanticCoreEntityId - override lazy val localizedUttEntity: Option[LocalizedEntity] = candidate.localizedUttEntity - override lazy val algorithmCR: Option[String] = candidate.algorithmCR - override lazy val isMrBackfillCR: Option[Boolean] = candidate.isMrBackfillCR - override lazy val tagsCR: Option[Seq[MetricTag]] = candidate.tagsCR -} - -case class OutOfNetworkTweetCandidatePredicates(override val config: Config) - extends OutOfNetworkTweetPredicates[OutOfNetworkTweetPushCandidate] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override def postCandidateSpecificPredicates: List[ - NamedPredicate[OutOfNetworkTweetPushCandidate] - ] = - List( - HealthPredicates.agathaAbusiveTweetAuthorPredicateMrTwistly(), - ) - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/PushTypes.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/PushTypes.scala deleted file mode 100644 index 83d5b67c3..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/PushTypes.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.candidate.UserLanguage -import com.twitter.frigate.common.candidate._ -import com.twitter.frigate.data_pipeline.features_common.RequestContextForFeatureStore -import com.twitter.frigate.pushservice.model.candidate.CopyInfo -import com.twitter.frigate.pushservice.model.candidate.MLScores -import com.twitter.frigate.pushservice.model.candidate.QualityScribing -import com.twitter.frigate.pushservice.model.candidate.Scriber -import com.twitter.frigate.pushservice.model.ibis.Ibis2HydratorForCandidate -import com.twitter.frigate.pushservice.model.ntab.NTabRequest -import com.twitter.frigate.pushservice.take.ChannelForCandidate -import com.twitter.frigate.pushservice.target._ -import com.twitter.util.Time - -object PushTypes { - - trait Target - extends TargetUser - with UserDetails - with TargetWithPushContext - with TargetDecider - with TargetABDecider - with FrigateHistory - with PushTargeting - with TargetScoringDetails - with TweetImpressionHistory - with CustomConfigForExpt - with CaretFeedbackHistory - with NotificationFeedbackHistory - with PromptFeedbackHistory - with HTLVisitHistory - with MaxTweetAge - with NewUserDetails - with ResurrectedUserDetails - with TargetWithSeedUsers - with MagicFanoutHistory - with OptOutUserInterests - with RequestContextForFeatureStore - with TargetAppPermissions - with UserLanguage - with InlineActionHistory - with TargetPlaces - - trait RawCandidate extends Candidate with TargetInfo[PushTypes.Target] with RecommendationType { - - val createdAt: Time = Time.now - } - - trait PushCandidate - extends RawCandidate - with CandidateScoringDetails - with MLScores - with QualityScribing - with CopyInfo - with Scriber - with Ibis2HydratorForCandidate - with NTabRequest - with ChannelForCandidate -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ScheduledSpaceSpeaker.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ScheduledSpaceSpeaker.scala deleted file mode 100644 index 6da10ed77..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ScheduledSpaceSpeaker.scala +++ /dev/null @@ -1,85 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.ScheduledSpaceSpeakerCandidate -import com.twitter.frigate.common.base.SpaceCandidateFanoutDetails -import com.twitter.frigate.common.util.FeatureSwitchParams -import com.twitter.frigate.magic_events.thriftscala.SpaceMetadata -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.ScheduledSpaceSpeakerIbis2Hydrator -import com.twitter.frigate.pushservice.model.ntab.ScheduledSpaceSpeakerNTabRequestHydrator -import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate -import com.twitter.frigate.pushservice.predicate.SpacePredicate -import com.twitter.frigate.pushservice.take.predicates.BasicSendHandlerPredicates -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.storehaus.ReadableStore -import com.twitter.ubs.thriftscala.AudioSpace -import com.twitter.util.Future - -class ScheduledSpaceSpeakerPushCandidate( - candidate: RawCandidate with ScheduledSpaceSpeakerCandidate, - hostUser: Option[User], - copyIds: CopyIds, - audioSpaceStore: ReadableStore[String, AudioSpace] -)( - implicit val statsScoped: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with ScheduledSpaceSpeakerCandidate - with ScheduledSpaceSpeakerIbis2Hydrator - with SpaceCandidateFanoutDetails - with ScheduledSpaceSpeakerNTabRequestHydrator { - - override val startTime: Long = candidate.startTime - - override val hydratedHost: Option[User] = hostUser - - override val spaceId: String = candidate.spaceId - - override val hostId: Option[Long] = candidate.hostId - - override val speakerIds: Option[Seq[Long]] = candidate.speakerIds - - override val listenerIds: Option[Seq[Long]] = candidate.listenerIds - - override val frigateNotification: FrigateNotification = candidate.frigateNotification - - override val pushCopyId: Option[Int] = copyIds.pushCopyId - - override val ntabCopyId: Option[Int] = copyIds.ntabCopyId - - override val copyAggregationId: Option[String] = copyIds.aggregationId - - override val target: Target = candidate.target - - override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - - override lazy val audioSpaceFut: Future[Option[AudioSpace]] = audioSpaceStore.get(spaceId) - - override val spaceFanoutMetadata: Option[SpaceMetadata] = None - - override val statsReceiver: StatsReceiver = - statsScoped.scope("ScheduledSpaceSpeakerCandidate") -} - -case class ScheduledSpaceSpeakerCandidatePredicates(config: Config) - extends BasicSendHandlerPredicates[ScheduledSpaceSpeakerPushCandidate] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override val preCandidateSpecificPredicates: List[ - NamedPredicate[ScheduledSpaceSpeakerPushCandidate] - ] = List( - SpacePredicate.scheduledSpaceStarted( - config.audioSpaceStore - ), - PredicatesForCandidate.paramPredicate(FeatureSwitchParams.EnableScheduledSpaceSpeakers) - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ScheduledSpaceSubscriber.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ScheduledSpaceSubscriber.scala deleted file mode 100644 index 78977ab5d..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ScheduledSpaceSubscriber.scala +++ /dev/null @@ -1,86 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.ScheduledSpaceSubscriberCandidate -import com.twitter.frigate.common.base.SpaceCandidateFanoutDetails -import com.twitter.frigate.common.util.FeatureSwitchParams -import com.twitter.frigate.magic_events.thriftscala.SpaceMetadata -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.ScheduledSpaceSubscriberIbis2Hydrator -import com.twitter.frigate.pushservice.model.ntab.ScheduledSpaceSubscriberNTabRequestHydrator -import com.twitter.frigate.pushservice.predicate._ -import com.twitter.frigate.pushservice.take.predicates.BasicSendHandlerPredicates -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.storehaus.ReadableStore -import com.twitter.ubs.thriftscala.AudioSpace -import com.twitter.util.Future - -class ScheduledSpaceSubscriberPushCandidate( - candidate: RawCandidate with ScheduledSpaceSubscriberCandidate, - hostUser: Option[User], - copyIds: CopyIds, - audioSpaceStore: ReadableStore[String, AudioSpace] -)( - implicit val statsScoped: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with ScheduledSpaceSubscriberCandidate - with SpaceCandidateFanoutDetails - with ScheduledSpaceSubscriberIbis2Hydrator - with ScheduledSpaceSubscriberNTabRequestHydrator { - - override val startTime: Long = candidate.startTime - - override val hydratedHost: Option[User] = hostUser - - override val spaceId: String = candidate.spaceId - - override val hostId: Option[Long] = candidate.hostId - - override val speakerIds: Option[Seq[Long]] = candidate.speakerIds - - override val listenerIds: Option[Seq[Long]] = candidate.listenerIds - - override val frigateNotification: FrigateNotification = candidate.frigateNotification - - override val pushCopyId: Option[Int] = copyIds.pushCopyId - - override val ntabCopyId: Option[Int] = copyIds.ntabCopyId - - override val copyAggregationId: Option[String] = copyIds.aggregationId - - override val target: Target = candidate.target - - override lazy val audioSpaceFut: Future[Option[AudioSpace]] = audioSpaceStore.get(spaceId) - - override val spaceFanoutMetadata: Option[SpaceMetadata] = None - - override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - - override val statsReceiver: StatsReceiver = - statsScoped.scope("ScheduledSpaceSubscriberCandidate") -} - -case class ScheduledSpaceSubscriberCandidatePredicates(config: Config) - extends BasicSendHandlerPredicates[ScheduledSpaceSubscriberPushCandidate] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override val preCandidateSpecificPredicates: List[ - NamedPredicate[ScheduledSpaceSubscriberPushCandidate] - ] = - List( - PredicatesForCandidate.paramPredicate(FeatureSwitchParams.EnableScheduledSpaceSubscribers), - SpacePredicate.narrowCastSpace, - SpacePredicate.targetInSpace(config.audioSpaceParticipantsStore), - SpacePredicate.spaceHostTargetUserBlocking(config.edgeStore), - PredicatesForCandidate.duplicateSpacesPredicate - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/SubscribedSearchTweetPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/SubscribedSearchTweetPushCandidate.scala deleted file mode 100644 index 4d71a0c75..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/SubscribedSearchTweetPushCandidate.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.SubscribedSearchTweetCandidate -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.SubscribedSearchTweetIbis2Hydrator -import com.twitter.frigate.pushservice.model.ntab.SubscribedSearchTweetNtabRequestHydrator -import com.twitter.frigate.pushservice.take.predicates.BasicTweetPredicatesForRFPH -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.stitch.tweetypie.TweetyPie -import com.twitter.util.Future - -class SubscribedSearchTweetPushCandidate( - candidate: RawCandidate with SubscribedSearchTweetCandidate, - author: Option[User], - copyIds: CopyIds -)( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with SubscribedSearchTweetCandidate - with TweetAuthorDetails - with SubscribedSearchTweetIbis2Hydrator - with SubscribedSearchTweetNtabRequestHydrator { - override def tweetAuthor: Future[Option[User]] = Future.value(author) - - override def weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - - override def tweetId: Long = candidate.tweetId - - override def pushCopyId: Option[Int] = copyIds.pushCopyId - - override def ntabCopyId: Option[Int] = copyIds.ntabCopyId - - override def copyAggregationId: Option[String] = copyIds.aggregationId - - override def target: PushTypes.Target = candidate.target - - override def searchTerm: String = candidate.searchTerm - - override def timeBoundedLandingUrl: Option[String] = None - - override def statsReceiver: StatsReceiver = stats - - override def tweetyPieResult: Option[TweetyPie.TweetyPieResult] = candidate.tweetyPieResult -} - -case class SubscribedSearchTweetCandidatePredicates(override val config: Config) - extends BasicTweetPredicatesForRFPH[SubscribedSearchTweetPushCandidate] { - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TopTweetImpressionsPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TopTweetImpressionsPushCandidate.scala deleted file mode 100644 index b04a16ac3..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TopTweetImpressionsPushCandidate.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.TopTweetImpressionsCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.TopTweetImpressionsCandidateIbis2Hydrator -import com.twitter.frigate.pushservice.model.ntab.TopTweetImpressionsNTabRequestHydrator -import com.twitter.frigate.pushservice.predicate.TopTweetImpressionsPredicates -import com.twitter.frigate.pushservice.take.predicates.BasicTweetPredicatesForRFPH -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.notificationservice.thriftscala.StoryContext -import com.twitter.notificationservice.thriftscala.StoryContextValue -import com.twitter.stitch.tweetypie.TweetyPie - -/** - * This class defines a hydrated [[TopTweetImpressionsCandidate]] - * - * @param candidate: [[TopTweetImpressionsCandidate]] for the candidate representing the user's Tweet with the most impressions - * @param copyIds: push and ntab notification copy - * @param stats: finagle scoped states receiver - * @param pushModelScorer: ML model score object for fetching prediction scores - */ -class TopTweetImpressionsPushCandidate( - candidate: RawCandidate with TopTweetImpressionsCandidate, - copyIds: CopyIds -)( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with TopTweetImpressionsCandidate - with TopTweetImpressionsNTabRequestHydrator - with TopTweetImpressionsCandidateIbis2Hydrator { - override val target: PushTypes.Target = candidate.target - override val commonRecType: CommonRecommendationType = candidate.commonRecType - override val tweetId: Long = candidate.tweetId - override lazy val tweetyPieResult: Option[TweetyPie.TweetyPieResult] = - candidate.tweetyPieResult - override val impressionsCount: Long = candidate.impressionsCount - - override val statsReceiver: StatsReceiver = stats.scope(getClass.getSimpleName) - override val pushCopyId: Option[Int] = copyIds.pushCopyId - override val ntabCopyId: Option[Int] = copyIds.ntabCopyId - override val copyAggregationId: Option[String] = copyIds.aggregationId - override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - override val storyContext: Option[StoryContext] = - Some(StoryContext(altText = "", value = Some(StoryContextValue.Tweets(Seq(tweetId))))) -} - -case class TopTweetImpressionsPushCandidatePredicates(config: Config) - extends BasicTweetPredicatesForRFPH[TopTweetImpressionsPushCandidate] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override val preCandidateSpecificPredicates: List[ - NamedPredicate[TopTweetImpressionsPushCandidate] - ] = List( - TopTweetImpressionsPredicates.topTweetImpressionsFatiguePredicate - ) - - override val postCandidateSpecificPredicates: List[ - NamedPredicate[TopTweetImpressionsPushCandidate] - ] = List( - TopTweetImpressionsPredicates.topTweetImpressionsThreshold() - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TopicProofTweetPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TopicProofTweetPushCandidate.scala deleted file mode 100644 index f89eb28bf..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TopicProofTweetPushCandidate.scala +++ /dev/null @@ -1,71 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.TopicProofTweetCandidate -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.TopicProofTweetIbis2Hydrator -import com.twitter.frigate.pushservice.model.ntab.TopicProofTweetNtabRequestHydrator -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate -import com.twitter.frigate.pushservice.take.predicates.BasicTweetPredicatesForRFPH -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.stitch.tweetypie.TweetyPie -import com.twitter.util.Future - -/** - * This class defines a hydrated [[TopicProofTweetCandidate]] - * - * @param candidate : [[TopicProofTweetCandidate]] for the candidate representint a Tweet recommendation for followed Topic - * @param author : Tweet author representated as Gizmoduck user object - * @param copyIds : push and ntab notification copy - * @param stats : finagle scoped states receiver - * @param pushModelScorer : ML model score object for fetching prediction scores - */ -class TopicProofTweetPushCandidate( - candidate: RawCandidate with TopicProofTweetCandidate, - author: Option[User], - copyIds: CopyIds -)( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with TopicProofTweetCandidate - with TweetAuthorDetails - with TopicProofTweetNtabRequestHydrator - with TopicProofTweetIbis2Hydrator { - override val statsReceiver: StatsReceiver = stats - override val target: PushTypes.Target = candidate.target - override val tweetId: Long = candidate.tweetId - override lazy val tweetyPieResult: Option[TweetyPie.TweetyPieResult] = candidate.tweetyPieResult - override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - override val pushCopyId: Option[Int] = copyIds.pushCopyId - override val ntabCopyId: Option[Int] = copyIds.ntabCopyId - override val copyAggregationId: Option[String] = copyIds.aggregationId - override val semanticCoreEntityId = candidate.semanticCoreEntityId - override val localizedUttEntity = candidate.localizedUttEntity - override val tweetAuthor = Future.value(author) - override val topicListingSetting = candidate.topicListingSetting - override val algorithmCR = candidate.algorithmCR - override val commonRecType: CommonRecommendationType = candidate.commonRecType - override val tagsCR = candidate.tagsCR - override val isOutOfNetwork = candidate.isOutOfNetwork -} - -case class TopicProofTweetCandidatePredicates(override val config: Config) - extends BasicTweetPredicatesForRFPH[TopicProofTweetPushCandidate] { - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override val preCandidateSpecificPredicates: List[NamedPredicate[TopicProofTweetPushCandidate]] = - List( - PredicatesForCandidate.paramPredicate( - PushFeatureSwitchParams.EnableTopicProofTweetRecs - ), - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TrendTweetPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TrendTweetPushCandidate.scala deleted file mode 100644 index ec580f629..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TrendTweetPushCandidate.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.events.recos.thriftscala.TrendsContext -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.TrendTweetCandidate -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.TrendTweetIbis2Hydrator -import com.twitter.frigate.pushservice.model.ntab.TrendTweetNtabHydrator -import com.twitter.frigate.pushservice.take.predicates.BasicTweetPredicatesForRFPH -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.stitch.tweetypie.TweetyPie -import com.twitter.util.Future - -class TrendTweetPushCandidate( - candidate: RawCandidate with TrendTweetCandidate, - author: Option[User], - copyIds: CopyIds -)( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with TrendTweetCandidate - with TweetAuthorDetails - with TrendTweetIbis2Hydrator - with TrendTweetNtabHydrator { - override val statsReceiver: StatsReceiver = stats - override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - override val tweetId: Long = candidate.tweetId - override lazy val tweetyPieResult: Option[TweetyPie.TweetyPieResult] = candidate.tweetyPieResult - override lazy val tweetAuthor: Future[Option[User]] = Future.value(author) - override val target: PushTypes.Target = candidate.target - override val landingUrl: String = candidate.landingUrl - override val timeBoundedLandingUrl: Option[String] = candidate.timeBoundedLandingUrl - override val pushCopyId: Option[Int] = copyIds.pushCopyId - override val ntabCopyId: Option[Int] = copyIds.ntabCopyId - override val trendId: String = candidate.trendId - override val trendName: String = candidate.trendName - override val copyAggregationId: Option[String] = copyIds.aggregationId - override val context: TrendsContext = candidate.context -} - -case class TrendTweetPredicates(override val config: Config) - extends BasicTweetPredicatesForRFPH[TrendTweetPushCandidate] { - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TripTweetPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TripTweetPushCandidate.scala deleted file mode 100644 index 1981e7bb5..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TripTweetPushCandidate.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.OutOfNetworkTweetCandidate -import com.twitter.frigate.common.base.TopicCandidate -import com.twitter.frigate.common.base.TripCandidate -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.OutOfNetworkTweetIbis2HydratorForCandidate -import com.twitter.frigate.pushservice.model.ntab.OutOfNetworkTweetNTabRequestHydrator -import com.twitter.frigate.pushservice.take.predicates.OutOfNetworkTweetPredicates -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.stitch.tweetypie.TweetyPie -import com.twitter.topiclisting.utt.LocalizedEntity -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain -import com.twitter.util.Future - -class TripTweetPushCandidate( - candidate: RawCandidate with OutOfNetworkTweetCandidate with TripCandidate, - author: Future[Option[User]], - copyIds: CopyIds -)( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with TripCandidate - with TopicCandidate - with OutOfNetworkTweetCandidate - with TweetAuthorDetails - with OutOfNetworkTweetNTabRequestHydrator - with OutOfNetworkTweetIbis2HydratorForCandidate { - override val statsReceiver: StatsReceiver = stats - override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - override val tweetId: Long = candidate.tweetId - override lazy val tweetyPieResult: Option[TweetyPie.TweetyPieResult] = - candidate.tweetyPieResult - override lazy val tweetAuthor: Future[Option[User]] = author - override val target: PushTypes.Target = candidate.target - override lazy val commonRecType: CommonRecommendationType = - candidate.commonRecType - override val pushCopyId: Option[Int] = copyIds.pushCopyId - override val ntabCopyId: Option[Int] = copyIds.ntabCopyId - override val copyAggregationId: Option[String] = copyIds.aggregationId - override lazy val semanticCoreEntityId: Option[Long] = None - override lazy val localizedUttEntity: Option[LocalizedEntity] = None - override lazy val algorithmCR: Option[String] = None - override val tripDomain: Option[collection.Set[TripDomain]] = candidate.tripDomain -} - -case class TripTweetCandidatePredicates(override val config: Config) - extends OutOfNetworkTweetPredicates[TripTweetPushCandidate] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetAction.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetAction.scala deleted file mode 100644 index 72453224d..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetAction.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.SocialContextActions -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.common.base.TweetDetails -import com.twitter.frigate.pushservice.model.PushTypes._ -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.predicate._ -import com.twitter.frigate.pushservice.take.predicates.BasicTweetPredicatesForRFPH - -case class TweetActionCandidatePredicates(override val config: Config) - extends BasicTweetPredicatesForRFPH[ - PushCandidate with TweetCandidate with TweetDetails with SocialContextActions - ] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override val preCandidateSpecificPredicates = List(PredicatesForCandidate.minSocialContext(1)) - - override val postCandidateSpecificPredicates = List( - PredicatesForCandidate.socialContextBeingFollowed(config.edgeStore), - PredicatesForCandidate.socialContextBlockingOrMuting(config.edgeStore), - PredicatesForCandidate.socialContextNotRetweetFollowing(config.edgeStore) - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetFavorite.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetFavorite.scala deleted file mode 100644 index ae31ddc6c..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetFavorite.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.SocialContextAction -import com.twitter.frigate.common.base.SocialContextUserDetails -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.base.TweetFavoriteCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.TweetFavoriteCandidateIbis2Hydrator -import com.twitter.frigate.pushservice.model.ntab.TweetFavoriteNTabRequestHydrator -import com.twitter.frigate.pushservice.util.CandidateHydrationUtil.TweetWithSocialContextTraits -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.stitch.tweetypie.TweetyPie -import com.twitter.util.Future - -class TweetFavoritePushCandidate( - candidate: RawCandidate with TweetWithSocialContextTraits, - socialContextUserMap: Future[Map[Long, Option[User]]], - author: Future[Option[User]], - copyIds: CopyIds -)( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with TweetFavoriteCandidate - with SocialContextUserDetails - with TweetAuthorDetails - with TweetFavoriteNTabRequestHydrator - with TweetFavoriteCandidateIbis2Hydrator { - override val statsReceiver: StatsReceiver = stats - override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - override val tweetId: Long = candidate.tweetId - override val socialContextActions: Seq[SocialContextAction] = - candidate.socialContextActions - - override val socialContextAllTypeActions: Seq[SocialContextAction] = - candidate.socialContextAllTypeActions - - override lazy val scUserMap: Future[Map[Long, Option[User]]] = socialContextUserMap - override lazy val tweetAuthor: Future[Option[User]] = author - override lazy val commonRecType: CommonRecommendationType = - candidate.commonRecType - override val target: PushTypes.Target = candidate.target - override lazy val tweetyPieResult: Option[TweetyPie.TweetyPieResult] = - candidate.tweetyPieResult - override val pushCopyId: Option[Int] = copyIds.pushCopyId - override val ntabCopyId: Option[Int] = copyIds.ntabCopyId - override val copyAggregationId: Option[String] = copyIds.aggregationId -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetRetweet.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetRetweet.scala deleted file mode 100644 index 61c8c6526..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetRetweet.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.SocialContextAction -import com.twitter.frigate.common.base.SocialContextUserDetails -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.base.TweetRetweetCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.TweetRetweetCandidateIbis2Hydrator -import com.twitter.frigate.pushservice.model.ntab.TweetRetweetNTabRequestHydrator -import com.twitter.frigate.pushservice.util.CandidateHydrationUtil.TweetWithSocialContextTraits -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.stitch.tweetypie.TweetyPie -import com.twitter.util.Future - -class TweetRetweetPushCandidate( - candidate: RawCandidate with TweetWithSocialContextTraits, - socialContextUserMap: Future[Map[Long, Option[User]]], - author: Future[Option[User]], - copyIds: CopyIds -)( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with TweetRetweetCandidate - with SocialContextUserDetails - with TweetAuthorDetails - with TweetRetweetNTabRequestHydrator - with TweetRetweetCandidateIbis2Hydrator { - override val statsReceiver: StatsReceiver = stats - override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - override val tweetId: Long = candidate.tweetId - override val socialContextActions: Seq[SocialContextAction] = - candidate.socialContextActions - - override val socialContextAllTypeActions: Seq[SocialContextAction] = - candidate.socialContextAllTypeActions - - override lazy val scUserMap: Future[Map[Long, Option[User]]] = socialContextUserMap - override lazy val tweetAuthor: Future[Option[User]] = author - override lazy val commonRecType: CommonRecommendationType = candidate.commonRecType - override val target: PushTypes.Target = candidate.target - override lazy val tweetyPieResult: Option[TweetyPie.TweetyPieResult] = candidate.tweetyPieResult - override val pushCopyId: Option[Int] = copyIds.pushCopyId - override val ntabCopyId: Option[Int] = copyIds.ntabCopyId - override val copyAggregationId: Option[String] = copyIds.aggregationId -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/CopyInfo.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/CopyInfo.scala deleted file mode 100644 index 11cc0617a..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/CopyInfo.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.frigate.pushservice.model.candidate - -import com.twitter.frigate.common.util.MRPushCopy -import com.twitter.frigate.common.util.MrPushCopyObjects -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.util.CandidateUtil - -case class CopyIds( - pushCopyId: Option[Int] = None, - ntabCopyId: Option[Int] = None, - aggregationId: Option[String] = None) - -trait CopyInfo { - self: PushCandidate => - - import com.twitter.frigate.data_pipeline.common.FrigateNotificationUtil._ - - def getPushCopy: Option[MRPushCopy] = - pushCopyId match { - case Some(pushCopyId) => MrPushCopyObjects.getCopyFromId(pushCopyId) - case _ => - crt2PushCopy( - commonRecType, - CandidateUtil.getSocialContextActionsFromCandidate(self).size - ) - } - - def pushCopyId: Option[Int] - - def ntabCopyId: Option[Int] - - def copyAggregationId: Option[String] -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/MLScores.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/MLScores.scala deleted file mode 100644 index 4ba79f485..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/MLScores.scala +++ /dev/null @@ -1,307 +0,0 @@ -package com.twitter.frigate.pushservice.model.candidate - -import com.twitter.frigate.common.base.FeatureMap -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.ml.HydrationContextBuilder -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushMLModel -import com.twitter.frigate.pushservice.params.WeightedOpenOrNtabClickModel -import com.twitter.nrel.hydration.push.HydrationContext -import com.twitter.timelines.configapi.FSParam -import com.twitter.util.Future -import java.util.concurrent.ConcurrentHashMap -import scala.collection.concurrent.{Map => CMap} -import scala.collection.convert.decorateAsScala._ - -trait MLScores { - - self: PushCandidate => - - lazy val candidateHydrationContext: Future[HydrationContext] = HydrationContextBuilder.build(self) - - def weightedOpenOrNtabClickModelScorer: PushMLModelScorer - - // Used to store the scores and avoid duplicate prediction - private val qualityModelScores: CMap[ - (PushMLModel.Value, WeightedOpenOrNtabClickModel.ModelNameType), - Future[Option[Double]] - ] = - new ConcurrentHashMap[(PushMLModel.Value, WeightedOpenOrNtabClickModel.ModelNameType), Future[ - Option[Double] - ]]().asScala - - def populateQualityModelScore( - pushMLModel: PushMLModel.Value, - modelVersion: WeightedOpenOrNtabClickModel.ModelNameType, - prob: Future[Option[Double]] - ) = { - val modelAndVersion = (pushMLModel, modelVersion) - if (!qualityModelScores.contains(modelAndVersion)) { - qualityModelScores += modelAndVersion -> prob - } - } - - // The ML scores that also depend on other candidates and are only available after all candidates are processed - // For example, the likelihood info for Importance Sampling - private lazy val crossCandidateMlScores: CMap[String, Double] = - new ConcurrentHashMap[String, Double]().asScala - - def populateCrossCandidateMlScores(scoreName: String, score: Double): Unit = { - if (crossCandidateMlScores.contains(scoreName)) { - throw new Exception( - s"$scoreName has been populated in the CrossCandidateMlScores!\n" + - s"Existing crossCandidateMlScores are ${crossCandidateMlScores}\n" - ) - } - crossCandidateMlScores += scoreName -> score - } - - def getMLModelScore( - pushMLModel: PushMLModel.Value, - modelVersion: WeightedOpenOrNtabClickModel.ModelNameType - ): Future[Option[Double]] = { - qualityModelScores.getOrElseUpdate( - (pushMLModel, modelVersion), - weightedOpenOrNtabClickModelScorer - .singlePredicationForModelVersion(modelVersion, self, Some(pushMLModel)) - ) - } - - def getMLModelScoreWithoutUpdate( - pushMLModel: PushMLModel.Value, - modelVersion: WeightedOpenOrNtabClickModel.ModelNameType - ): Future[Option[Double]] = { - qualityModelScores.getOrElse( - (pushMLModel, modelVersion), - Future.None - ) - } - - def getWeightedOpenOrNtabClickModelScore( - weightedOONCModelParam: FSParam[WeightedOpenOrNtabClickModel.ModelNameType] - ): Future[Option[Double]] = { - getMLModelScore( - PushMLModel.WeightedOpenOrNtabClickProbability, - target.params(weightedOONCModelParam) - ) - } - - /* After we unify the ranking and filtering models, we follow the iteration process below - When improving the WeightedOONC model, - 1) Run experiment which only replace the ranking model - 2) Make decisions according to the experiment results - 3) Use the ranking model for filtering - 4) Adjust percentile thresholds if necessary - */ - lazy val mrWeightedOpenOrNtabClickRankingProbability: Future[Option[Double]] = - target.rankingModelParam.flatMap { modelParam => - getWeightedOpenOrNtabClickModelScore(modelParam) - } - - def getBigFilteringScore( - pushMLModel: PushMLModel.Value, - modelVersion: WeightedOpenOrNtabClickModel.ModelNameType - ): Future[Option[Double]] = { - mrWeightedOpenOrNtabClickRankingProbability.flatMap { - case Some(rankingScore) => - // Adds ranking score to feature map (we must ensure the feature key is also in the feature context) - mergeFeatures( - FeatureMap( - numericFeatures = Map("scribe.WeightedOpenOrNtabClickProbability" -> rankingScore) - ) - ) - getMLModelScore(pushMLModel, modelVersion) - case _ => Future.None - } - } - - def getWeightedOpenOrNtabClickScoreForScribing(): Seq[Future[Map[String, Double]]] = { - Seq( - mrWeightedOpenOrNtabClickRankingProbability.map { - case Some(score) => Map(PushMLModel.WeightedOpenOrNtabClickProbability.toString -> score) - case _ => Map.empty[String, Double] - }, - Future - .join( - target.rankingModelParam, - mrWeightedOpenOrNtabClickRankingProbability - ).map { - case (rankingModelParam, Some(score)) => - Map(target.params(rankingModelParam).toString -> score) - case _ => Map.empty[String, Double] - } - ) - } - - def getNsfwScoreForScribing(): Seq[Future[Map[String, Double]]] = { - val nsfwScoreFut = getMLModelScoreWithoutUpdate( - PushMLModel.HealthNsfwProbability, - target.params(PushFeatureSwitchParams.BqmlHealthModelTypeParam)) - Seq(nsfwScoreFut.map { nsfwScoreOpt => - nsfwScoreOpt - .map(nsfwScore => Map(PushMLModel.HealthNsfwProbability.toString -> nsfwScore)).getOrElse( - Map.empty[String, Double]) - }) - } - - def getBigFilteringSupervisedScoresForScribing(): Seq[Future[Map[String, Double]]] = { - if (target.params( - PushFeatureSwitchParams.EnableMrRequestScribingBigFilteringSupervisedScores)) { - Seq( - mrBigFilteringSupervisedSendingScore.map { - case Some(score) => - Map(PushMLModel.BigFilteringSupervisedSendingModel.toString -> score) - case _ => Map.empty[String, Double] - }, - mrBigFilteringSupervisedWithoutSendingScore.map { - case Some(score) => - Map(PushMLModel.BigFilteringSupervisedWithoutSendingModel.toString -> score) - case _ => Map.empty[String, Double] - } - ) - } else Seq.empty[Future[Map[String, Double]]] - } - - def getBigFilteringRLScoresForScribing(): Seq[Future[Map[String, Double]]] = { - if (target.params(PushFeatureSwitchParams.EnableMrRequestScribingBigFilteringRLScores)) { - Seq( - mrBigFilteringRLSendingScore.map { - case Some(score) => Map(PushMLModel.BigFilteringRLSendingModel.toString -> score) - case _ => Map.empty[String, Double] - }, - mrBigFilteringRLWithoutSendingScore.map { - case Some(score) => Map(PushMLModel.BigFilteringRLWithoutSendingModel.toString -> score) - case _ => Map.empty[String, Double] - } - ) - } else Seq.empty[Future[Map[String, Double]]] - } - - def buildModelScoresSeqForScribing(): Seq[Future[Map[String, Double]]] = { - getWeightedOpenOrNtabClickScoreForScribing() ++ - getBigFilteringSupervisedScoresForScribing() ++ - getBigFilteringRLScoresForScribing() ++ - getNsfwScoreForScribing() - } - - lazy val mrBigFilteringSupervisedSendingScore: Future[Option[Double]] = - getBigFilteringScore( - PushMLModel.BigFilteringSupervisedSendingModel, - target.params(PushFeatureSwitchParams.BigFilteringSupervisedSendingModelParam) - ) - - lazy val mrBigFilteringSupervisedWithoutSendingScore: Future[Option[Double]] = - getBigFilteringScore( - PushMLModel.BigFilteringSupervisedWithoutSendingModel, - target.params(PushFeatureSwitchParams.BigFilteringSupervisedWithoutSendingModelParam) - ) - - lazy val mrBigFilteringRLSendingScore: Future[Option[Double]] = - getBigFilteringScore( - PushMLModel.BigFilteringRLSendingModel, - target.params(PushFeatureSwitchParams.BigFilteringRLSendingModelParam) - ) - - lazy val mrBigFilteringRLWithoutSendingScore: Future[Option[Double]] = - getBigFilteringScore( - PushMLModel.BigFilteringRLWithoutSendingModel, - target.params(PushFeatureSwitchParams.BigFilteringRLWithoutSendingModelParam) - ) - - lazy val mrWeightedOpenOrNtabClickFilteringProbability: Future[Option[Double]] = - getWeightedOpenOrNtabClickModelScore( - target.filteringModelParam - ) - - lazy val mrQualityUprankingProbability: Future[Option[Double]] = - getMLModelScore( - PushMLModel.FilteringProbability, - target.params(PushFeatureSwitchParams.QualityUprankingModelTypeParam) - ) - - lazy val mrNsfwScore: Future[Option[Double]] = - getMLModelScoreWithoutUpdate( - PushMLModel.HealthNsfwProbability, - target.params(PushFeatureSwitchParams.BqmlHealthModelTypeParam) - ) - - // MR quality upranking param - private val qualityUprankingBoost: String = "QualityUprankingBoost" - private val producerQualityUprankingBoost: String = "ProducerQualityUprankingBoost" - private val qualityUprankingInfo: CMap[String, Double] = - new ConcurrentHashMap[String, Double]().asScala - - lazy val mrQualityUprankingBoost: Option[Double] = - qualityUprankingInfo.get(qualityUprankingBoost) - lazy val mrProducerQualityUprankingBoost: Option[Double] = - qualityUprankingInfo.get(producerQualityUprankingBoost) - - def setQualityUprankingBoost(boost: Double) = - if (qualityUprankingInfo.contains(qualityUprankingBoost)) { - qualityUprankingInfo(qualityUprankingBoost) = boost - } else { - qualityUprankingInfo += qualityUprankingBoost -> boost - } - def setProducerQualityUprankingBoost(boost: Double) = - if (qualityUprankingInfo.contains(producerQualityUprankingBoost)) { - qualityUprankingInfo(producerQualityUprankingBoost) = boost - } else { - qualityUprankingInfo += producerQualityUprankingBoost -> boost - } - - private lazy val mrModelScoresFut: Future[Map[String, Double]] = { - if (self.target.isLoggedOutUser) { - Future.value(Map.empty[String, Double]) - } else { - Future - .collectToTry { - buildModelScoresSeqForScribing() - }.map { scoreTrySeq => - scoreTrySeq - .collect { - case result if result.isReturn => result.get() - }.reduce(_ ++ _) - } - } - } - - // Internal model scores (scores that are independent of other candidates) for scribing - lazy val modelScores: Future[Map[String, Double]] = - target.dauProbability.flatMap { dauProbabilityOpt => - val dauProbScoreMap = dauProbabilityOpt - .map(_.probability).map { dauProb => - PushMLModel.DauProbability.toString -> dauProb - }.toMap - - // Avoid unnecessary MR model scribing - if (target.isDarkWrite) { - mrModelScoresFut.map(dauProbScoreMap ++ _) - } else if (RecTypes.isSendHandlerType(commonRecType) && !RecTypes - .sendHandlerTypesUsingMrModel(commonRecType)) { - Future.value(dauProbScoreMap) - } else { - mrModelScoresFut.map(dauProbScoreMap ++ _) - } - } - - // We will scribe both internal ML scores and cross-Candidate scores - def getModelScoresforScribing(): Future[Map[String, Double]] = { - if (RecTypes.notEligibleForModelScoreTracking(commonRecType) || self.target.isLoggedOutUser) { - Future.value(Map.empty[String, Double]) - } else { - modelScores.map { internalScores => - if (internalScores.keySet.intersect(crossCandidateMlScores.keySet).nonEmpty) { - throw new Exception( - "crossCandidateMlScores overlap internalModelScores\n" + - s"internalScores keySet: ${internalScores.keySet}\n" + - s"crossCandidateScores keySet: ${crossCandidateMlScores.keySet}\n" - ) - } - - internalScores ++ crossCandidateMlScores - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/QualityScribing.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/QualityScribing.scala deleted file mode 100644 index 283c3d97c..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/QualityScribing.scala +++ /dev/null @@ -1,104 +0,0 @@ -package com.twitter.frigate.pushservice.model.candidate - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.HighQualityScribingScores -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushMLModel -import com.twitter.util.Future -import java.util.concurrent.ConcurrentHashMap -import scala.collection.concurrent.{Map => CMap} -import scala.collection.convert.decorateAsScala._ - -trait QualityScribing { - self: PushCandidate with MLScores => - - // Use to store other scores (to avoid duplicate queries to other services, e.g. HSS) - private val externalCachedScores: CMap[String, Future[Option[Double]]] = - new ConcurrentHashMap[String, Future[Option[Double]]]().asScala - - /** - * Retrieves the model version as specified by the corresponding FS param. - * This model version will be used for getting the cached score or triggering - * a prediction request. - * - * @param modelName The score we will like to scribe - */ - private def getModelVersion( - modelName: HighQualityScribingScores.Name - ): String = { - modelName match { - case HighQualityScribingScores.HeavyRankingScore => - target.params(PushFeatureSwitchParams.HighQualityCandidatesHeavyRankingModel) - case HighQualityScribingScores.NonPersonalizedQualityScoreUsingCnn => - target.params(PushFeatureSwitchParams.HighQualityCandidatesNonPersonalizedQualityCnnModel) - case HighQualityScribingScores.BqmlNsfwScore => - target.params(PushFeatureSwitchParams.HighQualityCandidatesBqmlNsfwModel) - case HighQualityScribingScores.BqmlReportScore => - target.params(PushFeatureSwitchParams.HighQualityCandidatesBqmlReportModel) - } - } - - /** - * Retrieves the score for scribing either from a cached value or - * by generating a prediction request. This will increase model QPS - * - * @param pushMLModel This represents the prefix of the model name (i.e. [pushMLModel]_[version]) - * @param scoreName The name to be use when scribing this score - */ - def getScribingScore( - pushMLModel: PushMLModel.Value, - scoreName: HighQualityScribingScores.Name - ): Future[(String, Option[Double])] = { - getMLModelScore( - pushMLModel, - getModelVersion(scoreName) - ).map { scoreOpt => - scoreName.toString -> scoreOpt - } - } - - /** - * Retrieves the score for scribing if it has been computed/cached before otherwise - * it will return Future.None - * - * @param pushMLModel This represents the prefix of the model name (i.e. [pushMLModel]_[version]) - * @param scoreName The name to be use when scribing this score - */ - def getScribingScoreWithoutUpdate( - pushMLModel: PushMLModel.Value, - scoreName: HighQualityScribingScores.Name - ): Future[(String, Option[Double])] = { - getMLModelScoreWithoutUpdate( - pushMLModel, - getModelVersion(scoreName) - ).map { scoreOpt => - scoreName.toString -> scoreOpt - } - } - - /** - * Caches the given score future - * - * @param scoreName The name to be use when scribing this score - * @param scoreFut Future mapping scoreName -> scoreOpt - */ - def cacheExternalScore(scoreName: String, scoreFut: Future[Option[Double]]) = { - if (!externalCachedScores.contains(scoreName)) { - externalCachedScores += scoreName -> scoreFut - } - } - - /** - * Returns all external scores future cached as a sequence - */ - def getExternalCachedScores: Seq[Future[(String, Option[Double])]] = { - externalCachedScores.map { - case (modelName, scoreFut) => - scoreFut.map { scoreOpt => modelName -> scoreOpt } - }.toSeq - } - - def getExternalCachedScoreByName(name: String): Future[Option[Double]] = { - externalCachedScores.getOrElse(name, Future.None) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/Scriber.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/Scriber.scala deleted file mode 100644 index a43530b44..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/Scriber.scala +++ /dev/null @@ -1,277 +0,0 @@ -package com.twitter.frigate.pushservice.model.candidate - -import com.twitter.frigate.data_pipeline.features_common.PushQualityModelFeatureContext.featureContext -import com.twitter.frigate.data_pipeline.features_common.PushQualityModelUtil -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.common.util.NotificationScribeUtil -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.OutOfNetworkTweetPushCandidate -import com.twitter.frigate.pushservice.model.TopicProofTweetPushCandidate -import com.twitter.frigate.pushservice.ml.HydrationContextBuilder -import com.twitter.frigate.pushservice.predicate.quality_model_predicate.PDauCohort -import com.twitter.frigate.pushservice.predicate.quality_model_predicate.PDauCohortUtil -import com.twitter.frigate.pushservice.util.Candidate2FrigateNotification -import com.twitter.frigate.pushservice.util.MediaAnnotationsUtil.sensitiveMediaCategoryFeatureName -import com.twitter.frigate.scribe.thriftscala.FrigateNotificationScribeType -import com.twitter.frigate.scribe.thriftscala.NotificationScribe -import com.twitter.frigate.scribe.thriftscala.PredicateDetailedInfo -import com.twitter.frigate.scribe.thriftscala.PushCapInfo -import com.twitter.frigate.thriftscala.ChannelName -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.frigate.thriftscala.OverrideInfo -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.hermit.model.user_state.UserState.UserState -import com.twitter.ibis2.service.thriftscala.Ibis2Response -import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions -import com.twitter.nrel.heavyranker.FeatureHydrator -import com.twitter.util.Future -import java.util.UUID -import java.util.concurrent.ConcurrentHashMap -import scala.collection.concurrent.{Map => CMap} -import scala.collection.Map -import scala.collection.convert.decorateAsScala._ - -trait Scriber { - self: PushCandidate => - - def statsReceiver: StatsReceiver - - def frigateNotification: FrigateNotification = Candidate2FrigateNotification - .getFrigateNotification(self)(statsReceiver) - .copy(copyAggregationId = self.copyAggregationId) - - lazy val impressionId: String = UUID.randomUUID.toString.replaceAll("-", "") - - // Used to store the score and threshold for predicates - // Map(predicate name, (score, threshold, filter?)) - private val predicateScoreAndThreshold: CMap[String, PredicateDetailedInfo] = - new ConcurrentHashMap[String, PredicateDetailedInfo]().asScala - - def cachePredicateInfo( - predName: String, - predScore: Double, - predThreshold: Double, - predResult: Boolean, - additionalInformation: Option[Map[String, Double]] = None - ) = { - if (!predicateScoreAndThreshold.contains(predName)) { - predicateScoreAndThreshold += predName -> PredicateDetailedInfo( - predName, - predScore, - predThreshold, - predResult, - additionalInformation) - } - } - - def getCachedPredicateInfo(): Seq[PredicateDetailedInfo] = predicateScoreAndThreshold.values.toSeq - - def frigateNotificationForPersistence( - channels: Seq[ChannelName], - isSilentPush: Boolean, - overrideInfoOpt: Option[OverrideInfo] = None, - copyFeaturesList: Set[String] - ): Future[FrigateNotification] = { - - // record display location for frigate notification - statsReceiver - .scope("FrigateNotificationForPersistence") - .scope("displayLocation") - .counter(frigateNotification.notificationDisplayLocation.name) - .incr() - - val getModelScores = self.getModelScoresforScribing() - - Future.join(getModelScores, self.target.targetMrUserState).map { - case (mlScores, mrUserState) => - frigateNotification.copy( - impressionId = Some(impressionId), - isSilentPush = Some(isSilentPush), - overrideInfo = overrideInfoOpt, - mlModelScores = Some(mlScores), - mrUserState = mrUserState.map(_.name), - copyFeatures = Some(copyFeaturesList.toSeq) - ) - } - } - // scribe data - private def getNotificationScribe( - notifForPersistence: FrigateNotification, - userState: Option[UserState], - dauCohort: PDauCohort.Value, - ibis2Response: Option[Ibis2Response], - tweetAuthorId: Option[Long], - recUserId: Option[Long], - modelScoresMap: Option[Map[String, Double]], - primaryClient: Option[String], - isMrBackfillCR: Option[Boolean] = None, - tagsCR: Option[Seq[String]] = None, - gizmoduckTargetUser: Option[User], - predicateDetailedInfoList: Option[Seq[PredicateDetailedInfo]] = None, - pushCapInfoList: Option[Seq[PushCapInfo]] = None - ): NotificationScribe = { - NotificationScribe( - FrigateNotificationScribeType.SendMessage, - System.currentTimeMillis(), - targetUserId = Some(self.target.targetId), - timestampKeyForHistoryV2 = Some(createdAt.inSeconds), - sendType = NotificationScribeUtil.convertToScribeDisplayLocation( - self.frigateNotification.notificationDisplayLocation - ), - recommendationType = NotificationScribeUtil.convertToScribeRecommendationType( - self.frigateNotification.commonRecommendationType - ), - commonRecommendationType = Some(self.frigateNotification.commonRecommendationType), - fromPushService = Some(true), - frigateNotification = Some(notifForPersistence), - impressionId = Some(impressionId), - skipModelInfo = target.skipModelInfo, - ibis2Response = ibis2Response, - tweetAuthorId = tweetAuthorId, - scribeFeatures = Some(target.noSkipButScribeFeatures), - userState = userState.map(_.toString), - pDauCohort = Some(dauCohort.toString), - recommendedUserId = recUserId, - modelScores = modelScoresMap, - primaryClient = primaryClient, - isMrBackfillCR = isMrBackfillCR, - tagsCR = tagsCR, - targetUserType = gizmoduckTargetUser.map(_.userType), - predicateDetailedInfoList = predicateDetailedInfoList, - pushCapInfoList = pushCapInfoList - ) - } - - def scribeData( - ibis2Response: Option[Ibis2Response] = None, - isSilentPush: Boolean = false, - overrideInfoOpt: Option[OverrideInfo] = None, - copyFeaturesList: Set[String] = Set.empty, - channels: Seq[ChannelName] = Seq.empty - ): Future[NotificationScribe] = { - - val recTweetAuthorId = self match { - case t: TweetCandidate with TweetAuthor => t.authorId - case _ => None - } - - val recUserId = self match { - case u: UserCandidate => Some(u.userId) - case _ => None - } - - val isMrBackfillCR = self match { - case t: OutOfNetworkTweetPushCandidate => t.isMrBackfillCR - case _ => None - } - - val tagsCR = self match { - case t: OutOfNetworkTweetPushCandidate => - t.tagsCR.map { tags => - tags.map(_.toString) - } - case t: TopicProofTweetPushCandidate => - t.tagsCR.map { tags => - tags.map(_.toString) - } - case _ => None - } - - Future - .join( - frigateNotificationForPersistence( - channels = channels, - isSilentPush = isSilentPush, - overrideInfoOpt = overrideInfoOpt, - copyFeaturesList = copyFeaturesList - ), - target.targetUserState, - PDauCohortUtil.getPDauCohort(target), - target.deviceInfo, - target.targetUser - ) - .flatMap { - case (notifForPersistence, userState, dauCohort, deviceInfo, gizmoduckTargetUserOpt) => - val primaryClient = deviceInfo.flatMap(_.guessedPrimaryClient).map(_.toString) - val cachedPredicateInfo = - if (self.target.params(PushParams.EnablePredicateDetailedInfoScribing)) { - Some(getCachedPredicateInfo()) - } else None - - val cachedPushCapInfo = - if (self.target - .params(PushParams.EnablePushCapInfoScribing)) { - Some(target.finalPushcapAndFatigue.values.toSeq) - } else None - - val data = getNotificationScribe( - notifForPersistence, - userState, - dauCohort, - ibis2Response, - recTweetAuthorId, - recUserId, - notifForPersistence.mlModelScores, - primaryClient, - isMrBackfillCR, - tagsCR, - gizmoduckTargetUserOpt, - cachedPredicateInfo, - cachedPushCapInfo - ) - //Don't scribe features for CRTs not eligible for ML Layer - if ((target.isModelTrainingData || target.scribeFeatureWithoutHydratingNewFeatures) - && !RecTypes.notEligibleForModelScoreTracking(self.commonRecType)) { - // scribe all the features for the model training data - self.getFeaturesForScribing.map { scribedFeatureMap => - if (target.params(PushParams.EnableScribingMLFeaturesAsDataRecord) && !target.params( - PushFeatureSwitchParams.EnableMrScribingMLFeaturesAsFeatureMapForStaging)) { - val scribedFeatureDataRecord = - ScalaToJavaDataRecordConversions.javaDataRecord2ScalaDataRecord( - PushQualityModelUtil.adaptToDataRecord(scribedFeatureMap, featureContext)) - data.copy( - featureDataRecord = Some(scribedFeatureDataRecord) - ) - } else { - data.copy(features = - Some(PushQualityModelUtil.convertFeatureMapToFeatures(scribedFeatureMap))) - } - } - } else Future.value(data) - } - } - - def getFeaturesForScribing: Future[FeatureMap] = { - target.featureMap - .flatMap { targetFeatureMap => - val onlineFeatureMap = targetFeatureMap ++ self - .candidateFeatureMap() // targetFeatureMap includes target core user history features - - val filteredFeatureMap = { - onlineFeatureMap.copy( - sparseContinuousFeatures = onlineFeatureMap.sparseContinuousFeatures.filterKeys( - !_.equals(sensitiveMediaCategoryFeatureName)) - ) - } - - val targetHydrationContext = HydrationContextBuilder.build(self.target) - val candidateHydrationContext = HydrationContextBuilder.build(self) - - val featureMapFut = targetHydrationContext.join(candidateHydrationContext).flatMap { - case (targetContext, candidateContext) => - FeatureHydrator.getFeatures( - candidateHydrationContext = candidateContext, - targetHydrationContext = targetContext, - onlineFeatures = filteredFeatureMap, - statsReceiver = statsReceiver) - } - - featureMapFut - } - } - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/CustomConfigurationMapForIbis.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/CustomConfigurationMapForIbis.scala deleted file mode 100644 index 75b00e346..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/CustomConfigurationMapForIbis.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.ibis2.lib.util.JsonMarshal -import com.twitter.util.Future - -trait CustomConfigurationMapForIbis { - self: PushCandidate => - - lazy val customConfigMapsJsonFut: Future[String] = { - customFieldsMapFut.map { customFields => - JsonMarshal.toJson(customFields) - } - } - - lazy val customConfigMapsFut: Future[Map[String, String]] = { - if (self.target.isLoggedOutUser) { - Future.value(Map.empty[String, String]) - } else { - customConfigMapsJsonFut.map { customConfigMapsJson => - Map("custom_config" -> customConfigMapsJson) - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/DiscoverTwitterPushIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/DiscoverTwitterPushIbis2Hydrator.scala deleted file mode 100644 index a3a48ff28..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/DiscoverTwitterPushIbis2Hydrator.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.common.base.DiscoverTwitterCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.util.PushIbisUtil.mergeFutModelValues -import com.twitter.util.Future - -trait DiscoverTwitterPushIbis2Hydrator extends Ibis2HydratorForCandidate { - self: PushCandidate with DiscoverTwitterCandidate => - - private lazy val targetModelValues: Map[String, String] = Map( - "target_user" -> target.targetId.toString - ) - - override lazy val modelValues: Future[Map[String, String]] = - mergeFutModelValues(super.modelValues, Future.value(targetModelValues)) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/F1FirstDegreeTweetIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/F1FirstDegreeTweetIbis2Hydrator.scala deleted file mode 100644 index 6ddaa49d1..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/F1FirstDegreeTweetIbis2Hydrator.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.F1FirstDegree -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.util.Future - -trait F1FirstDegreeTweetIbis2HydratorForCandidate - extends TweetCandidateIbis2Hydrator - with RankedSocialContextIbis2Hydrator { - self: PushCandidate with F1FirstDegree with TweetAuthorDetails => - - override lazy val scopedStats: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) - - override lazy val tweetModelValues: Future[Map[String, String]] = { - for { - superModelValues <- super.tweetModelValues - tweetInlineModelValues <- tweetInlineActionModelValue - } yield { - superModelValues ++ otherModelValues ++ mediaModelValue ++ tweetInlineModelValues ++ inlineVideoMediaMap - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/Ibis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/Ibis2Hydrator.scala deleted file mode 100644 index fd7d26186..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/Ibis2Hydrator.scala +++ /dev/null @@ -1,127 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.common.util.MRPushCopy -import com.twitter.frigate.common.util.MrPushCopyObjects -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} -import com.twitter.ibis2.service.thriftscala.Flags -import com.twitter.ibis2.service.thriftscala.Ibis2Request -import com.twitter.ibis2.service.thriftscala.RecipientSelector -import com.twitter.ibis2.service.thriftscala.ResponseFlags -import com.twitter.util.Future -import scala.util.control.NoStackTrace -import com.twitter.ni.lib.logged_out_transform.Ibis2RequestTransform - -class PushCopyIdNotFoundException(private val message: String) - extends Exception(message) - with NoStackTrace - -class InvalidPushCopyIdException(private val message: String) - extends Exception(message) - with NoStackTrace - -trait Ibis2HydratorForCandidate - extends CandidatePushCopy - with OverrideForIbis2Request - with CustomConfigurationMapForIbis { - self: PushCandidate => - - lazy val silentPushModelValue: Map[String, String] = - if (RecTypes.silentPushDefaultEnabledCrts.contains(commonRecType)) { - Map.empty - } else { - Map("is_silent_push" -> "true") - } - - private def transformRelevanceScore( - mlScore: Double, - scoreRange: Seq[Double] - ): Double = { - val (lowerBound, upperBound) = (scoreRange.head, scoreRange.last) - (mlScore * (upperBound - lowerBound)) + lowerBound - } - - private def getBoundedMlScore(mlScore: Double): Double = { - if (RecTypes.isMagicFanoutEventType(commonRecType)) { - val mfScoreRange = target.params(FS.MagicFanoutRelevanceScoreRange) - transformRelevanceScore(mlScore, mfScoreRange) - } else { - val mrScoreRange = target.params(FS.MagicRecsRelevanceScoreRange) - transformRelevanceScore(mlScore, mrScoreRange) - } - } - - lazy val relevanceScoreMapFut: Future[Map[String, String]] = { - mrWeightedOpenOrNtabClickRankingProbability.map { - case Some(mlScore) if target.params(FS.IncludeRelevanceScoreInIbis2Payload) => - val boundedMlScore = getBoundedMlScore(mlScore) - Map("relevance_score" -> boundedMlScore.toString) - case _ => Map.empty[String, String] - } - } - - def customFieldsMapFut: Future[Map[String, String]] = relevanceScoreMapFut - - //override is only enabled for RFPH CRT - def modelValues: Future[Map[String, String]] = { - Future.join(overrideModelValueFut, customConfigMapsFut).map { - case (overrideModelValue, customConfig) => - overrideModelValue ++ silentPushModelValue ++ customConfig - } - } - - def modelName: String = pushCopy.ibisPushModelName - - def senderId: Option[Long] = None - - def ibis2Request: Future[Option[Ibis2Request]] = { - Future.join(self.target.loggedOutMetadata, modelValues).map { - case (Some(metadata), modelVals) => - Some( - Ibis2RequestTransform - .apply(metadata, modelName, modelVals).copy( - senderId = senderId, - flags = Some(Flags( - darkWrite = Some(target.isDarkWrite), - skipDupcheck = target.pushContext.flatMap(_.useDebugHandler), - responseFlags = Some(ResponseFlags(stringTelemetry = Some(true))) - )) - )) - case (None, modelVals) => - Some( - Ibis2Request( - recipientSelector = RecipientSelector(Some(target.targetId)), - modelName = modelName, - modelValues = Some(modelVals), - senderId = senderId, - flags = Some( - Flags( - darkWrite = Some(target.isDarkWrite), - skipDupcheck = target.pushContext.flatMap(_.useDebugHandler), - responseFlags = Some(ResponseFlags(stringTelemetry = Some(true))) - ) - ) - )) - } - } -} - -trait CandidatePushCopy { - self: PushCandidate => - - final lazy val pushCopy: MRPushCopy = - pushCopyId match { - case Some(pushCopyId) => - MrPushCopyObjects - .getCopyFromId(pushCopyId) - .getOrElse( - throw new InvalidPushCopyIdException( - s"Invalid push copy id: $pushCopyId for ${self.commonRecType}")) - - case None => - throw new PushCopyIdNotFoundException( - s"PushCopy not found in frigateNotification for ${self.commonRecType}" - ) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/InlineActionIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/InlineActionIbis2Hydrator.scala deleted file mode 100644 index 8e927254a..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/InlineActionIbis2Hydrator.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.util.InlineActionUtil -import com.twitter.util.Future - -trait InlineActionIbis2Hydrator { - self: PushCandidate => - - lazy val tweetInlineActionModelValue: Future[Map[String, String]] = - InlineActionUtil.getTweetInlineActionValue(target) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ListIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ListIbis2Hydrator.scala deleted file mode 100644 index 57483c8ba..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ListIbis2Hydrator.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.pushservice.model.ListRecommendationPushCandidate -import com.twitter.util.Future - -trait ListIbis2Hydrator extends Ibis2HydratorForCandidate { - self: ListRecommendationPushCandidate => - - override lazy val senderId: Option[Long] = Some(0L) - - override lazy val modelValues: Future[Map[String, String]] = - Future.join(listName, listOwnerId).map { - case (nameOpt, authorId) => - Map( - "list" -> listId.toString, - "list_name" -> nameOpt - .getOrElse(""), - "list_author" -> s"${authorId.getOrElse(0L)}" - ) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutCreatorEventIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutCreatorEventIbis2Hydrator.scala deleted file mode 100644 index edb0aa51e..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutCreatorEventIbis2Hydrator.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.magic_events.thriftscala.CreatorFanoutType -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.MagicFanoutCreatorEventPushCandidate -import com.twitter.frigate.pushservice.util.PushIbisUtil.mergeModelValues -import com.twitter.util.Future - -trait MagicFanoutCreatorEventIbis2Hydrator - extends CustomConfigurationMapForIbis - with Ibis2HydratorForCandidate { - self: PushCandidate with MagicFanoutCreatorEventPushCandidate => - - val userMap = Map( - "handle" -> userProfile.screenName, - "display_name" -> userProfile.name - ) - - override val senderId = hydratedCreator.map(_.id) - - override lazy val modelValues: Future[Map[String, String]] = - mergeModelValues(super.modelValues, userMap) - - override val ibis2Request = creatorFanoutType match { - case CreatorFanoutType.UserSubscription => Future.None - case CreatorFanoutType.NewCreator => super.ibis2Request - case _ => super.ibis2Request - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutNewsEventIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutNewsEventIbis2Hydrator.scala deleted file mode 100644 index a1b073a38..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutNewsEventIbis2Hydrator.scala +++ /dev/null @@ -1,103 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.MagicFanoutEventHydratedCandidate -import com.twitter.frigate.pushservice.params.PushConstants -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutPredicatesUtil -import com.twitter.frigate.pushservice.util.PushIbisUtil._ -import com.twitter.util.Future - -trait MagicFanoutNewsEventIbis2Hydrator extends Ibis2HydratorForCandidate { - self: PushCandidate with MagicFanoutEventHydratedCandidate => - - override lazy val senderId: Option[Long] = { - val isUgmMoment = self.semanticCoreEntityTags.values.flatten.toSet - .contains(MagicFanoutPredicatesUtil.UgmMomentTag) - - owningTwitterUserIds.headOption match { - case Some(owningTwitterUserId) - if isUgmMoment && target.params( - PushFeatureSwitchParams.MagicFanoutNewsUserGeneratedEventsEnable) => - Some(owningTwitterUserId) - case _ => None - } - } - - lazy val stats = self.statsReceiver.scope("MagicFanout") - lazy val defaultImageCounter = stats.counter("default_image") - lazy val requestImageCounter = stats.counter("request_num") - lazy val noneImageCounter = stats.counter("none_num") - - private def getModelValueMediaUrl( - urlOpt: Option[String], - mapKey: String - ): Option[(String, String)] = { - requestImageCounter.incr() - urlOpt match { - case Some(PushConstants.DefaultEventMediaUrl) => - defaultImageCounter.incr() - None - case Some(url) => Some(mapKey -> url) - case None => - noneImageCounter.incr() - None - } - } - - private lazy val eventModelValuesFut: Future[Map[String, String]] = { - for { - title <- eventTitleFut - squareImageUrl <- squareImageUrlFut - primaryImageUrl <- primaryImageUrlFut - eventDescriptionOpt <- eventDescriptionFut - } yield { - - val authorId = owningTwitterUserIds.headOption match { - case Some(author) - if target.params(PushFeatureSwitchParams.MagicFanoutNewsUserGeneratedEventsEnable) => - Some("author" -> author.toString) - case _ => None - } - - val eventDescription = eventDescriptionOpt match { - case Some(description) - if target.params(PushFeatureSwitchParams.MagicFanoutNewsEnableDescriptionCopy) => - Some("event_description" -> description) - case _ => - None - } - - Map( - "event_id" -> s"$eventId", - "event_title" -> title - ) ++ - getModelValueMediaUrl(squareImageUrl, "square_media_url") ++ - getModelValueMediaUrl(primaryImageUrl, "media_url") ++ - authorId ++ - eventDescription - } - } - - private lazy val topicValuesFut: Future[Map[String, String]] = { - if (target.params(PushFeatureSwitchParams.EnableTopicCopyForMF)) { - followedTopicLocalizedEntities.map(_.headOption).flatMap { - case Some(localizedEntity) => - Future.value(Map("topic_name" -> localizedEntity.localizedNameForDisplay)) - case _ => - ergLocalizedEntities.map(_.headOption).map { - case Some(localizedEntity) - if target.params(PushFeatureSwitchParams.EnableTopicCopyForImplicitTopics) => - Map("topic_name" -> localizedEntity.localizedNameForDisplay) - case _ => Map.empty[String, String] - } - } - } else { - Future.value(Map.empty[String, String]) - } - } - - override lazy val modelValues: Future[Map[String, String]] = - mergeFutModelValues(super.modelValues, mergeFutModelValues(eventModelValuesFut, topicValuesFut)) - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutProductLaunchIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutProductLaunchIbis2Hydrator.scala deleted file mode 100644 index 3062a66d0..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutProductLaunchIbis2Hydrator.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.common.base.MagicFanoutProductLaunchCandidate -import com.twitter.frigate.magic_events.thriftscala.ProductInfo -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.util.PushIbisUtil.mergeModelValues -import com.twitter.util.Future - -trait MagicFanoutProductLaunchIbis2Hydrator - extends CustomConfigurationMapForIbis - with Ibis2HydratorForCandidate { - self: PushCandidate with MagicFanoutProductLaunchCandidate => - - private def getProductInfoMap(productInfo: ProductInfo): Map[String, String] = { - val titleMap = productInfo.title - .map { title => - Map("title" -> title) - }.getOrElse(Map.empty) - val bodyMap = productInfo.body - .map { body => - Map("body" -> body) - }.getOrElse(Map.empty) - val deeplinkMap = productInfo.deeplink - .map { deeplink => - Map("deeplink" -> deeplink) - }.getOrElse(Map.empty) - - titleMap ++ bodyMap ++ deeplinkMap - } - - private lazy val landingPage: Map[String, String] = { - val urlFromFS = target.params(PushFeatureSwitchParams.ProductLaunchLandingPageDeepLink) - Map("push_land_url" -> urlFromFS) - } - - private lazy val customProductLaunchPushDetails: Map[String, String] = { - frigateNotification.magicFanoutProductLaunchNotification match { - case Some(productLaunchNotif) => - productLaunchNotif.productInfo match { - case Some(productInfo) => - getProductInfoMap(productInfo) - case _ => Map.empty - } - case _ => Map.empty - } - } - - override lazy val customFieldsMapFut: Future[Map[String, String]] = - mergeModelValues(super.customFieldsMapFut, customProductLaunchPushDetails) - - override lazy val modelValues: Future[Map[String, String]] = - mergeModelValues(super.modelValues, landingPage) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutSportsEventIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutSportsEventIbis2Hydrator.scala deleted file mode 100644 index 811caa993..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutSportsEventIbis2Hydrator.scala +++ /dev/null @@ -1,89 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.common.base.BaseGameScore -import com.twitter.frigate.common.base.MagicFanoutSportsEventCandidate -import com.twitter.frigate.common.base.MagicFanoutSportsScoreInformation -import com.twitter.frigate.common.base.NflGameScore -import com.twitter.frigate.common.base.SoccerGameScore -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.MagicFanoutEventHydratedCandidate -import com.twitter.frigate.pushservice.params.PushConstants -import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutSportsUtil -import com.twitter.frigate.pushservice.util.PushIbisUtil._ -import com.twitter.util.Future - -trait MagicFanoutSportsEventIbis2Hydrator extends Ibis2HydratorForCandidate { - self: PushCandidate - with MagicFanoutEventHydratedCandidate - with MagicFanoutSportsEventCandidate - with MagicFanoutSportsScoreInformation => - - lazy val stats = self.statsReceiver.scope("MagicFanoutSportsEvent") - lazy val defaultImageCounter = stats.counter("default_image") - lazy val requestImageCounter = stats.counter("request_num") - lazy val noneImageCounter = stats.counter("none_num") - - override lazy val relevanceScoreMapFut = Future.value(Map.empty[String, String]) - - private def getModelValueMediaUrl( - urlOpt: Option[String], - mapKey: String - ): Option[(String, String)] = { - requestImageCounter.incr() - urlOpt match { - case Some(PushConstants.DefaultEventMediaUrl) => - defaultImageCounter.incr() - None - case Some(url) => Some(mapKey -> url) - case None => - noneImageCounter.incr() - None - } - } - - private lazy val eventModelValuesFut: Future[Map[String, String]] = { - for { - title <- eventTitleFut - squareImageUrl <- squareImageUrlFut - primaryImageUrl <- primaryImageUrlFut - } yield { - Map( - "event_id" -> s"$eventId", - "event_title" -> title - ) ++ - getModelValueMediaUrl(squareImageUrl, "square_media_url") ++ - getModelValueMediaUrl(primaryImageUrl, "media_url") - } - } - - private lazy val sportsScoreValues: Future[Map[String, String]] = { - for { - scores <- gameScores - homeName <- homeTeamInfo.map(_.map(_.name)) - awayName <- awayTeamInfo.map(_.map(_.name)) - } yield { - if (awayName.isDefined && homeName.isDefined && scores.isDefined) { - scores.get match { - case game: SoccerGameScore => - MagicFanoutSportsUtil.getSoccerIbisMap(game) ++ Map( - "away_team" -> awayName.get, - "home_team" -> homeName.get - ) - case game: NflGameScore => - MagicFanoutSportsUtil.getNflIbisMap(game) ++ Map( - "away_team" -> MagicFanoutSportsUtil.getNFLReadableName(awayName.get), - "home_team" -> MagicFanoutSportsUtil.getNFLReadableName(homeName.get) - ) - case baseGameScore: BaseGameScore => - Map.empty[String, String] - } - } else Map.empty[String, String] - } - } - - override lazy val customFieldsMapFut: Future[Map[String, String]] = - mergeFutModelValues(super.customFieldsMapFut, sportsScoreValues) - - override lazy val modelValues: Future[Map[String, String]] = - mergeFutModelValues(super.modelValues, eventModelValuesFut) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/OutOfNetworkTweetIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/OutOfNetworkTweetIbis2Hydrator.scala deleted file mode 100644 index c7bd051b7..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/OutOfNetworkTweetIbis2Hydrator.scala +++ /dev/null @@ -1,90 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.common.base.OutOfNetworkTweetCandidate -import com.twitter.frigate.common.base.TopicCandidate -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.rec_types.RecTypes._ -import com.twitter.frigate.common.util.MrPushCopyObjects -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.util.InlineActionUtil -import com.twitter.frigate.pushservice.util.PushIbisUtil.mergeModelValues -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.util.Future - -trait OutOfNetworkTweetIbis2HydratorForCandidate extends TweetCandidateIbis2Hydrator { - self: PushCandidate with OutOfNetworkTweetCandidate with TopicCandidate with TweetAuthorDetails => - - private lazy val useNewOonCopyValue = - if (target.params(PushFeatureSwitchParams.EnableNewMROONCopyForPush)) { - Map( - "use_new_oon_copy" -> "true" - ) - } else Map.empty[String, String] - - override lazy val tweetDynamicInlineActionsModelValues = - if (target.params(PushFeatureSwitchParams.EnableOONGeneratedInlineActions)) { - val actions = target.params(PushFeatureSwitchParams.OONTweetDynamicInlineActionsList) - InlineActionUtil.getGeneratedTweetInlineActions(target, statsReceiver, actions) - } else Map.empty[String, String] - - private lazy val ibtModelValues: Map[String, String] = - Map( - "is_tweet" -> s"${!(hasPhoto || hasVideo)}", - "is_photo" -> s"$hasPhoto", - "is_video" -> s"$hasVideo" - ) - - private lazy val launchVideosInImmersiveExploreValue = - Map( - "launch_videos_in_immersive_explore" -> s"${hasVideo && target.params(PushFeatureSwitchParams.EnableLaunchVideosInImmersiveExplore)}" - ) - - private lazy val oonTweetModelValues = - useNewOonCopyValue ++ ibtModelValues ++ tweetDynamicInlineActionsModelValues ++ launchVideosInImmersiveExploreValue - - lazy val useTopicCopyForMBCGIbis = mrModelingBasedTypes.contains(commonRecType) && target.params( - PushFeatureSwitchParams.EnableMrModelingBasedCandidatesTopicCopy) - lazy val useTopicCopyForFrsIbis = frsTypes.contains(commonRecType) && target.params( - PushFeatureSwitchParams.EnableFrsTweetCandidatesTopicCopy) - lazy val useTopicCopyForTagspaceIbis = tagspaceTypes.contains(commonRecType) && target.params( - PushFeatureSwitchParams.EnableHashspaceCandidatesTopicCopy) - - override lazy val modelName: String = { - if (localizedUttEntity.isDefined && - (useTopicCopyForMBCGIbis || useTopicCopyForFrsIbis || useTopicCopyForTagspaceIbis)) { - MrPushCopyObjects.TopicTweet.ibisPushModelName // uses topic copy - } else super.modelName - } - - lazy val exploreVideoParams: Map[String, String] = { - if (self.commonRecType == CommonRecommendationType.ExploreVideoTweet) { - Map( - "is_explore_video" -> "true" - ) - } else Map.empty[String, String] - } - - override lazy val customFieldsMapFut: Future[Map[String, String]] = - mergeModelValues(super.customFieldsMapFut, exploreVideoParams) - - override lazy val tweetModelValues: Future[Map[String, String]] = - if (localizedUttEntity.isDefined && - (useTopicCopyForMBCGIbis || useTopicCopyForFrsIbis || useTopicCopyForTagspaceIbis)) { - lazy val topicTweetModelValues: Map[String, String] = - Map("topic_name" -> s"${localizedUttEntity.get.localizedNameForDisplay}") - for { - superModelValues <- super.tweetModelValues - tweetInlineModelValue <- tweetInlineActionModelValue - } yield { - superModelValues ++ topicTweetModelValues ++ tweetInlineModelValue - } - } else { - for { - superModelValues <- super.tweetModelValues - tweetInlineModelValues <- tweetInlineActionModelValue - } yield { - superModelValues ++ mediaModelValue ++ oonTweetModelValues ++ tweetInlineModelValues ++ inlineVideoMediaMap - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/OverrideForIbis2Request.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/OverrideForIbis2Request.scala deleted file mode 100644 index e802a6421..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/OverrideForIbis2Request.scala +++ /dev/null @@ -1,210 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.common.store.deviceinfo.DeviceInfo -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FSParams} -import com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue.ContinuousFunction -import com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue.ContinuousFunctionParam -import com.twitter.frigate.pushservice.util.OverrideNotificationUtil -import com.twitter.frigate.pushservice.util.PushCapUtil -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.frigate.thriftscala.CommonRecommendationType.MagicFanoutSportsEvent -import com.twitter.ibis2.lib.util.JsonMarshal -import com.twitter.util.Future - -trait OverrideForIbis2Request { - self: PushCandidate => - - private lazy val overrideStats = self.statsReceiver.scope("override_for_ibis2") - - private lazy val addedOverrideAndroidCounter = - overrideStats.scope("android").counter("added_override_for_ibis2_request") - private lazy val addedSmartPushConfigAndroidCounter = - overrideStats.scope("android").counter("added_smart_push_config_for_ibis2_request") - private lazy val addedOverrideIosCounter = - overrideStats.scope("ios").counter("added_override_for_ibis2_request") - private lazy val noOverrideCounter = overrideStats.counter("no_override_for_ibis2_request") - private lazy val noOverrideDueToDeviceInfoCounter = - overrideStats.counter("no_override_due_to_device_info") - private lazy val addedMlScoreToPayloadAndroid = - overrideStats.scope("android").counter("added_ml_score") - private lazy val noMlScoreAddedToPayload = - overrideStats.counter("no_ml_score") - private lazy val addedNSlotsToPayload = - overrideStats.counter("added_n_slots") - private lazy val noNSlotsAddedToPayload = - overrideStats.counter("no_n_slots") - private lazy val addedCustomThreadIdToPayload = - overrideStats.counter("added_custom_thread_id") - private lazy val noCustomThreadIdAddedToPayload = - overrideStats.counter("no_custom_thread_id") - private lazy val enableTargetIdOverrideForMagicFanoutSportsEventCounter = - overrideStats.counter("enable_target_id_override_for_mf_sports_event") - - lazy val candidateModelScoreFut: Future[Option[Double]] = { - if (RecTypes.notEligibleForModelScoreTracking(commonRecType)) Future.None - else mrWeightedOpenOrNtabClickRankingProbability - } - - lazy val overrideModelValueFut: Future[Map[String, String]] = { - if (self.target.isLoggedOutUser) { - Future.value(Map.empty[String, String]) - } else { - Future - .join( - target.deviceInfo, - target.accountCountryCode, - OverrideNotificationUtil.getCollapseAndImpressionIdForOverride(self), - candidateModelScoreFut, - target.dynamicPushcap, - target.optoutAdjustedPushcap, - PushCapUtil.getDefaultPushCap(target) - ).map { - case ( - deviceInfoOpt, - countryCodeOpt, - Some((collapseId, impressionIds)), - mlScore, - dynamicPushcapOpt, - optoutAdjustedPushcapOpt, - defaultPushCap) => - val pushCap: Int = (dynamicPushcapOpt, optoutAdjustedPushcapOpt) match { - case (_, Some(optoutAdjustedPushcap)) => optoutAdjustedPushcap - case (Some(pushcapInfo), _) => pushcapInfo.pushcap - case _ => defaultPushCap - } - getClientSpecificOverrideModelValues( - target, - deviceInfoOpt, - countryCodeOpt, - collapseId, - impressionIds, - mlScore, - pushCap) - case _ => - noOverrideCounter.incr() - Map.empty[String, String] - } - } - } - - /** - * Determines the appropriate Override Notification model values based on the client - * @param target Target that will be receiving the push recommendation - * @param deviceInfoOpt Target's Device Info - * @param collapseId Collapse ID determined by OverrideNotificationUtil - * @param impressionIds Impression IDs of previously sent Override Notifications - * @param mlScore Open/NTab click ranking score of the current push candidate - * @param pushCap Push cap for the target - * @return Map consisting of the model values that need to be added to the Ibis2 Request - */ - def getClientSpecificOverrideModelValues( - target: Target, - deviceInfoOpt: Option[DeviceInfo], - countryCodeOpt: Option[String], - collapseId: String, - impressionIds: Seq[String], - mlScoreOpt: Option[Double], - pushCap: Int - ): Map[String, String] = { - - val primaryDeviceIos = PushDeviceUtil.isPrimaryDeviceIOS(deviceInfoOpt) - val primaryDeviceAndroid = PushDeviceUtil.isPrimaryDeviceAndroid(deviceInfoOpt) - - if (primaryDeviceIos || - (primaryDeviceAndroid && - target.params(FSParams.EnableOverrideNotificationsSmartPushConfigForAndroid))) { - - if (primaryDeviceIos) addedOverrideIosCounter.incr() - else addedSmartPushConfigAndroidCounter.incr() - - val impressionIdsSeq = { - if (target.params(FSParams.EnableTargetIdsInSmartPushPayload)) { - if (target.params(FSParams.EnableOverrideNotificationsMultipleTargetIds)) - impressionIds - else Seq(impressionIds.head) - } - // Explicitly enable targetId-based override for MagicFanoutSportsEvent candidates (live sport update notifications) - else if (self.commonRecType == MagicFanoutSportsEvent && target.params( - FSParams.EnableTargetIdInSmartPushPayloadForMagicFanoutSportsEvent)) { - enableTargetIdOverrideForMagicFanoutSportsEventCounter.incr() - Seq(impressionIds.head) - } else Seq.empty[String] - } - - val mlScoreMap = mlScoreOpt match { - case Some(mlScore) - if target.params(FSParams.EnableOverrideNotificationsScoreBasedOverride) => - addedMlScoreToPayloadAndroid.incr() - Map("score" -> mlScore) - case _ => - noMlScoreAddedToPayload.incr() - Map.empty - } - - val nSlotsMap = { - if (target.params(FSParams.EnableOverrideNotificationsNSlots)) { - if (target.params(FSParams.EnableOverrideMaxSlotFn)) { - val nslotFnParam = ContinuousFunctionParam( - target - .params(PushFeatureSwitchParams.OverrideMaxSlotFnPushCapKnobs), - target - .params(PushFeatureSwitchParams.OverrideMaxSlotFnNSlotKnobs), - target - .params(PushFeatureSwitchParams.OverrideMaxSlotFnPowerKnobs), - target - .params(PushFeatureSwitchParams.OverrideMaxSlotFnWeight), - target.params(FSParams.OverrideNotificationsMaxNumOfSlots) - ) - val numOfSlots = ContinuousFunction.safeEvaluateFn( - pushCap, - nslotFnParam, - overrideStats.scope("max_nslot_fn")) - overrideStats.counter("max_notification_slots_num_" + numOfSlots.toString).incr() - addedNSlotsToPayload.incr() - Map("max_notification_slots" -> numOfSlots) - } else { - addedNSlotsToPayload.incr() - val numOfSlots = target.params(FSParams.OverrideNotificationsMaxNumOfSlots) - Map("max_notification_slots" -> numOfSlots) - } - } else { - noNSlotsAddedToPayload.incr() - Map.empty - } - } - - val baseActionDetailsMap = Map("target_ids" -> impressionIdsSeq) - - val actionDetailsMap = - Map("action_details" -> (baseActionDetailsMap ++ nSlotsMap)) - - val baseSmartPushConfigMap = Map("notification_action" -> "REPLACE") - - val customThreadId = { - if (target.params(FSParams.EnableCustomThreadIdForOverride)) { - addedCustomThreadIdToPayload.incr() - Map("custom_thread_id" -> impressionId) - } else { - noCustomThreadIdAddedToPayload.incr() - Map.empty - } - } - - val smartPushConfigMap = - JsonMarshal.toJson( - baseSmartPushConfigMap ++ actionDetailsMap ++ mlScoreMap ++ customThreadId) - - Map("smart_notification_configuration" -> smartPushConfigMap) - } else if (primaryDeviceAndroid) { - addedOverrideAndroidCounter.incr() - Map("notification_id" -> collapseId, "overriding_impression_id" -> impressionIds.head) - } else { - noOverrideDueToDeviceInfoCounter.incr() - Map.empty[String, String] - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/PushOverrideInfo.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/PushOverrideInfo.scala deleted file mode 100644 index 359b876a1..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/PushOverrideInfo.scala +++ /dev/null @@ -1,246 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.history.History -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.frigate.thriftscala.OverrideInfo -import com.twitter.util.Duration -import com.twitter.util.Time - -object PushOverrideInfo { - - private val name: String = this.getClass.getSimpleName - - /** - * Gets all eligible time + override push notification pairs from a target's History - * - * @param history: history of push notifications - * @param lookbackDuration: duration to look back up in history for overriding notifications - * @return: list of notifications with send timestamps which are eligible for overriding - */ - def getOverrideEligibleHistory( - history: History, - lookbackDuration: Duration, - ): Seq[(Time, FrigateNotification)] = { - history.sortedHistory - .takeWhile { case (notifTimestamp, _) => lookbackDuration.ago < notifTimestamp } - .filter { - case (_, notification) => notification.overrideInfo.isDefined - } - } - - /** - * Gets all eligible override push notifications from a target's History - * - * @param history Target's History - * @param lookbackDuration Duration in which we would like to obtain the eligible push notifications - * @param stats StatsReceiver to track stats for this function - * @return Returns a list of FrigateNotification - */ - def getOverrideEligiblePushNotifications( - history: History, - lookbackDuration: Duration, - stats: StatsReceiver, - ): Seq[FrigateNotification] = { - val eligibleNotificationsDistribution = - stats.scope(name).stat("eligible_notifications_size_distribution") - val eligibleNotificationsSeq = - getOverrideEligibleHistory(history, lookbackDuration) - .collect { - case (_, notification) => notification - } - - eligibleNotificationsDistribution.add(eligibleNotificationsSeq.size) - eligibleNotificationsSeq - } - - /** - * Gets the OverrideInfo for the last eligible Override Notification FrigateNotification, if it exists - * @param history Target's History - * @param lookbackDuration Duration in which we would like to obtain the last override notification - * @param stats StatsReceiver to track stats for this function - * @return Returns OverrideInfo of the last MR push, else None - */ - def getOverrideInfoOfLastEligiblePushNotif( - history: History, - lookbackDuration: Duration, - stats: StatsReceiver - ): Option[OverrideInfo] = { - val overrideInfoEmptyOfLastPush = stats.scope(name).counter("override_info_empty_of_last_push") - val overrideInfoExistsForLastPush = - stats.scope(name).counter("override_info_exists_for_last_push") - val overrideHistory = - getOverrideEligiblePushNotifications(history, lookbackDuration, stats) - if (overrideHistory.isEmpty) { - overrideInfoEmptyOfLastPush.incr() - None - } else { - overrideInfoExistsForLastPush.incr() - overrideHistory.head.overrideInfo - } - } - - /** - * Gets all the MR Push Notifications in the specified override chain - * @param history Target's History - * @param overrideChainId Override Chain Identifier - * @param stats StatsReceiver to track stats for this function - * @return Returns a sequence of FrigateNotification that exist in the override chain - */ - def getMrPushNotificationsInOverrideChain( - history: History, - overrideChainId: String, - stats: StatsReceiver - ): Seq[FrigateNotification] = { - val notificationInOverrideChain = stats.scope(name).counter("notification_in_override_chain") - val notificationNotInOverrideChain = - stats.scope(name).counter("notification_not_in_override_chain") - history.sortedHistory.flatMap { - case (_, notification) - if isNotificationInOverrideChain(notification, overrideChainId, stats) => - notificationInOverrideChain.incr() - Some(notification) - case _ => - notificationNotInOverrideChain.incr() - None - } - } - - /** - * Gets the timestamp (in milliseconds) for the specified FrigateNotification - * @param notification The FrigateNotification that we would like the timestamp for - * @param history Target's History - * @param stats StatsReceiver to track stats for this function - * @return Returns the timestamp in milliseconds for the specified notification - * if it exists History, else None - */ - def getTimestampInMillisForFrigateNotification( - notification: FrigateNotification, - history: History, - stats: StatsReceiver - ): Option[Long] = { - val foundTimestampOfNotificationInHistory = - stats.scope(name).counter("found_timestamp_of_notification_in_history") - history.sortedHistory - .find(_._2.equals(notification)).map { - case (time, _) => - foundTimestampOfNotificationInHistory.incr() - time.inMilliseconds - } - } - - /** - * Gets the oldest frigate notification based on the user's NTab last read position - * @param overrideCandidatesMap All the NTab Notifications in the override chain - * @return Returns the oldest frigate notification in the chain - */ - def getOldestFrigateNotification( - overrideCandidatesMap: Map[Long, FrigateNotification], - ): FrigateNotification = { - overrideCandidatesMap.minBy(_._1)._2 - } - - /** - * Gets the impression ids of previous eligible push notification. - * @param history Target's History - * @param lookbackDuration Duration in which we would like to obtain previous impression ids - * @param stats StatsReceiver to track stats for this function - * @return Returns the impression identifier for the last eligible push notif. - * if it exists in the target's History, else None. - */ - def getImpressionIdsOfPrevEligiblePushNotif( - history: History, - lookbackDuration: Duration, - stats: StatsReceiver - ): Seq[String] = { - val foundImpressionIdOfLastEligiblePushNotif = - stats.scope(name).counter("found_impression_id_of_last_eligible_push_notif") - val overrideHistoryEmptyWhenFetchingImpressionId = - stats.scope(name).counter("override_history_empty_when_fetching_impression_id") - val overrideHistory = getOverrideEligiblePushNotifications(history, lookbackDuration, stats) - .filter(frigateNotification => - // Exclude notifications of nonGenericOverrideTypes from being overridden - !RecTypes.nonGenericOverrideTypes.contains(frigateNotification.commonRecommendationType)) - - if (overrideHistory.isEmpty) { - overrideHistoryEmptyWhenFetchingImpressionId.incr() - Seq.empty - } else { - foundImpressionIdOfLastEligiblePushNotif.incr() - overrideHistory.flatMap(_.impressionId) - } - } - - /** - * Gets the impressions ids by eventId, for MagicFanoutEvent candidates. - * - * @param history Target's History - * @param lookbackDuration Duration in which we would like to obtain previous impression ids - * @param stats StatsReceiver to track stats for this function - * @param overridableType Specific MagicFanoutEvent CRT - * @param eventId Event identifier for MagicFanoutEventCandidate. - * @return Returns the impression identifiers for the last eligible, eventId-matching - * MagicFanoutEvent push notifications if they exist in the target's history, else None. - */ - def getImpressionIdsForPrevEligibleMagicFanoutEventCandidates( - history: History, - lookbackDuration: Duration, - stats: StatsReceiver, - overridableType: CommonRecommendationType, - eventId: Long - ): Seq[String] = { - val foundImpressionIdOfMagicFanoutEventNotif = - stats.scope(name).counter("found_impression_id_of_magic_fanout_event_notif") - val overrideHistoryEmptyWhenFetchingImpressionId = - stats - .scope(name).counter( - "override_history_empty_when_fetching_impression_id_for_magic_fanout_event_notif") - - val overrideHistory = - getOverrideEligiblePushNotifications(history, lookbackDuration, stats) - .filter(frigateNotification => - // Only override notifications with same CRT and eventId - frigateNotification.commonRecommendationType == overridableType && - frigateNotification.magicFanoutEventNotification.exists(_.eventId == eventId)) - - if (overrideHistory.isEmpty) { - overrideHistoryEmptyWhenFetchingImpressionId.incr() - Seq.empty - } else { - foundImpressionIdOfMagicFanoutEventNotif.incr() - overrideHistory.flatMap(_.impressionId) - } - } - - /** - * Determines if the provided notification is part of the specified override chain - * @param notification FrigateNotification that we're trying to identify as within the override chain - * @param overrideChainId Override Chain Identifier - * @param stats StatsReceiver to track stats for this function - * @return Returns true if the provided FrigateNotification is within the override chain, else false - */ - private def isNotificationInOverrideChain( - notification: FrigateNotification, - overrideChainId: String, - stats: StatsReceiver - ): Boolean = { - val notifIsInOverrideChain = stats.scope(name).counter("notif_is_in_override_chain") - val notifNotInOverrideChain = stats.scope(name).counter("notif_not_in_override_chain") - notification.overrideInfo match { - case Some(overrideInfo) => - val isNotifInOverrideChain = overrideInfo.collapseInfo.overrideChainId == overrideChainId - if (isNotifInOverrideChain) { - notifIsInOverrideChain.incr() - true - } else { - notifNotInOverrideChain.incr() - false - } - case _ => - notifNotInOverrideChain.incr() - false - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/RankedSocialContextIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/RankedSocialContextIbis2Hydrator.scala deleted file mode 100644 index 479c230eb..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/RankedSocialContextIbis2Hydrator.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.common.base.SocialContextAction -import com.twitter.frigate.common.base.SocialContextActions -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.util.CandidateUtil -import com.twitter.frigate.pushservice.util.PushIbisUtil -import com.twitter.util.Future - -trait RankedSocialContextIbis2Hydrator { - self: PushCandidate with SocialContextActions => - - lazy val socialContextModelValues: Future[Map[String, String]] = - rankedSocialContextActionsFut.map(rankedSocialContextActions => - PushIbisUtil.getSocialContextModelValues(rankedSocialContextActions.map(_.userId))) - - lazy val rankedSocialContextActionsFut: Future[Seq[SocialContextAction]] = - CandidateUtil.getRankedSocialContext( - socialContextActions, - target.seedsWithWeight, - defaultToRecency = false) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ScheduledSpaceSpeakerIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ScheduledSpaceSpeakerIbis2Hydrator.scala deleted file mode 100644 index d1a439972..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ScheduledSpaceSpeakerIbis2Hydrator.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.pushservice.model.ScheduledSpaceSpeakerPushCandidate -import com.twitter.frigate.pushservice.util.PushIbisUtil._ -import com.twitter.frigate.thriftscala.SpaceNotificationType -import com.twitter.util.Future - -trait ScheduledSpaceSpeakerIbis2Hydrator extends Ibis2HydratorForCandidate { - self: ScheduledSpaceSpeakerPushCandidate => - - override lazy val senderId: Option[Long] = None - - private lazy val targetModelValues: Future[Map[String, String]] = { - hostId match { - case Some(spaceHostId) => - audioSpaceFut.map { audioSpace => - val isStartNow = frigateNotification.spaceNotification.exists( - _.spaceNotificationType.contains(SpaceNotificationType.AtSpaceBroadcast)) - - Map( - "host_id" -> s"$spaceHostId", - "space_id" -> spaceId, - "is_start_now" -> s"$isStartNow" - ) ++ audioSpace.flatMap(_.title.map("space_title" -> _)) - } - case _ => - Future.exception( - new IllegalStateException("Unable to get host id for ScheduledSpaceSpeakerIbis2Hydrator")) - } - } - - override lazy val modelValues: Future[Map[String, String]] = - mergeFutModelValues(super.modelValues, targetModelValues) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ScheduledSpaceSubscriberIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ScheduledSpaceSubscriberIbis2Hydrator.scala deleted file mode 100644 index b1486de3f..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ScheduledSpaceSubscriberIbis2Hydrator.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.pushservice.model.ScheduledSpaceSubscriberPushCandidate -import com.twitter.frigate.pushservice.util.PushIbisUtil._ -import com.twitter.util.Future - -trait ScheduledSpaceSubscriberIbis2Hydrator extends Ibis2HydratorForCandidate { - self: ScheduledSpaceSubscriberPushCandidate => - - override lazy val senderId: Option[Long] = hostId - - private lazy val targetModelValues: Future[Map[String, String]] = { - hostId match { - case Some(spaceHostId) => - audioSpaceFut.map { audioSpace => - Map( - "host_id" -> s"$spaceHostId", - "space_id" -> spaceId, - ) ++ audioSpace.flatMap(_.title.map("space_title" -> _)) - } - case _ => - Future.exception( - new RuntimeException("Unable to get host id for ScheduledSpaceSubscriberIbis2Hydrator")) - } - } - - override lazy val modelValues: Future[Map[String, String]] = - mergeFutModelValues(super.modelValues, targetModelValues) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/SubscribedSearchTweetIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/SubscribedSearchTweetIbis2Hydrator.scala deleted file mode 100644 index a61edc509..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/SubscribedSearchTweetIbis2Hydrator.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.pushservice.model.SubscribedSearchTweetPushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.util.InlineActionUtil -import com.twitter.util.Future - -trait SubscribedSearchTweetIbis2Hydrator extends TweetCandidateIbis2Hydrator { - self: SubscribedSearchTweetPushCandidate => - - override lazy val tweetDynamicInlineActionsModelValues = { - if (target.params(PushFeatureSwitchParams.EnableOONGeneratedInlineActions)) { - val actions = target.params(PushFeatureSwitchParams.TweetDynamicInlineActionsList) - InlineActionUtil.getGeneratedTweetInlineActions(target, statsReceiver, actions) - } else Map.empty[String, String] - } - - private lazy val searchTermValue: Map[String, String] = - Map( - "search_term" -> searchTerm, - "search_url" -> pushLandingUrl - ) - - private lazy val searchModelValues = searchTermValue ++ tweetDynamicInlineActionsModelValues - - override lazy val tweetModelValues: Future[Map[String, String]] = - for { - superModelValues <- super.tweetModelValues - tweetInlineModelValues <- tweetInlineActionModelValue - } yield { - superModelValues ++ mediaModelValue ++ searchModelValues ++ tweetInlineModelValues ++ inlineVideoMediaMap - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TopTweetImpressionsCandidateIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TopTweetImpressionsCandidateIbis2Hydrator.scala deleted file mode 100644 index e12733fb2..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TopTweetImpressionsCandidateIbis2Hydrator.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.common.base.TopTweetImpressionsCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.util.PushIbisUtil.mergeFutModelValues -import com.twitter.util.Future - -trait TopTweetImpressionsCandidateIbis2Hydrator extends Ibis2HydratorForCandidate { - self: PushCandidate with TopTweetImpressionsCandidate => - - private lazy val targetModelValues: Map[String, String] = { - Map( - "target_user" -> target.targetId.toString, - "tweet" -> tweetId.toString, - "impressions_count" -> impressionsCount.toString - ) - } - - override lazy val modelValues: Future[Map[String, String]] = - mergeFutModelValues(super.modelValues, Future.value(targetModelValues)) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TopicProofTweetIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TopicProofTweetIbis2Hydrator.scala deleted file mode 100644 index 6a187dfeb..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TopicProofTweetIbis2Hydrator.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.pushservice.model.TopicProofTweetPushCandidate -import com.twitter.frigate.pushservice.exception.UttEntityNotFoundException -import com.twitter.util.Future - -trait TopicProofTweetIbis2Hydrator extends TweetCandidateIbis2Hydrator { - self: TopicProofTweetPushCandidate => - - private lazy val implicitTopicTweetModelValues: Map[String, String] = { - val uttEntity = localizedUttEntity.getOrElse( - throw new UttEntityNotFoundException( - s"${getClass.getSimpleName} UttEntity missing for $tweetId")) - - Map( - "topic_name" -> uttEntity.localizedNameForDisplay, - "topic_id" -> uttEntity.entityId.toString - ) - } - - override lazy val modelName: String = pushCopy.ibisPushModelName - - override lazy val tweetModelValues: Future[Map[String, String]] = - for { - superModelValues <- super.tweetModelValues - tweetInlineModelValues <- tweetInlineActionModelValue - } yield { - superModelValues ++ - tweetInlineModelValues ++ - implicitTopicTweetModelValues - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TrendTweetIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TrendTweetIbis2Hydrator.scala deleted file mode 100644 index 1c3420df4..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TrendTweetIbis2Hydrator.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.common.base.TrendTweetCandidate -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate - -trait TrendTweetIbis2Hydrator extends TweetCandidateIbis2Hydrator { - self: PushCandidate with TrendTweetCandidate with TweetAuthorDetails => - - lazy val trendNameModelValue = Map("trend_name" -> trendName) - - override lazy val tweetModelValues = for { - tweetValues <- super.tweetModelValues - inlineActionValues <- tweetInlineActionModelValue - } yield tweetValues ++ inlineActionValues ++ trendNameModelValue -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetCandidateIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetCandidateIbis2Hydrator.scala deleted file mode 100644 index 0b0a5db05..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetCandidateIbis2Hydrator.scala +++ /dev/null @@ -1,166 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.common.base.TweetDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.SubtextForAndroidPushHeader -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} -import com.twitter.frigate.pushservice.util.CopyUtil -import com.twitter.frigate.pushservice.util.EmailLandingPageExperimentUtil -import com.twitter.frigate.pushservice.util.InlineActionUtil -import com.twitter.frigate.pushservice.util.PushToHomeUtil -import com.twitter.frigate.pushservice.util.PushIbisUtil.mergeFutModelValues -import com.twitter.util.Future - -trait TweetCandidateIbis2Hydrator - extends Ibis2HydratorForCandidate - with InlineActionIbis2Hydrator - with CustomConfigurationMapForIbis { - self: PushCandidate with TweetCandidate with TweetDetails with TweetAuthorDetails => - - lazy val scopedStats: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) - - lazy val tweetIdModelValue: Map[String, String] = - Map( - "tweet" -> tweetId.toString - ) - - lazy val authorModelValue: Map[String, String] = { - assert(authorId.isDefined) - Map( - "author" -> authorId.getOrElse(0L).toString - ) - } - - lazy val otherModelValues: Map[String, String] = - Map( - "show_explanatory_text" -> "true", - "show_negative_feedback" -> "true" - ) - - lazy val mediaModelValue: Map[String, String] = - Map( - "show_media" -> "true" - ) - - lazy val inlineVideoMediaMap: Map[String, String] = { - if (hasVideo) { - val isInlineVideoEnabled = target.params(FS.EnableInlineVideo) - val isAutoplayEnabled = target.params(FS.EnableAutoplayForInlineVideo) - Map( - "enable_inline_video_for_ios" -> isInlineVideoEnabled.toString, - "enable_autoplay_for_inline_video_ios" -> isAutoplayEnabled.toString - ) - } else Map.empty - } - - lazy val landingPageModelValues: Future[Map[String, String]] = { - for { - deviceInfoOpt <- target.deviceInfo - } yield { - PushToHomeUtil.getIbis2ModelValue(deviceInfoOpt, target, scopedStats) match { - case Some(pushToHomeModelValues) => pushToHomeModelValues - case _ => - EmailLandingPageExperimentUtil.getIbis2ModelValue( - deviceInfoOpt, - target, - tweetId - ) - } - } - } - - lazy val tweetDynamicInlineActionsModelValues = { - if (target.params(PushFeatureSwitchParams.EnableTweetDynamicInlineActions)) { - val actions = target.params(PushFeatureSwitchParams.TweetDynamicInlineActionsList) - InlineActionUtil.getGeneratedTweetInlineActions(target, statsReceiver, actions) - } else Map.empty[String, String] - } - - lazy val tweetDynamicInlineActionsModelValuesForWeb: Map[String, String] = { - if (target.isLoggedOutUser) { - Map.empty[String, String] - } else { - InlineActionUtil.getGeneratedTweetInlineActionsForWeb( - actions = target.params(PushFeatureSwitchParams.TweetDynamicInlineActionsListForWeb), - enableForDesktopWeb = - target.params(PushFeatureSwitchParams.EnableDynamicInlineActionsForDesktopWeb), - enableForMobileWeb = - target.params(PushFeatureSwitchParams.EnableDynamicInlineActionsForMobileWeb) - ) - } - } - - lazy val copyFeaturesFut: Future[Map[String, String]] = - CopyUtil.getCopyFeatures(self, scopedStats) - - private def getVerifiedSymbolModelValue: Future[Map[String, String]] = { - self.tweetAuthor.map { - case Some(author) => - if (author.safety.exists(_.verified)) { - scopedStats.counter("is_verified").incr() - if (target.params(FS.EnablePushPresentationVerifiedSymbol)) { - scopedStats.counter("is_verified_and_add").incr() - Map("is_author_verified" -> "true") - } else { - scopedStats.counter("is_verified_and_NOT_add").incr() - Map.empty - } - } else { - scopedStats.counter("is_NOT_verified").incr() - Map.empty - } - case _ => - scopedStats.counter("none_author").incr() - Map.empty - } - } - - private def subtextAndroidPushHeader: Map[String, String] = { - self.target.params(PushFeatureSwitchParams.SubtextInAndroidPushHeaderParam) match { - case SubtextForAndroidPushHeader.None => - Map.empty - case SubtextForAndroidPushHeader.TargetHandler => - Map("subtext_target_handler" -> "true") - case SubtextForAndroidPushHeader.TargetTagHandler => - Map("subtext_target_tag_handler" -> "true") - case SubtextForAndroidPushHeader.TargetName => - Map("subtext_target_name" -> "true") - case SubtextForAndroidPushHeader.AuthorTagHandler => - Map("subtext_author_tag_handler" -> "true") - case SubtextForAndroidPushHeader.AuthorName => - Map("subtext_author_name" -> "true") - case _ => - Map.empty - } - } - - lazy val bodyPushMap: Map[String, String] = { - if (self.target.params(PushFeatureSwitchParams.EnableEmptyBody)) { - Map("enable_empty_body" -> "true") - } else Map.empty[String, String] - } - - override def customFieldsMapFut: Future[Map[String, String]] = - for { - superModelValues <- super.customFieldsMapFut - copyFeaturesModelValues <- copyFeaturesFut - verifiedSymbolModelValue <- getVerifiedSymbolModelValue - } yield { - superModelValues ++ copyFeaturesModelValues ++ - verifiedSymbolModelValue ++ subtextAndroidPushHeader ++ bodyPushMap - } - - override lazy val senderId: Option[Long] = authorId - - def tweetModelValues: Future[Map[String, String]] = - landingPageModelValues.map { landingPageModelValues => - tweetIdModelValue ++ authorModelValue ++ landingPageModelValues ++ tweetDynamicInlineActionsModelValues ++ tweetDynamicInlineActionsModelValuesForWeb - } - - override lazy val modelValues: Future[Map[String, String]] = - mergeFutModelValues(super.modelValues, tweetModelValues) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetFavoriteIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetFavoriteIbis2Hydrator.scala deleted file mode 100644 index ae4cd9174..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetFavoriteIbis2Hydrator.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.base.TweetFavoriteCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.util.Future - -trait TweetFavoriteCandidateIbis2Hydrator - extends TweetCandidateIbis2Hydrator - with RankedSocialContextIbis2Hydrator { - self: PushCandidate with TweetFavoriteCandidate with TweetAuthorDetails => - - override lazy val tweetModelValues: Future[Map[String, String]] = - for { - socialContextModelValues <- socialContextModelValues - superModelValues <- super.tweetModelValues - tweetInlineModelValues <- tweetInlineActionModelValue - } yield { - superModelValues ++ mediaModelValue ++ otherModelValues ++ socialContextModelValues ++ tweetInlineModelValues - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetRetweetIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetRetweetIbis2Hydrator.scala deleted file mode 100644 index 2b665a8fa..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetRetweetIbis2Hydrator.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.base.TweetRetweetCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.util.PushIbisUtil.mergeModelValues - -import com.twitter.util.Future - -trait TweetRetweetCandidateIbis2Hydrator - extends TweetCandidateIbis2Hydrator - with RankedSocialContextIbis2Hydrator { - self: PushCandidate with TweetRetweetCandidate with TweetAuthorDetails => - - override lazy val tweetModelValues: Future[Map[String, String]] = - for { - socialContextModelValues <- socialContextModelValues - superModelValues <- super.tweetModelValues - tweetInlineModelValues <- tweetInlineActionModelValue - } yield { - superModelValues ++ mediaModelValue ++ otherModelValues ++ socialContextModelValues ++ tweetInlineModelValues ++ inlineVideoMediaMap - } - - lazy val socialContextForRetweetMap: Map[String, String] = - if (self.target.params(PushFeatureSwitchParams.EnableSocialContextForRetweet)) { - Map("enable_social_context_retweet" -> "true") - } else Map.empty[String, String] - - override lazy val customFieldsMapFut: Future[Map[String, String]] = - mergeModelValues(super.customFieldsMapFut, socialContextForRetweetMap) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/CandidateNTabCopy.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/CandidateNTabCopy.scala deleted file mode 100644 index ef80db5b5..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/CandidateNTabCopy.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.common.util.MRNtabCopy -import com.twitter.frigate.common.util.MrNtabCopyObjects -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.take.InvalidNtabCopyIdException -import com.twitter.frigate.pushservice.take.NtabCopyIdNotFoundException - -trait CandidateNTabCopy { - self: PushCandidate => - - def ntabCopy: MRNtabCopy = - ntabCopyId - .map(getNtabCopyFromCopyId).getOrElse( - throw new NtabCopyIdNotFoundException(s"NtabCopyId not found for $commonRecType")) - - private def getNtabCopyFromCopyId(ntabCopyId: Int): MRNtabCopy = - MrNtabCopyObjects - .getCopyFromId(ntabCopyId).getOrElse( - throw new InvalidNtabCopyIdException(s"Unknown NTab Copy ID: $ntabCopyId")) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/DiscoverTwitterNtabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/DiscoverTwitterNtabRequestHydrator.scala deleted file mode 100644 index 4d6d67893..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/DiscoverTwitterNtabRequestHydrator.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushConstants -import com.twitter.frigate.thriftscala.{CommonRecommendationType => CRT} -import com.twitter.notificationservice.thriftscala._ -import com.twitter.util.Future -import com.twitter.util.Time - -trait DiscoverTwitterNtabRequestHydrator extends NTabRequestHydrator { - self: PushCandidate => - - override val senderIdFut: Future[Long] = Future.value(0L) - - override val tapThroughFut: Future[String] = - commonRecType match { - case CRT.AddressBookUploadPush => Future.value(PushConstants.AddressBookUploadTapThrough) - case CRT.InterestPickerPush => Future.value(PushConstants.InterestPickerTapThrough) - case CRT.CompleteOnboardingPush => - Future.value(PushConstants.CompleteOnboardingInterestAddressTapThrough) - case _ => - Future.value(PushConstants.ConnectTabPushTapThrough) - } - - override val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = Future.Nil - - override val facepileUsersFut: Future[Seq[Long]] = Future.Nil - - override val storyContext: Option[StoryContext] = None - - override val inlineCard: Option[InlineCard] = None - - override val socialProofDisplayText: Option[DisplayText] = Some(DisplayText()) - - override lazy val ntabRequest: Future[Option[CreateGenericNotificationRequest]] = - if (self.commonRecType == CRT.ConnectTabPush || RecTypes.isOnboardingFlowType( - self.commonRecType)) { - Future.join(senderIdFut, displayTextEntitiesFut, facepileUsersFut, tapThroughFut).map { - case (senderId, displayTextEntities, facepileUsers, tapThrough) => - Some( - CreateGenericNotificationRequest( - userId = target.targetId, - senderId = senderId, - genericType = GenericType.RefreshableNotification, - displayText = DisplayText(values = displayTextEntities), - facepileUsers = facepileUsers, - timestampMillis = Time.now.inMillis, - tapThroughAction = Some(TapThroughAction(Some(tapThrough))), - impressionId = Some(impressionId), - socialProofText = socialProofDisplayText, - context = storyContext, - inlineCard = inlineCard, - refreshableType = refreshableType - )) - } - } else Future.None -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/EventNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/EventNTabRequestHydrator.scala deleted file mode 100644 index 082bc1742..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/EventNTabRequestHydrator.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.notificationservice.thriftscala.DisplayText -import com.twitter.notificationservice.thriftscala.InlineCard -import com.twitter.notificationservice.thriftscala.StoryContext -import com.twitter.util.Future - -trait EventNTabRequestHydrator extends NTabRequestHydrator { - self: PushCandidate => - - override def senderIdFut: Future[Long] = Future.value(0L) - - override def facepileUsersFut: Future[Seq[Long]] = Future.Nil - - override val storyContext: Option[StoryContext] = None - - override val inlineCard: Option[InlineCard] = None - - override val socialProofDisplayText: Option[DisplayText] = Some(DisplayText()) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/F1FirstDegreeTweetNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/F1FirstDegreeTweetNTabRequestHydrator.scala deleted file mode 100644 index 18662e257..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/F1FirstDegreeTweetNTabRequestHydrator.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.take.NotificationServiceSender -import com.twitter.notificationservice.thriftscala.DisplayTextEntity -import com.twitter.util.Future - -trait F1FirstDegreeTweetNTabRequestHydrator extends TweetNTabRequestHydrator { - self: PushCandidate with TweetCandidate with TweetAuthorDetails => - - override val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = - NotificationServiceSender.getDisplayTextEntityFromUser(tweetAuthor, "author", true).map(_.toSeq) - - override lazy val facepileUsersFut: Future[Seq[Long]] = senderIdFut.map(Seq(_)) - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/ListCandidateNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/ListCandidateNTabRequestHydrator.scala deleted file mode 100644 index 8475256ad..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/ListCandidateNTabRequestHydrator.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.pushservice.model.ListRecommendationPushCandidate -import com.twitter.notificationservice.thriftscala.DisplayText -import com.twitter.notificationservice.thriftscala.DisplayTextEntity -import com.twitter.notificationservice.thriftscala.InlineCard -import com.twitter.notificationservice.thriftscala.StoryContext -import com.twitter.notificationservice.thriftscala.TextValue -import com.twitter.util.Future - -trait ListCandidateNTabRequestHydrator extends NTabRequestHydrator { - - self: ListRecommendationPushCandidate => - - override lazy val senderIdFut: Future[Long] = - listOwnerId.map(_.getOrElse(0L)) - - override lazy val facepileUsersFut: Future[Seq[Long]] = Future.Nil - - override lazy val storyContext: Option[StoryContext] = None - - override lazy val inlineCard: Option[InlineCard] = None - - override lazy val tapThroughFut: Future[String] = Future.value(s"i/lists/${listId}") - - override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = listName.map { - listNameOpt => - listNameOpt.toSeq.map { name => - DisplayTextEntity(name = "title", value = TextValue.Text(name)) - } - } - - override val socialProofDisplayText: Option[DisplayText] = Some(DisplayText()) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutCreatorEventNtabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutCreatorEventNtabRequestHydrator.scala deleted file mode 100644 index a245769a6..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutCreatorEventNtabRequestHydrator.scala +++ /dev/null @@ -1,110 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.magic_events.thriftscala.CreatorFanoutType -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.MagicFanoutCreatorEventPushCandidate -import com.twitter.frigate.pushservice.take.NotificationServiceSender -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationRequest -import com.twitter.notificationservice.thriftscala.DisplayText -import com.twitter.notificationservice.thriftscala.DisplayTextEntity -import com.twitter.notificationservice.thriftscala.GenericType -import com.twitter.notificationservice.thriftscala.InlineCard -import com.twitter.notificationservice.thriftscala.StoryContext -import com.twitter.notificationservice.thriftscala.TextValue -import com.twitter.notificationservice.thriftscala.TapThroughAction -import com.twitter.util.Future -import com.twitter.util.Time - -trait MagicFanoutCreatorEventNtabRequestHydrator extends NTabRequestHydrator { - self: PushCandidate with MagicFanoutCreatorEventPushCandidate => - - override val senderIdFut: Future[Long] = Future.value(creatorId) - - override lazy val tapThroughFut: Future[String] = - Future.value(s"/${userProfile.screenName}/superfollows/subscribe") - - lazy val optionalTweetCountEntityFut: Future[Option[DisplayTextEntity]] = { - creatorFanoutType match { - case CreatorFanoutType.UserSubscription => - numberOfTweetsFut.map { - _.flatMap { - case numberOfTweets if numberOfTweets >= 10 => - Some( - DisplayTextEntity( - name = "tweet_count", - emphasis = true, - value = TextValue.Text(numberOfTweets.toString))) - case _ => None - } - } - case _ => Future.None - } - } - - override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = - optionalTweetCountEntityFut - .map { tweetCountOpt => - Seq( - NotificationServiceSender - .getDisplayTextEntityFromUser(hydratedCreator, "display_name", isBold = true), - tweetCountOpt).flatten - } - - override lazy val facepileUsersFut: Future[Seq[Long]] = Future.value(Seq(creatorId)) - - override val storyContext: Option[StoryContext] = None - - override val inlineCard: Option[InlineCard] = None - - lazy val refreshableTypeFut = { - creatorFanoutType match { - case CreatorFanoutType.UserSubscription => - numberOfTweetsFut.map { - _.flatMap { - case numberOfTweets if numberOfTweets >= 10 => - Some("MagicFanoutCreatorSubscriptionWithTweets") - case _ => super.refreshableType - } - } - case _ => Future.value(super.refreshableType) - } - } - - override lazy val socialProofDisplayText: Option[DisplayText] = { - creatorFanoutType match { - case CreatorFanoutType.UserSubscription => - Some( - DisplayText(values = Seq( - DisplayTextEntity(name = "handle", value = TextValue.Text(userProfile.screenName))))) - case CreatorFanoutType.NewCreator => None - case _ => None - } - } - - override lazy val ntabRequest = { - Future - .join( - senderIdFut, - displayTextEntitiesFut, - facepileUsersFut, - tapThroughFut, - refreshableTypeFut).map { - case (senderId, displayTextEntities, facepileUsers, tapThrough, refreshableTypeOpt) => - Some( - CreateGenericNotificationRequest( - userId = target.targetId, - senderId = senderId, - genericType = GenericType.RefreshableNotification, - displayText = DisplayText(values = displayTextEntities), - facepileUsers = facepileUsers, - timestampMillis = Time.now.inMillis, - tapThroughAction = Some(TapThroughAction(Some(tapThrough))), - impressionId = Some(impressionId), - socialProofText = socialProofDisplayText, - context = storyContext, - inlineCard = inlineCard, - refreshableType = refreshableTypeOpt - )) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutNewsEventNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutNewsEventNTabRequestHydrator.scala deleted file mode 100644 index 202533e3c..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutNewsEventNTabRequestHydrator.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.MagicFanoutEventHydratedCandidate -import com.twitter.notificationservice.thriftscala.DisplayTextEntity -import com.twitter.notificationservice.thriftscala.TextValue -import com.twitter.util.Future - -trait MagicFanoutNewsEventNTabRequestHydrator extends EventNTabRequestHydrator { - self: PushCandidate with MagicFanoutEventHydratedCandidate => - override lazy val tapThroughFut: Future[String] = Future.value(s"i/events/$eventId") - override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = - eventTitleFut.map { eventTitle => - Seq(DisplayTextEntity(name = "title", value = TextValue.Text(eventTitle))) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutProductLaunchNtabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutProductLaunchNtabRequestHydrator.scala deleted file mode 100644 index 797dbe890..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutProductLaunchNtabRequestHydrator.scala +++ /dev/null @@ -1,97 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.common.base.MagicFanoutProductLaunchCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.notificationservice.thriftscala._ -import com.twitter.util.Future -import com.twitter.util.Time - -trait MagicFanoutProductLaunchNtabRequestHydrator extends NTabRequestHydrator { - self: PushCandidate with MagicFanoutProductLaunchCandidate => - - override val senderIdFut: Future[Long] = Future.value(0L) - - override lazy val tapThroughFut: Future[String] = Future.value(getProductLaunchTapThrough()) - - override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = { - Future.value( - frigateNotification.magicFanoutProductLaunchNotification - .flatMap { - _.productInfo.flatMap { - _.body.map { body => - Seq( - DisplayTextEntity(name = "body", value = TextValue.Text(body)), - ) - } - } - }.getOrElse(Nil)) - } - - override lazy val facepileUsersFut: Future[Seq[Long]] = { - Future.value( - frigateNotification.magicFanoutProductLaunchNotification - .flatMap { - _.productInfo.flatMap { - _.facepileUsers - } - }.getOrElse(Nil)) - } - - override val storyContext: Option[StoryContext] = None - - override val inlineCard: Option[InlineCard] = None - - override lazy val socialProofDisplayText: Option[DisplayText] = { - frigateNotification.magicFanoutProductLaunchNotification.flatMap { - _.productInfo.flatMap { - _.title.map { title => - DisplayText(values = - Seq(DisplayTextEntity(name = "social_context", value = TextValue.Text(title)))) - } - } - } - } - - lazy val defaultTapThrough = target.params(PushFeatureSwitchParams.ProductLaunchTapThrough) - - private def getProductLaunchTapThrough(): String = { - frigateNotification.magicFanoutProductLaunchNotification match { - case Some(productLaunchNotif) => - productLaunchNotif.productInfo match { - case Some(productInfo) => productInfo.tapThrough.getOrElse(defaultTapThrough) - case _ => defaultTapThrough - } - case _ => defaultTapThrough - } - } - - private lazy val productLaunchNtabRequest: Future[Option[CreateGenericNotificationRequest]] = { - Future - .join(senderIdFut, displayTextEntitiesFut, facepileUsersFut, tapThroughFut) - .map { - case (senderId, displayTextEntities, facepileUsers, tapThrough) => - Some( - CreateGenericNotificationRequest( - userId = target.targetId, - senderId = senderId, - genericType = GenericType.RefreshableNotification, - displayText = DisplayText(values = displayTextEntities), - facepileUsers = facepileUsers, - timestampMillis = Time.now.inMillis, - tapThroughAction = Some(TapThroughAction(Some(tapThrough))), - impressionId = Some(impressionId), - socialProofText = socialProofDisplayText, - context = storyContext, - inlineCard = inlineCard, - refreshableType = refreshableType - )) - } - } - - override lazy val ntabRequest: Future[Option[CreateGenericNotificationRequest]] = { - if (target.params(PushFeatureSwitchParams.EnableNTabEntriesForProductLaunchNotifications)) { - productLaunchNtabRequest - } else Future.None - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutSportsEventNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutSportsEventNTabRequestHydrator.scala deleted file mode 100644 index ca3d9faf0..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutSportsEventNTabRequestHydrator.scala +++ /dev/null @@ -1,95 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.common.base.MagicFanoutSportsEventCandidate -import com.twitter.frigate.common.base.MagicFanoutSportsScoreInformation -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.MagicFanoutEventHydratedCandidate -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationRequest -import com.twitter.notificationservice.thriftscala.DisplayText -import com.twitter.notificationservice.thriftscala.DisplayTextEntity -import com.twitter.notificationservice.thriftscala.GenericType -import com.twitter.notificationservice.thriftscala.TextValue -import com.twitter.notificationservice.thriftscala.TapThroughAction -import com.twitter.util.Future -import com.twitter.util.Time - -trait MagicFanoutSportsEventNTabRequestHydrator extends EventNTabRequestHydrator { - self: PushCandidate - with MagicFanoutEventHydratedCandidate - with MagicFanoutSportsEventCandidate - with MagicFanoutSportsScoreInformation => - - lazy val stats = self.statsReceiver.scope("MagicFanoutSportsEventNtabHydrator") - lazy val inNetworkOnlyCounter = stats.counter("in_network_only") - lazy val facePilesEnabledCounter = stats.counter("face_piles_enabled") - lazy val facePilesDisabledCounter = stats.counter("face_piles_disabled") - lazy val filterPeopleWhoDontFollowMeCounter = stats.counter("pepole_who_dont_follow_me_counter") - - override lazy val tapThroughFut: Future[String] = { - Future.value(s"i/events/$eventId") - } - override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = - eventTitleFut.map { eventTitle => - Seq(DisplayTextEntity(name = "title", value = TextValue.Text(eventTitle))) - } - - override lazy val facepileUsersFut: Future[Seq[Long]] = - if (target.params(FS.EnableNTabFacePileForSportsEventNotifications)) { - Future - .join( - target.notificationsFromOnlyPeopleIFollow, - target.filterNotificationsFromPeopleThatDontFollowMe, - awayTeamInfo, - homeTeamInfo).map { - case (inNetworkOnly, filterPeopleWhoDontFollowMe, away, home) - if !(inNetworkOnly || filterPeopleWhoDontFollowMe) => - val awayTeamId = away.flatMap(_.twitterUserId) - val homeTeamId = home.flatMap(_.twitterUserId) - facePilesEnabledCounter.incr - Seq(awayTeamId, homeTeamId).flatten - case (inNetworkOnly, filterPeopleWhoDontFollowMe, _, _) => - facePilesDisabledCounter.incr - if (inNetworkOnly) inNetworkOnlyCounter.incr - if (filterPeopleWhoDontFollowMe) filterPeopleWhoDontFollowMeCounter.incr - Seq.empty[Long] - } - } else Future.Nil - - private lazy val sportsNtabRequest: Future[Option[CreateGenericNotificationRequest]] = { - Future - .join(senderIdFut, displayTextEntitiesFut, facepileUsersFut, tapThroughFut) - .map { - case (senderId, displayTextEntities, facepileUsers, tapThrough) => - Some( - CreateGenericNotificationRequest( - userId = target.targetId, - senderId = senderId, - genericType = GenericType.RefreshableNotification, - displayText = DisplayText(values = displayTextEntities), - facepileUsers = facepileUsers, - timestampMillis = Time.now.inMillis, - tapThroughAction = Some(TapThroughAction(Some(tapThrough))), - impressionId = Some(impressionId), - socialProofText = socialProofDisplayText, - context = storyContext, - inlineCard = inlineCard, - refreshableType = refreshableType - )) - } - } - - override lazy val ntabRequest: Future[Option[CreateGenericNotificationRequest]] = { - if (target.params(FS.EnableNTabEntriesForSportsEventNotifications)) { - self.target.history.flatMap { pushHistory => - val prevEventHistoryExists = pushHistory.sortedHistory.exists { - case (_, notification) => - notification.magicFanoutEventNotification.exists(_.eventId == self.eventId) - } - if (prevEventHistoryExists) { - Future.None - } else sportsNtabRequest - } - } else Future.None - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabRequest.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabRequest.scala deleted file mode 100644 index ea99ea68d..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabRequest.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationRequest -import com.twitter.util.Future - -trait NTabRequest { - - def ntabRequest: Future[Option[CreateGenericNotificationRequest]] - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabRequestHydrator.scala deleted file mode 100644 index 01df5365f..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabRequestHydrator.scala +++ /dev/null @@ -1,64 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationRequest -import com.twitter.notificationservice.thriftscala.DisplayText -import com.twitter.notificationservice.thriftscala.DisplayTextEntity -import com.twitter.notificationservice.thriftscala.GenericType -import com.twitter.notificationservice.thriftscala.InlineCard -import com.twitter.notificationservice.thriftscala.StoryContext -import com.twitter.notificationservice.thriftscala.TapThroughAction -import com.twitter.util.Future -import com.twitter.util.Time - -trait NTabRequestHydrator extends NTabRequest with CandidateNTabCopy { - self: PushCandidate => - - // Represents the sender of the recommendation - def senderIdFut: Future[Long] - - // Consists of a sequence representing the social context user ids. - def facepileUsersFut: Future[Seq[Long]] - - // Story Context is required for Tweet Recommendations - // Contains the Tweet ID of the recommended Tweet - def storyContext: Option[StoryContext] - - // Inline card used to render a generic notification. - def inlineCard: Option[InlineCard] - - // Represents where the recommendation should land when clicked - def tapThroughFut: Future[String] - - // Hydration for fields that are used within the NTab copy - def displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] - - // Represents the social proof text that is needed for specific NTab copies - def socialProofDisplayText: Option[DisplayText] - - // MagicRecs NTab entries always use RefreshableType as the Generic Type - final val genericType: GenericType = GenericType.RefreshableNotification - - def refreshableType: Option[String] = ntabCopy.refreshableType - - lazy val ntabRequest: Future[Option[CreateGenericNotificationRequest]] = { - Future.join(senderIdFut, displayTextEntitiesFut, facepileUsersFut, tapThroughFut).map { - case (senderId, displayTextEntities, facepileUsers, tapThrough) => - Some( - CreateGenericNotificationRequest( - userId = target.targetId, - senderId = senderId, - genericType = GenericType.RefreshableNotification, - displayText = DisplayText(values = displayTextEntities), - facepileUsers = facepileUsers, - timestampMillis = Time.now.inMillis, - tapThroughAction = Some(TapThroughAction(Some(tapThrough))), - impressionId = Some(impressionId), - socialProofText = socialProofDisplayText, - context = storyContext, - inlineCard = inlineCard, - refreshableType = refreshableType - )) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabSocialContext.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabSocialContext.scala deleted file mode 100644 index 17b43f457..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabSocialContext.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.common.base.SocialContextActions -import com.twitter.frigate.common.base.SocialContextUserDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.util.CandidateUtil -import com.twitter.util.Future - -trait NTabSocialContext { - self: PushCandidate with SocialContextActions with SocialContextUserDetails => - - private def ntabDisplayUserIds: Seq[Long] = - socialContextUserIds.take(ntabDisplayUserIdsLength) - - def ntabDisplayUserIdsLength: Int = - if (socialContextUserIds.size == 2) 2 else 1 - - def ntabDisplayNamesAndIds: Future[Seq[(String, Long)]] = - scUserMap.map { userObjMap => - ntabDisplayUserIds.flatMap { id => - userObjMap(id).flatMap(_.profile.map(_.name)).map { name => (name, id) } - } - } - - def rankedNtabDisplayNamesAndIds(defaultToRecency: Boolean): Future[Seq[(String, Long)]] = - scUserMap.flatMap { userObjMap => - val rankedSocialContextActivityFut = - CandidateUtil.getRankedSocialContext( - socialContextActions, - target.seedsWithWeight, - defaultToRecency) - rankedSocialContextActivityFut.map { rankedSocialContextActivity => - val ntabDisplayUserIds = - rankedSocialContextActivity.map(_.userId).take(ntabDisplayUserIdsLength) - ntabDisplayUserIds.flatMap { id => - userObjMap(id).flatMap(_.profile.map(_.name)).map { name => (name, id) } - } - } - } - - def otherCount: Future[Int] = - ntabDisplayNamesAndIds.map { - case namesWithIdSeq => - Math.max(0, socialContextUserIds.length - namesWithIdSeq.size) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/OutOfNetworkTweetNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/OutOfNetworkTweetNTabRequestHydrator.scala deleted file mode 100644 index a2b99d1af..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/OutOfNetworkTweetNTabRequestHydrator.scala +++ /dev/null @@ -1,78 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.common.base.TopicCandidate -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.common.base.TweetDetails -import com.twitter.frigate.common.rec_types.RecTypes._ -import com.twitter.frigate.common.util.MrNtabCopyObjects -import com.twitter.frigate.pushservice.exception.TweetNTabRequestHydratorException -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.take.NotificationServiceSender -import com.twitter.notificationservice.thriftscala.DisplayText -import com.twitter.notificationservice.thriftscala.DisplayTextEntity -import com.twitter.notificationservice.thriftscala.TextValue -import com.twitter.util.Future - -trait OutOfNetworkTweetNTabRequestHydrator extends TweetNTabRequestHydrator { - self: PushCandidate - with TweetCandidate - with TweetAuthorDetails - with TopicCandidate - with TweetDetails => - - lazy val useTopicCopyForMBCGNtab = mrModelingBasedTypes.contains(commonRecType) && target.params( - PushFeatureSwitchParams.EnableMrModelingBasedCandidatesTopicCopy) - lazy val useTopicCopyForFrsNtab = frsTypes.contains(commonRecType) && target.params( - PushFeatureSwitchParams.EnableFrsTweetCandidatesTopicCopy) - lazy val useTopicCopyForTagspaceNtab = tagspaceTypes.contains(commonRecType) && target.params( - PushFeatureSwitchParams.EnableHashspaceCandidatesTopicCopy) - - override lazy val tapThroughFut: Future[String] = { - if (hasVideo && self.target.params( - PushFeatureSwitchParams.EnableLaunchVideosInImmersiveExplore)) { - Future.value( - s"i/immersive_timeline?display_location=notification&include_pinned_tweet=true&pinned_tweet_id=${tweetId}&tl_type=imv") - } else { - tweetAuthor.map { - case Some(author) => - val authorProfile = author.profile.getOrElse( - throw new TweetNTabRequestHydratorException( - s"Unable to obtain author profile for: ${author.id}")) - s"${authorProfile.screenName}/status/${tweetId.toString}" - case _ => - throw new TweetNTabRequestHydratorException( - s"Unable to obtain author and target details to generate tap through for Tweet: $tweetId") - } - } - } - - override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = - if (localizedUttEntity.isDefined && - (useTopicCopyForMBCGNtab || useTopicCopyForFrsNtab || useTopicCopyForTagspaceNtab)) { - NotificationServiceSender - .getDisplayTextEntityFromUser(tweetAuthor, "tweetAuthorName", isBold = true).map(_.toSeq) - } else { - NotificationServiceSender - .getDisplayTextEntityFromUser(tweetAuthor, "author", isBold = true).map(_.toSeq) - } - - override lazy val refreshableType: Option[String] = { - if (localizedUttEntity.isDefined && - (useTopicCopyForMBCGNtab || useTopicCopyForFrsNtab || useTopicCopyForTagspaceNtab)) { - MrNtabCopyObjects.TopicTweet.refreshableType - } else ntabCopy.refreshableType - } - - override def socialProofDisplayText: Option[DisplayText] = { - if (localizedUttEntity.isDefined && - (useTopicCopyForMBCGNtab || useTopicCopyForFrsNtab || useTopicCopyForTagspaceNtab)) { - localizedUttEntity.map(uttEntity => - DisplayText(values = - Seq(DisplayTextEntity("topic_name", TextValue.Text(uttEntity.localizedNameForDisplay))))) - } else None - } - - override lazy val facepileUsersFut: Future[Seq[Long]] = senderIdFut.map(Seq(_)) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/ScheduledSpaceNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/ScheduledSpaceNTabRequestHydrator.scala deleted file mode 100644 index 4673a001e..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/ScheduledSpaceNTabRequestHydrator.scala +++ /dev/null @@ -1,106 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.common.base.SpaceCandidate -import com.twitter.frigate.common.util.MrNtabCopyObjects -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.ScheduledSpaceSpeakerPushCandidate -import com.twitter.frigate.pushservice.model.ScheduledSpaceSubscriberPushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.take.NotificationServiceSender -import com.twitter.frigate.thriftscala.SpaceNotificationType -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.notificationservice.thriftscala._ -import com.twitter.util.Future -import com.twitter.util.Time - -trait ScheduledSpaceSpeakerNTabRequestHydrator extends ScheduledSpaceNTabRequestHydrator { - self: PushCandidate with ScheduledSpaceSpeakerPushCandidate => - - override def refreshableType: Option[String] = { - frigateNotification.spaceNotification.flatMap { spaceNotification => - spaceNotification.spaceNotificationType.flatMap { - case SpaceNotificationType.PreSpaceBroadcast => - MrNtabCopyObjects.ScheduledSpaceSpeakerSoon.refreshableType - case SpaceNotificationType.AtSpaceBroadcast => - MrNtabCopyObjects.ScheduledSpaceSpeakerNow.refreshableType - case _ => - throw new IllegalStateException(s"Unexpected SpaceNotificationType") - } - } - } - - override lazy val facepileUsersFut: Future[Seq[Long]] = Future.Nil - - override val socialProofDisplayText: Option[DisplayText] = Some(DisplayText()) -} - -trait ScheduledSpaceSubscriberNTabRequestHydrator extends ScheduledSpaceNTabRequestHydrator { - self: PushCandidate with ScheduledSpaceSubscriberPushCandidate => - - override lazy val facepileUsersFut: Future[Seq[Long]] = { - hostId match { - case Some(spaceHostId) => Future.value(Seq(spaceHostId)) - case _ => - Future.exception( - new IllegalStateException( - "Unable to get host id for ScheduledSpaceSubscriberNTabRequestHydrator")) - } - } - - override val socialProofDisplayText: Option[DisplayText] = None -} - -trait ScheduledSpaceNTabRequestHydrator extends NTabRequestHydrator { - self: PushCandidate with SpaceCandidate => - - def hydratedHost: Option[User] - - override lazy val senderIdFut: Future[Long] = { - hostId match { - case Some(spaceHostId) => Future.value(spaceHostId) - case _ => throw new IllegalStateException(s"No Space Host Id") - } - } - - override lazy val tapThroughFut: Future[String] = Future.value(s"i/spaces/$spaceId") - - override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = - NotificationServiceSender - .getDisplayTextEntityFromUser( - Future.value(hydratedHost), - fieldName = "space_host_name", - isBold = true - ).map(_.toSeq) - - override val storyContext: Option[StoryContext] = None - - override val inlineCard: Option[InlineCard] = None - - override lazy val ntabRequest: Future[Option[CreateGenericNotificationRequest]] = { - Future.join(senderIdFut, displayTextEntitiesFut, facepileUsersFut, tapThroughFut).map { - case (senderId, displayTextEntities, facepileUsers, tapThrough) => - val expiryTimeMillis = if (target.params(PushFeatureSwitchParams.EnableSpacesTtlForNtab)) { - Some( - (Time.now + target.params( - PushFeatureSwitchParams.SpaceNotificationsTTLDurationForNTab)).inMillis) - } else None - - Some( - CreateGenericNotificationRequest( - userId = target.targetId, - senderId = senderId, - genericType = GenericType.RefreshableNotification, - displayText = DisplayText(values = displayTextEntities), - facepileUsers = facepileUsers, - timestampMillis = Time.now.inMillis, - tapThroughAction = Some(TapThroughAction(Some(tapThrough))), - impressionId = Some(impressionId), - socialProofText = socialProofDisplayText, - context = storyContext, - inlineCard = inlineCard, - refreshableType = refreshableType, - expiryTimeMillis = expiryTimeMillis - )) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/SubscribedSearchTweetNtabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/SubscribedSearchTweetNtabRequestHydrator.scala deleted file mode 100644 index caa2a8cd0..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/SubscribedSearchTweetNtabRequestHydrator.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.pushservice.model.SubscribedSearchTweetPushCandidate -import com.twitter.frigate.pushservice.take.NotificationServiceSender -import com.twitter.notificationservice.thriftscala.DisplayText -import com.twitter.notificationservice.thriftscala.DisplayTextEntity -import com.twitter.notificationservice.thriftscala.TextValue -import com.twitter.util.Future - -trait SubscribedSearchTweetNtabRequestHydrator extends TweetNTabRequestHydrator { - self: SubscribedSearchTweetPushCandidate => - override def displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = NotificationServiceSender - .getDisplayTextEntityFromUser(tweetAuthor, "tweetAuthor", isBold = true).map(_.toSeq) - - override def socialProofDisplayText: Option[DisplayText] = { - Some(DisplayText(values = Seq(DisplayTextEntity("search_query", TextValue.Text(searchTerm))))) - } - - override lazy val facepileUsersFut: Future[Seq[Long]] = senderIdFut.map(Seq(_)) - - override lazy val tapThroughFut: Future[String] = - Future.value(self.ntabLandingUrl) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TopTweetImpressionsNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TopTweetImpressionsNTabRequestHydrator.scala deleted file mode 100644 index a67dee399..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TopTweetImpressionsNTabRequestHydrator.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.common.base.TopTweetImpressionsCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.notificationservice.thriftscala.DisplayText -import com.twitter.notificationservice.thriftscala.DisplayTextEntity -import com.twitter.notificationservice.thriftscala.InlineCard -import com.twitter.notificationservice.thriftscala.StoryContext -import com.twitter.notificationservice.thriftscala.StoryContextValue -import com.twitter.notificationservice.thriftscala.TextValue -import com.twitter.util.Future - -trait TopTweetImpressionsNTabRequestHydrator extends NTabRequestHydrator { - self: PushCandidate with TopTweetImpressionsCandidate => - - override lazy val tapThroughFut: Future[String] = - Future.value(s"${target.targetId}/status/$tweetId") - - override val senderIdFut: Future[Long] = Future.value(0L) - - override val facepileUsersFut: Future[Seq[Long]] = Future.Nil - - override val storyContext: Option[StoryContext] = - Some(StoryContext(altText = "", value = Some(StoryContextValue.Tweets(Seq(tweetId))))) - - override val inlineCard: Option[InlineCard] = None - - override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = { - Future.value( - Seq( - DisplayTextEntity(name = "num_impressions", value = TextValue.Number(self.impressionsCount)) - ) - ) - } - - override def socialProofDisplayText: Option[DisplayText] = None -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TopicProofTweetNtabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TopicProofTweetNtabRequestHydrator.scala deleted file mode 100644 index 17519efda..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TopicProofTweetNtabRequestHydrator.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.pushservice.model.TopicProofTweetPushCandidate -import com.twitter.frigate.pushservice.exception.TweetNTabRequestHydratorException -import com.twitter.frigate.pushservice.exception.UttEntityNotFoundException -import com.twitter.frigate.pushservice.take.NotificationServiceSender -import com.twitter.notificationservice.thriftscala.DisplayText -import com.twitter.notificationservice.thriftscala.DisplayTextEntity -import com.twitter.notificationservice.thriftscala.StoryContext -import com.twitter.notificationservice.thriftscala.StoryContextValue -import com.twitter.notificationservice.thriftscala.TextValue -import com.twitter.util.Future - -trait TopicProofTweetNtabRequestHydrator extends NTabRequestHydrator { - self: TopicProofTweetPushCandidate => - - override def displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = NotificationServiceSender - .getDisplayTextEntityFromUser(tweetAuthor, "tweetAuthorName", true) - .map(_.toSeq) - - private lazy val uttEntity = localizedUttEntity.getOrElse( - throw new UttEntityNotFoundException( - s"${getClass.getSimpleName} UttEntity missing for $tweetId") - ) - - override lazy val tapThroughFut: Future[String] = { - tweetAuthor.map { - case Some(author) => - val authorProfile = author.profile.getOrElse( - throw new TweetNTabRequestHydratorException( - s"Unable to obtain author profile for: ${author.id}")) - s"${authorProfile.screenName}/status/${tweetId.toString}" - case _ => - throw new TweetNTabRequestHydratorException( - s"Unable to obtain author and target details to generate tap through for Tweet: $tweetId") - } - } - - override lazy val socialProofDisplayText: Option[DisplayText] = { - Some( - DisplayText(values = - Seq(DisplayTextEntity("topic_name", TextValue.Text(uttEntity.localizedNameForDisplay)))) - ) - } - - override lazy val facepileUsersFut: Future[Seq[Long]] = senderIdFut.map(Seq(_)) - - override val inlineCard = None - - override def storyContext: Option[StoryContext] = Some( - StoryContext("", Some(StoryContextValue.Tweets(Seq(tweetId))))) - - override def senderIdFut: Future[Long] = - tweetAuthor.map { - case Some(author) => author.id - case _ => - throw new TweetNTabRequestHydratorException( - s"Unable to obtain Author ID for: $commonRecType") - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TrendTweetNtabHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TrendTweetNtabHydrator.scala deleted file mode 100644 index 07946a220..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TrendTweetNtabHydrator.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.common.base.TrendTweetCandidate -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.pushservice.exception.TweetNTabRequestHydratorException -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.take.NotificationServiceSender -import com.twitter.frigate.pushservice.util.EmailLandingPageExperimentUtil -import com.twitter.notificationservice.thriftscala.DisplayText -import com.twitter.notificationservice.thriftscala.DisplayTextEntity -import com.twitter.notificationservice.thriftscala.TextValue -import com.twitter.util.Future - -trait TrendTweetNtabHydrator extends TweetNTabRequestHydrator { - self: PushCandidate with TrendTweetCandidate with TweetCandidate with TweetAuthorDetails => - - private lazy val trendTweetNtabStats = self.statsReceiver.scope("trend_tweet_ntab") - - private lazy val ruxLandingOnNtabCounter = - trendTweetNtabStats.counter("use_rux_landing_on_ntab") - - override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = - NotificationServiceSender - .getDisplayTextEntityFromUser(tweetAuthor, fieldName = "author_name", isBold = true) - .map( - _.toSeq :+ DisplayTextEntity( - name = "trend_name", - value = TextValue.Text(trendName), - emphasis = true) - ) - - override lazy val facepileUsersFut: Future[Seq[Long]] = senderIdFut.map(Seq(_)) - - override lazy val socialProofDisplayText: Option[DisplayText] = None - - override def refreshableType: Option[String] = ntabCopy.refreshableType - - override lazy val tapThroughFut: Future[String] = { - Future.join(tweetAuthor, target.deviceInfo).map { - case (Some(author), Some(deviceInfo)) => - val enableRuxLandingPage = deviceInfo.isRuxLandingPageEligible && target.params( - PushFeatureSwitchParams.EnableNTabRuxLandingPage) - val authorProfile = author.profile.getOrElse( - throw new TweetNTabRequestHydratorException( - s"Unable to obtain author profile for: ${author.id}")) - - if (enableRuxLandingPage) { - ruxLandingOnNtabCounter.incr() - EmailLandingPageExperimentUtil.createNTabRuxLandingURI(authorProfile.screenName, tweetId) - } else { - s"${authorProfile.screenName}/status/${tweetId.toString}" - } - - case _ => - throw new TweetNTabRequestHydratorException( - s"Unable to obtain author and target details to generate tap through for Tweet: $tweetId") - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetFavoriteNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetFavoriteNTabRequestHydrator.scala deleted file mode 100644 index 52a643b84..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetFavoriteNTabRequestHydrator.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.common.base.SocialContextActions -import com.twitter.frigate.common.base.SocialContextUserDetails -import com.twitter.frigate.common.base.TweetAuthor -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.take.NotificationServiceSender -import com.twitter.notificationservice.thriftscala.DisplayTextEntity -import com.twitter.util.Future - -trait TweetFavoriteNTabRequestHydrator extends TweetNTabRequestHydrator with NTabSocialContext { - self: PushCandidate - with TweetCandidate - with TweetAuthor - with TweetAuthorDetails - with SocialContextActions - with SocialContextUserDetails => - - override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = { - Future - .join( - NotificationServiceSender - .getDisplayTextEntityFromUser(tweetAuthor, "tweetAuthorName", isBold = false), - NotificationServiceSender - .generateSocialContextTextEntities( - rankedNtabDisplayNamesAndIds(defaultToRecency = false), - otherCount) - ) - .map { - case (authorDisplay, socialContextDisplay) => - socialContextDisplay ++ authorDisplay - } - } - - override lazy val facepileUsersFut: Future[Seq[Long]] = Future.value(socialContextUserIds) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetNTabRequestHydrator.scala deleted file mode 100644 index bfa8507f0..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetNTabRequestHydrator.scala +++ /dev/null @@ -1,55 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.pushservice.exception.TweetNTabRequestHydratorException -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.notificationservice.thriftscala.InlineCard -import com.twitter.notificationservice.thriftscala.StoryContext -import com.twitter.notificationservice.thriftscala.StoryContextValue -import com.twitter.frigate.pushservice.util.EmailLandingPageExperimentUtil -import com.twitter.notificationservice.thriftscala._ -import com.twitter.util.Future - -trait TweetNTabRequestHydrator extends NTabRequestHydrator { - self: PushCandidate with TweetCandidate with TweetAuthorDetails => - - override def senderIdFut: Future[Long] = - tweetAuthor.map { - case Some(author) => author.id - case _ => - throw new TweetNTabRequestHydratorException( - s"Unable to obtain Author ID for: $commonRecType") - } - - override def storyContext: Option[StoryContext] = Some( - StoryContext( - altText = "", - value = Some(StoryContextValue.Tweets(Seq(tweetId))), - details = None - )) - - override def inlineCard: Option[InlineCard] = Some(InlineCard.TweetCard(TweetCard(tweetId))) - - override lazy val tapThroughFut: Future[String] = { - Future.join(tweetAuthor, target.deviceInfo).map { - case (Some(author), Some(deviceInfo)) => - val enableRuxLandingPage = deviceInfo.isRuxLandingPageEligible && target.params( - PushFeatureSwitchParams.EnableNTabRuxLandingPage) - val authorProfile = author.profile.getOrElse( - throw new TweetNTabRequestHydratorException( - s"Unable to obtain author profile for: ${author.id}")) - if (enableRuxLandingPage) { - EmailLandingPageExperimentUtil.createNTabRuxLandingURI(authorProfile.screenName, tweetId) - } else { - s"${authorProfile.screenName}/status/${tweetId.toString}" - } - case _ => - throw new TweetNTabRequestHydratorException( - s"Unable to obtain author and target details to generate tap through for Tweet: $tweetId") - } - } - - override def socialProofDisplayText: Option[DisplayText] = None -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetRetweetNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetRetweetNTabRequestHydrator.scala deleted file mode 100644 index c142fbfba..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetRetweetNTabRequestHydrator.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.common.base.SocialContextActions -import com.twitter.frigate.common.base.SocialContextUserDetails -import com.twitter.frigate.common.base.TweetAuthor -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.take.NotificationServiceSender -import com.twitter.notificationservice.thriftscala.DisplayTextEntity -import com.twitter.util.Future - -trait TweetRetweetNTabRequestHydrator extends TweetNTabRequestHydrator with NTabSocialContext { - self: PushCandidate - with TweetCandidate - with TweetAuthor - with TweetAuthorDetails - with SocialContextActions - with SocialContextUserDetails => - - override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = { - Future - .join( - NotificationServiceSender - .getDisplayTextEntityFromUser(tweetAuthor, "tweetAuthorName", isBold = false), - NotificationServiceSender - .generateSocialContextTextEntities( - rankedNtabDisplayNamesAndIds(defaultToRecency = false), - otherCount) - ) - .map { - case (authorDisplay, socialContextDisplay) => - socialContextDisplay ++ authorDisplay - } - } - - override lazy val facepileUsersFut: Future[Seq[Long]] = Future.value(socialContextUserIds) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/DeployConfigModule.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/DeployConfigModule.scala deleted file mode 100644 index 238efe0bb..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/DeployConfigModule.scala +++ /dev/null @@ -1,68 +0,0 @@ -package com.twitter.frigate.pushservice.module - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.abdecider.LoggingABDecider -import com.twitter.decider.Decider -import com.twitter.featureswitches.v2.FeatureSwitches -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.tunable.StandardTunableMap -import com.twitter.frigate.pushservice.config.DeployConfig -import com.twitter.frigate.pushservice.config.ProdConfig -import com.twitter.frigate.pushservice.config.StagingConfig -import com.twitter.frigate.pushservice.params.ShardParams -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.product_mixer.core.module.product_mixer_flags.ProductMixerFlagModule.ConfigRepoLocalPath -import com.twitter.product_mixer.core.module.product_mixer_flags.ProductMixerFlagModule.ServiceLocal - -object DeployConfigModule extends TwitterModule { - - @Provides - @Singleton - def providesDeployConfig( - @Flag(FlagName.numShards) numShards: Int, - @Flag(FlagName.shardId) shardId: Int, - @Flag(FlagName.isInMemCacheOff) inMemCacheOff: Boolean, - @Flag(ServiceLocal) isServiceLocal: Boolean, - @Flag(ConfigRepoLocalPath) localConfigRepoPath: String, - serviceIdentifier: ServiceIdentifier, - decider: Decider, - abDecider: LoggingABDecider, - featureSwitches: FeatureSwitches, - statsReceiver: StatsReceiver - ): DeployConfig = { - val tunableMap = if (serviceIdentifier.service.contains("canary")) { - StandardTunableMap(id = "frigate-pushservice-canary") - } else { StandardTunableMap(id = serviceIdentifier.service) } - val shardParams = ShardParams(numShards, shardId) - serviceIdentifier.environment match { - case "devel" | "staging" => - StagingConfig( - isServiceLocal = isServiceLocal, - localConfigRepoPath = localConfigRepoPath, - inMemCacheOff = inMemCacheOff, - decider = decider, - abDecider = abDecider, - featureSwitches = featureSwitches, - serviceIdentifier = serviceIdentifier, - tunableMap = tunableMap, - shardParams = shardParams - )(statsReceiver) - case "prod" => - ProdConfig( - isServiceLocal = isServiceLocal, - localConfigRepoPath = localConfigRepoPath, - inMemCacheOff = inMemCacheOff, - decider = decider, - abDecider = abDecider, - featureSwitches = featureSwitches, - serviceIdentifier = serviceIdentifier, - tunableMap = tunableMap, - shardParams = shardParams - )(statsReceiver) - case env => throw new Exception(s"Unknown environment $env") - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/FilterModule.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/FilterModule.scala deleted file mode 100644 index 579f65acf..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/FilterModule.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.frigate.pushservice.module - -import com.google.inject.Provides -import javax.inject.Singleton -import com.twitter.discovery.common.nackwarmupfilter.NackWarmupFilter -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.util.Duration - -object FilterModule extends TwitterModule { - @Singleton - @Provides - def providesNackWarmupFilter( - @Flag(FlagName.nackWarmupDuration) warmupDuration: Duration - ): NackWarmupFilter = new NackWarmupFilter(warmupDuration) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/FlagModule.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/FlagModule.scala deleted file mode 100644 index 4306e47ca..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/FlagModule.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.frigate.pushservice.module - -import com.twitter.app.Flag -import com.twitter.inject.TwitterModule -import com.twitter.util.Duration -import com.twitter.conversions.DurationOps._ - -object FlagName { - final val shardId = "service.shard" - final val numShards = "service.num_shards" - final val nackWarmupDuration = "service.nackWarmupDuration" - final val isInMemCacheOff = "service.isInMemCacheOff" -} - -object FlagModule extends TwitterModule { - - val shardId: Flag[Int] = flag[Int]( - name = FlagName.shardId, - help = "Service shard id" - ) - - val numShards: Flag[Int] = flag[Int]( - name = FlagName.numShards, - help = "Number of shards" - ) - - val mrLoggerIsTraceAll: Flag[Boolean] = flag[Boolean]( - name = "service.isTraceAll", - help = "atraceflag", - default = false - ) - - val mrLoggerNthLog: Flag[Boolean] = flag[Boolean]( - name = "service.nthLog", - help = "nthlog", - default = false - ) - - val inMemCacheOff: Flag[Boolean] = flag[Boolean]( - name = FlagName.isInMemCacheOff, - help = "is inMemCache Off (currently only applies for user_health_model_score_store_cache)", - default = false - ) - - val mrLoggerNthVal: Flag[Long] = flag[Long]( - name = "service.nthVal", - help = "nthlogval", - default = 0, - ) - - val nackWarmupDuration: Flag[Duration] = flag[Duration]( - name = FlagName.nackWarmupDuration, - help = "duration to nack at startup", - default = 0.seconds - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/LoggedOutPushTargetUserBuilderModule.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/LoggedOutPushTargetUserBuilderModule.scala deleted file mode 100644 index d4bceb549..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/LoggedOutPushTargetUserBuilderModule.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.frigate.pushservice.module - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.decider.Decider -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.target.LoggedOutPushTargetUserBuilder -import com.twitter.frigate.pushservice.config.DeployConfig -import com.twitter.inject.TwitterModule - -object LoggedOutPushTargetUserBuilderModule extends TwitterModule { - - @Provides - @Singleton - def providesLoggedOutPushTargetUserBuilder( - decider: Decider, - config: DeployConfig, - statsReceiver: StatsReceiver - ): LoggedOutPushTargetUserBuilder = { - LoggedOutPushTargetUserBuilder( - historyStore = config.loggedOutHistoryStore, - inputDecider = decider, - inputAbDecider = config.abDecider, - loggedOutPushInfoStore = config.loggedOutPushInfoStore - )(statsReceiver) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushHandlerModule.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushHandlerModule.scala deleted file mode 100644 index c71ff24dd..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushHandlerModule.scala +++ /dev/null @@ -1,78 +0,0 @@ -package com.twitter.frigate.pushservice.module - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.target.LoggedOutPushTargetUserBuilder -import com.twitter.frigate.pushservice.refresh_handler.RefreshForPushHandler -import com.twitter.frigate.pushservice.config.DeployConfig -import com.twitter.frigate.pushservice.send_handler.SendHandler -import com.twitter.frigate.pushservice.take.candidate_validator.RFPHCandidateValidator -import com.twitter.frigate.pushservice.take.candidate_validator.SendHandlerPostCandidateValidator -import com.twitter.frigate.pushservice.take.candidate_validator.SendHandlerPreCandidateValidator -import com.twitter.frigate.pushservice.refresh_handler.LoggedOutRefreshForPushHandler -import com.twitter.frigate.pushservice.take.SendHandlerNotifier -import com.twitter.frigate.pushservice.target.PushTargetUserBuilder -import com.twitter.inject.TwitterModule - -object PushHandlerModule extends TwitterModule { - - @Provides - @Singleton - def providesRefreshForPushHandler( - pushTargetUserBuilder: PushTargetUserBuilder, - config: DeployConfig, - statsReceiver: StatsReceiver - ): RefreshForPushHandler = { - new RefreshForPushHandler( - pushTargetUserBuilder = pushTargetUserBuilder, - candSourceGenerator = config.candidateSourceGenerator, - rfphRanker = config.rfphRanker, - candidateHydrator = config.candidateHydrator, - candidateValidator = new RFPHCandidateValidator(config), - rfphTakeStepUtil = config.rfphTakeStepUtil, - rfphRestrictStep = config.rfphRestrictStep, - rfphNotifier = config.rfphNotifier, - rfphStatsRecorder = config.rfphStatsRecorder, - mrRequestScriberNode = config.mrRequestScriberNode, - rfphFeatureHydrator = config.rfphFeatureHydrator, - rfphPrerankFilter = config.rfphPrerankFilter, - rfphLightRanker = config.rfphLightRanker - )(statsReceiver) - } - - @Provides - @Singleton - def providesSendHandler( - pushTargetUserBuilder: PushTargetUserBuilder, - config: DeployConfig, - statsReceiver: StatsReceiver - ): SendHandler = { - new SendHandler( - pushTargetUserBuilder, - new SendHandlerPreCandidateValidator(config), - new SendHandlerPostCandidateValidator(config), - new SendHandlerNotifier(config.candidateNotifier, statsReceiver.scope("SendHandlerNotifier")), - config.sendHandlerCandidateHydrator, - config.featureHydrator, - config.sendHandlerPredicateUtil, - config.mrRequestScriberNode)(statsReceiver, config) - } - - @Provides - @Singleton - def providesLoggedOutRefreshForPushHandler( - loPushTargetUserBuilder: LoggedOutPushTargetUserBuilder, - config: DeployConfig, - statsReceiver: StatsReceiver - ): LoggedOutRefreshForPushHandler = { - new LoggedOutRefreshForPushHandler( - loPushTargetUserBuilder = loPushTargetUserBuilder, - loPushCandidateSourceGenerator = config.loCandidateSourceGenerator, - candidateHydrator = config.candidateHydrator, - loRanker = config.loggedOutRFPHRanker, - loRfphNotifier = config.loRfphNotifier, - loMrRequestScriberNode = config.loggedOutMrRequestScriberNode, - )(statsReceiver) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushServiceDarkTrafficModule.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushServiceDarkTrafficModule.scala deleted file mode 100644 index 97e484492..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushServiceDarkTrafficModule.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.frigate.pushservice.module - -import com.google.inject.Singleton -import com.twitter.decider.Decider -import com.twitter.decider.RandomRecipient -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.frigate.pushservice.thriftscala.PushService -import com.twitter.inject.Injector -import com.twitter.inject.thrift.modules.ReqRepDarkTrafficFilterModule - -/** - * The darkTraffic filter sample all requests by default - and set the diffy dest to nil for non prod environments - */ -@Singleton -object PushServiceDarkTrafficModule - extends ReqRepDarkTrafficFilterModule[PushService.ReqRepServicePerEndpoint] - with MtlsClient { - - override def label: String = "frigate-pushservice-diffy-proxy" - - /** - * Function to determine if the request should be "sampled", e.g. - * sent to the dark service. - * - * @param injector the [[com.twitter.inject.Injector]] for use in determining if a given request - * should be forwarded or not. - */ - override protected def enableSampling(injector: Injector): Any => Boolean = { - val decider = injector.instance[Decider] - _ => decider.isAvailable("frigate_pushservice_dark_traffic_percent", Some(RandomRecipient)) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushTargetUserBuilderModule.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushTargetUserBuilderModule.scala deleted file mode 100644 index ccdd1f110..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushTargetUserBuilderModule.scala +++ /dev/null @@ -1,64 +0,0 @@ -package com.twitter.frigate.pushservice.module - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.decider.Decider -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.config.DeployConfig -import com.twitter.frigate.pushservice.target.PushTargetUserBuilder -import com.twitter.inject.TwitterModule - -object PushTargetUserBuilderModule extends TwitterModule { - - @Provides - @Singleton - def providesPushTargetUserBuilder( - decider: Decider, - config: DeployConfig, - statsReceiver: StatsReceiver - ): PushTargetUserBuilder = { - PushTargetUserBuilder( - historyStore = config.historyStore, - emailHistoryStore = config.emailHistoryStore, - labeledPushRecsStore = config.labeledPushRecsDecideredStore, - onlineUserHistoryStore = config.onlineUserHistoryStore, - pushRecItemsStore = config.pushRecItemStore, - userStore = config.safeUserStore, - pushInfoStore = config.pushInfoStore, - userCountryStore = config.userCountryStore, - userUtcOffsetStore = config.userUtcOffsetStore, - dauProbabilityStore = config.dauProbabilityStore, - nsfwConsumerStore = config.nsfwConsumerStore, - genericNotificationFeedbackStore = config.genericNotificationFeedbackStore, - userFeatureStore = config.userFeaturesStore, - mrUserStateStore = config.mrUserStatePredictionStore, - tweetImpressionStore = config.tweetImpressionStore, - timelinesUserSessionStore = config.timelinesUserSessionStore, - cachedTweetyPieStore = config.cachedTweetyPieStoreV2, - strongTiesStore = config.strongTiesStore, - userHTLLastVisitStore = config.userHTLLastVisitStore, - userLanguagesStore = config.userLanguagesStore, - inputDecider = decider, - inputAbDecider = config.abDecider, - realGraphScoresTop500InStore = config.realGraphScoresTop500InStore, - recentFollowsStore = config.recentFollowsStore, - resurrectedUserStore = config.reactivatedUserInfoStore, - configParamsBuilder = config.configParamsBuilder, - optOutUserInterestsStore = config.optOutUserInterestsStore, - deviceInfoStore = config.deviceInfoStore, - pushcapDynamicPredictionStore = config.pushcapDynamicPredictionStore, - appPermissionStore = config.appPermissionStore, - optoutModelScorer = config.optoutModelScorer, - userTargetingPropertyStore = config.userTargetingPropertyStore, - ntabCaretFeedbackStore = config.ntabCaretFeedbackStore, - genericFeedbackStore = config.genericFeedbackStore, - inlineActionHistoryStore = config.inlineActionHistoryStore, - featureHydrator = config.featureHydrator, - openAppUserStore = config.openAppUserStore, - openedPushByHourAggregatedStore = config.openedPushByHourAggregatedStore, - geoduckStoreV2 = config.geoDuckV2Store, - superFollowEligibilityUserStore = config.superFollowEligibilityUserStore, - superFollowApplicationStatusStore = config.superFollowApplicationStatusStore - )(statsReceiver) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/ThriftWebFormsModule.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/ThriftWebFormsModule.scala deleted file mode 100644 index 049d731a3..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/ThriftWebFormsModule.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.frigate.pushservice.module - -import com.twitter.finatra.mtls.thriftmux.modules.MtlsThriftWebFormsModule -import com.twitter.finatra.thrift.ThriftServer -import com.twitter.frigate.pushservice.thriftscala.PushService - -class ThriftWebFormsModule(server: ThriftServer) - extends MtlsThriftWebFormsModule[PushService.MethodPerEndpoint](server) { -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/DeciderKey.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/DeciderKey.scala deleted file mode 100644 index 9c17ea5f2..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/DeciderKey.scala +++ /dev/null @@ -1,210 +0,0 @@ -package com.twitter.frigate.pushservice.params - -import com.twitter.servo.decider.DeciderKeyEnum - -object DeciderKey extends DeciderKeyEnum { - val disableAllRelevance = Value("frigate_pushservice_disable_all_relevance") - val disableHeavyRanking = Value("frigate_pushservice_disable_heavy_ranking") - val restrictLightRanking = Value("frigate_pushservice_restrict_light_ranking") - val downSampleLightRankingScribeCandidates = Value( - "frigate_pushservice_down_sample_light_ranking_scribe_candidates") - val entityGraphTweetRecsDeciderKey = Value("user_tweet_entity_graph_tweet_recs") - val enablePushserviceWritesToNotificationServiceDeciderKey = Value( - "frigate_pushservice_enable_writes_to_notification_service") - val enablePushserviceWritesToNotificationServiceForAllEmployeesDeciderKey = Value( - "frigate_pushservice_enable_writes_to_notification_service_for_employees") - val enablePushserviceWritesToNotificationServiceForEveryoneDeciderKey = Value( - "frigate_pushservice_enable_writes_to_notification_service_for_everyone") - val enablePromptFeedbackFatigueResponseNoPredicateDeciderKey = Value( - "frigate_pushservice_enable_ntab_feedback_prompt_response_no_filter_predicate") - val enablePushserviceDeepbirdv2CanaryClusterDeciderKey = Value( - "frigate_pushservice_canary_enable_deepbirdv2_canary_cluster") - val enableUTEGSCForEarlybirdTweetsDecider = Value( - "frigate_pushservice_enable_uteg_sc_for_eb_tweets") - val enableTweetFavRecs = Value("frigate_pushservice_enable_tweet_fav_recs") - val enableTweetRetweetRecs = Value("frigate_pushservice_enable_tweet_retweet_recs") - val enablePushSendEventBus = Value("frigate_pushservice_enable_push_send_eventbus") - - val enableModelBasedPushcapAssignments = Value( - "frigate_pushservice_enable_model_based_pushcap_assignments") - - val enableTweetAnnotationFeatureHydration = Value( - "frigate_pushservice_enable_tweet_annotation_features_hydration") - val enableMrRequestScribing = Value("frigate_pushservice_enable_mr_request_scribing") - val enableHighQualityCandidateScoresScribing = Value( - "frigate_pushservice_enable_high_quality_candidate_scribing") - val enableHtlUserAuthorRealTimeAggregateFeatureHydration = Value( - "frigate_pushservice_enable_htl_new_user_user_author_rta_hydration") - val enableMrUserSemanticCoreFeaturesHydration = Value( - "frigate_pushservice_enable_mr_user_semantic_core_feature_hydration") - val enableMrUserSemanticCoreNoZeroFeaturesHydration = Value( - "frigate_pushservice_enable_mr_user_semantic_core_no_zero_feature_hydration") - val enableHtlOfflineUserAggregateExtendedFeaturesHydration = Value( - "frigate_pushservice_enable_htl_offline_user_aggregate_extended_features_hydration") - val enableNerErgFeaturesHydration = Value("frigate_pushservice_enable_ner_erg_features_hydration") - val enableDaysSinceRecentResurrectionFeatureHydration = Value( - "frigate_pushservice_enable_days_since_recent_resurrection_features_hydration") - val enableUserPastAggregatesFeatureHydration = Value( - "frigate_pushservice_enable_user_past_aggregates_features_hydration") - val enableUserSignalLanguageFeatureHydration = Value( - "frigate_pushservice_enable_user_signal_language_features_hydration") - val enableUserPreferredLanguageFeatureHydration = Value( - "frigate_pushservice_enable_user_preferred_language_features_hydration") - val enablePredicateDetailedInfoScribing = Value( - "frigate_pushservice_enable_predicate_detailed_info_scribing") - val enablePushCapInfoScribing = Value("frigate_pushservice_enable_push_cap_info_scribing") - val disableMLInFiltering = Value("frigate_pushservice_disable_ml_in_filtering") - val useHydratedLabeledSendsForFeaturesDeciderKey = Value( - "use_hydrated_labeled_sends_for_features") - val verifyHydratedLabeledSendsForFeaturesDeciderKey = Value( - "verify_hydrated_labeled_sends_for_features") - val trainingDataDeciderKey = Value("frigate_notifier_quality_model_training_data") - val skipMlModelPredicateDeciderKey = Value("skip_ml_model_predicate") - val scribeModelFeaturesDeciderKey = Value("scribe_model_features") - val scribeModelFeaturesWithoutHydratingNewFeaturesDeciderKey = Value( - "scribe_model_features_without_hydrating_new_features") - val scribeModelFeaturesForRequestScribe = Value("scribe_model_features_for_request_scribe") - val enableMrUserSimclusterV2020FeaturesHydration = Value( - "frigate_pushservice_enable_mr_user_simcluster_v2020_hydration") - val enableMrUserSimclusterV2020NoZeroFeaturesHydration = Value( - "frigate_pushservice_enable_mr_user_simcluster_v2020_no_zero_feature_hydration") - val enableMrUserEngagedTweetTokensFeaturesHydration = Value( - "frigate_pushservice_enable_mr_user_engaged_tweet_tokens_feature_hydration") - val enableMrCandidateTweetTokensFeaturesHydration = Value( - "frigate_pushservice_enable_mr_candidate_tweet_tokens_feature_hydration") - val enableTopicEngagementRealTimeAggregatesFeatureHydration = Value( - "frigate_pushservice_enable_topic_engagement_real_time_aggregates_feature_hydration" - ) - val enableUserTopicAggregatesFeatureHydration = Value( - "frigate_pushservice_enable_user_topic_aggregates_feature_hydration" - ) - val enableDurationSinceLastVisitFeatureHydration = Value( - "frigate_pushservice_enable_duration_since_last_visit_features_hydration" - ) - val enableTwistlyAggregatesFeatureHydration = Value( - "frigate_pushservice_enable_twistly_agg_feature_hydration" - ) - val enableTwHINUserEngagementFeaturesHydration = Value( - "frigate_pushservice_enable_twhin_user_engagement_features_hydration" - ) - val enableTwHINUserFollowFeaturesHydration = Value( - "frigate_pushservice_enable_twhin_user_follow_features_hydration" - ) - val enableTwHINAuthorFollowFeaturesHydration = Value( - "frigate_pushservice_enable_twhin_author_follow_features_hydration" - ) - val enableTweetTwHINFavFeaturesHydration = Value( - "frigate_pushservice_enable_tweet_twhin_fav_features_hydration" - ) - val enableSpaceVisibilityLibraryFiltering = Value( - "frigate_pushservice_enable_space_visibility_library_filtering" - ) - val enableVfFeatureHydrationSpaceShim = Value( - "frigate_pushservice_enable_visibility_filtering_feature_hydration_in_space_shim") - val enableUserTopicFollowFeatureSet = Value( - "frigate_pushservice_enable_user_topic_follow_feature_hydration") - val enableOnboardingNewUserFeatureSet = Value( - "frigate_pushservice_enable_onboarding_new_user_feature_hydration") - val enableMrUserTopicSparseContFeatureSet = Value( - "frigate_pushservice_enable_mr_user_topic_sparse_cont_feature_hydration" - ) - val enableUserPenguinLanguageFeatureSet = Value( - "frigate_pushservice_enable_user_penguin_language_feature_hydration") - val enableMrUserHashspaceEmbeddingFeatureSet = Value( - "frigate_pushservice_enable_mr_user_hashspace_embedding_feature_hydration") - val enableMrUserAuthorSparseContFeatureSet = Value( - "frigate_pushservice_enable_mr_user_author_sparse_cont_feature_hydration" - ) - val enableMrTweetSentimentFeatureSet = Value( - "frigate_pushservice_enable_mr_tweet_sentiment_feature_hydration" - ) - val enableMrTweetAuthorAggregatesFeatureSet = Value( - "frigate_pushservice_enable_mr_tweet_author_aggregates_feature_hydration" - ) - val enableUserGeoFeatureSet = Value("frigate_pushservice_enable_user_geo_feature_hydration") - val enableAuthorGeoFeatureSet = Value("frigate_pushservice_enable_author_geo_feature_hydration") - - val rampupUserGeoFeatureSet = Value("frigate_pushservice_ramp_up_user_geo_feature_hydration") - val rampupAuthorGeoFeatureSet = Value("frigate_pushservice_ramp_up_author_geo_feature_hydration") - - val enablePopGeoTweets = Value("frigate_pushservice_enable_pop_geo_tweets") - val enableTrendsTweets = Value("frigate_pushservice_enable_trends_tweets") - val enableTripGeoTweetCandidates = Value("frigate_pushservice_enable_trip_geo_tweets") - val enableContentRecommenderMixerAdaptor = Value( - "frigate_pushservice_enable_content_recommender_mixer_adaptor") - val enableGenericCandidateAdaptor = Value("frigate_pushservice_enable_generic_candidate_adaptor") - val enableTripGeoTweetContentMixerDarkTraffic = Value( - "frigate_pushservice_enable_trip_geo_tweets_content_mixer_dark_traffic") - - val enableInsTraffic = Value("frigate_pushservice_enable_ins_traffic") - val enableIsTweetTranslatable = Value("frigate_pushservice_enable_is_tweet_translatable") - - val enableMrTweetSimClusterFeatureSet = Value( - "frigate_pushservice_enable_mr_tweet_simcluster_feature_hydration") - - val enableMrOfflineUserTweetTopicAggregate = Value( - "frigate_pushservice_enable_mr_offline_user_tweet_topic_aggregate_hydration") - - val enableMrOfflineUserTweetSimClusterAggregate = Value( - "frigate_pushservice_enable_mr_offline_user_tweet_simcluster_aggregate_hydration" - ) - val enableRealGraphV2FeatureHydration = Value( - "frigate_pushservice_enable_real_graph_v2_features_hydration") - - val enableTweetBeTFeatureHydration = Value( - "frigate_pushservice_enable_tweet_bet_features_hydration") - - val enableInvalidatingCachedHistoryStoreAfterWrites = Value( - "frigate_pushservice_enable_invalidating_cached_history_store_after_writes") - - val enableInvalidatingCachedLoggedOutHistoryStoreAfterWrites = Value( - "frigate_pushservice_enable_invalidating_cached_logged_out_history_store_after_writes") - - val enableUserSendTimeFeatureHydration = Value( - "frigate_pushservice_enable_user_send_time_feature_hydration" - ) - - val enablePnegMultimodalPredictionForF1Tweets = Value( - "frigate_pushservice_enable_pneg_multimodal_prediction_for_f1_tweets" - ) - - val enableScribingOonFavScoreForF1Tweets = Value( - "frigate_pushservice_enable_oon_fav_scribe_for_f1_tweets" - ) - - val enableMrUserUtcSendTimeAggregateFeaturesHydration = Value( - "frigate_pushservice_enable_mr_user_utc_send_time_aggregate_hydration" - ) - - val enableMrUserLocalSendTimeAggregateFeaturesHydration = Value( - "frigate_pushservice_enable_mr_user_local_send_time_aggregate_hydration" - ) - - val enableBqmlReportModelPredictionForF1Tweets = Value( - "frigate_pushservice_enable_bqml_report_model_prediction_for_f1_tweets" - ) - - val enableUserTwhinEmbeddingFeatureHydration = Value( - "frigate_pushservice_enable_user_twhin_embedding_feature_hydration" - ) - - val enableAuthorFollowTwhinEmbeddingFeatureHydration = Value( - "frigate_pushservice_enable_author_follow_twhin_embedding_feature_hydration" - ) - - val enableScribingMLFeaturesAsDataRecord = Value( - "frigate_pushservice_enable_scribing_ml_features_as_datarecord" - ) - - val enableDirectHydrationForUserFeatures = Value( - "frigate_pushservice_enable_direct_hydration_for_user_features" - ) - - val enableAuthorVerifiedFeatureHydration = Value( - "frigate_pushservice_enable_author_verified_feature_hydration" - ) - - val enableAuthorCreatorSubscriptionFeatureHydration = Value( - "frigate_pushservice_enable_author_creator_subscription_feature_hydration" - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushConstants.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushConstants.scala deleted file mode 100644 index cd96b4934..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushConstants.scala +++ /dev/null @@ -1,126 +0,0 @@ -package com.twitter.frigate.pushservice.params - -import com.twitter.conversions.DurationOps._ -import com.twitter.frigate.user_states.thriftscala.UserState -import java.util.Locale - -object PushConstants { - - final val ServiceProdEnvironmentName = "prod" - - final val RestrictLightRankingCandidatesThreshold = 1 - - final val DownSampleLightRankingScribeCandidatesRate = 1 - - final val NewUserLookbackWindow = 1.days - - final val PushCapInactiveUserAndroid = 1 - final val PushCapInactiveUserIos = 1 - final val PushCapLightOccasionalOpenerUserAndroid = 1 - final val PushCapLightOccasionalOpenerUserIos = 1 - - final val UserStateToPushCapIos = Map( - UserState.Inactive.name -> PushCapInactiveUserIos, - UserState.LightOccasionalOpener.name -> PushCapLightOccasionalOpenerUserIos - ) - final val UserStateToPushCapAndroid = Map( - UserState.Inactive.name -> PushCapInactiveUserAndroid, - UserState.LightOccasionalOpener.name -> PushCapLightOccasionalOpenerUserAndroid - ) - - final val AcceptableTimeSinceLastNegativeResponse = 1.days - - final val DefaultLookBackForHistory = 1.hours - - final val DefaultEventMediaUrl = "" - - final val ConnectTabPushTapThrough = "i/connect_people" - - final val AddressBookUploadTapThrough = "i/flow/mr-address-book-upload" - final val InterestPickerTapThrough = "i/flow/mr-interest-picker" - final val CompleteOnboardingInterestAddressTapThrough = "i/flow/mr-interest-address" - - final val IndiaCountryCode = "IN" - final val JapanCountryCode = Locale.JAPAN.getCountry.toUpperCase - final val UKCountryCode = Locale.UK.getCountry.toUpperCase - - final val IndiaTimeZoneCode = "Asia/Kolkata" - final val JapanTimeZoneCode = "Asia/Tokyo" - final val UKTimeZoneCode = "Europe/London" - - final val countryCodeToTimeZoneMap = Map( - IndiaCountryCode -> IndiaTimeZoneCode, - JapanCountryCode -> JapanTimeZoneCode, - UKCountryCode -> UKTimeZoneCode - ) - - final val AbuseStrike_Top2Percent_Id = "AbuseStrike_Top2Percent_Id" - final val AbuseStrike_Top1Percent_Id = "AbuseStrike_Top1Percent_Id" - final val AbuseStrike_Top05Percent_Id = "AbuseStrike_Top05Percent_Id" - final val AbuseStrike_Top025Percent_Id = "AbuseStrike_Top025Percent_Id" - final val AllSpamReportsPerFav_Top1Percent_Id = "AllSpamReportsPerFav_Top1Percent_Id" - final val ReportsPerFav_Top1Percent_Id = "ReportsPerFav_Top1Percent_Id" - final val ReportsPerFav_Top2Percent_Id = "ReportsPerFav_Top2Percent_Id" - final val MediaUnderstanding_Nudity_Id = "MediaUnderstanding_Nudity_Id" - final val MediaUnderstanding_Beauty_Id = "MediaUnderstanding_Beauty_Id" - final val MediaUnderstanding_SinglePerson_Id = "MediaUnderstanding_SinglePerson_Id" - final val PornList_Id = "PornList_Id" - final val PornographyAndNsfwContent_Id = "PornographyAndNsfwContent_Id" - final val SexLife_Id = "SexLife_Id" - final val SexLifeOrSexualOrientation_Id = "SexLifeOrSexualOrientation_Id" - final val ProfanityFilter_Id = "ProfanityFilter_Id" - final val TweetSemanticCoreIdFeature = "tweet.core.tweet.semantic_core_annotations" - final val targetUserGenderFeatureName = "Target.User.Gender" - final val targetUserAgeFeatureName = "Target.User.AgeBucket" - final val targetUserPreferredLanguage = "user.language.user.preferred_contents" - final val tweetAgeInHoursFeatureName = "RecTweet.TweetyPieResult.TweetAgeInHrs" - final val authorActiveFollowerFeatureName = "RecTweetAuthor.User.ActiveFollowers" - final val favFeatureName = "tweet.core.tweet_counts.favorite_count" - final val sentFeatureName = - "tweet.magic_recs_tweet_real_time_aggregates_v2.pair.v2.magicrecs.realtime.is_sent.any_feature.Duration.Top.count" - final val authorSendCountFeatureName = - "tweet_author_aggregate.pair.any_label.any_feature.28.days.count" - final val authorReportCountFeatureName = - "tweet_author_aggregate.pair.label.reportTweetDone.any_feature.28.days.count" - final val authorDislikeCountFeatureName = - "tweet_author_aggregate.pair.label.ntab.isDisliked.any_feature.28.days.count" - final val TweetLikesFeatureName = "tweet.core.tweet_counts.favorite_count" - final val TweetRepliesFeatureName = "tweet.core.tweet_counts.reply_count" - - final val EnableCopyFeaturesForIbis2ModelValues = "has_copy_features" - - final val EmojiFeatureNameForIbis2ModelValues = "emoji" - - final val TargetFeatureNameForIbis2ModelValues = "target" - - final val CopyBodyExpIbisModelValues = "enable_body_exp" - - final val TweetMediaEmbeddingBQKeyIds = Seq( - 230, 110, 231, 111, 232, 233, 112, 113, 234, 235, 114, 236, 115, 237, 116, 117, 238, 118, 239, - 119, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 240, 120, 241, 121, 242, 0, 1, 122, 243, 244, 123, - 2, 124, 245, 3, 4, 246, 125, 5, 126, 247, 127, 248, 6, 128, 249, 7, 8, 129, 9, 20, 21, 22, 23, - 24, 25, 26, 27, 28, 29, 250, 130, 251, 252, 131, 132, 253, 133, 254, 134, 255, 135, 136, 137, - 138, 139, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 140, 141, 142, 143, 144, 145, 146, 147, 148, - 149, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, - 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 60, - 61, 62, 63, 64, 65, 66, 67, 68, 69, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 70, 71, - 72, 73, 74, 75, 76, 77, 78, 79, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 80, 81, 82, - 83, 84, 85, 86, 87, 88, 89, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 90, 91, 92, 93, - 94, 95, 96, 97, 98, 99, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, - 214, 215, 216, 217, 218, 219, 220, 100, 221, 101, 222, 223, 102, 224, 103, 104, 225, 105, 226, - 227, 106, 107, 228, 108, 229, 109 - ) - - final val SportsEventDomainId = 6L - - final val OoncQualityCombinedScore = "OoncQualityCombinedScore" -} - -object PushQPSLimitConstants { - - final val PerspectiveStoreQPS = 100000 - - final val IbisOrNTabQPSForRFPH = 100000 - - final val SocialGraphServiceBatchSize = 100000 -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushEnums.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushEnums.scala deleted file mode 100644 index 370e40076..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushEnums.scala +++ /dev/null @@ -1,135 +0,0 @@ -package com.twitter.frigate.pushservice.params - -/** - * Enum for naming scores we will scribe for non-personalized high quality candidate generation - */ -object HighQualityScribingScores extends Enumeration { - type Name = Value - val HeavyRankingScore = Value - val NonPersonalizedQualityScoreUsingCnn = Value - val BqmlNsfwScore = Value - val BqmlReportScore = Value -} - -/** - * Enum for quality upranking transform - */ -object MrQualityUprankingTransformTypeEnum extends Enumeration { - val Linear = Value - val Sigmoid = Value -} - -/** - * Enum for quality partial upranking transform - */ -object MrQualityUprankingPartialTypeEnum extends Enumeration { - val All = Value - val Oon = Value -} - -/** - * Enum for bucket membership in DDG 10220 Mr Bold Title Favorite Retweet Notification experiment - */ -object MRBoldTitleFavoriteAndRetweetExperimentEnum extends Enumeration { - val ShortTitle = Value -} - -/** - * Enum for ML filtering predicates - */ -object QualityPredicateEnum extends Enumeration { - val WeightedOpenOrNtabClick = Value - val ExplicitOpenOrNtabClickFilter = Value - val AlwaysTrue = Value // Disable ML filtering -} - -/** - * Enum to specify normalization used in BigFiltering experiments - */ -object BigFilteringNormalizationEnum extends Enumeration { - val NormalizationDisabled = Value - val NormalizeByNotSendingScore = Value -} - -/** - * Enum for inline actions - */ -object InlineActionsEnum extends Enumeration { - val Favorite = Value - val Follow = Value - val Reply = Value - val Retweet = Value -} - -/** - * Enum for template format - */ -object IbisTemplateFormatEnum extends Enumeration { - val template1 = Value -} - -/** - * Enum for Store name for Top Tweets By Geo - */ -object TopTweetsForGeoCombination extends Enumeration { - val Default = Value - val AccountsTweetFavAsBackfill = Value - val AccountsTweetFavIntermixed = Value -} - -/** - * Enum for scoring function for Top Tweets By Geo - */ -object TopTweetsForGeoRankingFunction extends Enumeration { - val Score = Value - val GeohashLengthAndThenScore = Value -} - -/** - * Enum for which version of popgeo tweets to be using - */ -object PopGeoTweetVersion extends Enumeration { - val Prod = Value -} - -/** - * Enum for Subtext in Android header - */ -object SubtextForAndroidPushHeader extends Enumeration { - val None = Value - val TargetHandler = Value - val TargetTagHandler = Value - val TargetName = Value - val AuthorTagHandler = Value - val AuthorName = Value -} - -object NsfwTextDetectionModel extends Enumeration { - val ProdModel = Value - val RetrainedModel = Value -} - -object HighQualityCandidateGroupEnum extends Enumeration { - val AgeBucket = Value - val Language = Value - val Topic = Value - val Country = Value - val Admin0 = Value - val Admin1 = Value -} - -object CrtGroupEnum extends Enumeration { - val Twistly = Value - val Frs = Value - val F1 = Value - val Topic = Value - val Trip = Value - val GeoPop = Value - val Other = Value - val None = Value -} - -object SportGameEnum extends Enumeration { - val Soccer = Value - val Nfl = Value -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushFeatureSwitchParams.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushFeatureSwitchParams.scala deleted file mode 100644 index 262b3a8b7..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushFeatureSwitchParams.scala +++ /dev/null @@ -1,5043 +0,0 @@ -package com.twitter.frigate.pushservice.params - -import com.twitter.conversions.DurationOps._ -import com.twitter.frigate.pushservice.params.InlineActionsEnum._ -import com.twitter.frigate.pushservice.params.HighQualityCandidateGroupEnum._ -import com.twitter.timelines.configapi.DurationConversion -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSEnumParam -import com.twitter.timelines.configapi.FSEnumSeqParam -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.util.Duration - -object PushFeatureSwitchParams { - - /** - * List of CRTs to uprank. Last CRT in sequence ends up on top of list - */ - object ListOfCrtsToUpRank - extends FSParam[Seq[String]]("rerank_candidates_crt_to_top", default = Seq.empty[String]) - - object ListOfCrtsForOpenApp - extends FSParam[Seq[String]]( - "open_app_allowed_crts", - default = Seq( - "f1firstdegreetweet", - "f1firstdegreephoto", - "f1firstdegreevideo", - "geopoptweet", - "frstweet", - "trendtweet", - "hermituser", - "triangularloopuser" - )) - - /** - * List of CRTs to downrank. Last CRT in sequence ends up on bottom of list - */ - object ListOfCrtsToDownRank - extends FSParam[Seq[String]]( - name = "rerank_candidates_crt_to_downrank", - default = Seq.empty[String]) - - /** - * Param to enable VF filtering in Tweetypie (vs using VisibilityLibrary) - */ - object EnableVFInTweetypie - extends FSParam[Boolean]( - name = "visibility_filtering_enable_vf_in_tweetypie", - default = true - ) - - /** - * Number of max earlybird candidates - */ - object NumberOfMaxEarlybirdInNetworkCandidatesParam - extends FSBoundedParam( - name = "frigate_push_max_earlybird_in_network_candidates", - default = 100, - min = 0, - max = 800 - ) - - /** - * Number of max UserTweetEntityGraph candidates to query - */ - object NumberOfMaxUTEGCandidatesQueriedParam - extends FSBoundedParam( - name = "frigate_push_max_uteg_candidates_queried", - default = 30, - min = 0, - max = 300 - ) - - /** - * Param to control the max tweet age for users - */ - object MaxTweetAgeParam - extends FSBoundedParam[Duration]( - name = "tweet_age_max_hours", - default = 24.hours, - min = 1.hours, - max = 72.hours - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to control the max tweet age for modeling-based candidates - */ - object ModelingBasedCandidateMaxTweetAgeParam - extends FSBoundedParam[Duration]( - name = "tweet_age_candidate_generation_model_max_hours", - default = 24.hours, - min = 1.hours, - max = 72.hours - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to control the max tweet age for simcluster-based candidates - */ - object GeoPopTweetMaxAgeInHours - extends FSBoundedParam[Duration]( - name = "tweet_age_geo_pop_max_hours", - default = 24.hours, - min = 1.hours, - max = 120.hours - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to control the max tweet age for simcluster-based candidates - */ - object SimclusterBasedCandidateMaxTweetAgeParam - extends FSBoundedParam[Duration]( - name = "tweet_age_simcluster_max_hours", - default = 24.hours, - min = 24.hours, - max = 48.hours - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to control the max tweet age for Detopic-based candidates - */ - object DetopicBasedCandidateMaxTweetAgeParam - extends FSBoundedParam[Duration]( - name = "tweet_age_detopic_max_hours", - default = 24.hours, - min = 24.hours, - max = 48.hours - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to control the max tweet age for F1 candidates - */ - object F1CandidateMaxTweetAgeParam - extends FSBoundedParam[Duration]( - name = "tweet_age_f1_max_hours", - default = 24.hours, - min = 1.hours, - max = 96.hours - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to control the max tweet age for Explore Video Tweet - */ - object ExploreVideoTweetAgeParam - extends FSBoundedParam[Duration]( - name = "explore_video_tweets_age_max_hours", - default = 48.hours, - min = 1.hours, - max = 336.hours // Two weeks - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to no send for new user playbook push if user login for past hours - */ - object NewUserPlaybookAllowedLastLoginHours - extends FSBoundedParam[Duration]( - name = "new_user_playbook_allowed_last_login_hours", - default = 0.hours, - min = 0.hours, - max = 72.hours - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * The batch size of RefreshForPushHandler's Take step - */ - object NumberOfMaxCandidatesToBatchInRFPHTakeStep - extends FSBoundedParam( - name = "frigate_push_rfph_batch_take_max_size", - default = 1, - min = 1, - max = 10 - ) - - /** - * The maximum number of candidates to batch for Importance Sampling - */ - object NumberOfMaxCandidatesToBatchForImportanceSampling - extends FSBoundedParam( - name = "frigate_push_rfph_max_candidates_to_batch_for_importance_sampling", - default = 65, - min = 1, - max = 500 - ) - - /** - * Maximum number of regular MR push in 24.hours/daytime/nighttime - */ - object MaxMrPushSends24HoursParam - extends FSBoundedParam( - name = "pushcap_max_sends_24hours", - default = 5, - min = 0, - max = 12 - ) - - /** - * Maximum number of regular MR ntab only channel in 24.hours/daytime/nighttime - */ - object MaxMrNtabOnlySends24HoursParamV3 - extends FSBoundedParam( - name = "pushcap_max_sends_24hours_ntabonly_v3", - default = 5, - min = 0, - max = 12 - ) - - /** - * Maximum number of regular MR ntab only in 24.hours/daytime/nighttime - */ - object MaxMrPushSends24HoursNtabOnlyUsersParam - extends FSBoundedParam( - name = "pushcap_max_sends_24hours_ntab_only", - default = 5, - min = 0, - max = 10 - ) - - /** - * Customized PushCap offset (e.g., to the predicted value) - */ - object CustomizedPushCapOffset - extends FSBoundedParam[Int]( - name = "pushcap_customized_offset", - default = 0, - min = -2, - max = 4 - ) - - /** - * Param to enable restricting minimum pushcap assigned with ML models - * */ - object EnableRestrictedMinModelPushcap - extends FSParam[Boolean]( - name = "pushcap_restricted_model_min_enable", - default = false - ) - - /** - * Param to specify the minimum pushcap allowed to be assigned with ML models - * */ - object RestrictedMinModelPushcap - extends FSBoundedParam[Int]( - name = "pushcap_restricted_model_min_value", - default = 1, - min = 0, - max = 9 - ) - - object EnablePushcapRefactor - extends FSParam[Boolean]( - name = "pushcap_enable_refactor", - default = false - ) - - /** - * Enables the restrict step in pushservice for a given user - * - * Setting this to false may cause a large number of candidates to be passed on to filtering/take - * step in RefreshForPushHandler, increasing the service latency significantly - */ - object EnableRestrictStep extends FSParam[Boolean]("frigate_push_rfph_restrict_step_enable", true) - - /** - * The number of candidates that are able to pass through the restrict step. - */ - object RestrictStepSize - extends FSBoundedParam( - name = "frigate_push_rfph_restrict_step_size", - default = 65, - min = 65, - max = 200 - ) - - /** - * Number of max crMixer candidates to send. - */ - object NumberOfMaxCrMixerCandidatesParam - extends FSBoundedParam( - name = "cr_mixer_migration_max_num_of_candidates_to_return", - default = 400, - min = 0, - max = 2000 - ) - - /** - * Duration between two MR pushes - */ - object MinDurationSincePushParam - extends FSBoundedParam[Duration]( - name = "pushcap_min_duration_since_push_hours", - default = 4.hours, - min = 0.hours, - max = 72.hours - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Each Phase duration to gradually ramp up MagicRecs for new users - */ - object GraduallyRampUpPhaseDurationDays - extends FSBoundedParam[Duration]( - name = "pushcap_gradually_ramp_up_phase_duration_days", - default = 3.days, - min = 2.days, - max = 7.days - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Param to specify interval for target pushcap fatigue - */ - object TargetPushCapFatigueIntervalHours - extends FSBoundedParam[Duration]( - name = "pushcap_fatigue_interval_hours", - default = 24.hours, - min = 1.hour, - max = 240.hours - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to specify interval for target ntabOnly fatigue - */ - object TargetNtabOnlyCapFatigueIntervalHours - extends FSBoundedParam[Duration]( - name = "pushcap_ntabonly_fatigue_interval_hours", - default = 24.hours, - min = 1.hour, - max = 240.hours - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to use completely explicit push cap instead of LTV/modeling-based - */ - object EnableExplicitPushCap - extends FSParam[Boolean]( - name = "pushcap_explicit_enable", - default = false - ) - - /** - * Param to control explicit push cap (non-LTV) - */ - object ExplicitPushCap - extends FSBoundedParam[Int]( - name = "pushcap_explicit_value", - default = 1, - min = 0, - max = 20 - ) - - /** - * Parameters for percentile thresholds of OpenOrNtabClick model in MR filtering model refreshing DDG - */ - object PercentileThresholdCohort1 - extends FSBoundedParam[Double]( - name = "frigate_push_modeling_percentile_threshold_cohort1", - default = 0.65, - min = 0.0, - max = 1.0 - ) - - object PercentileThresholdCohort2 - extends FSBoundedParam[Double]( - name = "frigate_push_modeling_percentile_threshold_cohort2", - default = 0.03, - min = 0.0, - max = 1.0 - ) - object PercentileThresholdCohort3 - extends FSBoundedParam[Double]( - name = "frigate_push_modeling_percentile_threshold_cohort3", - default = 0.03, - min = 0.0, - max = 1.0 - ) - object PercentileThresholdCohort4 - extends FSBoundedParam[Double]( - name = "frigate_push_modeling_percentile_threshold_cohort4", - default = 0.06, - min = 0.0, - max = 1.0 - ) - object PercentileThresholdCohort5 - extends FSBoundedParam[Double]( - name = "frigate_push_modeling_percentile_threshold_cohort5", - default = 0.06, - min = 0.0, - max = 1.0 - ) - object PercentileThresholdCohort6 - extends FSBoundedParam[Double]( - name = "frigate_push_modeling_percentile_threshold_cohort6", - default = 0.8, - min = 0.0, - max = 1.0 - ) - - /** - * Parameters for percentile threshold list of OpenOrNtabCLick model in MR percentile grid search experiments - */ - object MrPercentileGridSearchThresholdsCohort1 - extends FSParam[Seq[Double]]( - name = "frigate_push_modeling_percentile_grid_search_thresholds_cohort1", - default = Seq(0.8, 0.75, 0.65, 0.55, 0.45, 0.35, 0.25) - ) - object MrPercentileGridSearchThresholdsCohort2 - extends FSParam[Seq[Double]]( - name = "frigate_push_modeling_percentile_grid_search_thresholds_cohort2", - default = Seq(0.15, 0.12, 0.1, 0.08, 0.06, 0.045, 0.03) - ) - object MrPercentileGridSearchThresholdsCohort3 - extends FSParam[Seq[Double]]( - name = "frigate_push_modeling_percentile_grid_search_thresholds_cohort3", - default = Seq(0.15, 0.12, 0.1, 0.08, 0.06, 0.045, 0.03) - ) - object MrPercentileGridSearchThresholdsCohort4 - extends FSParam[Seq[Double]]( - name = "frigate_push_modeling_percentile_grid_search_thresholds_cohort4", - default = Seq(0.15, 0.12, 0.1, 0.08, 0.06, 0.045, 0.03) - ) - object MrPercentileGridSearchThresholdsCohort5 - extends FSParam[Seq[Double]]( - name = "frigate_push_modeling_percentile_grid_search_thresholds_cohort5", - default = Seq(0.3, 0.2, 0.15, 0.1, 0.08, 0.06, 0.05) - ) - object MrPercentileGridSearchThresholdsCohort6 - extends FSParam[Seq[Double]]( - name = "frigate_push_modeling_percentile_grid_search_thresholds_cohort6", - default = Seq(0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2) - ) - - /** - * Parameters for threshold list of OpenOrNtabClick model in MF grid search experiments - */ - object MfGridSearchThresholdsCohort1 - extends FSParam[Seq[Double]]( - name = "frigate_push_modeling_mf_grid_search_thresholds_cohort1", - default = Seq(0.030, 0.040, 0.050, 0.062, 0.070, 0.080, 0.090) // default: 0.062 - ) - object MfGridSearchThresholdsCohort2 - extends FSParam[Seq[Double]]( - name = "frigate_push_modeling_mf_grid_search_thresholds_cohort2", - default = Seq(0.005, 0.010, 0.015, 0.020, 0.030, 0.040, 0.050) // default: 0.020 - ) - object MfGridSearchThresholdsCohort3 - extends FSParam[Seq[Double]]( - name = "frigate_push_modeling_mf_grid_search_thresholds_cohort3", - default = Seq(0.010, 0.015, 0.020, 0.025, 0.035, 0.045, 0.055) // default: 0.025 - ) - object MfGridSearchThresholdsCohort4 - extends FSParam[Seq[Double]]( - name = "frigate_push_modeling_mf_grid_search_thresholds_cohort4", - default = Seq(0.015, 0.020, 0.025, 0.030, 0.040, 0.050, 0.060) // default: 0.030 - ) - object MfGridSearchThresholdsCohort5 - extends FSParam[Seq[Double]]( - name = "frigate_push_modeling_mf_grid_search_thresholds_cohort5", - default = Seq(0.035, 0.040, 0.045, 0.050, 0.060, 0.070, 0.080) // default: 0.050 - ) - object MfGridSearchThresholdsCohort6 - extends FSParam[Seq[Double]]( - name = "frigate_push_modeling_mf_grid_search_thresholds_cohort6", - default = Seq(0.040, 0.045, 0.050, 0.055, 0.065, 0.075, 0.085) // default: 0.055 - ) - - /** - * Param to specify which global optout models to use to first predict the global scores for users - */ - object GlobalOptoutModelParam - extends FSParam[Seq[OptoutModel.ModelNameType]]( - name = "optout_model_global_model_ids", - default = Seq.empty[OptoutModel.ModelNameType] - ) - - /** - * Param to specify which optout model to use according to the experiment bucket - */ - object BucketOptoutModelParam - extends FSParam[OptoutModel.ModelNameType]( - name = "optout_model_bucket_model_id", - default = OptoutModel.D0_has_realtime_features - ) - - /* - * Param to enable candidate generation model - * */ - object EnableCandidateGenerationModelParam - extends FSParam[Boolean]( - name = "candidate_generation_model_enable", - default = false - ) - - object EnableOverrideForSportsCandidates - extends FSParam[Boolean](name = "magicfanout_sports_event_enable_override", default = true) - - object EnableEventIdBasedOverrideForSportsCandidates - extends FSParam[Boolean]( - name = "magicfanout_sports_event_enable_event_id_based_override", - default = true) - - /** - * Param to specify the threshold to determine if a user’s optout score is high enough to enter the experiment. - */ - object GlobalOptoutThresholdParam - extends FSParam[Seq[Double]]( - name = "optout_model_global_thresholds", - default = Seq(1.0, 1.0) - ) - - /** - * Param to specify the threshold to determine if a user’s optout score is high enough to be assigned - * with a reduced pushcap based on the bucket membership. - */ - object BucketOptoutThresholdParam - extends FSBoundedParam[Double]( - name = "optout_model_bucket_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Param to specify the reduced pushcap value if the optout probability predicted by the bucket - * optout model is higher than the specified bucket optout threshold. - */ - object OptoutExptPushCapParam - extends FSBoundedParam[Int]( - name = "optout_model_expt_push_cap", - default = 10, - min = 0, - max = 10 - ) - - /** - * Param to specify the thresholds to determine which push cap slot the user should be assigned to - * according to the optout score. For example,the slot thresholds are [0.1, 0.2, ..., 1.0], the user - * is assigned to the second slot if the optout score is in (0.1, 0.2]. - */ - object BucketOptoutSlotThresholdParam - extends FSParam[Seq[Double]]( - name = "optout_model_bucket_slot_thresholds", - default = Seq.empty[Double] - ) - - /** - * Param to specify the adjusted push cap of each slot. For example, if the slot push caps are [1, 2, ..., 10] - * and the user is assigned to the 2nd slot according to the optout score, the push cap of the user - * will be adjusted to 2. - */ - object BucketOptoutSlotPushcapParam - extends FSParam[Seq[Int]]( - name = "optout_model_bucket_slot_pushcaps", - default = Seq.empty[Int] - ) - - /** - * Param to specify if the optout score based push cap adjustment is enabled - */ - object EnableOptoutAdjustedPushcap - extends FSParam[Boolean]( - "optout_model_enable_optout_adjusted_pushcap", - false - ) - - /** - * Param to specify which weighted open or ntab click model to use - */ - object WeightedOpenOrNtabClickRankingModelParam - extends FSParam[WeightedOpenOrNtabClickModel.ModelNameType]( - name = "frigate_push_modeling_oonc_ranking_model_id", - default = WeightedOpenOrNtabClickModel.Periodically_Refreshed_Prod_Model - ) - - /** - * Param to disable heavy ranker - */ - object DisableHeavyRankingModelFSParam - extends FSParam[Boolean]( - name = "frigate_push_modeling_disable_heavy_ranking", - default = false - ) - - /** - * Param to specify which weighted open or ntab click model to use for Android modelling experiment - */ - object WeightedOpenOrNtabClickRankingModelForAndroidParam - extends FSParam[WeightedOpenOrNtabClickModel.ModelNameType]( - name = "frigate_push_modeling_oonc_ranking_model_for_android_id", - default = WeightedOpenOrNtabClickModel.Periodically_Refreshed_Prod_Model - ) - - /** - * Param to specify which weighted open or ntab click model to use for FILTERING - */ - object WeightedOpenOrNtabClickFilteringModelParam - extends FSParam[WeightedOpenOrNtabClickModel.ModelNameType]( - name = "frigate_push_modeling_oonc_filtering_model_id", - default = WeightedOpenOrNtabClickModel.Periodically_Refreshed_Prod_Model - ) - - /** - * Param to specify which quality predicate to use for ML filtering - */ - object QualityPredicateIdParam - extends FSEnumParam[QualityPredicateEnum.type]( - name = "frigate_push_modeling_quality_predicate_id", - default = QualityPredicateEnum.WeightedOpenOrNtabClick, - enum = QualityPredicateEnum - ) - - /** - * Param to control threshold for any quality predicates using explicit thresholds - */ - object QualityPredicateExplicitThresholdParam - extends FSBoundedParam[Double]( - name = "frigate_push_modeling_quality_predicate_explicit_threshold", - default = 0.1, - min = 0, - max = 1) - - /** - * MagicFanout relaxed eventID fatigue interval (when we want to enable multiple updates for the same event) - */ - object MagicFanoutRelaxedEventIdFatigueIntervalInHours - extends FSBoundedParam[Int]( - name = "frigate_push_magicfanout_relaxed_event_id_fatigue_interval_in_hours", - default = 24, - min = 0, - max = 720 - ) - - /** - * MagicFanout DenyListed Countries - */ - object MagicFanoutDenyListedCountries - extends FSParam[Seq[String]]( - "frigate_push_magicfanout_denylisted_countries", - Seq.empty[String]) - - object MagicFanoutSportsEventDenyListedCountries - extends FSParam[Seq[String]]( - "magicfanout_sports_event_denylisted_countries", - Seq.empty[String]) - - /** - * MagicFanout maximum erg rank for a given push event for non heavy users - */ - object MagicFanoutRankErgThresholdNonHeavy - extends FSBoundedParam[Int]( - name = "frigate_push_magicfanout_erg_rank_threshold_non_heavy", - default = 25, - min = 1, - max = 50 - ) - - /** - * MagicFanout maximum erg rank for a given push event for heavy users - */ - object MagicFanoutRankErgThresholdHeavy - extends FSBoundedParam[Int]( - name = "frigate_push_magicfanout_erg_rank_threshold_heavy", - default = 20, - min = 1, - max = 50 - ) - - object EnablePushMixerReplacingAllSources - extends FSParam[Boolean]( - name = "push_mixer_enable_replacing_all_sources", - default = false - ) - - object EnablePushMixerReplacingAllSourcesWithControl - extends FSParam[Boolean]( - name = "push_mixer_enable_replacing_all_sources_with_control", - default = false - ) - - object EnablePushMixerReplacingAllSourcesWithExtra - extends FSParam[Boolean]( - name = "push_mixer_enable_replacing_all_sources_with_extra", - default = false - ) - - object EnablePushMixerSource - extends FSParam[Boolean]( - name = "push_mixer_enable_source", - default = false - ) - - object PushMixerMaxResults - extends FSBoundedParam[Int]( - name = "push_mixer_max_results", - default = 10, - min = 1, - max = 5000 - ) - - /** - * Enable tweets from trends that have been annotated by curators - */ - object EnableCuratedTrendTweets - extends FSParam[Boolean](name = "trend_tweet_curated_trends_enable", default = false) - - /** - * Enable tweets from trends that haven't been annotated by curators - */ - object EnableNonCuratedTrendTweets - extends FSParam[Boolean](name = "trend_tweet_non_curated_trends_enable", default = false) - - /** - * Maximum trend tweet notifications in fixed duration - */ - object MaxTrendTweetNotificationsInDuration - extends FSBoundedParam[Int]( - name = "trend_tweet_max_notifications_in_duration", - min = 0, - default = 0, - max = 20) - - /** - * Duration in days over which trend tweet notifications fatigue is applied - */ - object TrendTweetNotificationsFatigueDuration - extends FSBoundedParam[Duration]( - name = "trend_tweet_notifications_fatigue_in_days", - default = 1.day, - min = Duration.Bottom, - max = Duration.Top - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Maximum number of trends candidates to query from event-recos endpoint - */ - object MaxRecommendedTrendsToQuery - extends FSBoundedParam[Int]( - name = "trend_tweet_max_trends_to_query", - min = 0, - default = 0, - max = 100) - - /** - * Fix missing event-associated interests in MagicFanoutNoOptoutInterestsPredicate - */ - object MagicFanoutFixNoOptoutInterestsBugParam - extends FSParam[Boolean]("frigate_push_magicfanout_fix_no_optout_interests", default = true) - - object EnableSimclusterOfflineAggFeatureForExpt - extends FSParam[Boolean]("frigate_enable_simcluster_offline_agg_feature", false) - - /** - * Param to enable removal of UTT domain for - */ - object ApplyMagicFanoutBroadEntityInterestRankThresholdPredicate - extends FSParam[Boolean]( - "frigate_push_magicfanout_broad_entity_interest_rank_threshold_predicate", - false - ) - - object HydrateEventReasonsFeatures - extends FSParam[Boolean]( - name = "frigate_push_magicfanout_hydrate_event_reasons_features", - false - ) - - /** - * Param to enable online MR history features - */ - object EnableHydratingOnlineMRHistoryFeatures - extends FSParam[Boolean]( - name = "feature_hydration_online_mr_history", - default = false - ) - - /** - * Param to enable bold title on favorite and retweet push copy for Android in DDG 10220 - */ - object MRBoldTitleFavoriteAndRetweetParam - extends FSEnumParam[MRBoldTitleFavoriteAndRetweetExperimentEnum.type]( - name = "frigate_push_bold_title_favorite_and_retweet_id", - default = MRBoldTitleFavoriteAndRetweetExperimentEnum.ShortTitle, - enum = MRBoldTitleFavoriteAndRetweetExperimentEnum - ) - - /** - * Param to enable high priority push - */ - object EnableHighPriorityPush - extends FSParam[Boolean]("frigate_push_magicfanout_enable_high_priority_push", false) - - /** - * Param to redirect sports crt event to a custom url - */ - object EnableSearchURLRedirectForSportsFanout - extends FSParam[Boolean]("magicfanout_sports_event_enable_search_url_redirect", false) - - /** - * Param to enable score fanout notification for sports - */ - object EnableScoreFanoutNotification - extends FSParam[Boolean]("magicfanout_sports_event_enable_score_fanout", false) - - /** - * Param to add custom search url for sports crt event - */ - object SearchURLRedirectForSportsFanout - extends FSParam[String]( - name = "magicfanout_sports_event_search_url_redirect", - default = "https://twitter.com/explore/tabs/ipl", - ) - - /** - * Param to enable high priority sports push - */ - object EnableHighPrioritySportsPush - extends FSParam[Boolean]("magicfanout_sports_event_enable_high_priority_push", false) - - /** - * Param to control rank threshold for magicfanout user follow - */ - object MagicFanoutRealgraphRankThreshold - extends FSBoundedParam[Int]( - name = "magicfanout_realgraph_threshold", - default = 500, - max = 500, - min = 100 - ) - - /** - * Topic score threshold for topic proof tweet candidates topic annotations - * */ - object TopicProofTweetCandidatesTopicScoreThreshold - extends FSBoundedParam[Double]( - name = "topics_as_social_proof_topic_score_threshold", - default = 0.0, - min = 0.0, - max = 100.0 - ) - - /** - * Enable Topic Proof Tweet Recs - */ - object EnableTopicProofTweetRecs - extends FSParam[Boolean](name = "topics_as_social_proof_enable", default = true) - - /** - * Enable health filters for topic tweet notifications - */ - object EnableHealthFiltersForTopicProofTweet - extends FSParam[Boolean]( - name = "topics_as_social_proof_enable_health_filters", - default = false) - - /** - * Disable health filters for CrMixer candidates - */ - object DisableHealthFiltersForCrMixerCandidates - extends FSParam[Boolean]( - name = "health_and_quality_filter_disable_for_crmixer_candidates", - default = false) - - object EnableMagicFanoutNewsForYouNtabCopy - extends FSParam[Boolean](name = "send_handler_enable_nfy_ntab_copy", default = false) - - /** - * Param to enable semi-personalized high quality candidates in pushservice - * */ - object HighQualityCandidatesEnableCandidateSource - extends FSParam[Boolean]( - name = "high_quality_candidates_enable_candidate_source", - default = false - ) - - /** - * Param to decide semi-personalized high quality candidates - * */ - object HighQualityCandidatesEnableGroups - extends FSEnumSeqParam[HighQualityCandidateGroupEnum.type]( - name = "high_quality_candidates_enable_groups_ids", - default = Seq(AgeBucket, Language), - enum = HighQualityCandidateGroupEnum - ) - - /** - * Param to decide semi-personalized high quality candidates - * */ - object HighQualityCandidatesNumberOfCandidates - extends FSBoundedParam[Int]( - name = "high_quality_candidates_number_of_candidates", - default = 0, - min = 0, - max = Int.MaxValue - ) - - /** - * Param to enable small domain falling back to bigger domains for high quality candidates in pushservice - * */ - object HighQualityCandidatesEnableFallback - extends FSParam[Boolean]( - name = "high_quality_candidates_enable_fallback", - default = false - ) - - /** - * Param to decide whether to fallback to bigger domain for high quality candidates - * */ - object HighQualityCandidatesMinNumOfCandidatesToFallback - extends FSBoundedParam[Int]( - name = "high_quality_candidates_min_num_of_candidates_to_fallback", - default = 50, - min = 0, - max = Int.MaxValue - ) - - /** - * Param to specific source ids for high quality candidates - * */ - object HighQualityCandidatesFallbackSourceIds - extends FSParam[Seq[String]]( - name = "high_quality_candidates_fallback_source_ids", - default = Seq("HQ_C_COUNT_PASS_QUALITY_SCORES")) - - /** - * Param to decide groups for semi-personalized high quality candidates - * */ - object HighQualityCandidatesFallbackEnabledGroups - extends FSEnumSeqParam[HighQualityCandidateGroupEnum.type]( - name = "high_quality_candidates_fallback_enabled_groups_ids", - default = Seq(Country), - enum = HighQualityCandidateGroupEnum - ) - - /** - * Param to control what heavy ranker model to use for scribing scores - */ - object HighQualityCandidatesHeavyRankingModel - extends FSParam[String]( - name = "high_quality_candidates_heavy_ranking_model", - default = "Periodically_Refreshed_Prod_Model_V11" - ) - - /** - * Param to control what non personalized quality "Cnn" model to use for scribing scores - */ - object HighQualityCandidatesNonPersonalizedQualityCnnModel - extends FSParam[String]( - name = "high_quality_candidates_non_personalized_quality_cnn_model", - default = "Q1_2023_Mr_Tf_Quality_Model_cnn" - ) - - /** - * Param to control what nsfw health model to use for scribing scores - */ - object HighQualityCandidatesBqmlNsfwModel - extends FSParam[String]( - name = "high_quality_candidates_bqml_nsfw_model", - default = "Q2_2022_Mr_Bqml_Health_Model_NsfwV0" - ) - - /** - * Param to control what reportodel to use for scribing scores - */ - object HighQualityCandidatesBqmlReportModel - extends FSParam[String]( - name = "high_quality_candidates_bqml_report_model", - default = "Q3_2022_15266_Mr_Bqml_Non_Personalized_Report_Model_with_Media_Embeddings" - ) - - /** - * Param to specify the threshold to determine if a tweet contains nudity media - */ - object TweetMediaSensitiveCategoryThresholdParam - extends FSBoundedParam[Double]( - name = "tweet_media_sensitive_category_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Param to boost candidates from subscription creators - */ - object BoostCandidatesFromSubscriptionCreators - extends FSParam[Boolean]( - name = "subscription_enable_boost_candidates_from_active_creators", - default = false - ) - - /** - * Param to soft rank candidates from subscription creators - */ - object SoftRankCandidatesFromSubscriptionCreators - extends FSParam[Boolean]( - name = "subscription_enable_soft_rank_candidates_from_active_creators", - default = false - ) - - /** - * Param as factor to control how much we want to boost creator tweets - */ - object SoftRankFactorForSubscriptionCreators - extends FSBoundedParam[Double]( - name = "subscription_soft_rank_factor_for_boost", - default = 1.0, - min = 0.0, - max = Double.MaxValue - ) - - /** - * Param to enable new OON copy for Push Notifications - */ - object EnableNewMROONCopyForPush - extends FSParam[Boolean]( - name = "mr_copy_enable_new_mr_oon_copy_push", - default = true - ) - - /** - * Param to enable generated inline actions on OON Notifications - */ - object EnableOONGeneratedInlineActions - extends FSParam[Boolean]( - name = "mr_inline_enable_oon_generated_actions", - default = false - ) - - /** - * Param to control dynamic inline actions for Out-of-Network copies - */ - object OONTweetDynamicInlineActionsList - extends FSEnumSeqParam[InlineActionsEnum.type]( - name = "mr_inline_oon_tweet_dynamic_action_ids", - default = Seq(Follow, Retweet, Favorite), - enum = InlineActionsEnum - ) - - object HighOONCTweetFormat - extends FSEnumParam[IbisTemplateFormatEnum.type]( - name = "mr_copy_high_oonc_format_id", - default = IbisTemplateFormatEnum.template1, - enum = IbisTemplateFormatEnum - ) - - object LowOONCTweetFormat - extends FSEnumParam[IbisTemplateFormatEnum.type]( - name = "mr_copy_low_oonc_format_id", - default = IbisTemplateFormatEnum.template1, - enum = IbisTemplateFormatEnum - ) - - /** - * Param to enable dynamic inline actions based on FSParams for Tweet copies (not OON) - */ - object EnableTweetDynamicInlineActions - extends FSParam[Boolean]( - name = "mr_inline_enable_tweet_dynamic_actions", - default = false - ) - - /** - * Param to control dynamic inline actions for Tweet copies (not OON) - */ - object TweetDynamicInlineActionsList - extends FSEnumSeqParam[InlineActionsEnum.type]( - name = "mr_inline_tweet_dynamic_action_ids", - default = Seq(Reply, Retweet, Favorite), - enum = InlineActionsEnum - ) - - object UseInlineActionsV1 - extends FSParam[Boolean]( - name = "mr_inline_use_inline_action_v1", - default = true - ) - - object UseInlineActionsV2 - extends FSParam[Boolean]( - name = "mr_inline_use_inline_action_v2", - default = false - ) - - object EnableInlineFeedbackOnPush - extends FSParam[Boolean]( - name = "mr_inline_enable_inline_feedback_on_push", - default = false - ) - - object InlineFeedbackSubstitutePosition - extends FSBoundedParam[Int]( - name = "mr_inline_feedback_substitute_position", - min = 0, - max = 2, - default = 2, // default to substitute or append last inline action - ) - - /** - * Param to control dynamic inline actions for web notifications - */ - object EnableDynamicInlineActionsForDesktopWeb - extends FSParam[Boolean]( - name = "mr_inline_enable_dynamic_actions_for_desktop_web", - default = false - ) - - object EnableDynamicInlineActionsForMobileWeb - extends FSParam[Boolean]( - name = "mr_inline_enable_dynamic_actions_for_mobile_web", - default = false - ) - - /** - * Param to define dynamic inline action types for web notifications (both desktop web + mobile web) - */ - object TweetDynamicInlineActionsListForWeb - extends FSEnumSeqParam[InlineActionsEnum.type]( - name = "mr_inline_tweet_dynamic_action_for_web_ids", - default = Seq(Retweet, Favorite), - enum = InlineActionsEnum - ) - - /** - * Param to enable MR Override Notifications for Android - */ - object EnableOverrideNotificationsForAndroid - extends FSParam[Boolean]( - name = "mr_override_enable_override_notifications_for_android", - default = false - ) - - /** - * Param to enable MR Override Notifications for iOS - */ - object EnableOverrideNotificationsForIos - extends FSParam[Boolean]( - name = "mr_override_enable_override_notifications_for_ios", - default = false - ) - - /** - * Param to enable gradually ramp up notification - */ - object EnableGraduallyRampUpNotification - extends FSParam[Boolean]( - name = "pushcap_gradually_ramp_up_enable", - default = false - ) - - /** - * Param to control the minInrerval for fatigue between consecutive MFNFY pushes - */ - object MFMinIntervalFatigue - extends FSBoundedParam[Duration]( - name = "frigate_push_magicfanout_fatigue_min_interval_consecutive_pushes_minutes", - default = 240.minutes, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromMinutes - } - - /** - * Param to control the interval for MFNFY pushes - */ - object MFPushIntervalInHours - extends FSBoundedParam[Duration]( - name = "frigate_push_magicfanout_fatigue_push_interval_in_hours", - default = 24.hours, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to control the maximum number of Sports MF pushes in a period of time - */ - object SportsMaxNumberOfPushesInInterval - extends FSBoundedParam[Int]( - name = "magicfanout_sports_event_fatigue_max_pushes_in_interval", - default = 2, - min = 0, - max = 6) - - /** - * Param to control the minInterval for fatigue between consecutive sports pushes - */ - object SportsMinIntervalFatigue - extends FSBoundedParam[Duration]( - name = "magicfanout_sports_event_fatigue_min_interval_consecutive_pushes_minutes", - default = 240.minutes, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromMinutes - } - - /** - * Param to control the interval for sports pushes - */ - object SportsPushIntervalInHours - extends FSBoundedParam[Duration]( - name = "magicfanout_sports_event_fatigue_push_interval_in_hours", - default = 24.hours, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to control the maximum number of same event sports MF pushes in a period of time - */ - object SportsMaxNumberOfPushesInIntervalPerEvent - extends FSBoundedParam[Int]( - name = "magicfanout_sports_event_fatigue_max_pushes_in_per_event_interval", - default = 2, - min = 0, - max = 6) - - /** - * Param to control the minInterval for fatigue between consecutive same event sports pushes - */ - object SportsMinIntervalFatiguePerEvent - extends FSBoundedParam[Duration]( - name = "magicfanout_sports_event_fatigue_min_interval_consecutive_pushes_per_event_minutes", - default = 240.minutes, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromMinutes - } - - /** - * Param to control the interval for same event sports pushes - */ - object SportsPushIntervalInHoursPerEvent - extends FSBoundedParam[Duration]( - name = "magicfanout_sports_event_fatigue_push_interval_per_event_in_hours", - default = 24.hours, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to control the maximum number of MF pushes in a period of time - */ - object MFMaxNumberOfPushesInInterval - extends FSBoundedParam[Int]( - name = "frigate_push_magicfanout_fatigue_max_pushes_in_interval", - default = 2, - min = 0, - max = 6) - - /** - * Param to enable custom duration for fatiguing - */ - object GPEnableCustomMagicFanoutCricketFatigue - extends FSParam[Boolean]( - name = "global_participation_cricket_magicfanout_enable_custom_fatigue", - default = false - ) - - /** - * Param to enable e2e scribing for target filtering step - */ - object EnableMrRequestScribingForTargetFiltering - extends FSParam[Boolean]( - name = "mr_request_scribing_enable_for_target_filtering", - default = false - ) - - /** - * Param to enable e2e scribing for candidate filtering step - */ - object EnableMrRequestScribingForCandidateFiltering - extends FSParam[Boolean]( - name = "mr_request_scribing_enable_for_candidate_filtering", - default = false - ) - - /** - * Param to enable e2e scribing with feature hydrating - */ - object EnableMrRequestScribingWithFeatureHydrating - extends FSParam[Boolean]( - name = "mr_request_scribing_enable_with_feature_hydrating", - default = false - ) - - /* - * TargetLevel Feature list for Mr request scribing - */ - object TargetLevelFeatureListForMrRequestScribing - extends FSParam[Seq[String]]( - name = "mr_request_scribing_target_level_feature_list", - default = Seq.empty - ) - - /** - * Param to enable \eps-greedy exploration for BigFiltering/LTV-based filtering - */ - object EnableMrRequestScribingForEpsGreedyExploration - extends FSParam[Boolean]( - name = "mr_request_scribing_eps_greedy_exploration_enable", - default = false - ) - - /** - * Param to control epsilon in \eps-greedy exploration for BigFiltering/LTV-based filtering - */ - object MrRequestScribingEpsGreedyExplorationRatio - extends FSBoundedParam[Double]( - name = "mr_request_scribing_eps_greedy_exploration_ratio", - default = 0.0, - min = 0.0, - max = 1.0 - ) - - /** - * Param to enable scribing dismiss model score - */ - object EnableMrRequestScribingDismissScore - extends FSParam[Boolean]( - name = "mr_request_scribing_dismiss_score_enable", - default = false - ) - - /** - * Param to enable scribing BigFiltering supervised model(s) score(s) - */ - object EnableMrRequestScribingBigFilteringSupervisedScores - extends FSParam[Boolean]( - name = "mr_request_scribing_bigfiltering_supervised_scores_enable", - default = false - ) - - /** - * Param to enable scribing BigFiltering RL model(s) score(s) - */ - object EnableMrRequestScribingBigFilteringRLScores - extends FSParam[Boolean]( - name = "mr_request_scribing_bigfiltering_rl_scores_enable", - default = false - ) - - /** - * Param to flatten mr request scribe - */ - object EnableFlattenMrRequestScribing - extends FSParam[Boolean]( - name = "mr_request_scribing_enable_flatten", - default = false - ) - - /** - * Param to enable NSFW token based filtering - */ - object EnableNsfwTokenBasedFiltering - extends FSParam[Boolean]( - name = "health_and_quality_filter_enable_nsfw_token_based_filtering", - default = false - ) - - object NsfwTokensParam - extends FSParam[Seq[String]]( - name = "health_and_quality_filter_nsfw_tokens", - default = Seq("nsfw", "18+", "\uD83D\uDD1E")) - - object MinimumAllowedAuthorAccountAgeInHours - extends FSBoundedParam[Int]( - name = "health_and_quality_filter_minimum_allowed_author_account_age_in_hours", - default = 0, - min = 0, - max = 168 - ) - - /** - * Param to enable the profanity filter - */ - object EnableProfanityFilterParam - extends FSParam[Boolean]( - name = "health_and_quality_filter_enable_profanity_filter", - default = false - ) - - /** - * Param to enable query the author media representation store - */ - object EnableQueryAuthorMediaRepresentationStore - extends FSParam[Boolean]( - name = "health_and_quality_filter_enable_query_author_media_representation_store", - default = false - ) - - /** - * Threshold to filter a tweet based on the author sensitive media score - */ - object AuthorSensitiveMediaFilteringThreshold - extends FSBoundedParam[Double]( - name = "health_and_quality_filter_author_sensitive_media_filtering_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Threshold to filter a tweet based on the author sensitive media score - */ - object AuthorSensitiveMediaFilteringThresholdForMrTwistly - extends FSBoundedParam[Double]( - name = "health_and_quality_filter_author_sensitive_media_filtering_threshold_for_mrtwistly", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Param to enable filtering the SimCluster tweet if it has AbuseStrike_Top2Percent entitiy - */ - object EnableAbuseStrikeTop2PercentFilterSimCluster - extends FSParam[Boolean]( - name = "health_signal_store_enable_abuse_strike_top_2_percent_filter_sim_cluster", - default = false - ) - - /** - * Param to enable filtering the SimCluster tweet if it has AbuseStrike_Top1Percent entitiy - */ - object EnableAbuseStrikeTop1PercentFilterSimCluster - extends FSParam[Boolean]( - name = "health_signal_store_enable_abuse_strike_top_1_percent_filter_sim_cluster", - default = false - ) - - /** - * Param to enable filtering the SimCluster tweet if it has AbuseStrike_Top0.5Percent entitiy - */ - object EnableAbuseStrikeTop05PercentFilterSimCluster - extends FSParam[Boolean]( - name = "health_signal_store_enable_abuse_strike_top_05_percent_filter_sim_cluster", - default = false - ) - - object EnableAgathaUserHealthModelPredicate - extends FSParam[Boolean]( - name = "health_signal_store_enable_agatha_user_health_model_predicate", - default = false - ) - - /** - * Threshold to filter a tweet based on the agatha_calibrated_nsfw score of its author for MrTwistly - */ - object AgathaCalibratedNSFWThresholdForMrTwistly - extends FSBoundedParam[Double]( - name = "health_signal_store_agatha_calibrated_nsfw_threshold_for_mrtwistly", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Threshold to filter a tweet based on the agatha_calibrated_nsfw score of its author - */ - object AgathaCalibratedNSFWThreshold - extends FSBoundedParam[Double]( - name = "health_signal_store_agatha_calibrated_nsfw_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Threshold to filter a tweet based on the agatha_nsfw_text_user score of its author for MrTwistly - */ - object AgathaTextNSFWThresholdForMrTwistly - extends FSBoundedParam[Double]( - name = "health_signal_store_agatha_text_nsfw_threshold_for_mrtwistly", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Threshold to filter a tweet based on the agatha_nsfw_text_user score of its author - */ - object AgathaTextNSFWThreshold - extends FSBoundedParam[Double]( - name = "health_signal_store_agatha_text_nsfw_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Threshold to bucket a user based on the agatha_calibrated_nsfw score of the tweet author - */ - object AgathaCalibratedNSFWBucketThreshold - extends FSBoundedParam[Double]( - name = "health_signal_store_agatha_calibrated_nsfw_bucket_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Threshold to bucket a user based on the agatha_nsfw_text_user score of the tweet author - */ - object AgathaTextNSFWBucketThreshold - extends FSBoundedParam[Double]( - name = "health_signal_store_agatha_text_nsfw_bucket_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Param to enable filtering using pnsfw_text_tweet model. - */ - object EnableHealthSignalStorePnsfwTweetTextPredicate - extends FSParam[Boolean]( - name = "health_signal_store_enable_pnsfw_tweet_text_predicate", - default = false - ) - - /** - * Threshold score for filtering based on pnsfw_text_tweet Model. - */ - object PnsfwTweetTextThreshold - extends FSBoundedParam[Double]( - name = "health_signal_store_pnsfw_tweet_text_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Threshold score for bucketing based on pnsfw_text_tweet Model. - */ - object PnsfwTweetTextBucketingThreshold - extends FSBoundedParam[Double]( - name = "health_signal_store_pnsfw_tweet_text_bucketing_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Enable filtering tweets with media based on pnsfw_media_tweet Model for OON tweets only. - */ - object PnsfwTweetMediaFilterOonOnly - extends FSParam[Boolean]( - name = "health_signal_store_pnsfw_tweet_media_filter_oon_only", - default = true - ) - - /** - * Threshold score for filtering tweets with media based on pnsfw_media_tweet Model. - */ - object PnsfwTweetMediaThreshold - extends FSBoundedParam[Double]( - name = "health_signal_store_pnsfw_tweet_media_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Threshold score for filtering tweets with images based on pnsfw_media_tweet Model. - */ - object PnsfwTweetImageThreshold - extends FSBoundedParam[Double]( - name = "health_signal_store_pnsfw_tweet_image_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Threshold score for filtering quote/reply tweets based on source tweet's media - */ - object PnsfwQuoteTweetThreshold - extends FSBoundedParam[Double]( - name = "health_signal_store_pnsfw_quote_tweet_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Threshold score for bucketing based on pnsfw_media_tweet Model. - */ - object PnsfwTweetMediaBucketingThreshold - extends FSBoundedParam[Double]( - name = "health_signal_store_pnsfw_tweet_media_bucketing_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Param to enable filtering using multilingual psnfw predicate - */ - object EnableHealthSignalStoreMultilingualPnsfwTweetTextPredicate - extends FSParam[Boolean]( - name = "health_signal_store_enable_multilingual_pnsfw_tweet_text_predicate", - default = false - ) - - /** - * Language sequence we will query pnsfw scores for - */ - object MultilingualPnsfwTweetTextSupportedLanguages - extends FSParam[Seq[String]]( - name = "health_signal_store_multilingual_pnsfw_tweet_supported_languages", - default = Seq.empty[String], - ) - - /** - * Threshold score per language for bucketing based on pnsfw scores. - */ - object MultilingualPnsfwTweetTextBucketingThreshold - extends FSParam[Seq[Double]]( - name = "health_signal_store_multilingual_pnsfw_tweet_text_bucketing_thresholds", - default = Seq.empty[Double], - ) - - /** - * Threshold score per language for filtering based on pnsfw scores. - */ - object MultilingualPnsfwTweetTextFilteringThreshold - extends FSParam[Seq[Double]]( - name = "health_signal_store_multilingual_pnsfw_tweet_text_filtering_thresholds", - default = Seq.empty[Double], - ) - - /** - * List of models to threshold scores for bucketing purposes - */ - object MultilingualPnsfwTweetTextBucketingModelList - extends FSEnumSeqParam[NsfwTextDetectionModel.type]( - name = "health_signal_store_multilingual_pnsfw_tweet_text_bucketing_models_ids", - default = Seq(NsfwTextDetectionModel.ProdModel), - enum = NsfwTextDetectionModel - ) - - object MultilingualPnsfwTweetTextModel - extends FSEnumParam[NsfwTextDetectionModel.type]( - name = "health_signal_store_multilingual_pnsfw_tweet_text_model", - default = NsfwTextDetectionModel.ProdModel, - enum = NsfwTextDetectionModel - ) - - /** - * Param to determine media should be enabled for android - */ - object EnableEventSquareMediaAndroid - extends FSParam[Boolean]( - name = "mr_enable_event_media_square_android", - default = false - ) - - /** - * Param to determine expanded media should be enabled for android - */ - object EnableEventPrimaryMediaAndroid - extends FSParam[Boolean]( - name = "mr_enable_event_media_primary_android", - default = false - ) - - /** - * Param to determine media should be enabled for ios for MagicFanout - */ - object EnableEventSquareMediaIosMagicFanoutNewsEvent - extends FSParam[Boolean]( - name = "mr_enable_event_media_square_ios_mf", - default = false - ) - - /** - * Param to configure HTL Visit fatigue - */ - object HTLVisitFatigueTime - extends FSBoundedParam[Int]( - name = "frigate_push_htl_visit_fatigue_time", - default = 20, - min = 0, - max = 72) { - - // Fatigue duration for HTL visit - final val DefaultHoursToFatigueAfterHtlVisit = 20 - final val OldHoursToFatigueAfterHtlVisit = 8 - } - - object MagicFanoutNewsUserGeneratedEventsEnable - extends FSParam[Boolean]( - name = "magicfanout_news_user_generated_events_enable", - default = false) - - object MagicFanoutSkipAccountCountryPredicate - extends FSParam[Boolean]("magicfanout_news_skip_account_country_predicate", false) - - object MagicFanoutNewsEnableDescriptionCopy - extends FSParam[Boolean](name = "magicfanout_news_enable_description_copy", default = false) - - /** - * Enables Custom Targeting for MagicFnaout News events in Pushservice - */ - object MagicFanoutEnableCustomTargetingNewsEvent - extends FSParam[Boolean]("magicfanout_news_event_custom_targeting_enable", false) - - /** - * Enable Topic Copy in MF - */ - object EnableTopicCopyForMF - extends FSParam[Boolean]( - name = "magicfanout_enable_topic_copy", - default = false - ) - - /** - * Enable Topic Copy in MF for implicit topics - */ - object EnableTopicCopyForImplicitTopics - extends FSParam[Boolean]( - name = "magicfanout_enable_topic_copy_erg_interests", - default = false - ) - - /** - * Enable NewCreator push - */ - object EnableNewCreatorPush - extends FSParam[Boolean]( - name = "new_creator_enable_push", - default = false - ) - - /** - * Enable CreatorSubscription push - */ - object EnableCreatorSubscriptionPush - extends FSParam[Boolean]( - name = "creator_subscription_enable_push", - default = false - ) - - /** - * Featureswitch param to enable/disable push recommendations - */ - object EnablePushRecommendationsParam - extends FSParam[Boolean](name = "push_recommendations_enabled", default = false) - - object DisableMlInFilteringFeatureSwitchParam - extends FSParam[Boolean]( - name = "frigate_push_modeling_disable_ml_in_filtering", - default = false - ) - - object EnableMinDurationModifier - extends FSParam[Boolean]( - name = "min_duration_modifier_enable_hour_modifier", - default = false - ) - - object EnableMinDurationModifierV2 - extends FSParam[Boolean]( - name = "min_duration_modifier_enable_hour_modifier_v2", - default = false - ) - - object MinDurationModifierStartHourList - extends FSParam[Seq[Int]]( - name = "min_duration_modifier_start_time_list", - default = Seq(), - ) - - object MinDurationModifierEndHourList - extends FSParam[Seq[Int]]( - name = "min_duration_modifier_start_end_list", - default = Seq(), - ) - - object MinDurationTimeModifierConst - extends FSParam[Seq[Int]]( - name = "min_duration_modifier_const_list", - default = Seq(), - ) - - object EnableQueryUserOpenedHistory - extends FSParam[Boolean]( - name = "min_duration_modifier_enable_query_user_opened_history", - default = false - ) - - object EnableMinDurationModifierByUserHistory - extends FSParam[Boolean]( - name = "min_duration_modifier_enable_hour_modifier_by_user_history", - default = false - ) - - object EnableRandomHourForQuickSend - extends FSParam[Boolean]( - name = "min_duration_modifier_enable_random_hour_for_quick_send", - default = false - ) - - object SendTimeByUserHistoryMaxOpenedThreshold - extends FSBoundedParam[Int]( - name = "min_duration_modifier_max_opened_threshold", - default = 4, - min = 0, - max = 100) - - object SendTimeByUserHistoryNoSendsHours - extends FSBoundedParam[Int]( - name = "min_duration_modifier_no_sends_hours", - default = 1, - min = 0, - max = 24) - - object SendTimeByUserHistoryQuickSendBeforeHours - extends FSBoundedParam[Int]( - name = "min_duration_modifier_quick_send_before_hours", - default = 0, - min = 0, - max = 24) - - object SendTimeByUserHistoryQuickSendAfterHours - extends FSBoundedParam[Int]( - name = "min_duration_modifier_quick_send_after_hours", - default = 0, - min = 0, - max = 24) - - object SendTimeByUserHistoryQuickSendMinDurationInMinute - extends FSBoundedParam[Int]( - name = "min_duration_modifier_quick_send_min_duration", - default = 0, - min = 0, - max = 1440) - - object SendTimeByUserHistoryNoSendMinDuration - extends FSBoundedParam[Int]( - name = "min_duration_modifier_no_send_min_duration", - default = 24, - min = 0, - max = 24) - - object EnableMfGeoTargeting - extends FSParam[Boolean]( - name = "frigate_push_magicfanout_geo_targeting_enable", - default = false) - - /** - * Enable RUX Tweet landing page for push open. When this param is enabled, user will go to RUX - * landing page instead of Tweet details page when opening MagicRecs push. - */ - object EnableRuxLandingPage - extends FSParam[Boolean](name = "frigate_push_enable_rux_landing_page", default = false) - - /** - * Enable RUX Tweet landing page for Ntab Click. When this param is enabled, user will go to RUX - * landing page instead of Tweet details page when click MagicRecs entry on Ntab. - */ - object EnableNTabRuxLandingPage - extends FSParam[Boolean](name = "frigate_push_enable_ntab_rux_landing_page", default = false) - - /** - * Param to enable Onboarding Pushes - */ - object EnableOnboardingPushes - extends FSParam[Boolean]( - name = "onboarding_push_enable", - default = false - ) - - /** - * Param to enable Address Book Pushes - */ - object EnableAddressBookPush - extends FSParam[Boolean]( - name = "onboarding_push_enable_address_book_push", - default = false - ) - - /** - * Param to enable Complete Onboarding Pushes - */ - object EnableCompleteOnboardingPush - extends FSParam[Boolean]( - name = "onboarding_push_enable_complete_onboarding_push", - default = false - ) - - /** - * Param to enable Smart Push Config for MR Override Notifs on Android - */ - object EnableOverrideNotificationsSmartPushConfigForAndroid - extends FSParam[Boolean]( - name = "mr_override_enable_smart_push_config_for_android", - default = false) - - /** - * Param to control the min duration since last MR push for Onboarding Pushes - */ - object MrMinDurationSincePushForOnboardingPushes - extends FSBoundedParam[Duration]( - name = "onboarding_push_min_duration_since_push_days", - default = 4.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Param to control the push fatigue for Onboarding Pushes - */ - object FatigueForOnboardingPushes - extends FSBoundedParam[Duration]( - name = "onboarding_push_fatigue_days", - default = 30.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Param to specify the maximum number of Onboarding Push Notifs in a specified period of time - */ - object MaxOnboardingPushInInterval - extends FSBoundedParam[Int]( - name = "onboarding_push_max_in_interval", - default = 1, - min = 0, - max = 10 - ) - - /** - * Param to disable the Onboarding Push Notif Fatigue - */ - object DisableOnboardingPushFatigue - extends FSParam[Boolean]( - name = "onboarding_push_disable_push_fatigue", - default = false - ) - - /** - * Param to control the inverter for fatigue between consecutive TopTweetsByGeoPush - */ - object TopTweetsByGeoPushInterval - extends FSBoundedParam[Duration]( - name = "top_tweets_by_geo_interval_days", - default = 0.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Param to control the inverter for fatigue between consecutive TripTweets - */ - object HighQualityTweetsPushInterval - extends FSBoundedParam[Duration]( - name = "high_quality_candidates_push_interval_days", - default = 1.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Expiry TTL duration for Tweet Notification types written to history store - */ - object FrigateHistoryTweetNotificationWriteTtl - extends FSBoundedParam[Duration]( - name = "frigate_notification_history_tweet_write_ttl_days", - default = 60.days, - min = Duration.Bottom, - max = Duration.Top - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Expiry TTL duration for Notification written to history store - */ - object FrigateHistoryOtherNotificationWriteTtl - extends FSBoundedParam[Duration]( - name = "frigate_notification_history_other_write_ttl_days", - default = 90.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Param to control maximum number of TopTweetsByGeoPush pushes to receive in an interval - */ - object MaxTopTweetsByGeoPushGivenInterval - extends FSBoundedParam[Int]( - name = "top_tweets_by_geo_push_given_interval", - default = 1, - min = 0, - max = 10 - ) - - /** - * Param to control maximum number of HighQualityTweet pushes to receive in an interval - */ - object MaxHighQualityTweetsPushGivenInterval - extends FSBoundedParam[Int]( - name = "high_quality_candidates_max_push_given_interval", - default = 3, - min = 0, - max = 10 - ) - - /** - * Param to downrank/backfill top tweets by geo candidates - */ - object BackfillRankTopTweetsByGeoCandidates - extends FSParam[Boolean]( - name = "top_tweets_by_geo_backfill_rank", - default = false - ) - - /** - * Determine whether to use aggressive thresholds for Health filtering on SearchTweet - */ - object PopGeoTweetEnableAggressiveThresholds - extends FSParam[Boolean]( - name = "top_tweets_by_geo_enable_aggressive_health_thresholds", - default = false - ) - - /** - * Param to apply different scoring functions to select top tweets by geo candidates - */ - object ScoringFuncForTopTweetsByGeo - extends FSParam[String]( - name = "top_tweets_by_geo_scoring_function", - default = "Pop8H", - ) - - /** - * Param to query different stores in pop geo service. - */ - object TopTweetsByGeoCombinationParam - extends FSEnumParam[TopTweetsForGeoCombination.type]( - name = "top_tweets_by_geo_combination_id", - default = TopTweetsForGeoCombination.Default, - enum = TopTweetsForGeoCombination - ) - - /** - * Param for popgeo tweet version - */ - object PopGeoTweetVersionParam - extends FSEnumParam[PopGeoTweetVersion.type]( - name = "top_tweets_by_geo_version_id", - default = PopGeoTweetVersion.Prod, - enum = PopGeoTweetVersion - ) - - /** - * Param to query what length of hash for geoh store - */ - object GeoHashLengthList - extends FSParam[Seq[Int]]( - name = "top_tweets_by_geo_hash_length_list", - default = Seq(4), - ) - - /** - * Param to include country code results as back off . - */ - object EnableCountryCodeBackoffTopTweetsByGeo - extends FSParam[Boolean]( - name = "top_tweets_by_geo_enable_country_code_backoff", - default = false, - ) - - /** - * Param to decide ranking function for fetched top tweets by geo - */ - object RankingFunctionForTopTweetsByGeo - extends FSEnumParam[TopTweetsForGeoRankingFunction.type]( - name = "top_tweets_by_geo_ranking_function_id", - default = TopTweetsForGeoRankingFunction.Score, - enum = TopTweetsForGeoRankingFunction - ) - - /** - * Param to enable top tweets by geo candidates - */ - object EnableTopTweetsByGeoCandidates - extends FSParam[Boolean]( - name = "top_tweets_by_geo_enable_candidate_source", - default = false - ) - - /** - * Param to enable top tweets by geo candidates for dormant users - */ - object EnableTopTweetsByGeoCandidatesForDormantUsers - extends FSParam[Boolean]( - name = "top_tweets_by_geo_enable_candidate_source_dormant_users", - default = false - ) - - /** - * Param to specify the maximum number of Top Tweets by Geo candidates to take - */ - object MaxTopTweetsByGeoCandidatesToTake - extends FSBoundedParam[Int]( - name = "top_tweets_by_geo_candidates_to_take", - default = 10, - min = 0, - max = 100 - ) - - /** - * Param to min duration since last MR push for top tweets by geo pushes - */ - object MrMinDurationSincePushForTopTweetsByGeoPushes - extends FSBoundedParam[Duration]( - name = "top_tweets_by_geo_min_duration_since_last_mr_days", - default = 3.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Param to enable FRS candidate tweets - */ - object EnableFrsCandidates - extends FSParam[Boolean]( - name = "frs_tweet_candidate_enable_adaptor", - default = false - ) - - /** - * Param to enable FRSTweet candidates for topic setting users - * */ - object EnableFrsTweetCandidatesTopicSetting - extends FSParam[Boolean]( - name = "frs_tweet_candidate_enable_adaptor_for_topic_setting", - default = false - ) - - /** - * Param to enable topic annotations for FRSTweet candidates tweets - * */ - object EnableFrsTweetCandidatesTopicAnnotation - extends FSParam[Boolean]( - name = "frs_tweet_candidate_enable_topic_annotation", - default = false - ) - - /** - * Param to enable topic copy for FRSTweet candidates tweets - * */ - object EnableFrsTweetCandidatesTopicCopy - extends FSParam[Boolean]( - name = "frs_tweet_candidate_enable_topic_copy", - default = false - ) - - /** - * Topic score threshold for FRSTweet candidates topic annotations - * */ - object FrsTweetCandidatesTopicScoreThreshold - extends FSBoundedParam[Double]( - name = "frs_tweet_candidate_topic_score_threshold", - default = 0.0, - min = 0.0, - max = 100.0 - ) - - /** - * Param to enable mr modeling-based candidates tweets - * */ - object EnableMrModelingBasedCandidates - extends FSParam[Boolean]( - name = "candidate_generation_model_enable_adaptor", - default = false - ) - - /** - Param to enable mr modeling-based candidates tweets for topic setting users - * */ - object EnableMrModelingBasedCandidatesTopicSetting - extends FSParam[Boolean]( - name = "candidate_generation_model_enable_adaptor_for_topic_setting", - default = false - ) - - /** - * Param to enable topic annotations for mr modeling-based candidates tweets - * */ - object EnableMrModelingBasedCandidatesTopicAnnotation - extends FSParam[Boolean]( - name = "candidate_generation_model_enable_adaptor_topic_annotation", - default = false - ) - - /** - * Topic score threshold for mr modeling based candidates topic annotations - * */ - object MrModelingBasedCandidatesTopicScoreThreshold - extends FSBoundedParam[Double]( - name = "candidate_generation_model_topic_score_threshold", - default = 0.0, - min = 0.0, - max = 100.0 - ) - - /** - * Param to enable topic copy for mr modeling-based candidates tweets - * */ - object EnableMrModelingBasedCandidatesTopicCopy - extends FSParam[Boolean]( - name = "candidate_generation_model_enable_topic_copy", - default = false - ) - - /** - * Number of max mr modeling based candidates - * */ - object NumberOfMaxMrModelingBasedCandidates - extends FSBoundedParam[Int]( - name = "candidate_generation_model_max_mr_modeling_based_candidates", - default = 200, - min = 0, - max = 1000 - ) - - /** - * Enable the traffic to use fav threshold - * */ - object EnableThresholdOfFavMrModelingBasedCandidates - extends FSParam[Boolean]( - name = "candidate_generation_model_enable_fav_threshold", - default = false - ) - - /** - * Threshold of fav for mr modeling based candidates - * */ - object ThresholdOfFavMrModelingBasedCandidates - extends FSBoundedParam[Int]( - name = "candidate_generation_model_fav_threshold", - default = 0, - min = 0, - max = 500 - ) - - /** - * Filtered threshold for mr modeling based candidates - * */ - object CandidateGenerationModelCosineThreshold - extends FSBoundedParam[Double]( - name = "candidate_generation_model_cosine_threshold", - default = 0.9, - min = 0.0, - max = 1.0 - ) - - /* - * ANN hyparameters - * */ - object ANNEfQuery - extends FSBoundedParam[Int]( - name = "candidate_generation_model_ann_ef_query", - default = 300, - min = 50, - max = 1500 - ) - - /** - * Param to do real A/B impression for FRS candidates to avoid dilution - */ - object EnableResultFromFrsCandidates - extends FSParam[Boolean]( - name = "frs_tweet_candidate_enable_returned_result", - default = false - ) - - /** - * Param to enable hashspace candidate tweets - */ - object EnableHashspaceCandidates - extends FSParam[Boolean]( - name = "hashspace_candidate_enable_adaptor", - default = false - ) - - /** - * Param to enable hashspace candidates tweets for topic setting users - * */ - object EnableHashspaceCandidatesTopicSetting - extends FSParam[Boolean]( - name = "hashspace_candidate_enable_adaptor_for_topic_setting", - default = false - ) - - /** - * Param to enable topic annotations for hashspace candidates tweets - * */ - object EnableHashspaceCandidatesTopicAnnotation - extends FSParam[Boolean]( - name = "hashspace_candidate_enable_topic_annotation", - default = false - ) - - /** - * Param to enable topic copy for hashspace candidates tweets - * */ - object EnableHashspaceCandidatesTopicCopy - extends FSParam[Boolean]( - name = "hashspace_candidate_enable_topic_copy", - default = false - ) - - /** - * Topic score threshold for hashspace candidates topic annotations - * */ - object HashspaceCandidatesTopicScoreThreshold - extends FSBoundedParam[Double]( - name = "hashspace_candidate_topic_score_threshold", - default = 0.0, - min = 0.0, - max = 100.0 - ) - - /** - * Param to do real A/B impression for hashspace candidates to avoid dilution - */ - object EnableResultFromHashspaceCandidates - extends FSParam[Boolean]( - name = "hashspace_candidate_enable_returned_result", - default = false - ) - - /** - * Param to enable detopic tweet candidates in adaptor - */ - object EnableDeTopicTweetCandidates - extends FSParam[Boolean]( - name = "detopic_tweet_candidate_enable_adaptor", - default = false - ) - - /** - * Param to enable detopic tweet candidates results (to avoid dilution) - */ - object EnableDeTopicTweetCandidateResults - extends FSParam[Boolean]( - name = "detopic_tweet_candidate_enable_results", - default = false - ) - - /** - * Param to specify whether to provide a custom list of topics in request - */ - object EnableDeTopicTweetCandidatesCustomTopics - extends FSParam[Boolean]( - name = "detopic_tweet_candidate_enable_custom_topics", - default = false - ) - - /** - * Param to specify whether to provide a custom language in request - */ - object EnableDeTopicTweetCandidatesCustomLanguages - extends FSParam[Boolean]( - name = "detopic_tweet_candidate_enable_custom_languages", - default = false - ) - - /** - * Number of detopic tweet candidates in the request - * */ - object NumberOfDeTopicTweetCandidates - extends FSBoundedParam[Int]( - name = "detopic_tweet_candidate_num_candidates_in_request", - default = 600, - min = 0, - max = 3000 - ) - - /** - * Max Number of detopic tweet candidates returned in adaptor - * */ - object NumberOfMaxDeTopicTweetCandidatesReturned - extends FSBoundedParam[Int]( - name = "detopic_tweet_candidate_max_num_candidates_returned", - default = 200, - min = 0, - max = 3000 - ) - - /** - * Param to enable F1 from protected Authors - */ - object EnableF1FromProtectedTweetAuthors - extends FSParam[Boolean]( - "f1_enable_protected_tweets", - false - ) - - /** - * Param to enable safe user tweet tweetypie store - */ - object EnableSafeUserTweetTweetypieStore - extends FSParam[Boolean]( - "mr_infra_enable_use_safe_user_tweet_tweetypie", - false - ) - - /** - * Param to min duration since last MR push for top tweets by geo pushes - */ - object EnableMrMinDurationSinceMrPushFatigue - extends FSParam[Boolean]( - name = "top_tweets_by_geo_enable_min_duration_since_mr_fatigue", - default = false - ) - - /** - * Param to check time since last time user logged in for geo top tweets by geo push - */ - object TimeSinceLastLoginForGeoPopTweetPush - extends FSBoundedParam[Duration]( - name = "top_tweets_by_geo_time_since_last_login_in_days", - default = 14.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Param to check time since last time user logged in for geo top tweets by geo push - */ - object MinimumTimeSinceLastLoginForGeoPopTweetPush - extends FSBoundedParam[Duration]( - name = "top_tweets_by_geo_minimum_time_since_last_login_in_days", - default = 14.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** How long we wait after a user visited the app before sending them a space fanout rec */ - object SpaceRecsAppFatigueDuration - extends FSBoundedParam[Duration]( - name = "space_recs_app_fatigue_duration_hours", - default = 4.hours, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** The fatigue time-window for OON space fanout recs, e.g. 1 push every 3 days */ - object OONSpaceRecsFatigueDuration - extends FSBoundedParam[Duration]( - name = "space_recs_oon_fatigue_duration_days", - default = 1.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** The global fatigue time-window for space fanout recs, e.g. 1 push every 3 days */ - object SpaceRecsGlobalFatigueDuration - extends FSBoundedParam[Duration]( - name = "space_recs_global_fatigue_duration_days", - default = 1.day, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** The min-interval between space fanout recs. - * After receiving a space fanout rec, they must wait a minimum of this - * interval before eligibile for another */ - object SpaceRecsFatigueMinIntervalDuration - extends FSBoundedParam[Duration]( - name = "space_recs_fatigue_mininterval_duration_minutes", - default = 30.minutes, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromMinutes - } - - /** Space fanout user-follow rank threshold. - * Users targeted by a follow that is above this threshold will be filtered */ - object SpaceRecsRealgraphThreshold - extends FSBoundedParam[Int]( - name = "space_recs_realgraph_threshold", - default = 50, - max = 500, - min = 0 - ) - - object EnableHydratingRealGraphTargetUserFeatures - extends FSParam[Boolean]( - name = "frigate_push_modeling_enable_hydrating_real_graph_target_user_feature", - default = true - ) - - /** Param to reduce dillution when checking if a space is featured or not */ - object CheckFeaturedSpaceOON - extends FSParam[Boolean](name = "space_recs_check_if_its_featured_space", default = false) - - /** Enable Featured Spaces Rules for OON spaces */ - object EnableFeaturedSpacesOON - extends FSParam[Boolean](name = "space_recs_enable_featured_spaces_oon", default = false) - - /** Enable Geo Targeting */ - object EnableGeoTargetingForSpaces - extends FSParam[Boolean](name = "space_recs_enable_geo_targeting", default = false) - - /** Number of max pushes within the fatigue duration for OON Space Recs */ - object OONSpaceRecsPushLimit - extends FSBoundedParam[Int]( - name = "space_recs_oon_push_limit", - default = 1, - max = 3, - min = 0 - ) - - /** Space fanout recs, number of max pushes within the fatigue duration */ - object SpaceRecsGlobalPushLimit - extends FSBoundedParam[Int]( - name = "space_recs_global_push_limit", - default = 3, - max = 50, - min = 0 - ) - - /** - * Param to enable score based override. - */ - object EnableOverrideNotificationsScoreBasedOverride - extends FSParam[Boolean]( - name = "mr_override_enable_score_ranking", - default = false - ) - - /** - * Param to determine the lookback duration when searching for override info. - */ - object OverrideNotificationsLookbackDurationForOverrideInfo - extends FSBoundedParam[Duration]( - name = "mr_override_lookback_duration_override_info_in_days", - default = 30.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Param to determine the lookback duration when searching for impression ids. - */ - object OverrideNotificationsLookbackDurationForImpressionId - extends FSBoundedParam[Duration]( - name = "mr_override_lookback_duration_impression_id_in_days", - default = 30.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Param to enable sending multiple target ids in the payload. - */ - object EnableOverrideNotificationsMultipleTargetIds - extends FSParam[Boolean]( - name = "mr_override_enable_multiple_target_ids", - default = false - ) - - /** - * Param for MR Web Notifications holdback - */ - object MRWebHoldbackParam - extends FSParam[Boolean]( - name = "mr_web_notifications_holdback", - default = false - ) - - object CommonRecommendationTypeDenyListPushHoldbacks - extends FSParam[Seq[String]]( - name = "crt_to_exclude_from_holdbacks_push_holdbacks", - default = Seq.empty[String] - ) - - /** - * Param to enable sending number of slots to maintain in the payload. - */ - object EnableOverrideNotificationsNSlots - extends FSParam[Boolean]( - name = "mr_override_enable_n_slots", - default = false - ) - - /** - * Enable down ranking of NUPS and pop geo topic follow candidates for new user playbook. - */ - object EnableDownRankOfNewUserPlaybookTopicFollowPush - extends FSParam[Boolean]( - name = "topic_follow_new_user_playbook_enable_down_rank", - default = false - ) - - /** - * Enable down ranking of NUPS and pop geo topic tweet candidates for new user playbook. - */ - object EnableDownRankOfNewUserPlaybookTopicTweetPush - extends FSParam[Boolean]( - name = "topic_tweet_new_user_playbook_enable_down_rank", - default = false - ) - - /** - * Param to enable/disable employee only spaces for fanout of notifications - */ - object EnableEmployeeOnlySpaceNotifications - extends FSParam[Boolean](name = "space_recs_employee_only_enable", default = false) - - /** - * NTab spaces ttl experiments - */ - object EnableSpacesTtlForNtab - extends FSParam[Boolean]( - name = "ntab_spaces_ttl_enable", - default = false - ) - - /** - * Param to determine the ttl duration for space notifications on NTab. - */ - object SpaceNotificationsTTLDurationForNTab - extends FSBoundedParam[Duration]( - name = "ntab_spaces_ttl_hours", - default = 1.hour, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /* - * NTab override experiments - * see go/ntab-override experiment brief for more details - */ - - /** - * Override notifications for Spaces on lockscreen. - */ - object EnableOverrideForSpaces - extends FSParam[Boolean]( - name = "mr_override_spaces", - default = false - ) - - /** - * Param to enable storing the Generic Notification Key. - */ - object EnableStoringNtabGenericNotifKey - extends FSParam[Boolean]( - name = "ntab_enable_storing_generic_notif_key", - default = false - ) - - /** - * Param to enable deleting the Target's timeline. - */ - object EnableDeletingNtabTimeline - extends FSParam[Boolean]( - name = "ntab_enable_delete_timeline", - default = false - ) - - /** - * Param to enable sending the overrideId - * to NTab which enables override support in NTab-api - */ - object EnableOverrideIdNTabRequest - extends FSParam[Boolean]( - name = "ntab_enable_override_id_in_request", - default = false - ) - - /** - * [Override Workstream] Param to enable NTab override n-slot feature. - */ - object EnableNslotsForOverrideOnNtab - extends FSParam[Boolean]( - name = "ntab_enable_override_max_count", - default = false - ) - - /** - * Param to determine the lookback duration for override candidates on NTab. - */ - object OverrideNotificationsLookbackDurationForNTab - extends FSBoundedParam[Duration]( - name = "ntab_override_lookback_duration_days", - default = 30.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Param to determine the max count for candidates on NTab. - */ - object OverrideNotificationsMaxCountForNTab - extends FSBoundedParam[Int]( - name = "ntab_override_limit", - min = 0, - max = Int.MaxValue, - default = 4) - - //// end override experiments //// - /** - * Param to enable top tweet impressions notification - */ - object EnableTopTweetImpressionsNotification - extends FSParam[Boolean]( - name = "top_tweet_impressions_notification_enable", - default = false - ) - - /** - * Param to control the inverter for fatigue between consecutive TweetImpressions - */ - object TopTweetImpressionsNotificationInterval - extends FSBoundedParam[Duration]( - name = "top_tweet_impressions_notification_interval_days", - default = 7.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * The min-interval between TweetImpressions notifications. - * After receiving a TweetImpressions notif, they must wait a minimum of this - * interval before being eligible for another - */ - object TopTweetImpressionsFatigueMinIntervalDuration - extends FSBoundedParam[Duration]( - name = "top_tweet_impressions_fatigue_mininterval_duration_days", - default = 1.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Maximum number of top tweet impressions notifications to receive in an interval - */ - object MaxTopTweetImpressionsNotifications - extends FSBoundedParam( - name = "top_tweet_impressions_fatigue_max_in_interval", - default = 0, - min = 0, - max = 10 - ) - - /** - * Param for min number of impressions counts to be eligible for lonely_birds_tweet_impressions model - */ - object TopTweetImpressionsMinRequired - extends FSBoundedParam[Int]( - name = "top_tweet_impressions_min_required", - default = 25, - min = 0, - max = Int.MaxValue - ) - - /** - * Param for threshold of impressions counts to notify for lonely_birds_tweet_impressions model - */ - object TopTweetImpressionsThreshold - extends FSBoundedParam[Int]( - name = "top_tweet_impressions_threshold", - default = 25, - min = 0, - max = Int.MaxValue - ) - - /** - * Param for the number of days to search up to for a user's original tweets - */ - object TopTweetImpressionsOriginalTweetsNumDaysSearch - extends FSBoundedParam[Int]( - name = "top_tweet_impressions_original_tweets_num_days_search", - default = 3, - min = 0, - max = 21 - ) - - /** - * Param for the minimum number of original tweets a user needs to be considered an original author - */ - object TopTweetImpressionsMinNumOriginalTweets - extends FSBoundedParam[Int]( - name = "top_tweet_impressions_num_original_tweets", - default = 3, - min = 0, - max = Int.MaxValue - ) - - /** - * Param for the max number of favorites any original Tweet can have - */ - object TopTweetImpressionsMaxFavoritesPerTweet - extends FSBoundedParam[Int]( - name = "top_tweet_impressions_max_favorites_per_tweet", - default = 3, - min = 0, - max = Int.MaxValue - ) - - /** - * Param for the max number of total inbound favorites for a user's tweets - */ - object TopTweetImpressionsTotalInboundFavoritesLimit - extends FSBoundedParam[Int]( - name = "top_tweet_impressions_total_inbound_favorites_limit", - default = 60, - min = 0, - max = Int.MaxValue - ) - - /** - * Param for the number of days to search for tweets to count the total inbound favorites - */ - object TopTweetImpressionsTotalFavoritesLimitNumDaysSearch - extends FSBoundedParam[Int]( - name = "top_tweet_impressions_total_favorites_limit_num_days_search", - default = 7, - min = 0, - max = 21 - ) - - /** - * Param for the max number of recent tweets Tflock should return - */ - object TopTweetImpressionsRecentTweetsByAuthorStoreMaxResults - extends FSBoundedParam[Int]( - name = "top_tweet_impressions_recent_tweets_by_author_store_max_results", - default = 50, - min = 0, - max = 1000 - ) - - /* - * Param to represent the max number of slots to maintain for Override Notifications - */ - object OverrideNotificationsMaxNumOfSlots - extends FSBoundedParam[Int]( - name = "mr_override_max_num_slots", - default = 1, - max = 10, - min = 1 - ) - - object EnableOverrideMaxSlotFn - extends FSParam[Boolean]( - name = "mr_override_enable_max_num_slots_fn", - default = false - ) - - object OverrideMaxSlotFnPushCapKnobs - extends FSParam[Seq[Double]]("mr_override_fn_pushcap_knobs", default = Seq.empty[Double]) - - object OverrideMaxSlotFnNSlotKnobs - extends FSParam[Seq[Double]]("mr_override_fn_nslot_knobs", default = Seq.empty[Double]) - - object OverrideMaxSlotFnPowerKnobs - extends FSParam[Seq[Double]]("mr_override_fn_power_knobs", default = Seq.empty[Double]) - - object OverrideMaxSlotFnWeight - extends FSBoundedParam[Double]( - "mr_override_fn_weight", - default = 1.0, - min = 0.0, - max = Double.MaxValue) - - /** - * Use to enable sending target ids in the Smart Push Payload - */ - object EnableTargetIdsInSmartPushPayload - extends FSParam[Boolean](name = "mr_override_enable_target_ids", default = true) - - /** - * Param to enable override by target id for MagicFanoutSportsEvent candidates - */ - object EnableTargetIdInSmartPushPayloadForMagicFanoutSportsEvent - extends FSParam[Boolean]( - name = "mr_override_enable_target_id_for_magic_fanout_sports_event", - default = true) - - /** - * Param to enable secondary account predicate on MF NFY - */ - object EnableSecondaryAccountPredicateMF - extends FSParam[Boolean]( - name = "frigate_push_magicfanout_secondary_account_predicate", - default = false - ) - - /** - * Enables showing our customers videos on their notifications - */ - object EnableInlineVideo - extends FSParam[Boolean](name = "mr_inline_enable_inline_video", default = false) - - /** - * Enables autoplay for inline videos - */ - object EnableAutoplayForInlineVideo - extends FSParam[Boolean](name = "mr_inline_enable_autoplay_for_inline_video", default = false) - - /** - * Enable OON filtering based on MentionFilter. - */ - object EnableOONFilteringBasedOnUserSettings - extends FSParam[Boolean](name = "oon_filtering_enable_based_on_user_settings", false) - - /** - * Enables Custom Thread Ids which is used to ungroup notifications for N-slots on iOS - */ - object EnableCustomThreadIdForOverride - extends FSParam[Boolean](name = "mr_override_enable_custom_thread_id", default = false) - - /** - * Enables showing verified symbol in the push presentation - */ - object EnablePushPresentationVerifiedSymbol - extends FSParam[Boolean](name = "push_presentation_enable_verified_symbol", default = false) - - /** - * Decide subtext in Android push header - */ - object SubtextInAndroidPushHeaderParam - extends FSEnumParam[SubtextForAndroidPushHeader.type]( - name = "push_presentation_subtext_in_android_push_header_id", - default = SubtextForAndroidPushHeader.None, - enum = SubtextForAndroidPushHeader) - - /** - * Enable SimClusters Targeting For Spaces. If false we just drop all candidates with such targeting reason - */ - object EnableSimClusterTargetingSpaces - extends FSParam[Boolean](name = "space_recs_send_simcluster_recommendations", default = false) - - /** - * Param to control threshold for dot product of simcluster based targeting on Spaces - */ - object SpacesTargetingSimClusterDotProductThreshold - extends FSBoundedParam[Double]( - "space_recs_simclusters_dot_product_threshold", - default = 0.0, - min = 0.0, - max = 10.0) - - /** - * Param to control top-k clusters simcluster based targeting on Spaces - */ - object SpacesTopKSimClusterCount - extends FSBoundedParam[Int]( - "space_recs_simclusters_top_k_count", - default = 1, - min = 1, - max = 50) - - /** SimCluster users host/speaker must meet this follower count minimum threshold to be considered for sends */ - object SpaceRecsSimClusterUserMinimumFollowerCount - extends FSBoundedParam[Int]( - name = "space_recs_simcluster_user_min_follower_count", - default = 5000, - max = Int.MaxValue, - min = 0 - ) - - /** - * Target has been bucketed into the Inline Action App Visit Fatigue Experiment - */ - object TargetInInlineActionAppVisitFatigue - extends FSParam[Boolean](name = "inline_action_target_in_app_visit_fatigue", default = false) - - /** - * Enables Inline Action App Visit Fatigue - */ - object EnableInlineActionAppVisitFatigue - extends FSParam[Boolean](name = "inline_action_enable_app_visit_fatigue", default = false) - - /** - * Determines the fatigue that we should apply when the target user has performed an inline action - */ - object InlineActionAppVisitFatigue - extends FSBoundedParam[Duration]( - name = "inline_action_app_visit_fatigue_hours", - default = 8.hours, - min = 1.hour, - max = 48.hours) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Weight for reranking(oonc - weight * nudityRate) - */ - object AuthorSensitiveScoreWeightInReranking - extends FSBoundedParam[Double]( - name = "rerank_candidates_author_sensitive_score_weight_in_reranking", - default = 0.0, - min = -100.0, - max = 100.0 - ) - - /** - * Param to control the last active space listener threshold to filter out based on that - */ - object SpaceParticipantHistoryLastActiveThreshold - extends FSBoundedParam[Duration]( - name = "space_recs_last_active_space_listener_threshold_in_hours", - default = 0.hours, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /* - * Param to enable mr user simcluster feature set (v2020) hydration for modeling-based candidate generation - * */ - object HydrateMrUserSimclusterV2020InModelingBasedCG - extends FSParam[Boolean]( - name = "candidate_generation_model_hydrate_mr_user_simcluster_v2020", - default = false) - - /* - * Param to enable mr semantic core feature set hydration for modeling-based candidate generation - * */ - object HydrateMrUserSemanticCoreInModelingBasedCG - extends FSParam[Boolean]( - name = "candidate_generation_model_hydrate_mr_user_semantic_core", - default = false) - - /* - * Param to enable mr semantic core feature set hydration for modeling-based candidate generation - * */ - object HydrateOnboardingInModelingBasedCG - extends FSParam[Boolean]( - name = "candidate_generation_model_hydrate_onboarding", - default = false) - - /* - * Param to enable mr topic follow feature set hydration for modeling-based candidate generation - * */ - object HydrateTopicFollowInModelingBasedCG - extends FSParam[Boolean]( - name = "candidate_generation_model_hydrate_topic_follow", - default = false) - - /* - * Param to enable mr user topic feature set hydration for modeling-based candidate generation - * */ - object HydrateMrUserTopicInModelingBasedCG - extends FSParam[Boolean]( - name = "candidate_generation_model_hydrate_mr_user_topic", - default = false) - - /* - * Param to enable mr user topic feature set hydration for modeling-based candidate generation - * */ - object HydrateMrUserAuthorInModelingBasedCG - extends FSParam[Boolean]( - name = "candidate_generation_model_hydrate_mr_user_author", - default = false) - - /* - * Param to enable user penguin language feature set hydration for modeling-based candidate generation - * */ - object HydrateUserPenguinLanguageInModelingBasedCG - extends FSParam[Boolean]( - name = "candidate_generation_model_hydrate_user_penguin_language", - default = false) - /* - * Param to enable user geo feature set hydration for modeling-based candidate generation - * */ - object HydrateUseGeoInModelingBasedCG - extends FSParam[Boolean]( - name = "candidate_generation_model_hydrate_user_geo", - default = false) - - /* - * Param to enable mr user hashspace embedding feature set hydration for modeling-based candidate generation - * */ - object HydrateMrUserHashspaceEmbeddingInModelingBasedCG - extends FSParam[Boolean]( - name = "candidate_generation_model_hydrate_mr_user_hashspace_embedding", - default = false) - /* - * Param to enable user tweet text feature hydration - * */ - object EnableMrUserEngagedTweetTokensFeature - extends FSParam[Boolean]( - name = "feature_hydration_mr_user_engaged_tweet_tokens", - default = false) - - /** - * Params for CRT based see less often fatigue rules - */ - object EnableF1TriggerSeeLessOftenFatigue - extends FSParam[Boolean]( - name = "seelessoften_enable_f1_trigger_fatigue", - default = false - ) - - object EnableNonF1TriggerSeeLessOftenFatigue - extends FSParam[Boolean]( - name = "seelessoften_enable_nonf1_trigger_fatigue", - default = false - ) - - /** - * Adjust the NtabCaretClickFatigue for candidates if it is triggered by - * TripHqTweet candidates - */ - object AdjustTripHqTweetTriggeredNtabCaretClickFatigue - extends FSParam[Boolean]( - name = "seelessoften_adjust_trip_hq_tweet_triggered_fatigue", - default = false - ) - - object NumberOfDaysToFilterForSeeLessOftenForF1TriggerF1 - extends FSBoundedParam[Duration]( - name = "seelessoften_for_f1_trigger_f1_tofiltermr_days", - default = 7.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - object NumberOfDaysToReducePushCapForSeeLessOftenForF1TriggerF1 - extends FSBoundedParam[Duration]( - name = "seelessoften_for_f1_trigger_f1_toreduce_pushcap_days", - default = 30.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - object NumberOfDaysToFilterForSeeLessOftenForF1TriggerNonF1 - extends FSBoundedParam[Duration]( - name = "seelessoften_for_f1_trigger_nonf1_tofiltermr_days", - default = 7.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - object NumberOfDaysToReducePushCapForSeeLessOftenForF1TriggerNonF1 - extends FSBoundedParam[Duration]( - name = "seelessoften_for_f1_trigger_non_f1_toreduce_pushcap_days", - default = 30.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - object NumberOfDaysToFilterForSeeLessOftenForNonF1TriggerF1 - extends FSBoundedParam[Duration]( - name = "seelessoften_for_nonf1_trigger_f1_tofiltermr_days", - default = 7.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - object NumberOfDaysToReducePushCapForSeeLessOftenForNonF1TriggerF1 - extends FSBoundedParam[Duration]( - name = "seelessoften_for_nonf1_trigger_f1_toreduce_pushcap_days", - default = 30.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - object NumberOfDaysToFilterForSeeLessOftenForNonF1TriggerNonF1 - extends FSBoundedParam[Duration]( - name = "seelessoften_for_nonf1_trigger_nonf1_tofiltermr_days", - default = 7.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - object NumberOfDaysToReducePushCapForSeeLessOftenForNonF1TriggerNonF1 - extends FSBoundedParam[Duration]( - name = "seelessoften_for_nonf1_trigger_nonf1_toreduce_pushcap_days", - default = 30.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - object EnableContFnF1TriggerSeeLessOftenFatigue - extends FSParam[Boolean]( - name = "seelessoften_fn_enable_f1_trigger_fatigue", - default = false - ) - - object EnableContFnNonF1TriggerSeeLessOftenFatigue - extends FSParam[Boolean]( - name = "seelessoften_fn_enable_nonf1_trigger_fatigue", - default = false - ) - - object SeeLessOftenListOfDayKnobs - extends FSParam[Seq[Double]]("seelessoften_fn_day_knobs", default = Seq.empty[Double]) - - object SeeLessOftenListOfPushCapWeightKnobs - extends FSParam[Seq[Double]]("seelessoften_fn_pushcap_knobs", default = Seq.empty[Double]) - - object SeeLessOftenListOfPowerKnobs - extends FSParam[Seq[Double]]("seelessoften_fn_power_knobs", default = Seq.empty[Double]) - - object SeeLessOftenF1TriggerF1PushCapWeight - extends FSBoundedParam[Double]( - "seelessoften_fn_f1_trigger_f1_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - object SeeLessOftenF1TriggerNonF1PushCapWeight - extends FSBoundedParam[Double]( - "seelessoften_fn_f1_trigger_nonf1_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - object SeeLessOftenNonF1TriggerF1PushCapWeight - extends FSBoundedParam[Double]( - "seelessoften_fn_nonf1_trigger_f1_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - object SeeLessOftenNonF1TriggerNonF1PushCapWeight - extends FSBoundedParam[Double]( - "seelessoften_fn_nonf1_trigger_nonf1_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - object SeeLessOftenTripHqTweetTriggerF1PushCapWeight - extends FSBoundedParam[Double]( - "seelessoften_fn_trip_hq_tweet_trigger_f1_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - object SeeLessOftenTripHqTweetTriggerNonF1PushCapWeight - extends FSBoundedParam[Double]( - "seelessoften_fn_trip_hq_tweet_trigger_nonf1_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - object SeeLessOftenTripHqTweetTriggerTripHqTweetPushCapWeight - extends FSBoundedParam[Double]( - "seelessoften_fn_trip_hq_tweet_trigger_trip_hq_tweet_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - object SeeLessOftenTopicTriggerTopicPushCapWeight - extends FSBoundedParam[Double]( - "seelessoften_fn_topic_trigger_topic_weight", - default = 1.0, - min = 0.0, - max = Double.MaxValue) - - object SeeLessOftenTopicTriggerF1PushCapWeight - extends FSBoundedParam[Double]( - "seelessoften_fn_topic_trigger_f1_weight", - default = 100000.0, - min = 0.0, - max = Double.MaxValue) - - object SeeLessOftenTopicTriggerOONPushCapWeight - extends FSBoundedParam[Double]( - "seelessoften_fn_topic_trigger_oon_weight", - default = 100000.0, - min = 0.0, - max = Double.MaxValue) - - object SeeLessOftenF1TriggerTopicPushCapWeight - extends FSBoundedParam[Double]( - "seelessoften_fn_f1_trigger_topic_weight", - default = 100000.0, - min = 0.0, - max = Double.MaxValue) - - object SeeLessOftenOONTriggerTopicPushCapWeight - extends FSBoundedParam[Double]( - "seelessoften_fn_oon_trigger_topic_weight", - default = 1.0, - min = 0.0, - max = Double.MaxValue) - - object SeeLessOftenDefaultPushCapWeight - extends FSBoundedParam[Double]( - "seelessoften_fn_default_weight", - default = 100000.0, - min = 0.0, - max = Double.MaxValue) - - object SeeLessOftenNtabOnlyNotifUserPushCapWeight - extends FSBoundedParam[Double]( - "seelessoften_fn_ntab_only_user_weight", - default = 1.0, - min = 0.0, - max = Double.MaxValue) - - // Params for inline feedback fatigue - object EnableContFnF1TriggerInlineFeedbackFatigue - extends FSParam[Boolean]( - name = "feedback_inline_fn_enable_f1_trigger_fatigue", - default = false - ) - - object EnableContFnNonF1TriggerInlineFeedbackFatigue - extends FSParam[Boolean]( - name = "feedback_inline_fn_enable_nonf1_trigger_fatigue", - default = false - ) - - object UseInlineDislikeForFatigue - extends FSParam[Boolean]( - name = "feedback_inline_fn_use_dislike", - default = true - ) - object UseInlineDismissForFatigue - extends FSParam[Boolean]( - name = "feedback_inline_fn_use_dismiss", - default = false - ) - object UseInlineSeeLessForFatigue - extends FSParam[Boolean]( - name = "feedback_inline_fn_use_see_less", - default = false - ) - object UseInlineNotRelevantForFatigue - extends FSParam[Boolean]( - name = "feedback_inline_fn_use_not_relevant", - default = false - ) - object InlineFeedbackListOfDayKnobs - extends FSParam[Seq[Double]]("feedback_inline_fn_day_knobs", default = Seq.empty[Double]) - - object InlineFeedbackListOfPushCapWeightKnobs - extends FSParam[Seq[Double]]("feedback_inline_fn_pushcap_knobs", default = Seq.empty[Double]) - - object InlineFeedbackListOfPowerKnobs - extends FSParam[Seq[Double]]("feedback_inline_fn_power_knobs", default = Seq.empty[Double]) - - object InlineFeedbackF1TriggerF1PushCapWeight - extends FSBoundedParam[Double]( - "feedback_inline_fn_f1_trigger_f1_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - object InlineFeedbackF1TriggerNonF1PushCapWeight - extends FSBoundedParam[Double]( - "feedback_inline_fn_f1_trigger_nonf1_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - object InlineFeedbackNonF1TriggerF1PushCapWeight - extends FSBoundedParam[Double]( - "feedback_inline_fn_nonf1_trigger_f1_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - object InlineFeedbackNonF1TriggerNonF1PushCapWeight - extends FSBoundedParam[Double]( - "feedback_inline_fn_nonf1_trigger_nonf1_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - // Params for prompt feedback - object EnableContFnF1TriggerPromptFeedbackFatigue - extends FSParam[Boolean]( - name = "feedback_prompt_fn_enable_f1_trigger_fatigue", - default = false - ) - - object EnableContFnNonF1TriggerPromptFeedbackFatigue - extends FSParam[Boolean]( - name = "feedback_prompt_fn_enable_nonf1_trigger_fatigue", - default = false - ) - object PromptFeedbackListOfDayKnobs - extends FSParam[Seq[Double]]("feedback_prompt_fn_day_knobs", default = Seq.empty[Double]) - - object PromptFeedbackListOfPushCapWeightKnobs - extends FSParam[Seq[Double]]("feedback_prompt_fn_pushcap_knobs", default = Seq.empty[Double]) - - object PromptFeedbackListOfPowerKnobs - extends FSParam[Seq[Double]]("feedback_prompt_fn_power_knobs", default = Seq.empty[Double]) - - object PromptFeedbackF1TriggerF1PushCapWeight - extends FSBoundedParam[Double]( - "feedback_prompt_fn_f1_trigger_f1_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - object PromptFeedbackF1TriggerNonF1PushCapWeight - extends FSBoundedParam[Double]( - "feedback_prompt_fn_f1_trigger_nonf1_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - object PromptFeedbackNonF1TriggerF1PushCapWeight - extends FSBoundedParam[Double]( - "feedback_prompt_fn_nonf1_trigger_f1_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - object PromptFeedbackNonF1TriggerNonF1PushCapWeight - extends FSBoundedParam[Double]( - "feedback_prompt_fn_nonf1_trigger_nonf1_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - /* - * Param to enable cohost join event notif - */ - object EnableSpaceCohostJoinEvent - extends FSParam[Boolean](name = "space_recs_cohost_join_enable", default = true) - - /* - * Param to bypass global push cap when target is device following host/speaker. - */ - object BypassGlobalSpacePushCapForSoftDeviceFollow - extends FSParam[Boolean](name = "space_recs_bypass_global_pushcap_for_soft_follow", false) - - /* - * Param to bypass active listener predicate when target is device following host/speaker. - */ - object CheckActiveListenerPredicateForSoftDeviceFollow - extends FSParam[Boolean](name = "space_recs_check_active_listener_for_soft_follow", false) - - object SpreadControlRatioParam - extends FSBoundedParam[Double]( - name = "oon_spread_control_ratio", - default = 1000.0, - min = 0.0, - max = 100000.0 - ) - - object FavOverSendThresholdParam - extends FSBoundedParam[Double]( - name = "oon_spread_control_fav_over_send_threshold", - default = 0.14, - min = 0.0, - max = 1000.0 - ) - - object AuthorReportRateThresholdParam - extends FSBoundedParam[Double]( - name = "oon_spread_control_author_report_rate_threshold", - default = 7.4e-6, - min = 0.0, - max = 1000.0 - ) - - object AuthorDislikeRateThresholdParam - extends FSBoundedParam[Double]( - name = "oon_spread_control_author_dislike_rate_threshold", - default = 1.0, - min = 0.0, - max = 1000.0 - ) - - object MinTweetSendsThresholdParam - extends FSBoundedParam[Double]( - name = "oon_spread_control_min_tweet_sends_threshold", - default = 10000000000.0, - min = 0.0, - max = 10000000000.0 - ) - - object MinAuthorSendsThresholdParam - extends FSBoundedParam[Double]( - name = "oon_spread_control_min_author_sends_threshold", - default = 10000000000.0, - min = 0.0, - max = 10000000000.0 - ) - - /* - * Tweet Ntab-dislike predicate related params - */ - object TweetNtabDislikeCountThresholdParam - extends FSBoundedParam[Double]( - name = "oon_tweet_ntab_dislike_count_threshold", - default = 10000.0, - min = 0.0, - max = 10000.0 - ) - object TweetNtabDislikeRateThresholdParam - extends FSBoundedParam[Double]( - name = "oon_tweet_ntab_dislike_rate_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Param for tweet language feature name - */ - object TweetLanguageFeatureNameParam - extends FSParam[String]( - name = "language_tweet_language_feature_name", - default = "tweet.language.tweet.identified") - - /** - * Threshold for user inferred language filtering - */ - object UserInferredLanguageThresholdParam - extends FSBoundedParam[Double]( - name = "language_user_inferred_language_threshold", - default = 0.0, - min = 0.0, - max = 1.0 - ) - - /** - * Threshold for user device language filtering - */ - object UserDeviceLanguageThresholdParam - extends FSBoundedParam[Double]( - name = "language_user_device_language_threshold", - default = 0.0, - min = 0.0, - max = 1.0 - ) - - /** - * Param to enable/disable tweet language filter - */ - object EnableTweetLanguageFilter - extends FSParam[Boolean]( - name = "language_enable_tweet_language_filter", - default = false - ) - - /** - * Param to skip language filter for media tweets - */ - object SkipLanguageFilterForMediaTweets - extends FSParam[Boolean]( - name = "language_skip_language_filter_for_media_tweets", - default = false - ) - - /* - * Tweet Ntab-dislike predicate related params for MrTwistly - */ - object TweetNtabDislikeCountThresholdForMrTwistlyParam - extends FSBoundedParam[Double]( - name = "oon_tweet_ntab_dislike_count_threshold_for_mrtwistly", - default = 10000.0, - min = 0.0, - max = 10000.0 - ) - object TweetNtabDislikeRateThresholdForMrTwistlyParam - extends FSBoundedParam[Double]( - name = "oon_tweet_ntab_dislike_rate_threshold_for_mrtwistly", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - object TweetNtabDislikeCountBucketThresholdParam - extends FSBoundedParam[Double]( - name = "oon_tweet_ntab_dislike_count_bucket_threshold", - default = 10.0, - min = 0.0, - max = 10000.0 - ) - - /* - * Tweet engagement ratio predicate related params - */ - object TweetQTtoNtabClickRatioThresholdParam - extends FSBoundedParam[Double]( - name = "oon_tweet_engagement_filter_qt_to_ntabclick_ratio_threshold", - default = 0.0, - min = 0.0, - max = 100000.0 - ) - - /** - * Lower bound threshold to filter a tweet based on its reply to like ratio - */ - object TweetReplytoLikeRatioThresholdLowerBound - extends FSBoundedParam[Double]( - name = "oon_tweet_engagement_filter_reply_to_like_ratio_threshold_lower_bound", - default = Double.MaxValue, - min = 0.0, - max = Double.MaxValue - ) - - /** - * Upper bound threshold to filter a tweet based on its reply to like ratio - */ - object TweetReplytoLikeRatioThresholdUpperBound - extends FSBoundedParam[Double]( - name = "oon_tweet_engagement_filter_reply_to_like_ratio_threshold_upper_bound", - default = 0.0, - min = 0.0, - max = Double.MaxValue - ) - - /** - * Upper bound threshold to filter a tweet based on its reply to like ratio - */ - object TweetReplytoLikeRatioReplyCountThreshold - extends FSBoundedParam[Int]( - name = "oon_tweet_engagement_filter_reply_count_threshold", - default = Int.MaxValue, - min = 0, - max = Int.MaxValue - ) - - /* - * oonTweetLengthBasedPrerankingPredicate related params - */ - object OonTweetLengthPredicateUpdatedMediaLogic - extends FSParam[Boolean]( - name = "oon_quality_filter_tweet_length_updated_media_logic", - default = false - ) - - object OonTweetLengthPredicateUpdatedQuoteTweetLogic - extends FSParam[Boolean]( - name = "oon_quality_filter_tweet_length_updated_quote_tweet_logic", - default = false - ) - - object OonTweetLengthPredicateMoreStrictForUndefinedLanguages - extends FSParam[Boolean]( - name = "oon_quality_filter_tweet_length_more_strict_for_undefined_languages", - default = false - ) - - object EnablePrerankingTweetLengthPredicate - extends FSParam[Boolean]( - name = "oon_quality_filter_enable_preranking_filter", - default = false - ) - - /* - * LengthLanguageBasedOONTweetCandidatesQualityPredicate related params - */ - object SautOonWithMediaTweetLengthThresholdParam - extends FSBoundedParam[Double]( - name = "oon_quality_filter_tweet_length_threshold_for_saut_oon_with_media", - default = 0.0, - min = 0.0, - max = 70.0 - ) - object NonSautOonWithMediaTweetLengthThresholdParam - extends FSBoundedParam[Double]( - name = "oon_quality_filter_tweet_length_threshold_for_non_saut_oon_with_media", - default = 0.0, - min = 0.0, - max = 70.0 - ) - object SautOonWithoutMediaTweetLengthThresholdParam - extends FSBoundedParam[Double]( - name = "oon_quality_filter_tweet_length_threshold_for_saut_oon_without_media", - default = 0.0, - min = 0.0, - max = 70.0 - ) - object NonSautOonWithoutMediaTweetLengthThresholdParam - extends FSBoundedParam[Double]( - name = "oon_quality_filter_tweet_length_threshold_for_non_saut_oon_without_media", - default = 0.0, - min = 0.0, - max = 70.0 - ) - - object ArgfOonWithMediaTweetWordLengthThresholdParam - extends FSBoundedParam[Double]( - name = "oon_quality_filter_tweet_word_length_threshold_for_argf_oon_with_media", - default = 0.0, - min = 0.0, - max = 18.0 - ) - object EsfthOonWithMediaTweetWordLengthThresholdParam - extends FSBoundedParam[Double]( - name = "oon_quality_filter_tweet_word_length_threshold_for_esfth_oon_with_media", - default = 0.0, - min = 0.0, - max = 10.0 - ) - - /** - * Param to enable/disable sentiment feature hydration - */ - object EnableMrTweetSentimentFeatureHydrationFS - extends FSParam[Boolean]( - name = "feature_hydration_enable_mr_tweet_sentiment_feature", - default = false - ) - - /** - * Param to enable/disable feature map scribing for staging test log - */ - object EnableMrScribingMLFeaturesAsFeatureMapForStaging - extends FSParam[Boolean]( - name = "frigate_pushservice_enable_scribing_ml_features_as_featuremap_for_staging", - default = false - ) - - /** - * Param to enable timeline health signal hydration - * */ - object EnableTimelineHealthSignalHydration - extends FSParam[Boolean]( - name = "timeline_health_signal_hydration", - default = false - ) - - /** - * Param to enable timeline health signal hydration for model training - * */ - object EnableTimelineHealthSignalHydrationForModelTraining - extends FSParam[Boolean]( - name = "timeline_health_signal_hydration_for_model_training", - default = false - ) - - /** - * Param to enable/disable mr user social context agg feature hydration - */ - object EnableMrUserSocialContextAggregateFeatureHydration - extends FSParam[Boolean]( - name = "frigate_push_modeling_hydrate_mr_user_social_context_agg_feature", - default = true - ) - - /** - * Param to enable/disable mr user semantic core agg feature hydration - */ - object EnableMrUserSemanticCoreAggregateFeatureHydration - extends FSParam[Boolean]( - name = "frigate_push_modeling_hydrate_mr_user_semantic_core_agg_feature", - default = true - ) - - /** - * Param to enable/disable mr user candidate sparse agg feature hydration - */ - object EnableMrUserCandidateSparseOfflineAggregateFeatureHydration - extends FSParam[Boolean]( - name = "frigate_push_modeling_hydrate_mr_user_candidate_sparse_agg_feature", - default = true - ) - - /** - * Param to enable/disable mr user candidate agg feature hydration - */ - object EnableMrUserCandidateOfflineAggregateFeatureHydration - extends FSParam[Boolean]( - name = "frigate_push_modeling_hydrate_mr_user_candidate_agg_feature", - default = true - ) - - /** - * Param to enable/disable mr user candidate compact agg feature hydration - */ - object EnableMrUserCandidateOfflineCompactAggregateFeatureHydration - extends FSParam[Boolean]( - name = "frigate_push_modeling_hydrate_mr_user_candidate_compact_agg_feature", - default = false - ) - - /** - * Param to enable/disable mr real graph user-author/social-context feature hydration - */ - object EnableRealGraphUserAuthorAndSocialContxtFeatureHydration - extends FSParam[Boolean]( - name = "frigate_push_modeling_hydrate_real_graph_user_social_feature", - default = true - ) - - /** - * Param to enable/disable mr user author agg feature hydration - */ - object EnableMrUserAuthorOfflineAggregateFeatureHydration - extends FSParam[Boolean]( - name = "frigate_push_modeling_hydrate_mr_user_author_agg_feature", - default = true - ) - - /** - * Param to enable/disable mr user author compact agg feature hydration - */ - object EnableMrUserAuthorOfflineCompactAggregateFeatureHydration - extends FSParam[Boolean]( - name = "frigate_push_modeling_hydrate_mr_user_author_compact_agg_feature", - default = false - ) - - /** - * Param to enable/disable mr user compact agg feature hydration - */ - object EnableMrUserOfflineCompactAggregateFeatureHydration - extends FSParam[Boolean]( - name = "frigate_push_modeling_hydrate_mr_user_compact_agg_feature", - default = false - ) - - /** - * Param to enable/disable mr user simcluster agg feature hydration - */ - object EnableMrUserSimcluster2020AggregateFeatureHydration - extends FSParam[Boolean]( - name = "frigate_push_modeling_hydrate_mr_user_simcluster_agg_feature", - default = true - ) - - /** - * Param to enable/disable mr user agg feature hydration - */ - object EnableMrUserOfflineAggregateFeatureHydration - extends FSParam[Boolean]( - name = "frigate_push_modeling_hydrate_mr_user_agg_feature", - default = true - ) - - /** - * Param to enable/disable topic engagement RTA in the ranking model - */ - object EnableTopicEngagementRealTimeAggregatesFS - extends FSParam[Boolean]( - "feature_hydration_enable_htl_topic_engagement_real_time_agg_feature", - false) - - /* - * Param to enable mr user semantic core feature hydration for heavy ranker - * */ - object EnableMrUserSemanticCoreFeatureForExpt - extends FSParam[Boolean]( - name = "frigate_push_modeling_hydrate_mr_user_semantic_core", - default = false) - - /** - * Param to enable hydrating user duration since last visit features - */ - object EnableHydratingUserDurationSinceLastVisitFeatures - extends FSParam[Boolean]( - name = "feature_hydration_user_duration_since_last_visit", - default = false) - - /** - Param to enable/disable user-topic aggregates in the ranking model - */ - object EnableUserTopicAggregatesFS - extends FSParam[Boolean]("feature_hydration_enable_htl_topic_user_agg_feature", false) - - /* - * PNegMultimodalPredicate related params - */ - object EnablePNegMultimodalPredicateParam - extends FSParam[Boolean]( - name = "pneg_multimodal_filter_enable_param", - default = false - ) - object PNegMultimodalPredicateModelThresholdParam - extends FSBoundedParam[Double]( - name = "pneg_multimodal_filter_model_threshold_param", - default = 1.0, - min = 0.0, - max = 1.0 - ) - object PNegMultimodalPredicateBucketThresholdParam - extends FSBoundedParam[Double]( - name = "pneg_multimodal_filter_bucket_threshold_param", - default = 0.4, - min = 0.0, - max = 1.0 - ) - - /* - * NegativeKeywordsPredicate related params - */ - object EnableNegativeKeywordsPredicateParam - extends FSParam[Boolean]( - name = "negative_keywords_filter_enable_param", - default = false - ) - object NegativeKeywordsPredicateDenylist - extends FSParam[Seq[String]]( - name = "negative_keywords_filter_denylist", - default = Seq.empty[String] - ) - /* - * LightRanking related params - */ - object EnableLightRankingParam - extends FSParam[Boolean]( - name = "light_ranking_enable_param", - default = false - ) - object LightRankingNumberOfCandidatesParam - extends FSBoundedParam[Int]( - name = "light_ranking_number_of_candidates_param", - default = 100, - min = 0, - max = 1000 - ) - object LightRankingModelTypeParam - extends FSParam[String]( - name = "light_ranking_model_type_param", - default = "WeightedOpenOrNtabClickProbability_Q4_2021_13172_Mr_Light_Ranker_Dbv2_Top3") - object EnableRandomBaselineLightRankingParam - extends FSParam[Boolean]( - name = "light_ranking_random_baseline_enable_param", - default = false - ) - - object LightRankingScribeCandidatesDownSamplingParam - extends FSBoundedParam[Double]( - name = "light_ranking_scribe_candidates_down_sampling_param", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /* - * Quality Upranking related params - */ - object EnableProducersQualityBoostingForHeavyRankingParam - extends FSParam[Boolean]( - name = "quality_upranking_enable_producers_quality_boosting_for_heavy_ranking_param", - default = false - ) - - object QualityUprankingBoostForHighQualityProducersParam - extends FSBoundedParam[Double]( - name = "quality_upranking_boost_for_high_quality_producers_param", - default = 1.0, - min = 0.0, - max = 10000.0 - ) - - object QualityUprankingDownboostForLowQualityProducersParam - extends FSBoundedParam[Double]( - name = "quality_upranking_downboost_for_low_quality_producers_param", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - object EnableQualityUprankingForHeavyRankingParam - extends FSParam[Boolean]( - name = "quality_upranking_enable_for_heavy_ranking_param", - default = false - ) - object QualityUprankingModelTypeParam - extends FSParam[WeightedOpenOrNtabClickModel.ModelNameType]( - name = "quality_upranking_model_id", - default = "Q4_2022_Mr_Bqml_Quality_Model_wALL" - ) - object QualityUprankingTransformTypeParam - extends FSEnumParam[MrQualityUprankingTransformTypeEnum.type]( - name = "quality_upranking_transform_id", - default = MrQualityUprankingTransformTypeEnum.Sigmoid, - enum = MrQualityUprankingTransformTypeEnum - ) - - object QualityUprankingBoostForHeavyRankingParam - extends FSBoundedParam[Double]( - name = "quality_upranking_boost_for_heavy_ranking_param", - default = 1.0, - min = -10.0, - max = 10.0 - ) - object QualityUprankingSigmoidBiasForHeavyRankingParam - extends FSBoundedParam[Double]( - name = "quality_upranking_sigmoid_bias_for_heavy_ranking_param", - default = 0.0, - min = -10.0, - max = 10.0 - ) - object QualityUprankingSigmoidWeightForHeavyRankingParam - extends FSBoundedParam[Double]( - name = "quality_upranking_sigmoid_weight_for_heavy_ranking_param", - default = 1.0, - min = -10.0, - max = 10.0 - ) - object QualityUprankingLinearBarForHeavyRankingParam - extends FSBoundedParam[Double]( - name = "quality_upranking_linear_bar_for_heavy_ranking_param", - default = 1.0, - min = 0.0, - max = 10.0 - ) - object EnableQualityUprankingCrtScoreStatsForHeavyRankingParam - extends FSParam[Boolean]( - name = "quality_upranking_enable_crt_score_stats_for_heavy_ranking_param", - default = false - ) - /* - * BQML Health Model related params - */ - object EnableBqmlHealthModelPredicateParam - extends FSParam[Boolean]( - name = "bqml_health_model_filter_enable_param", - default = false - ) - - object EnableBqmlHealthModelPredictionForInNetworkCandidatesParam - extends FSParam[Boolean]( - name = "bqml_health_model_enable_prediction_for_in_network_candidates_param", - default = false - ) - - object BqmlHealthModelTypeParam - extends FSParam[HealthNsfwModel.ModelNameType]( - name = "bqml_health_model_id", - default = HealthNsfwModel.Q2_2022_Mr_Bqml_Health_Model_NsfwV0 - ) - object BqmlHealthModelPredicateFilterThresholdParam - extends FSBoundedParam[Double]( - name = "bqml_health_model_filter_threshold_param", - default = 1.0, - min = 0.0, - max = 1.0 - ) - object BqmlHealthModelPredicateBucketThresholdParam - extends FSBoundedParam[Double]( - name = "bqml_health_model_bucket_threshold_param", - default = 0.005, - min = 0.0, - max = 1.0 - ) - - object EnableBqmlHealthModelScoreHistogramParam - extends FSParam[Boolean]( - name = "bqml_health_model_score_histogram_enable_param", - default = false - ) - - /* - * BQML Quality Model related params - */ - object EnableBqmlQualityModelPredicateParam - extends FSParam[Boolean]( - name = "bqml_quality_model_filter_enable_param", - default = false - ) - object EnableBqmlQualityModelScoreHistogramParam - extends FSParam[Boolean]( - name = "bqml_quality_model_score_histogram_enable_param", - default = false - ) - object BqmlQualityModelTypeParam - extends FSParam[WeightedOpenOrNtabClickModel.ModelNameType]( - name = "bqml_quality_model_id", - default = "Q1_2022_13562_Mr_Bqml_Quality_Model_V2" - ) - - /** - * Param to specify which quality models to use to get the scores for determining - * whether to bucket a user for the DDG - */ - object BqmlQualityModelBucketModelIdListParam - extends FSParam[Seq[WeightedOpenOrNtabClickModel.ModelNameType]]( - name = "bqml_quality_model_bucket_model_id_list", - default = Seq( - "Q1_2022_13562_Mr_Bqml_Quality_Model_V2", - "Q2_2022_DDG14146_Mr_Personalised_BQML_Quality_Model", - "Q2_2022_DDG14146_Mr_NonPersonalised_BQML_Quality_Model" - ) - ) - - object BqmlQualityModelPredicateThresholdParam - extends FSBoundedParam[Double]( - name = "bqml_quality_model_filter_threshold_param", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Param to specify the threshold to determine if a user’s quality score is high enough to enter the experiment. - */ - object BqmlQualityModelBucketThresholdListParam - extends FSParam[Seq[Double]]( - name = "bqml_quality_model_bucket_threshold_list", - default = Seq(0.7, 0.7, 0.7) - ) - - /* - * TweetAuthorAggregates related params - */ - object EnableTweetAuthorAggregatesFeatureHydrationParam - extends FSParam[Boolean]( - name = "tweet_author_aggregates_feature_hydration_enable_param", - default = false - ) - - /** - * Param to determine if we should include the relevancy score of candidates in the Ibis payload - */ - object IncludeRelevanceScoreInIbis2Payload - extends FSParam[Boolean]( - name = "relevance_score_include_in_ibis2_payload", - default = false - ) - - /** - * Param to specify supervised model to predict score by sending the notification - */ - object BigFilteringSupervisedSendingModelParam - extends FSParam[BigFilteringSupervisedModel.ModelNameType]( - name = "ltv_filtering_bigfiltering_supervised_sending_model_param", - default = BigFilteringSupervisedModel.V0_0_BigFiltering_Supervised_Sending_Model - ) - - /** - * Param to specify supervised model to predict score by not sending the notification - */ - object BigFilteringSupervisedWithoutSendingModelParam - extends FSParam[BigFilteringSupervisedModel.ModelNameType]( - name = "ltv_filtering_bigfiltering_supervised_without_sending_model_param", - default = BigFilteringSupervisedModel.V0_0_BigFiltering_Supervised_Without_Sending_Model - ) - - /** - * Param to specify RL model to predict score by sending the notification - */ - object BigFilteringRLSendingModelParam - extends FSParam[BigFilteringSupervisedModel.ModelNameType]( - name = "ltv_filtering_bigfiltering_rl_sending_model_param", - default = BigFilteringRLModel.V0_0_BigFiltering_Rl_Sending_Model - ) - - /** - * Param to specify RL model to predict score by not sending the notification - */ - object BigFilteringRLWithoutSendingModelParam - extends FSParam[BigFilteringSupervisedModel.ModelNameType]( - name = "ltv_filtering_bigfiltering_rl_without_sending_model_param", - default = BigFilteringRLModel.V0_0_BigFiltering_Rl_Without_Sending_Model - ) - - /** - * Param to specify the threshold (send notification if score >= threshold) - */ - object BigFilteringThresholdParam - extends FSBoundedParam[Double]( - name = "ltv_filtering_bigfiltering_threshold_param", - default = 0.0, - min = Double.MinValue, - max = Double.MaxValue - ) - - /** - * Param to specify normalization used for BigFiltering - */ - object BigFilteringNormalizationTypeIdParam - extends FSEnumParam[BigFilteringNormalizationEnum.type]( - name = "ltv_filtering_bigfiltering_normalization_type_id", - default = BigFilteringNormalizationEnum.NormalizationDisabled, - enum = BigFilteringNormalizationEnum - ) - - /** - * Param to specify histograms of model scores in BigFiltering - */ - object BigFilteringEnableHistogramsParam - extends FSParam[Boolean]( - name = "ltv_filtering_bigfiltering_enable_histograms_param", - default = false - ) - - /* - * Param to enable sending requests to Ins Sender - */ - object EnableInsSender extends FSParam[Boolean](name = "ins_enable_dark_traffic", default = false) - - /** - * Param to specify the range of relevance scores for MagicFanout types. - */ - object MagicFanoutRelevanceScoreRange - extends FSParam[Seq[Double]]( - name = "relevance_score_mf_range", - default = Seq(0.75, 1.0) - ) - - /** - * Param to specify the range of relevance scores for MR types. - */ - object MagicRecsRelevanceScoreRange - extends FSParam[Seq[Double]]( - name = "relevance_score_mr_range", - default = Seq(0.25, 0.5) - ) - - /** - * Param to enable backfilling OON candidates if number of F1 candidates is greater than a threshold K. - */ - object EnableOONBackfillBasedOnF1Candidates - extends FSParam[Boolean](name = "oon_enable_backfill_based_on_f1", default = false) - - /** - * Threshold for the minimum number of F1 candidates required to enable backfill of OON candidates. - */ - object NumberOfF1CandidatesThresholdForOONBackfill - extends FSBoundedParam[Int]( - name = "oon_enable_backfill_f1_threshold", - min = 0, - default = 5000, - max = 5000) - - /** - * Event ID allowlist to skip account country predicate - */ - object MagicFanoutEventAllowlistToSkipAccountCountryPredicate - extends FSParam[Seq[Long]]( - name = "magicfanout_event_allowlist_skip_account_country_predicate", - default = Seq.empty[Long] - ) - - /** - * MagicFanout Event Semantic Core Domain Ids - */ - object ListOfEventSemanticCoreDomainIds - extends FSParam[Seq[Long]]( - name = "magicfanout_automated_events_semantic_core_domain_ids", - default = Seq()) - - /** - * Adhoc id for detailed rank flow stats - */ - object ListOfAdhocIdsForStatsTracking - extends FSParam[Set[Long]]( - name = "stats_enable_detailed_stats_tracking_ids", - default = Set.empty[Long] - ) - - object EnableGenericCRTBasedFatiguePredicate - extends FSParam[Boolean]( - name = "seelessoften_enable_generic_crt_based_fatigue_predicate", - default = false) - - /** - * Param to enable copy features such as Emojis and Target Name - */ - object EnableCopyFeaturesForF1 - extends FSParam[Boolean](name = "mr_copy_enable_features_f1", default = false) - - /** - * Param to enable copy features such as Emojis and Target Name - */ - object EnableCopyFeaturesForOon - extends FSParam[Boolean](name = "mr_copy_enable_features_oon", default = false) - - /** - * Param to enable Emoji in F1 Copy - */ - object EnableEmojiInF1Copy - extends FSParam[Boolean](name = "mr_copy_enable_f1_emoji", default = false) - - /** - * Param to enable Target in F1 Copy - */ - object EnableTargetInF1Copy - extends FSParam[Boolean](name = "mr_copy_enable_f1_target", default = false) - - /** - * Param to enable Emoji in OON Copy - */ - object EnableEmojiInOonCopy - extends FSParam[Boolean](name = "mr_copy_enable_oon_emoji", default = false) - - /** - * Param to enable Target in OON Copy - */ - object EnableTargetInOonCopy - extends FSParam[Boolean](name = "mr_copy_enable_oon_target", default = false) - - /** - * Param to enable split fatigue for Target and Emoji copy for OON and F1 - */ - object EnableTargetAndEmojiSplitFatigue - extends FSParam[Boolean](name = "mr_copy_enable_target_emoji_split_fatigue", default = false) - - /** - * Param to enable experimenting string on the body - */ - object EnableF1CopyBody extends FSParam[Boolean](name = "mr_copy_f1_enable_body", default = false) - - object EnableOONCopyBody - extends FSParam[Boolean](name = "mr_copy_oon_enable_body", default = false) - - object EnableIosCopyBodyTruncate - extends FSParam[Boolean](name = "mr_copy_enable_body_truncate", default = false) - - object EnableNsfwCopy extends FSParam[Boolean](name = "mr_copy_enable_nsfw", default = false) - - /** - * Param to determine F1 candidate nsfw score threshold - */ - object NsfwScoreThresholdForF1Copy - extends FSBoundedParam[Double]( - name = "mr_copy_nsfw_threshold_f1", - default = 0.3, - min = 0.0, - max = 1.0 - ) - - /** - * Param to determine OON candidate nsfw score threshold - */ - object NsfwScoreThresholdForOONCopy - extends FSBoundedParam[Double]( - name = "mr_copy_nsfw_threshold_oon", - default = 0.2, - min = 0.0, - max = 1.0 - ) - - /** - * Param to determine the lookback duration when searching for prev copy features. - */ - object CopyFeaturesHistoryLookbackDuration - extends FSBoundedParam[Duration]( - name = "mr_copy_history_lookback_duration_in_days", - default = 30.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Param to determine the F1 emoji copy fatigue in # of hours. - */ - object F1EmojiCopyFatigueDuration - extends FSBoundedParam[Duration]( - name = "mr_copy_f1_emoji_copy_fatigue_in_hours", - default = 24.hours, - min = 0.hours, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to determine the F1 target copy fatigue in # of hours. - */ - object F1TargetCopyFatigueDuration - extends FSBoundedParam[Duration]( - name = "mr_copy_f1_target_copy_fatigue_in_hours", - default = 24.hours, - min = 0.hours, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to determine the OON emoji copy fatigue in # of hours. - */ - object OonEmojiCopyFatigueDuration - extends FSBoundedParam[Duration]( - name = "mr_copy_oon_emoji_copy_fatigue_in_hours", - default = 24.hours, - min = 0.hours, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to determine the OON target copy fatigue in # of hours. - */ - object OonTargetCopyFatigueDuration - extends FSBoundedParam[Duration]( - name = "mr_copy_oon_target_copy_fatigue_in_hours", - default = 24.hours, - min = 0.hours, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to turn on/off home timeline based fatigue rule, where once last home timeline visit - * is larger than the specified will evalute to not fatigue - */ - object EnableHTLBasedFatigueBasicRule - extends FSParam[Boolean]( - name = "mr_copy_enable_htl_based_fatigue_basic_rule", - default = false) - - /** - * Param to determine f1 emoji copy fatigue in # of pushes - */ - object F1EmojiCopyNumOfPushesFatigue - extends FSBoundedParam[Int]( - name = "mr_copy_f1_emoji_copy_number_of_pushes_fatigue", - default = 0, - min = 0, - max = 200 - ) - - /** - * Param to determine oon emoji copy fatigue in # of pushes - */ - object OonEmojiCopyNumOfPushesFatigue - extends FSBoundedParam[Int]( - name = "mr_copy_oon_emoji_copy_number_of_pushes_fatigue", - default = 0, - min = 0, - max = 200 - ) - - /** - * If user haven't visited home timeline for certain duration, we will - * exempt user from feature copy fatigue. This param is used to control - * how long it is before we enter exemption. - */ - object MinFatigueDurationSinceLastHTLVisit - extends FSBoundedParam[Duration]( - name = "mr_copy_min_duration_since_last_htl_visit_hours", - default = Duration.Top, - min = 0.hour, - max = Duration.Top, - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * If a user haven't visit home timeline very long, the user will return - * to fatigue state under the home timeline based fatigue rule. There will - * only be a window, where the user is out of fatigue state under the rule. - * This param control the length of the non fatigue period. - */ - object LastHTLVisitBasedNonFatigueWindow - extends FSBoundedParam[Duration]( - name = "mr_copy_last_htl_visit_based_non_fatigue_window_hours", - default = 48.hours, - min = 0.hour, - max = Duration.Top, - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - object EnableOONCBasedCopy - extends FSParam[Boolean]( - name = "mr_copy_enable_oonc_based_copy", - default = false - ) - - object HighOONCThresholdForCopy - extends FSBoundedParam[Double]( - name = "mr_copy_high_oonc_threshold_for_copy", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - object LowOONCThresholdForCopy - extends FSBoundedParam[Double]( - name = "mr_copy_low_oonc_threshold_for_copy", - default = 0.0, - min = 0.0, - max = 1.0 - ) - - object EnableTweetTranslation - extends FSParam[Boolean](name = "tweet_translation_enable", default = false) - - object TripTweetCandidateReturnEnable - extends FSParam[Boolean](name = "trip_tweet_candidate_enable", default = false) - - object TripTweetCandidateSourceIds - extends FSParam[Seq[String]]( - name = "trip_tweet_candidate_source_ids", - default = Seq("TOP_GEO_V3")) - - object TripTweetMaxTotalCandidates - extends FSBoundedParam[Int]( - name = "trip_tweet_max_total_candidates", - default = 500, - min = 10, - max = 1000) - - object EnableEmptyBody - extends FSParam[Boolean](name = "push_presentation_enable_empty_body", default = false) - - object EnableSocialContextForRetweet - extends FSParam[Boolean](name = "push_presentation_social_context_retweet", default = false) - - /** - * Param to enable/disable simcluster feature hydration - */ - object EnableMrTweetSimClusterFeatureHydrationFS - extends FSParam[Boolean]( - name = "feature_hydration_enable_mr_tweet_simcluster_feature", - default = false - ) - - /** - * Param to disable OON candidates based on tweetAuthor - */ - object DisableOutNetworkTweetCandidatesFS - extends FSParam[Boolean](name = "oon_filtering_disable_oon_candidates", default = false) - - /** - * Param to enable Local Viral Tweets - */ - object EnableLocalViralTweets - extends FSParam[Boolean](name = "local_viral_tweets_enable", default = true) - - /** - * Param to enable Explore Video Tweets - */ - object EnableExploreVideoTweets - extends FSParam[Boolean](name = "explore_video_tweets_enable", default = false) - - /** - * Param to enable List Recommendations - */ - object EnableListRecommendations - extends FSParam[Boolean](name = "list_recommendations_enable", default = false) - - /** - * Param to enable IDS List Recommendations - */ - object EnableIDSListRecommendations - extends FSParam[Boolean](name = "list_recommendations_ids_enable", default = false) - - /** - * Param to enable PopGeo List Recommendations - */ - object EnablePopGeoListRecommendations - extends FSParam[Boolean](name = "list_recommendations_pop_geo_enable", default = false) - - /** - * Param to control the inverter for fatigue between consecutive ListRecommendations - */ - object ListRecommendationsPushInterval - extends FSBoundedParam[Duration]( - name = "list_recommendations_interval_days", - default = 24.hours, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Param to control the granularity of GeoHash for ListRecommendations - */ - object ListRecommendationsGeoHashLength - extends FSBoundedParam[Int]( - name = "list_recommendations_geo_hash_length", - default = 5, - min = 3, - max = 5) - - /** - * Param to control maximum number of ListRecommendation pushes to receive in an interval - */ - object MaxListRecommendationsPushGivenInterval - extends FSBoundedParam[Int]( - name = "list_recommendations_push_given_interval", - default = 1, - min = 0, - max = 10 - ) - - /** - * Param to control the subscriber count for list recommendation - */ - object ListRecommendationsSubscriberCount - extends FSBoundedParam[Int]( - name = "list_recommendations_subscriber_count", - default = 0, - min = 0, - max = Integer.MAX_VALUE) - - /** - * Param to define dynamic inline action types for web notifications (both desktop web + mobile web) - */ - object LocalViralTweetsBucket - extends FSParam[String]( - name = "local_viral_tweets_bucket", - default = "high", - ) - - /** - * List of CrTags to disable - */ - object OONCandidatesDisabledCrTagParam - extends FSParam[Seq[String]]( - name = "oon_enable_oon_candidates_disabled_crtag", - default = Seq.empty[String] - ) - - /** - * List of Crt groups to disable - */ - object OONCandidatesDisabledCrtGroupParam - extends FSEnumSeqParam[CrtGroupEnum.type]( - name = "oon_enable_oon_candidates_disabled_crt_group_ids", - default = Seq.empty[CrtGroupEnum.Value], - enum = CrtGroupEnum - ) - - /** - * Param to enable launching video tweets in the Immersive Explore timeline - */ - object EnableLaunchVideosInImmersiveExplore - extends FSParam[Boolean](name = "launch_videos_in_immersive_explore", default = false) - - /** - * Param to enable Ntab Entries for Sports Event Notifications - */ - object EnableNTabEntriesForSportsEventNotifications - extends FSParam[Boolean]( - name = "magicfanout_sports_event_enable_ntab_entries", - default = false) - - /** - * Param to enable Ntab Facepiles for teams in Sport Notifs - */ - object EnableNTabFacePileForSportsEventNotifications - extends FSParam[Boolean]( - name = "magicfanout_sports_event_enable_ntab_facepiles", - default = false) - - /** - * Param to enable Ntab Override for Sports Event Notifications - */ - object EnableNTabOverrideForSportsEventNotifications - extends FSParam[Boolean]( - name = "magicfanout_sports_event_enable_ntab_override", - default = false) - - /** - * Param to control the interval for MF Product Launch Notifs - */ - object ProductLaunchPushIntervalInHours - extends FSBoundedParam[Duration]( - name = "product_launch_fatigue_push_interval_in_hours", - default = 24.hours, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to control the maximum number of MF Product Launch Notifs in a period of time - */ - object ProductLaunchMaxNumberOfPushesInInterval - extends FSBoundedParam[Int]( - name = "product_launch_fatigue_max_pushes_in_interval", - default = 1, - min = 0, - max = 10) - - /** - * Param to control the minInterval for fatigue between consecutive MF Product Launch Notifs - */ - object ProductLaunchMinIntervalFatigue - extends FSBoundedParam[Duration]( - name = "product_launch_fatigue_min_interval_consecutive_pushes_in_hours", - default = 24.hours, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to control the interval for MF New Creator Notifs - */ - object NewCreatorPushIntervalInHours - extends FSBoundedParam[Duration]( - name = "new_creator_fatigue_push_interval_in_hours", - default = 24.hours, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to control the maximum number of MF New Creator Notifs in a period of time - */ - object NewCreatorPushMaxNumberOfPushesInInterval - extends FSBoundedParam[Int]( - name = "new_creator_fatigue_max_pushes_in_interval", - default = 1, - min = 0, - max = 10) - - /** - * Param to control the minInterval for fatigue between consecutive MF New Creator Notifs - */ - object NewCreatorPushMinIntervalFatigue - extends FSBoundedParam[Duration]( - name = "new_creator_fatigue_min_interval_consecutive_pushes_in_hours", - default = 24.hours, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to control the interval for MF New Creator Notifs - */ - object CreatorSubscriptionPushIntervalInHours - extends FSBoundedParam[Duration]( - name = "creator_subscription_fatigue_push_interval_in_hours", - default = 24.hours, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to control the maximum number of MF New Creator Notifs in a period of time - */ - object CreatorSubscriptionPushMaxNumberOfPushesInInterval - extends FSBoundedParam[Int]( - name = "creator_subscription_fatigue_max_pushes_in_interval", - default = 1, - min = 0, - max = 10) - - /** - * Param to control the minInterval for fatigue between consecutive MF New Creator Notifs - */ - object CreatorSubscriptionPushhMinIntervalFatigue - extends FSBoundedParam[Duration]( - name = "creator_subscription_fatigue_min_interval_consecutive_pushes_in_hours", - default = 24.hours, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to define the landing page deeplink of product launch notifications - */ - object ProductLaunchLandingPageDeepLink - extends FSParam[String]( - name = "product_launch_landing_page_deeplink", - default = "" - ) - - /** - * Param to define the tap through of product launch notifications - */ - object ProductLaunchTapThrough - extends FSParam[String]( - name = "product_launch_tap_through", - default = "" - ) - - /** - * Param to skip checking isTargetBlueVerified - */ - object DisableIsTargetBlueVerifiedPredicate - extends FSParam[Boolean]( - name = "product_launch_disable_is_target_blue_verified_predicate", - default = false - ) - - /** - * Param to enable Ntab Entries for Sports Event Notifications - */ - object EnableNTabEntriesForProductLaunchNotifications - extends FSParam[Boolean](name = "product_launch_enable_ntab_entry", default = true) - - /** - * Param to skip checking isTargetLegacyVerified - */ - object DisableIsTargetLegacyVerifiedPredicate - extends FSParam[Boolean]( - name = "product_launch_disable_is_target_legacy_verified_predicate", - default = false - ) - - /** - * Param to enable checking isTargetSuperFollowCreator - */ - object EnableIsTargetSuperFollowCreatorPredicate - extends FSParam[Boolean]( - name = "product_launch_is_target_super_follow_creator_predicate_enabled", - default = false - ) - - /** - * Param to enable Spammy Tweet filter - */ - object EnableSpammyTweetFilter - extends FSParam[Boolean]( - name = "health_signal_store_enable_spammy_tweet_filter", - default = false) - - /** - * Param to enable Push to Home Android - */ - object EnableTweetPushToHomeAndroid - extends FSParam[Boolean](name = "push_to_home_tweet_recs_android", default = false) - - /** - * Param to enable Push to Home iOS - */ - object EnableTweetPushToHomeiOS - extends FSParam[Boolean](name = "push_to_home_tweet_recs_iOS", default = false) - - /** - * Param to set Spammy Tweet score threshold for OON candidates - */ - object SpammyTweetOonThreshold - extends FSBoundedParam[Double]( - name = "health_signal_store_spammy_tweet_oon_threshold", - default = 1.1, - min = 0.0, - max = 1.1 - ) - - object NumFollowerThresholdForHealthAndQualityFilters - extends FSBoundedParam[Double]( - name = "health_signal_store_num_follower_threshold_for_health_and_quality_filters", - default = 10000000000.0, - min = 0.0, - max = 10000000000.0 - ) - - object NumFollowerThresholdForHealthAndQualityFiltersPreranking - extends FSBoundedParam[Double]( - name = - "health_signal_store_num_follower_threshold_for_health_and_quality_filters_preranking", - default = 10000000.0, - min = 0.0, - max = 10000000000.0 - ) - - /** - * Param to set Spammy Tweet score threshold for IN candidates - */ - object SpammyTweetInThreshold - extends FSBoundedParam[Double]( - name = "health_signal_store_spammy_tweet_in_threshold", - default = 1.1, - min = 0.0, - max = 1.1 - ) - - /** - * Param to control bucketing for the Spammy Tweet score - */ - object SpammyTweetBucketingThreshold - extends FSBoundedParam[Double]( - name = "health_signal_store_spammy_tweet_bucketing_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Param to specify the maximum number of Explore Video Tweets to request - */ - object MaxExploreVideoTweets - extends FSBoundedParam[Int]( - name = "explore_video_tweets_max_candidates", - default = 100, - min = 0, - max = 500 - ) - - /** - * Param to enable social context feature set - */ - object EnableBoundedFeatureSetForSocialContext - extends FSParam[Boolean]( - name = "feature_hydration_user_social_context_bounded_feature_set_enable", - default = true) - - /** - * Param to enable stp user social context feature set - */ - object EnableStpBoundedFeatureSetForUserSocialContext - extends FSParam[Boolean]( - name = "feature_hydration_stp_social_context_bounded_feature_set_enable", - default = true) - - /** - * Param to enable core user history social context feature set - */ - object EnableCoreUserHistoryBoundedFeatureSetForSocialContext - extends FSParam[Boolean]( - name = "feature_hydration_core_user_history_social_context_bounded_feature_set_enable", - default = true) - - /** - * Param to enable skipping post-ranking filters - */ - object SkipPostRankingFilters - extends FSParam[Boolean]( - name = "frigate_push_modeling_skip_post_ranking_filters", - default = false) - - object MagicFanoutSimClusterDotProductNonHeavyUserThreshold - extends FSBoundedParam[Double]( - name = "frigate_push_magicfanout_simcluster_non_heavy_user_dot_product_threshold", - default = 0.0, - min = 0.0, - max = 100.0 - ) - - object MagicFanoutSimClusterDotProductHeavyUserThreshold - extends FSBoundedParam[Double]( - name = "frigate_push_magicfanout_simcluster_heavy_user_dot_product_threshold", - default = 10.0, - min = 0.0, - max = 100.0 - ) - - object EnableReducedFatigueRulesForSeeLessOften - extends FSParam[Boolean]( - name = "seelessoften_enable_reduced_fatigue", - default = false - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushFeatureSwitches.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushFeatureSwitches.scala deleted file mode 100644 index 96167c134..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushFeatureSwitches.scala +++ /dev/null @@ -1,751 +0,0 @@ -package com.twitter.frigate.pushservice.params - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.{FeatureSwitchParams => Common} -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => Pushservice} -import com.twitter.logging.Logger -import com.twitter.servo.decider.DeciderGateBuilder -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.decider.DeciderUtils - -case class PushFeatureSwitches( - deciderGateBuilder: DeciderGateBuilder, - statsReceiver: StatsReceiver) { - - private[this] val logger = Logger(classOf[PushFeatureSwitches]) - private[this] val stat = statsReceiver.scope("PushFeatureSwitches") - - private val booleanDeciderOverrides = DeciderUtils.getBooleanDeciderOverrides( - deciderGateBuilder, - PushParams.DisableAllRelevanceParam, - PushParams.DisableHeavyRankingParam, - PushParams.RestrictLightRankingParam, - PushParams.UTEGTweetCandidateSourceParam, - PushParams.EnableWritesToNotificationServiceParam, - PushParams.EnableWritesToNotificationServiceForAllEmployeesParam, - PushParams.EnableWritesToNotificationServiceForEveryoneParam, - PushParams.EnablePromptFeedbackFatigueResponseNoPredicate, - PushParams.EarlyBirdSCBasedCandidatesParam, - PushParams.MRTweetFavRecsParam, - PushParams.MRTweetRetweetRecsParam, - PushParams.EnablePushSendEventBus, - PushParams.DisableMlInFilteringParam, - PushParams.DownSampleLightRankingScribeCandidatesParam, - PushParams.EnableMrRequestScribing, - PushParams.EnableHighQualityCandidateScoresScribing, - PushParams.EnablePnegMultimodalPredictionForF1Tweets, - PushParams.EnableScribeOonFavScoreForF1Tweets, - PushParams.EnableMrUserSemanticCoreFeaturesHydration, - PushParams.EnableMrUserSemanticCoreNoZeroFeaturesHydration, - PushParams.EnableHtlOfflineUserAggregatesExtendedHydration, - PushParams.EnableNerErgFeatureHydration, - PushParams.EnableDaysSinceRecentResurrectionFeatureHydration, - PushParams.EnableUserPastAggregatesFeatureHydration, - PushParams.EnableMrUserSimclusterV2020FeaturesHydration, - PushParams.EnableMrUserSimclusterV2020NoZeroFeaturesHydration, - PushParams.EnableTopicEngagementRealTimeAggregatesFeatureHydration, - PushParams.EnableUserTopicAggregatesFeatureHydration, - PushParams.EnableHtlUserAuthorRTAFeaturesFromFeatureStoreHydration, - PushParams.EnableDurationSinceLastVisitFeatures, - PushParams.EnableTweetAnnotationFeaturesHydration, - PushParams.EnableSpaceVisibilityLibraryFiltering, - PushParams.EnableUserTopicFollowFeatureSetHydration, - PushParams.EnableOnboardingNewUserFeatureSetHydration, - PushParams.EnableMrUserAuthorSparseContFeatureSetHydration, - PushParams.EnableMrUserTopicSparseContFeatureSetHydration, - PushParams.EnableUserPenguinLanguageFeatureSetHydration, - PushParams.EnableMrUserHashspaceEmbeddingFeatureHydration, - PushParams.EnableMrUserEngagedTweetTokensFeatureHydration, - PushParams.EnableMrCandidateTweetTokensFeatureHydration, - PushParams.EnableMrTweetSentimentFeatureHydration, - PushParams.EnableMrTweetAuthorAggregatesFeatureHydration, - PushParams.EnableUserGeoFeatureSetHydration, - PushParams.EnableAuthorGeoFeatureSetHydration, - PushParams.EnableTwHINUserEngagementFeaturesHydration, - PushParams.EnableTwHINUserFollowFeaturesHydration, - PushParams.EnableTwHINAuthorFollowFeaturesHydration, - PushParams.EnableAuthorFollowTwhinEmbeddingFeatureHydration, - PushParams.RampupUserGeoFeatureSetHydration, - PushParams.RampupAuthorGeoFeatureSetHydration, - PushParams.EnablePredicateDetailedInfoScribing, - PushParams.EnablePushCapInfoScribing, - PushParams.EnableUserSignalLanguageFeatureHydration, - PushParams.EnableUserPreferredLanguageFeatureHydration, - PushParams.PopGeoCandidatesDecider, - PushParams.TrendsCandidateDecider, - PushParams.EnableInsTrafficDecider, - PushParams.EnableModelBasedPushcapAssignments, - PushParams.TripGeoTweetCandidatesDecider, - PushParams.ContentRecommenderMixerAdaptorDecider, - PushParams.GenericCandidateAdaptorDecider, - PushParams.TripGeoTweetContentMixerDarkTrafficDecider, - PushParams.EnableIsTweetTranslatableCheck, - PushParams.EnableMrTweetSimClusterFeatureHydration, - PushParams.EnableTwistlyAggregatesFeatureHydration, - PushParams.EnableTweetTwHINFavFeatureHydration, - PushParams.EnableRealGraphV2FeatureHydration, - PushParams.EnableTweetBeTFeatureHydration, - PushParams.EnableMrOfflineUserTweetTopicAggregateHydration, - PushParams.EnableMrOfflineUserTweetSimClusterAggregateHydration, - PushParams.EnableUserSendTimeFeatureHydration, - PushParams.EnableMrUserUtcSendTimeAggregateFeaturesHydration, - PushParams.EnableMrUserLocalSendTimeAggregateFeaturesHydration, - PushParams.EnableBqmlReportModelPredictionForF1Tweets, - PushParams.EnableUserTwhinEmbeddingFeatureHydration, - PushParams.EnableScribingMLFeaturesAsDataRecord, - PushParams.EnableAuthorVerifiedFeatureHydration, - PushParams.EnableAuthorCreatorSubscriptionFeatureHydration, - PushParams.EnableDirectHydrationForUserFeatures - ) - - private val intFeatureSwitchOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( - Pushservice.SportsMaxNumberOfPushesInIntervalPerEvent, - Pushservice.SportsMaxNumberOfPushesInInterval, - Pushservice.PushMixerMaxResults, - Pushservice.MaxTrendTweetNotificationsInDuration, - Pushservice.MaxRecommendedTrendsToQuery, - Pushservice.NumberOfMaxEarlybirdInNetworkCandidatesParam, - Pushservice.NumberOfMaxCandidatesToBatchInRFPHTakeStep, - Pushservice.MaxMrPushSends24HoursParam, - Pushservice.MaxMrPushSends24HoursNtabOnlyUsersParam, - Pushservice.NumberOfMaxCrMixerCandidatesParam, - Pushservice.RestrictStepSize, - Pushservice.MagicFanoutRankErgThresholdHeavy, - Pushservice.MagicFanoutRankErgThresholdNonHeavy, - Pushservice.MagicFanoutRelaxedEventIdFatigueIntervalInHours, - Pushservice.NumberOfMaxUTEGCandidatesQueriedParam, - Pushservice.HTLVisitFatigueTime, - Pushservice.MaxOnboardingPushInInterval, - Pushservice.MaxTopTweetsByGeoPushGivenInterval, - Pushservice.MaxHighQualityTweetsPushGivenInterval, - Pushservice.MaxTopTweetsByGeoCandidatesToTake, - Pushservice.SpaceRecsRealgraphThreshold, - Pushservice.SpaceRecsGlobalPushLimit, - Pushservice.OptoutExptPushCapParam, - Pushservice.MaxTopTweetImpressionsNotifications, - Pushservice.TopTweetImpressionsMinRequired, - Pushservice.TopTweetImpressionsThreshold, - Pushservice.TopTweetImpressionsOriginalTweetsNumDaysSearch, - Pushservice.TopTweetImpressionsMinNumOriginalTweets, - Pushservice.TopTweetImpressionsMaxFavoritesPerTweet, - Pushservice.TopTweetImpressionsTotalInboundFavoritesLimit, - Pushservice.TopTweetImpressionsTotalFavoritesLimitNumDaysSearch, - Pushservice.TopTweetImpressionsRecentTweetsByAuthorStoreMaxResults, - Pushservice.ANNEfQuery, - Pushservice.NumberOfMaxMrModelingBasedCandidates, - Pushservice.ThresholdOfFavMrModelingBasedCandidates, - Pushservice.LightRankingNumberOfCandidatesParam, - Pushservice.NumberOfDeTopicTweetCandidates, - Pushservice.NumberOfMaxDeTopicTweetCandidatesReturned, - Pushservice.OverrideNotificationsMaxNumOfSlots, - Pushservice.OverrideNotificationsMaxCountForNTab, - Pushservice.MFMaxNumberOfPushesInInterval, - Pushservice.SpacesTopKSimClusterCount, - Pushservice.SpaceRecsSimClusterUserMinimumFollowerCount, - Pushservice.OONSpaceRecsPushLimit, - Pushservice.MagicFanoutRealgraphRankThreshold, - Pushservice.CustomizedPushCapOffset, - Pushservice.NumberOfF1CandidatesThresholdForOONBackfill, - Pushservice.MinimumAllowedAuthorAccountAgeInHours, - Pushservice.RestrictedMinModelPushcap, - Pushservice.ListRecommendationsGeoHashLength, - Pushservice.ListRecommendationsSubscriberCount, - Pushservice.MaxListRecommendationsPushGivenInterval, - Pushservice.SendTimeByUserHistoryMaxOpenedThreshold, - Pushservice.SendTimeByUserHistoryNoSendsHours, - Pushservice.SendTimeByUserHistoryQuickSendBeforeHours, - Pushservice.SendTimeByUserHistoryQuickSendAfterHours, - Pushservice.SendTimeByUserHistoryQuickSendMinDurationInMinute, - Pushservice.SendTimeByUserHistoryNoSendMinDuration, - Pushservice.F1EmojiCopyNumOfPushesFatigue, - Pushservice.OonEmojiCopyNumOfPushesFatigue, - Pushservice.TripTweetMaxTotalCandidates, - Pushservice.InlineFeedbackSubstitutePosition, - Pushservice.HighQualityCandidatesNumberOfCandidates, - Pushservice.HighQualityCandidatesMinNumOfCandidatesToFallback, - Pushservice.ProductLaunchMaxNumberOfPushesInInterval, - Pushservice.CreatorSubscriptionPushMaxNumberOfPushesInInterval, - Pushservice.NewCreatorPushMaxNumberOfPushesInInterval, - Pushservice.TweetReplytoLikeRatioReplyCountThreshold, - Pushservice.MaxExploreVideoTweets, - ) - - private val doubleFeatureSwitchOverrides = - FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides( - Pushservice.PercentileThresholdCohort1, - Pushservice.PercentileThresholdCohort2, - Pushservice.PercentileThresholdCohort3, - Pushservice.PercentileThresholdCohort4, - Pushservice.PercentileThresholdCohort5, - Pushservice.PercentileThresholdCohort6, - Pushservice.PnsfwTweetTextThreshold, - Pushservice.PnsfwTweetTextBucketingThreshold, - Pushservice.PnsfwTweetMediaThreshold, - Pushservice.PnsfwTweetImageThreshold, - Pushservice.PnsfwQuoteTweetThreshold, - Pushservice.PnsfwTweetMediaBucketingThreshold, - Pushservice.AgathaCalibratedNSFWThreshold, - Pushservice.AgathaCalibratedNSFWThresholdForMrTwistly, - Pushservice.AgathaTextNSFWThreshold, - Pushservice.AgathaTextNSFWThresholdForMrTwistly, - Pushservice.AgathaCalibratedNSFWBucketThreshold, - Pushservice.AgathaTextNSFWBucketThreshold, - Pushservice.BucketOptoutThresholdParam, - Pushservice.TweetMediaSensitiveCategoryThresholdParam, - Pushservice.CandidateGenerationModelCosineThreshold, - Pushservice.MrModelingBasedCandidatesTopicScoreThreshold, - Pushservice.HashspaceCandidatesTopicScoreThreshold, - Pushservice.FrsTweetCandidatesTopicScoreThreshold, - Pushservice.TopicProofTweetCandidatesTopicScoreThreshold, - Pushservice.SpacesTargetingSimClusterDotProductThreshold, - Pushservice.SautOonWithMediaTweetLengthThresholdParam, - Pushservice.NonSautOonWithMediaTweetLengthThresholdParam, - Pushservice.SautOonWithoutMediaTweetLengthThresholdParam, - Pushservice.NonSautOonWithoutMediaTweetLengthThresholdParam, - Pushservice.ArgfOonWithMediaTweetWordLengthThresholdParam, - Pushservice.EsfthOonWithMediaTweetWordLengthThresholdParam, - Pushservice.BqmlQualityModelPredicateThresholdParam, - Pushservice.LightRankingScribeCandidatesDownSamplingParam, - Pushservice.QualityUprankingBoostForHeavyRankingParam, - Pushservice.QualityUprankingSigmoidBiasForHeavyRankingParam, - Pushservice.QualityUprankingSigmoidWeightForHeavyRankingParam, - Pushservice.QualityUprankingLinearBarForHeavyRankingParam, - Pushservice.QualityUprankingBoostForHighQualityProducersParam, - Pushservice.QualityUprankingDownboostForLowQualityProducersParam, - Pushservice.BqmlHealthModelPredicateFilterThresholdParam, - Pushservice.BqmlHealthModelPredicateBucketThresholdParam, - Pushservice.PNegMultimodalPredicateModelThresholdParam, - Pushservice.PNegMultimodalPredicateBucketThresholdParam, - Pushservice.SeeLessOftenF1TriggerF1PushCapWeight, - Pushservice.SeeLessOftenF1TriggerNonF1PushCapWeight, - Pushservice.SeeLessOftenNonF1TriggerF1PushCapWeight, - Pushservice.SeeLessOftenNonF1TriggerNonF1PushCapWeight, - Pushservice.SeeLessOftenTripHqTweetTriggerF1PushCapWeight, - Pushservice.SeeLessOftenTripHqTweetTriggerNonF1PushCapWeight, - Pushservice.SeeLessOftenTripHqTweetTriggerTripHqTweetPushCapWeight, - Pushservice.SeeLessOftenNtabOnlyNotifUserPushCapWeight, - Pushservice.PromptFeedbackF1TriggerF1PushCapWeight, - Pushservice.PromptFeedbackF1TriggerNonF1PushCapWeight, - Pushservice.PromptFeedbackNonF1TriggerF1PushCapWeight, - Pushservice.PromptFeedbackNonF1TriggerNonF1PushCapWeight, - Pushservice.InlineFeedbackF1TriggerF1PushCapWeight, - Pushservice.InlineFeedbackF1TriggerNonF1PushCapWeight, - Pushservice.InlineFeedbackNonF1TriggerF1PushCapWeight, - Pushservice.InlineFeedbackNonF1TriggerNonF1PushCapWeight, - Pushservice.TweetNtabDislikeCountThresholdParam, - Pushservice.TweetNtabDislikeRateThresholdParam, - Pushservice.TweetNtabDislikeCountThresholdForMrTwistlyParam, - Pushservice.TweetNtabDislikeRateThresholdForMrTwistlyParam, - Pushservice.TweetNtabDislikeCountBucketThresholdParam, - Pushservice.MinAuthorSendsThresholdParam, - Pushservice.MinTweetSendsThresholdParam, - Pushservice.AuthorDislikeRateThresholdParam, - Pushservice.AuthorReportRateThresholdParam, - Pushservice.FavOverSendThresholdParam, - Pushservice.SpreadControlRatioParam, - Pushservice.TweetQTtoNtabClickRatioThresholdParam, - Pushservice.TweetReplytoLikeRatioThresholdLowerBound, - Pushservice.TweetReplytoLikeRatioThresholdUpperBound, - Pushservice.AuthorSensitiveMediaFilteringThreshold, - Pushservice.AuthorSensitiveMediaFilteringThresholdForMrTwistly, - Pushservice.MrRequestScribingEpsGreedyExplorationRatio, - Pushservice.SeeLessOftenTopicTriggerTopicPushCapWeight, - Pushservice.SeeLessOftenTopicTriggerF1PushCapWeight, - Pushservice.SeeLessOftenTopicTriggerOONPushCapWeight, - Pushservice.SeeLessOftenF1TriggerTopicPushCapWeight, - Pushservice.SeeLessOftenOONTriggerTopicPushCapWeight, - Pushservice.SeeLessOftenDefaultPushCapWeight, - Pushservice.OverrideMaxSlotFnWeight, - Pushservice.QualityPredicateExplicitThresholdParam, - Pushservice.AuthorSensitiveScoreWeightInReranking, - Pushservice.BigFilteringThresholdParam, - Pushservice.NsfwScoreThresholdForF1Copy, - Pushservice.NsfwScoreThresholdForOONCopy, - Pushservice.HighOONCThresholdForCopy, - Pushservice.LowOONCThresholdForCopy, - Pushservice.UserDeviceLanguageThresholdParam, - Pushservice.UserInferredLanguageThresholdParam, - Pushservice.SpammyTweetOonThreshold, - Pushservice.SpammyTweetInThreshold, - Pushservice.SpammyTweetBucketingThreshold, - Pushservice.NumFollowerThresholdForHealthAndQualityFilters, - Pushservice.NumFollowerThresholdForHealthAndQualityFiltersPreranking, - Pushservice.SoftRankFactorForSubscriptionCreators, - Pushservice.MagicFanoutSimClusterDotProductHeavyUserThreshold, - Pushservice.MagicFanoutSimClusterDotProductNonHeavyUserThreshold - ) - - private val doubleSeqFeatureSwitchOverrides = - FeatureSwitchOverrideUtil.getDoubleSeqFSOverrides( - Pushservice.MfGridSearchThresholdsCohort1, - Pushservice.MfGridSearchThresholdsCohort2, - Pushservice.MfGridSearchThresholdsCohort3, - Pushservice.MfGridSearchThresholdsCohort4, - Pushservice.MfGridSearchThresholdsCohort5, - Pushservice.MfGridSearchThresholdsCohort6, - Pushservice.MrPercentileGridSearchThresholdsCohort1, - Pushservice.MrPercentileGridSearchThresholdsCohort2, - Pushservice.MrPercentileGridSearchThresholdsCohort3, - Pushservice.MrPercentileGridSearchThresholdsCohort4, - Pushservice.MrPercentileGridSearchThresholdsCohort5, - Pushservice.MrPercentileGridSearchThresholdsCohort6, - Pushservice.GlobalOptoutThresholdParam, - Pushservice.BucketOptoutSlotThresholdParam, - Pushservice.BqmlQualityModelBucketThresholdListParam, - Pushservice.SeeLessOftenListOfDayKnobs, - Pushservice.SeeLessOftenListOfPushCapWeightKnobs, - Pushservice.SeeLessOftenListOfPowerKnobs, - Pushservice.PromptFeedbackListOfDayKnobs, - Pushservice.PromptFeedbackListOfPushCapWeightKnobs, - Pushservice.PromptFeedbackListOfPowerKnobs, - Pushservice.InlineFeedbackListOfDayKnobs, - Pushservice.InlineFeedbackListOfPushCapWeightKnobs, - Pushservice.InlineFeedbackListOfPowerKnobs, - Pushservice.OverrideMaxSlotFnPushCapKnobs, - Pushservice.OverrideMaxSlotFnPowerKnobs, - Pushservice.OverrideMaxSlotFnPushCapKnobs, - Pushservice.MagicRecsRelevanceScoreRange, - Pushservice.MagicFanoutRelevanceScoreRange, - Pushservice.MultilingualPnsfwTweetTextBucketingThreshold, - Pushservice.MultilingualPnsfwTweetTextFilteringThreshold, - ) - - private val booleanFeatureSwitchOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - Pushservice.EnablePushRecommendationsParam, - Pushservice.DisableHeavyRankingModelFSParam, - Pushservice.EnablePushMixerReplacingAllSources, - Pushservice.EnablePushMixerReplacingAllSourcesWithControl, - Pushservice.EnablePushMixerReplacingAllSourcesWithExtra, - Pushservice.EnablePushMixerSource, - Common.EnableScheduledSpaceSpeakers, - Common.EnableScheduledSpaceSubscribers, - Pushservice.MagicFanoutNewsUserGeneratedEventsEnable, - Pushservice.MagicFanoutSkipAccountCountryPredicate, - Pushservice.MagicFanoutNewsEnableDescriptionCopy, - Pushservice.EnableF1TriggerSeeLessOftenFatigue, - Pushservice.EnableNonF1TriggerSeeLessOftenFatigue, - Pushservice.AdjustTripHqTweetTriggeredNtabCaretClickFatigue, - Pushservice.EnableCuratedTrendTweets, - Pushservice.EnableNonCuratedTrendTweets, - Pushservice.DisableMlInFilteringFeatureSwitchParam, - Pushservice.EnableTopicCopyForMF, - Pushservice.EnableTopicCopyForImplicitTopics, - Pushservice.EnableRestrictStep, - Pushservice.EnableHighPriorityPush, - Pushservice.BoostCandidatesFromSubscriptionCreators, - Pushservice.SoftRankCandidatesFromSubscriptionCreators, - Pushservice.EnableNewMROONCopyForPush, - Pushservice.EnableQueryAuthorMediaRepresentationStore, - Pushservice.EnableProfanityFilterParam, - Pushservice.EnableAbuseStrikeTop2PercentFilterSimCluster, - Pushservice.EnableAbuseStrikeTop1PercentFilterSimCluster, - Pushservice.EnableAbuseStrikeTop05PercentFilterSimCluster, - Pushservice.EnableAgathaUserHealthModelPredicate, - Pushservice.PnsfwTweetMediaFilterOonOnly, - Pushservice.EnableHealthSignalStorePnsfwTweetTextPredicate, - Pushservice.EnableHealthSignalStoreMultilingualPnsfwTweetTextPredicate, - Pushservice.DisableHealthFiltersForCrMixerCandidates, - Pushservice.EnableOverrideNotificationsForAndroid, - Pushservice.EnableOverrideNotificationsForIos, - Pushservice.EnableMrRequestScribingForTargetFiltering, - Pushservice.EnableMrRequestScribingForCandidateFiltering, - Pushservice.EnableMrRequestScribingWithFeatureHydrating, - Pushservice.EnableFlattenMrRequestScribing, - Pushservice.EnableMrRequestScribingForEpsGreedyExploration, - Pushservice.EnableMrRequestScribingDismissScore, - Pushservice.EnableMrRequestScribingBigFilteringSupervisedScores, - Pushservice.EnableMrRequestScribingBigFilteringRLScores, - Pushservice.EnableEventPrimaryMediaAndroid, - Pushservice.EnableEventSquareMediaIosMagicFanoutNewsEvent, - Pushservice.EnableEventSquareMediaAndroid, - Pushservice.EnableMagicFanoutNewsForYouNtabCopy, - Pushservice.EnableMfGeoTargeting, - Pushservice.EnableRuxLandingPage, - Pushservice.EnableNTabRuxLandingPage, - Pushservice.EnableGraduallyRampUpNotification, - Pushservice.EnableOnboardingPushes, - Pushservice.EnableAddressBookPush, - Pushservice.EnableCompleteOnboardingPush, - Pushservice.EnableOverrideNotificationsSmartPushConfigForAndroid, - Pushservice.DisableOnboardingPushFatigue, - Pushservice.EnableTopTweetsByGeoCandidates, - Pushservice.BackfillRankTopTweetsByGeoCandidates, - Pushservice.PopGeoTweetEnableAggressiveThresholds, - Pushservice.EnableMrMinDurationSinceMrPushFatigue, - Pushservice.EnableF1FromProtectedTweetAuthors, - Pushservice.MagicFanoutEnableCustomTargetingNewsEvent, - Pushservice.EnableSafeUserTweetTweetypieStore, - Pushservice.EnableMrMinDurationSinceMrPushFatigue, - Pushservice.EnableHydratingOnlineMRHistoryFeatures, - Common.SpaceRecsEnableHostNotifs, - Common.SpaceRecsEnableSpeakerNotifs, - Common.SpaceRecsEnableListenerNotifs, - Common.EnableMagicFanoutProductLaunch, - Pushservice.EnableTopTweetsByGeoCandidatesForDormantUsers, - Pushservice.EnableOverrideNotificationsScoreBasedOverride, - Pushservice.EnableOverrideNotificationsMultipleTargetIds, - Pushservice.EnableMinDurationModifier, - Pushservice.EnableMinDurationModifierV2, - Pushservice.EnableMinDurationModifierByUserHistory, - Pushservice.EnableQueryUserOpenedHistory, - Pushservice.EnableRandomHourForQuickSend, - Pushservice.EnableFrsCandidates, - Pushservice.EnableFrsTweetCandidatesTopicSetting, - Pushservice.EnableFrsTweetCandidatesTopicAnnotation, - Pushservice.EnableFrsTweetCandidatesTopicCopy, - Pushservice.EnableCandidateGenerationModelParam, - Pushservice.EnableOverrideForSportsCandidates, - Pushservice.EnableEventIdBasedOverrideForSportsCandidates, - Pushservice.EnableMrModelingBasedCandidates, - Pushservice.EnableMrModelingBasedCandidatesTopicSetting, - Pushservice.EnableMrModelingBasedCandidatesTopicAnnotation, - Pushservice.EnableMrModelingBasedCandidatesTopicCopy, - Pushservice.EnableResultFromFrsCandidates, - Pushservice.EnableHashspaceCandidates, - Pushservice.EnableHashspaceCandidatesTopicSetting, - Pushservice.EnableHashspaceCandidatesTopicAnnotation, - Pushservice.EnableHashspaceCandidatesTopicCopy, - Pushservice.EnableResultFromHashspaceCandidates, - Pushservice.EnableDownRankOfNewUserPlaybookTopicFollowPush, - Pushservice.EnableDownRankOfNewUserPlaybookTopicTweetPush, - Pushservice.EnableTopTweetImpressionsNotification, - Pushservice.EnableLightRankingParam, - Pushservice.EnableRandomBaselineLightRankingParam, - Pushservice.EnableQualityUprankingForHeavyRankingParam, - Pushservice.EnableQualityUprankingCrtScoreStatsForHeavyRankingParam, - Pushservice.EnableProducersQualityBoostingForHeavyRankingParam, - Pushservice.EnableMrScribingMLFeaturesAsFeatureMapForStaging, - Pushservice.EnableMrTweetSentimentFeatureHydrationFS, - Pushservice.EnableTimelineHealthSignalHydration, - Pushservice.EnableTopicEngagementRealTimeAggregatesFS, - Pushservice.EnableMrUserSemanticCoreFeatureForExpt, - Pushservice.EnableHydratingRealGraphTargetUserFeatures, - Pushservice.EnableHydratingUserDurationSinceLastVisitFeatures, - Pushservice.EnableRealGraphUserAuthorAndSocialContxtFeatureHydration, - Pushservice.EnableUserTopicAggregatesFS, - Pushservice.EnableTimelineHealthSignalHydrationForModelTraining, - Pushservice.EnableMrUserSocialContextAggregateFeatureHydration, - Pushservice.EnableMrUserSemanticCoreAggregateFeatureHydration, - Pushservice.EnableMrUserCandidateSparseOfflineAggregateFeatureHydration, - Pushservice.EnableMrUserCandidateOfflineAggregateFeatureHydration, - Pushservice.EnableMrUserCandidateOfflineCompactAggregateFeatureHydration, - Pushservice.EnableMrUserAuthorOfflineAggregateFeatureHydration, - Pushservice.EnableMrUserAuthorOfflineCompactAggregateFeatureHydration, - Pushservice.EnableMrUserOfflineCompactAggregateFeatureHydration, - Pushservice.EnableMrUserSimcluster2020AggregateFeatureHydration, - Pushservice.EnableMrUserOfflineAggregateFeatureHydration, - Pushservice.EnableBqmlQualityModelPredicateParam, - Pushservice.EnableBqmlQualityModelScoreHistogramParam, - Pushservice.EnableBqmlHealthModelPredicateParam, - Pushservice.EnableBqmlHealthModelPredictionForInNetworkCandidatesParam, - Pushservice.EnableBqmlHealthModelScoreHistogramParam, - Pushservice.EnablePNegMultimodalPredicateParam, - Pushservice.EnableNegativeKeywordsPredicateParam, - Pushservice.EnableTweetAuthorAggregatesFeatureHydrationParam, - Pushservice.OonTweetLengthPredicateUpdatedMediaLogic, - Pushservice.OonTweetLengthPredicateUpdatedQuoteTweetLogic, - Pushservice.OonTweetLengthPredicateMoreStrictForUndefinedLanguages, - Pushservice.EnablePrerankingTweetLengthPredicate, - Pushservice.EnableDeTopicTweetCandidates, - Pushservice.EnableDeTopicTweetCandidateResults, - Pushservice.EnableDeTopicTweetCandidatesCustomTopics, - Pushservice.EnableDeTopicTweetCandidatesCustomLanguages, - Pushservice.EnableMrTweetSimClusterFeatureHydrationFS, - Pushservice.DisableOutNetworkTweetCandidatesFS, - Pushservice.EnableLaunchVideosInImmersiveExplore, - Pushservice.EnableStoringNtabGenericNotifKey, - Pushservice.EnableDeletingNtabTimeline, - Pushservice.EnableOverrideNotificationsNSlots, - Pushservice.EnableNslotsForOverrideOnNtab, - Pushservice.EnableOverrideMaxSlotFn, - Pushservice.EnableTargetIdInSmartPushPayloadForMagicFanoutSportsEvent, - Pushservice.EnableOverrideIdNTabRequest, - Pushservice.EnableOverrideForSpaces, - Pushservice.EnableTopicProofTweetRecs, - Pushservice.EnableHealthFiltersForTopicProofTweet, - Pushservice.EnableTargetIdsInSmartPushPayload, - Pushservice.EnableSecondaryAccountPredicateMF, - Pushservice.EnableInlineVideo, - Pushservice.EnableAutoplayForInlineVideo, - Pushservice.EnableOONGeneratedInlineActions, - Pushservice.EnableInlineFeedbackOnPush, - Pushservice.UseInlineActionsV1, - Pushservice.UseInlineActionsV2, - Pushservice.EnableFeaturedSpacesOON, - Pushservice.CheckFeaturedSpaceOON, - Pushservice.EnableGeoTargetingForSpaces, - Pushservice.EnableEmployeeOnlySpaceNotifications, - Pushservice.EnableSpacesTtlForNtab, - Pushservice.EnableCustomThreadIdForOverride, - Pushservice.EnableSimClusterTargetingSpaces, - Pushservice.TargetInInlineActionAppVisitFatigue, - Pushservice.EnableInlineActionAppVisitFatigue, - Pushservice.EnableThresholdOfFavMrModelingBasedCandidates, - Pushservice.HydrateMrUserSimclusterV2020InModelingBasedCG, - Pushservice.HydrateMrUserSemanticCoreInModelingBasedCG, - Pushservice.HydrateOnboardingInModelingBasedCG, - Pushservice.HydrateTopicFollowInModelingBasedCG, - Pushservice.HydrateMrUserTopicInModelingBasedCG, - Pushservice.HydrateMrUserAuthorInModelingBasedCG, - Pushservice.HydrateUserPenguinLanguageInModelingBasedCG, - Pushservice.EnableMrUserEngagedTweetTokensFeature, - Pushservice.HydrateMrUserHashspaceEmbeddingInModelingBasedCG, - Pushservice.HydrateUseGeoInModelingBasedCG, - Pushservice.EnableSpaceCohostJoinEvent, - Pushservice.EnableOONFilteringBasedOnUserSettings, - Pushservice.EnableContFnF1TriggerSeeLessOftenFatigue, - Pushservice.EnableContFnNonF1TriggerSeeLessOftenFatigue, - Pushservice.EnableContFnF1TriggerPromptFeedbackFatigue, - Pushservice.EnableContFnNonF1TriggerPromptFeedbackFatigue, - Pushservice.EnableContFnF1TriggerInlineFeedbackFatigue, - Pushservice.EnableContFnNonF1TriggerInlineFeedbackFatigue, - Pushservice.UseInlineDislikeForFatigue, - Pushservice.UseInlineDismissForFatigue, - Pushservice.UseInlineSeeLessForFatigue, - Pushservice.UseInlineNotRelevantForFatigue, - Pushservice.GPEnableCustomMagicFanoutCricketFatigue, - Pushservice.IncludeRelevanceScoreInIbis2Payload, - Pushservice.BypassGlobalSpacePushCapForSoftDeviceFollow, - Pushservice.EnableCountryCodeBackoffTopTweetsByGeo, - Pushservice.EnableNewCreatorPush, - Pushservice.EnableCreatorSubscriptionPush, - Pushservice.EnableInsSender, - Pushservice.EnableOptoutAdjustedPushcap, - Pushservice.EnableOONBackfillBasedOnF1Candidates, - Pushservice.EnableVFInTweetypie, - Pushservice.EnablePushPresentationVerifiedSymbol, - Pushservice.EnableHighPrioritySportsPush, - Pushservice.EnableSearchURLRedirectForSportsFanout, - Pushservice.EnableScoreFanoutNotification, - Pushservice.EnableExplicitPushCap, - Pushservice.EnableNsfwTokenBasedFiltering, - Pushservice.EnableRestrictedMinModelPushcap, - Pushservice.EnableGenericCRTBasedFatiguePredicate, - Pushservice.EnableCopyFeaturesForF1, - Pushservice.EnableEmojiInF1Copy, - Pushservice.EnableTargetInF1Copy, - Pushservice.EnableCopyFeaturesForOon, - Pushservice.EnableEmojiInOonCopy, - Pushservice.EnableTargetInOonCopy, - Pushservice.EnableF1CopyBody, - Pushservice.EnableOONCopyBody, - Pushservice.EnableIosCopyBodyTruncate, - Pushservice.EnableHTLBasedFatigueBasicRule, - Pushservice.EnableTargetAndEmojiSplitFatigue, - Pushservice.EnableNsfwCopy, - Pushservice.EnableOONCopyBody, - Pushservice.EnableTweetDynamicInlineActions, - Pushservice.EnablePushcapRefactor, - Pushservice.BigFilteringEnableHistogramsParam, - Pushservice.EnableTweetTranslation, - Pushservice.TripTweetCandidateReturnEnable, - Pushservice.EnableSocialContextForRetweet, - Pushservice.EnableEmptyBody, - Pushservice.EnableLocalViralTweets, - Pushservice.EnableExploreVideoTweets, - Pushservice.EnableDynamicInlineActionsForDesktopWeb, - Pushservice.EnableDynamicInlineActionsForMobileWeb, - Pushservice.EnableNTabEntriesForSportsEventNotifications, - Pushservice.EnableNTabFacePileForSportsEventNotifications, - Pushservice.DisableIsTargetBlueVerifiedPredicate, - Pushservice.EnableNTabEntriesForProductLaunchNotifications, - Pushservice.DisableIsTargetLegacyVerifiedPredicate, - Pushservice.EnableNTabOverrideForSportsEventNotifications, - Pushservice.EnableOONCBasedCopy, - Pushservice.HighQualityCandidatesEnableCandidateSource, - Pushservice.HighQualityCandidatesEnableFallback, - Pushservice.EnableTweetLanguageFilter, - Pushservice.EnableListRecommendations, - Pushservice.EnableIDSListRecommendations, - Pushservice.EnablePopGeoListRecommendations, - Pushservice.SkipLanguageFilterForMediaTweets, - Pushservice.EnableSpammyTweetFilter, - Pushservice.EnableTweetPushToHomeAndroid, - Pushservice.EnableTweetPushToHomeiOS, - Pushservice.EnableBoundedFeatureSetForSocialContext, - Pushservice.EnableStpBoundedFeatureSetForUserSocialContext, - Pushservice.EnableCoreUserHistoryBoundedFeatureSetForSocialContext, - Pushservice.SkipPostRankingFilters, - Pushservice.MRWebHoldbackParam, - Pushservice.EnableIsTargetSuperFollowCreatorPredicate - ) - - private val longSeqFeatureSwitchOverrides = - FeatureSwitchOverrideUtil.getLongSeqFSOverrides( - Pushservice.MagicFanoutEventAllowlistToSkipAccountCountryPredicate - ) - - private val longSetFeatureSwitchOverrides = - FeatureSwitchOverrideUtil.getLongSetFSOverrides( - Pushservice.ListOfAdhocIdsForStatsTracking - ) - - private val stringSeqFeatureSwitchOverrides = - FeatureSwitchOverrideUtil.getStringSeqFSOverrides( - Pushservice.ListOfCrtsForOpenApp, - Pushservice.ListOfCrtsToUpRank, - Pushservice.OONCandidatesDisabledCrTagParam, - Pushservice.ListOfCrtsToDownRank, - Pushservice.MagicFanoutDenyListedCountries, - Pushservice.GlobalOptoutModelParam, - Pushservice.BqmlQualityModelBucketModelIdListParam, - Pushservice.CommonRecommendationTypeDenyListPushHoldbacks, - Pushservice.TargetLevelFeatureListForMrRequestScribing, - Pushservice.MagicFanoutSportsEventDenyListedCountries, - Pushservice.MultilingualPnsfwTweetTextSupportedLanguages, - Pushservice.NegativeKeywordsPredicateDenylist, - Pushservice.TripTweetCandidateSourceIds, - Pushservice.NsfwTokensParam, - Pushservice.HighQualityCandidatesFallbackSourceIds - ) - - private val intSeqFeatureSwitchOverrides = - FeatureSwitchOverrideUtil.getIntSeqFSOverrides( - Pushservice.BucketOptoutSlotPushcapParam, - Pushservice.GeoHashLengthList, - Pushservice.MinDurationModifierStartHourList, - Pushservice.MinDurationModifierEndHourList, - Pushservice.MinDurationTimeModifierConst - ) - - private val enumFeatureSwitchOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( - stat, - logger, - Pushservice.MRBoldTitleFavoriteAndRetweetParam, - Pushservice.QualityUprankingTransformTypeParam, - Pushservice.QualityPredicateIdParam, - Pushservice.BigFilteringNormalizationTypeIdParam, - Common.PushcapModelType, - Common.MFCricketTargetingPredicate, - Pushservice.RankingFunctionForTopTweetsByGeo, - Pushservice.TopTweetsByGeoCombinationParam, - Pushservice.PopGeoTweetVersionParam, - Pushservice.SubtextInAndroidPushHeaderParam, - Pushservice.HighOONCTweetFormat, - Pushservice.LowOONCTweetFormat, - ) - - private val enumSeqFeatureSwitchOverrides = FeatureSwitchOverrideUtil.getEnumSeqFSOverrides( - stat, - logger, - Pushservice.OONTweetDynamicInlineActionsList, - Pushservice.TweetDynamicInlineActionsList, - Pushservice.TweetDynamicInlineActionsListForWeb, - Pushservice.HighQualityCandidatesEnableGroups, - Pushservice.HighQualityCandidatesFallbackEnabledGroups, - Pushservice.OONCandidatesDisabledCrtGroupParam, - Pushservice.MultilingualPnsfwTweetTextBucketingModelList, - ) - - private val stringFeatureSwitchOverrides = FeatureSwitchOverrideUtil.getStringFSOverrides( - Common.PushcapModelPredictionVersion, - Pushservice.WeightedOpenOrNtabClickRankingModelParam, - Pushservice.WeightedOpenOrNtabClickFilteringModelParam, - Pushservice.BucketOptoutModelParam, - Pushservice.ScoringFuncForTopTweetsByGeo, - Pushservice.LightRankingModelTypeParam, - Pushservice.BigFilteringSupervisedSendingModelParam, - Pushservice.BigFilteringSupervisedWithoutSendingModelParam, - Pushservice.BigFilteringRLSendingModelParam, - Pushservice.BigFilteringRLWithoutSendingModelParam, - Pushservice.BqmlQualityModelTypeParam, - Pushservice.BqmlHealthModelTypeParam, - Pushservice.QualityUprankingModelTypeParam, - Pushservice.SearchURLRedirectForSportsFanout, - Pushservice.LocalViralTweetsBucket, - Pushservice.HighQualityCandidatesHeavyRankingModel, - Pushservice.HighQualityCandidatesNonPersonalizedQualityCnnModel, - Pushservice.HighQualityCandidatesBqmlNsfwModel, - Pushservice.HighQualityCandidatesBqmlReportModel, - Pushservice.ProductLaunchLandingPageDeepLink, - Pushservice.ProductLaunchTapThrough, - Pushservice.TweetLanguageFeatureNameParam - ) - - private val durationFeatureSwitchOverrides = - FeatureSwitchOverrideUtil.getBoundedDurationFSOverrides( - Common.NumberOfDaysToFilterMRForSeeLessOften, - Common.NumberOfDaysToReducePushCapForSeeLessOften, - Pushservice.NumberOfDaysToFilterForSeeLessOftenForF1TriggerF1, - Pushservice.NumberOfDaysToReducePushCapForSeeLessOftenForF1TriggerF1, - Pushservice.NumberOfDaysToFilterForSeeLessOftenForF1TriggerNonF1, - Pushservice.NumberOfDaysToReducePushCapForSeeLessOftenForF1TriggerNonF1, - Pushservice.NumberOfDaysToFilterForSeeLessOftenForNonF1TriggerF1, - Pushservice.NumberOfDaysToReducePushCapForSeeLessOftenForNonF1TriggerF1, - Pushservice.NumberOfDaysToFilterForSeeLessOftenForNonF1TriggerNonF1, - Pushservice.NumberOfDaysToReducePushCapForSeeLessOftenForNonF1TriggerNonF1, - Pushservice.TrendTweetNotificationsFatigueDuration, - Pushservice.MinDurationSincePushParam, - Pushservice.MFMinIntervalFatigue, - Pushservice.SimclusterBasedCandidateMaxTweetAgeParam, - Pushservice.DetopicBasedCandidateMaxTweetAgeParam, - Pushservice.F1CandidateMaxTweetAgeParam, - Pushservice.MaxTweetAgeParam, - Pushservice.ModelingBasedCandidateMaxTweetAgeParam, - Pushservice.GeoPopTweetMaxAgeInHours, - Pushservice.MinDurationSincePushParam, - Pushservice.GraduallyRampUpPhaseDurationDays, - Pushservice.MrMinDurationSincePushForOnboardingPushes, - Pushservice.FatigueForOnboardingPushes, - Pushservice.FrigateHistoryOtherNotificationWriteTtl, - Pushservice.FrigateHistoryTweetNotificationWriteTtl, - Pushservice.TopTweetsByGeoPushInterval, - Pushservice.HighQualityTweetsPushInterval, - Pushservice.MrMinDurationSincePushForTopTweetsByGeoPushes, - Pushservice.TimeSinceLastLoginForGeoPopTweetPush, - Pushservice.NewUserPlaybookAllowedLastLoginHours, - Pushservice.SpaceRecsAppFatigueDuration, - Pushservice.OONSpaceRecsFatigueDuration, - Pushservice.SpaceRecsFatigueMinIntervalDuration, - Pushservice.SpaceRecsGlobalFatigueDuration, - Pushservice.MinimumTimeSinceLastLoginForGeoPopTweetPush, - Pushservice.MinFatigueDurationSinceLastHTLVisit, - Pushservice.LastHTLVisitBasedNonFatigueWindow, - Pushservice.SpaceNotificationsTTLDurationForNTab, - Pushservice.OverrideNotificationsLookbackDurationForOverrideInfo, - Pushservice.OverrideNotificationsLookbackDurationForImpressionId, - Pushservice.OverrideNotificationsLookbackDurationForNTab, - Pushservice.TopTweetImpressionsNotificationInterval, - Pushservice.TopTweetImpressionsFatigueMinIntervalDuration, - Pushservice.MFPushIntervalInHours, - Pushservice.InlineActionAppVisitFatigue, - Pushservice.SpaceParticipantHistoryLastActiveThreshold, - Pushservice.SportsMinIntervalFatigue, - Pushservice.SportsPushIntervalInHours, - Pushservice.SportsMinIntervalFatiguePerEvent, - Pushservice.SportsPushIntervalInHoursPerEvent, - Pushservice.TargetNtabOnlyCapFatigueIntervalHours, - Pushservice.TargetPushCapFatigueIntervalHours, - Pushservice.CopyFeaturesHistoryLookbackDuration, - Pushservice.F1EmojiCopyFatigueDuration, - Pushservice.F1TargetCopyFatigueDuration, - Pushservice.OonEmojiCopyFatigueDuration, - Pushservice.OonTargetCopyFatigueDuration, - Pushservice.ProductLaunchPushIntervalInHours, - Pushservice.ExploreVideoTweetAgeParam, - Pushservice.ListRecommendationsPushInterval, - Pushservice.ProductLaunchMinIntervalFatigue, - Pushservice.NewCreatorPushIntervalInHours, - Pushservice.NewCreatorPushMinIntervalFatigue, - Pushservice.CreatorSubscriptionPushIntervalInHours, - Pushservice.CreatorSubscriptionPushhMinIntervalFatigue - ) - - private[params] val allFeatureSwitchOverrides = - booleanDeciderOverrides ++ - booleanFeatureSwitchOverrides ++ - intFeatureSwitchOverrides ++ - doubleFeatureSwitchOverrides ++ - doubleSeqFeatureSwitchOverrides ++ - enumFeatureSwitchOverrides ++ - stringSeqFeatureSwitchOverrides ++ - stringFeatureSwitchOverrides ++ - durationFeatureSwitchOverrides ++ - intSeqFeatureSwitchOverrides ++ - longSeqFeatureSwitchOverrides ++ - enumSeqFeatureSwitchOverrides ++ - longSetFeatureSwitchOverrides - - val config = BaseConfigBuilder(allFeatureSwitchOverrides).build() -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushMLModelParams.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushMLModelParams.scala deleted file mode 100644 index c451a61bc..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushMLModelParams.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.frigate.pushservice.params - -/** - * This enum defines ML models for push - */ -object PushMLModel extends Enumeration { - type PushMLModel = Value - - val WeightedOpenOrNtabClickProbability = Value - val DauProbability = Value - val OptoutProbability = Value - val FilteringProbability = Value - val BigFilteringSupervisedSendingModel = Value - val BigFilteringSupervisedWithoutSendingModel = Value - val BigFilteringRLSendingModel = Value - val BigFilteringRLWithoutSendingModel = Value - val HealthNsfwProbability = Value -} - -object WeightedOpenOrNtabClickModel { - type ModelNameType = String - - // MR models - val Periodically_Refreshed_Prod_Model = - "Periodically_Refreshed_Prod_Model" // used in DBv2 service, needed for gradually migrate via feature switch -} - - -object OptoutModel { - type ModelNameType = String - val D0_has_realtime_features = "D0_has_realtime_features" - val D0_no_realtime_features = "D0_no_realtime_features" -} - -object HealthNsfwModel { - type ModelNameType = String - val Q2_2022_Mr_Bqml_Health_Model_NsfwV0 = "Q2_2022_Mr_Bqml_Health_Model_NsfwV0" -} - -object BigFilteringSupervisedModel { - type ModelNameType = String - val V0_0_BigFiltering_Supervised_Sending_Model = "Q3_2022_bigfiltering_supervised_send_model_v0" - val V0_0_BigFiltering_Supervised_Without_Sending_Model = - "Q3_2022_bigfiltering_supervised_not_send_model_v0" -} - -object BigFilteringRLModel { - type ModelNameType = String - val V0_0_BigFiltering_Rl_Sending_Model = "Q3_2022_bigfiltering_rl_send_model_dqn_dau_15_open" - val V0_0_BigFiltering_Rl_Without_Sending_Model = - "Q3_2022_bigfiltering_rl_not_send_model_dqn_dau_15_open" -} - -case class PushModelName( - modelType: PushMLModel.Value, - version: WeightedOpenOrNtabClickModel.ModelNameType) { - override def toString: String = { - modelType.toString + "_" + version - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushParams.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushParams.scala deleted file mode 100644 index 5e5f6af6a..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushParams.scala +++ /dev/null @@ -1,534 +0,0 @@ -package com.twitter.frigate.pushservice.params - -import com.twitter.rux.common.context.thriftscala.ExperimentKey -import com.twitter.timelines.configapi.Param -import com.twitter.timelines.configapi.decider.BooleanDeciderParam - -object PushParams { - - /** - * Disable ML models in filtering - */ - object DisableMlInFilteringParam extends BooleanDeciderParam(DeciderKey.disableMLInFiltering) - - /** - * Disable ML models in ranking, use random ranking instead - * This param is used for ML holdback and training data collection - */ - object UseRandomRankingParam extends Param(false) - - /** - * Disable feature hydration, ML ranking, and ML filtering - * Use default order from candidate source - * This param is for service continuity - */ - object DisableAllRelevanceParam extends BooleanDeciderParam(DeciderKey.disableAllRelevance) - - /** - * Disable ML heavy ranking - * Use default order from candidate source - * This param is for service continuity - */ - object DisableHeavyRankingParam extends BooleanDeciderParam(DeciderKey.disableHeavyRanking) - - /** - * Restrict ML light ranking by selecting top3 candidates - * Use default order from candidate source - * This param is for service continuity - */ - object RestrictLightRankingParam extends BooleanDeciderParam(DeciderKey.restrictLightRanking) - - /** - * Downsample ML light ranking scribed candidates - */ - object DownSampleLightRankingScribeCandidatesParam - extends BooleanDeciderParam(DeciderKey.downSampleLightRankingScribeCandidates) - - /** - * Set it to true only for Android only ranking experiments - */ - object AndroidOnlyRankingExperimentParam extends Param(false) - - /** - * Enable the user_tweet_entity_graph tweet candidate source. - */ - object UTEGTweetCandidateSourceParam - extends BooleanDeciderParam(DeciderKey.entityGraphTweetRecsDeciderKey) - - /** - * Enable writes to Notification Service - */ - object EnableWritesToNotificationServiceParam - extends BooleanDeciderParam(DeciderKey.enablePushserviceWritesToNotificationServiceDeciderKey) - - /** - * Enable writes to Notification Service for all employees - */ - object EnableWritesToNotificationServiceForAllEmployeesParam - extends BooleanDeciderParam( - DeciderKey.enablePushserviceWritesToNotificationServiceForAllEmployeesDeciderKey) - - /** - * Enable writes to Notification Service for everyone - */ - object EnableWritesToNotificationServiceForEveryoneParam - extends BooleanDeciderParam( - DeciderKey.enablePushserviceWritesToNotificationServiceForEveryoneDeciderKey) - - /** - * Enable fatiguing MR for Ntab caret click - */ - object EnableFatigueNtabCaretClickingParam extends Param(true) - - /** - * Param for disabling in-network Tweet candidates - */ - object DisableInNetworkTweetCandidatesParam extends Param(false) - - /** - * Decider controlled param to enable prompt feedback response NO predicate - */ - object EnablePromptFeedbackFatigueResponseNoPredicate - extends BooleanDeciderParam( - DeciderKey.enablePromptFeedbackFatigueResponseNoPredicateDeciderKey) - - /** - * Enable hydration and generation of Social context (TF, TR) based candidates for Earlybird Tweets - */ - object EarlyBirdSCBasedCandidatesParam - extends BooleanDeciderParam(DeciderKey.enableUTEGSCForEarlybirdTweetsDecider) - - /** - * Param to allow reduce to one social proof for tweet param in UTEG - */ - object AllowOneSocialProofForTweetInUTEGParam extends Param(true) - - /** - * Param to query UTEG for out network tweets only - */ - object OutNetworkTweetsOnlyForUTEGParam extends Param(false) - - object EnablePushSendEventBus extends BooleanDeciderParam(DeciderKey.enablePushSendEventBus) - - /** - * Enable RUX Tweet landing page for push open on iOS - */ - object EnableRuxLandingPageIOSParam extends Param[Boolean](true) - - /** - * Enable RUX Tweet landing page for push open on Android - */ - object EnableRuxLandingPageAndroidParam extends Param[Boolean](true) - - /** - * Param to decide which ExperimentKey to be encoded into Rux landing page context object. - * The context object is sent to rux-api and rux-api applies logic (e.g. show reply module on - * rux landing page or not) accordingly based on the experiment key. - */ - object RuxLandingPageExperimentKeyIOSParam extends Param[Option[ExperimentKey]](None) - object RuxLandingPageExperimentKeyAndroidParam extends Param[Option[ExperimentKey]](None) - - /** - * Param to enable MR Tweet Fav Recs - */ - object MRTweetFavRecsParam extends BooleanDeciderParam(DeciderKey.enableTweetFavRecs) - - /** - * Param to enable MR Tweet Retweet Recs - */ - object MRTweetRetweetRecsParam extends BooleanDeciderParam(DeciderKey.enableTweetRetweetRecs) - - /** - * Param to disable writing to NTAB - * */ - object DisableWritingToNTAB extends Param[Boolean](default = false) - - /** - * Param to show RUX landing page as a modal on iOS - */ - object ShowRuxLandingPageAsModalOnIOS extends Param[Boolean](default = false) - - /** - * Param to enable mr end to end scribing - */ - object EnableMrRequestScribing extends BooleanDeciderParam(DeciderKey.enableMrRequestScribing) - - /** - * Param to enable scribing of high quality candidate scores - */ - object EnableHighQualityCandidateScoresScribing - extends BooleanDeciderParam(DeciderKey.enableHighQualityCandidateScoresScribing) - - /** - * Decider controlled param to pNeg multimodal predictions for F1 tweets - */ - object EnablePnegMultimodalPredictionForF1Tweets - extends BooleanDeciderParam(DeciderKey.enablePnegMultimodalPredictionForF1Tweets) - - /** - * Decider controlled param to scribe oonFav score for F1 tweets - */ - object EnableScribeOonFavScoreForF1Tweets - extends BooleanDeciderParam(DeciderKey.enableScribingOonFavScoreForF1Tweets) - - /** - * Param to enable htl user aggregates extended hydration - */ - object EnableHtlOfflineUserAggregatesExtendedHydration - extends BooleanDeciderParam(DeciderKey.enableHtlOfflineUserAggregateExtendedFeaturesHydration) - - /** - * Param to enable predicate detailed info scribing - */ - object EnablePredicateDetailedInfoScribing - extends BooleanDeciderParam(DeciderKey.enablePredicateDetailedInfoScribing) - - /** - * Param to enable predicate detailed info scribing - */ - object EnablePushCapInfoScribing - extends BooleanDeciderParam(DeciderKey.enablePredicateDetailedInfoScribing) - - /** - * Param to enable user signal language feature hydration - */ - object EnableUserSignalLanguageFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableUserSignalLanguageFeatureHydration) - - /** - * Param to enable user preferred language feature hydration - */ - object EnableUserPreferredLanguageFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableUserPreferredLanguageFeatureHydration) - - /** - * Param to enable ner erg feature hydration - */ - object EnableNerErgFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableNerErgFeaturesHydration) - - /** - * Param to enable inline action on push copy for Android - */ - object MRAndroidInlineActionOnPushCopyParam extends Param[Boolean](default = true) - - /** - * Param to enable hydrating mr user semantic core embedding features - * */ - object EnableMrUserSemanticCoreFeaturesHydration - extends BooleanDeciderParam(DeciderKey.enableMrUserSemanticCoreFeaturesHydration) - - /** - * Param to enable hydrating mr user semantic core embedding features filtered by 0.0000001 - * */ - object EnableMrUserSemanticCoreNoZeroFeaturesHydration - extends BooleanDeciderParam(DeciderKey.enableMrUserSemanticCoreNoZeroFeaturesHydration) - - /* - * Param to enable days since user's recent resurrection features hydration - */ - object EnableDaysSinceRecentResurrectionFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableDaysSinceRecentResurrectionFeatureHydration) - - /* - * Param to enable days since user past aggregates features hydration - */ - object EnableUserPastAggregatesFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableUserPastAggregatesFeatureHydration) - - /* - * Param to enable mr user simcluster features (v2020) hydration - * */ - object EnableMrUserSimclusterV2020FeaturesHydration - extends BooleanDeciderParam(DeciderKey.enableMrUserSimclusterV2020FeaturesHydration) - - /* - * Param to enable mr user simcluster features (v2020) hydration - * */ - object EnableMrUserSimclusterV2020NoZeroFeaturesHydration - extends BooleanDeciderParam(DeciderKey.enableMrUserSimclusterV2020NoZeroFeaturesHydration) - - /* - * Param to enable HTL topic engagement realtime aggregate features - * */ - object EnableTopicEngagementRealTimeAggregatesFeatureHydration - extends BooleanDeciderParam( - DeciderKey.enableTopicEngagementRealTimeAggregatesFeatureHydration) - - object EnableUserTopicAggregatesFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableUserTopicAggregatesFeatureHydration) - - /** - * Param to enable user author RTA feature hydration - */ - object EnableHtlUserAuthorRTAFeaturesFromFeatureStoreHydration - extends BooleanDeciderParam(DeciderKey.enableHtlUserAuthorRealTimeAggregateFeatureHydration) - - /** - * Param to enable duration since last visit features - */ - object EnableDurationSinceLastVisitFeatures - extends BooleanDeciderParam(DeciderKey.enableDurationSinceLastVisitFeatureHydration) - - object EnableTweetAnnotationFeaturesHydration - extends BooleanDeciderParam(DeciderKey.enableTweetAnnotationFeatureHydration) - - /** - * Param to Enable visibility filtering through SpaceVisibilityLibrary from SpacePredicate - */ - object EnableSpaceVisibilityLibraryFiltering - extends BooleanDeciderParam(DeciderKey.enableSpaceVisibilityLibraryFiltering) - - /* - * Param to enable user topic follow feature set hydration - * */ - object EnableUserTopicFollowFeatureSetHydration - extends BooleanDeciderParam(DeciderKey.enableUserTopicFollowFeatureSet) - - /* - * Param to enable onboarding new user feature set hydration - * */ - object EnableOnboardingNewUserFeatureSetHydration - extends BooleanDeciderParam(DeciderKey.enableOnboardingNewUserFeatureSet) - - /* - * Param to enable mr user author sparse continuous feature set hydration - * */ - object EnableMrUserAuthorSparseContFeatureSetHydration - extends BooleanDeciderParam(DeciderKey.enableMrUserAuthorSparseContFeatureSet) - - /* - * Param to enable mr user topic sparse continuous feature set hydration - * */ - object EnableMrUserTopicSparseContFeatureSetHydration - extends BooleanDeciderParam(DeciderKey.enableMrUserTopicSparseContFeatureSet) - - /* - * Param to enable penguin language feature set hydration - * */ - object EnableUserPenguinLanguageFeatureSetHydration - extends BooleanDeciderParam(DeciderKey.enableUserPenguinLanguageFeatureSet) - - /* - * Param to enable user engaged tweet tokens feature hydration - * */ - object EnableMrUserEngagedTweetTokensFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableMrUserEngagedTweetTokensFeaturesHydration) - - /* - * Param to enable candidate tweet tokens feature hydration - * */ - object EnableMrCandidateTweetTokensFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableMrCandidateTweetTokensFeaturesHydration) - - /* - * Param to enable mr user hashspace embedding feature set hydration - * */ - object EnableMrUserHashspaceEmbeddingFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableMrUserHashspaceEmbeddingFeatureSet) - - /* - * Param to enable mr tweet sentiment feature set hydration - * */ - object EnableMrTweetSentimentFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableMrTweetSentimentFeatureSet) - - /* - * Param to enable mr tweet_author aggregates feature set hydration - * */ - object EnableMrTweetAuthorAggregatesFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableMrTweetAuthorAggregatesFeatureSet) - - /** - * Param to enable twistly aggregated features - */ - object EnableTwistlyAggregatesFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableTwistlyAggregatesFeatureHydration) - - /** - * Param to enable tweet twhin favoriate features - */ - object EnableTweetTwHINFavFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableTweetTwHINFavFeaturesHydration) - - /* - * Param to enable mr user geo feature set hydration - * */ - object EnableUserGeoFeatureSetHydration - extends BooleanDeciderParam(DeciderKey.enableUserGeoFeatureSet) - - /* - * Param to enable mr author geo feature set hydration - * */ - object EnableAuthorGeoFeatureSetHydration - extends BooleanDeciderParam(DeciderKey.enableAuthorGeoFeatureSet) - - /* - * Param to ramp up mr user geo feature set hydration - * */ - object RampupUserGeoFeatureSetHydration - extends BooleanDeciderParam(DeciderKey.rampupUserGeoFeatureSet) - - /* - * Param to ramp up mr author geo feature set hydration - * */ - object RampupAuthorGeoFeatureSetHydration - extends BooleanDeciderParam(DeciderKey.rampupAuthorGeoFeatureSet) - - /* - * Decider controlled param to enable Pop Geo Tweets - * */ - object PopGeoCandidatesDecider extends BooleanDeciderParam(DeciderKey.enablePopGeoTweets) - - /** - * Decider controlled param to enable Trip Geo Tweets - */ - object TripGeoTweetCandidatesDecider - extends BooleanDeciderParam(DeciderKey.enableTripGeoTweetCandidates) - - /** - * Decider controlled param to enable ContentRecommenderMixerAdaptor - */ - object ContentRecommenderMixerAdaptorDecider - extends BooleanDeciderParam(DeciderKey.enableContentRecommenderMixerAdaptor) - - /** - * Decider controlled param to enable GenericCandidateAdaptor - */ - object GenericCandidateAdaptorDecider - extends BooleanDeciderParam(DeciderKey.enableGenericCandidateAdaptor) - - /** - * Decider controlled param to enable dark traffic to ContentMixer for Trip Geo Tweets - */ - object TripGeoTweetContentMixerDarkTrafficDecider - extends BooleanDeciderParam(DeciderKey.enableTripGeoTweetContentMixerDarkTraffic) - - /* - * Decider controlled param to enable Pop Geo Tweets - * */ - object TrendsCandidateDecider extends BooleanDeciderParam(DeciderKey.enableTrendsTweets) - - /* - * Decider controlled param to enable INS Traffic - **/ - object EnableInsTrafficDecider extends BooleanDeciderParam(DeciderKey.enableInsTraffic) - - /** - * Param to enable assigning pushcap with ML predictions (read from MH table). - * Disabling will fallback to only use heuristics and default values. - */ - object EnableModelBasedPushcapAssignments - extends BooleanDeciderParam(DeciderKey.enableModelBasedPushcapAssignments) - - /** - * Param to enable twhin user engagement feature hydration - */ - object EnableTwHINUserEngagementFeaturesHydration - extends BooleanDeciderParam(DeciderKey.enableTwHINUserEngagementFeaturesHydration) - - /** - * Param to enable twhin user follow feature hydration - */ - object EnableTwHINUserFollowFeaturesHydration - extends BooleanDeciderParam(DeciderKey.enableTwHINUserFollowFeaturesHydration) - - /** - * Param to enable twhin author follow feature hydration - */ - object EnableTwHINAuthorFollowFeaturesHydration - extends BooleanDeciderParam(DeciderKey.enableTwHINAuthorFollowFeaturesHydration) - - /** - * Param to enable calls to the IsTweetTranslatable strato column - */ - object EnableIsTweetTranslatableCheck - extends BooleanDeciderParam(DeciderKey.enableIsTweetTranslatable) - - /** - * Decider controlled param to enable mr tweet simcluster feature set hydration - */ - object EnableMrTweetSimClusterFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableMrTweetSimClusterFeatureSet) - - /** - * Decider controlled param to enable real graph v2 feature set hydration - */ - object EnableRealGraphV2FeatureHydration - extends BooleanDeciderParam(DeciderKey.enableRealGraphV2FeatureHydration) - - /** - * Decider controlled param to enable Tweet BeT feature set hydration - */ - object EnableTweetBeTFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableTweetBeTFeatureHydration) - - /** - * Decider controlled param to enable mr user tweet topic feature set hydration - */ - object EnableMrOfflineUserTweetTopicAggregateHydration - extends BooleanDeciderParam(DeciderKey.enableMrOfflineUserTweetTopicAggregate) - - /** - * Decider controlled param to enable mr tweet simcluster feature set hydration - */ - object EnableMrOfflineUserTweetSimClusterAggregateHydration - extends BooleanDeciderParam(DeciderKey.enableMrOfflineUserTweetSimClusterAggregate) - - /** - * Decider controlled param to enable user send time features - */ - object EnableUserSendTimeFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableUserSendTimeFeatureHydration) - - /** - * Decider controlled param to enable mr user utc send time aggregate features - */ - object EnableMrUserUtcSendTimeAggregateFeaturesHydration - extends BooleanDeciderParam(DeciderKey.enableMrUserUtcSendTimeAggregateFeaturesHydration) - - /** - * Decider controlled param to enable mr user local send time aggregate features - */ - object EnableMrUserLocalSendTimeAggregateFeaturesHydration - extends BooleanDeciderParam(DeciderKey.enableMrUserLocalSendTimeAggregateFeaturesHydration) - - /** - * Decider controlled param to enable BQML report model predictions for F1 tweets - */ - object EnableBqmlReportModelPredictionForF1Tweets - extends BooleanDeciderParam(DeciderKey.enableBqmlReportModelPredictionForF1Tweets) - - /** - * Decider controlled param to enable user Twhin embedding feature hydration - */ - object EnableUserTwhinEmbeddingFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableUserTwhinEmbeddingFeatureHydration) - - /** - * Decider controlled param to enable author follow Twhin embedding feature hydration - */ - object EnableAuthorFollowTwhinEmbeddingFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableAuthorFollowTwhinEmbeddingFeatureHydration) - - object EnableScribingMLFeaturesAsDataRecord - extends BooleanDeciderParam(DeciderKey.enableScribingMLFeaturesAsDataRecord) - - /** - * Decider controlled param to enable feature hydration for Verified related feature - */ - object EnableAuthorVerifiedFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableAuthorVerifiedFeatureHydration) - - /** - * Decider controlled param to enable feature hydration for creator subscription related feature - */ - object EnableAuthorCreatorSubscriptionFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableAuthorCreatorSubscriptionFeatureHydration) - - /** - * Decider controlled param to direct MH+Memcache hydration for the UserFeaturesDataset - */ - object EnableDirectHydrationForUserFeatures - extends BooleanDeciderParam(DeciderKey.enableDirectHydrationForUserFeatures) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushServiceTunableKeys.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushServiceTunableKeys.scala deleted file mode 100644 index 7920bb6cd..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushServiceTunableKeys.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.frigate.pushservice.params - -import com.twitter.util.tunable.TunableMap - -object PushServiceTunableKeys { - final val IbisQpsLimitTunableKey = TunableMap.Key[Int]("ibis2.qps.limit") - final val NtabQpsLimitTunableKey = TunableMap.Key[Int]("ntab.qps.limit") - final val TweetPerspectiveStoreQpsLimit = TunableMap.Key[Int]("tweetperspective.qps.limit") -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/ShardParams.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/ShardParams.scala deleted file mode 100644 index c0a68c939..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/ShardParams.scala +++ /dev/null @@ -1,3 +0,0 @@ -package com.twitter.frigate.pushservice.params - -case class ShardParams(numShards: Int, shardId: Int) diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BigFilteringEpsilonGreedyExplorationPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BigFilteringEpsilonGreedyExplorationPredicate.scala deleted file mode 100644 index 67a117cc5..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BigFilteringEpsilonGreedyExplorationPredicate.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.tracing.Trace -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.hashing.KeyHasher -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.util.Future - -/* - * A predicate for epsilon-greedy exploration; - * We defined it as a candidate level predicate to avoid changing the predicate and scribing pipeline, - * but it is actually a post-ranking target level predicate: - * if a target user IS ENABLED for \epsilon-greedy exploration, - * then with probability epsilon, the user (and thus all candidates) will be blocked - */ -object BigFilteringEpsilonGreedyExplorationPredicate { - - val name = "BigFilteringEpsilonGreedyExplorationPredicate" - - private def shouldFilterBasedOnEpsilonGreedyExploration( - target: Target - ): Boolean = { - val seed = KeyHasher.FNV1A_64.hashKey(s"${target.targetId}".getBytes("UTF8")) - val hashKey = KeyHasher.FNV1A_64 - .hashKey( - s"${Trace.id.traceId.toString}:${seed.toString}".getBytes("UTF8") - ) - - math.abs(hashKey).toDouble / Long.MaxValue < - target.params(PushFeatureSwitchParams.MrRequestScribingEpsGreedyExplorationRatio) - } - - def apply()(implicit statsReceiver: StatsReceiver): NamedPredicate[PushCandidate] = { - val stats = statsReceiver.scope(s"predicate_$name") - - val enabledForEpsilonGreedyCounter = stats.counter("enabled_for_eps_greedy") - - new Predicate[PushCandidate] { - def apply(candidates: Seq[PushCandidate]): Future[Seq[Boolean]] = { - val results = candidates.map { candidate => - if (!candidate.target.skipFilters && candidate.target.params( - PushFeatureSwitchParams.EnableMrRequestScribingForEpsGreedyExploration)) { - enabledForEpsilonGreedyCounter.incr() - !shouldFilterBasedOnEpsilonGreedyExploration(candidate.target) - } else { - true - } - } - Future.value(results) - } - }.withStats(stats) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BqmlHealthModelPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BqmlHealthModelPredicates.scala deleted file mode 100644 index f7ff95c9b..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BqmlHealthModelPredicates.scala +++ /dev/null @@ -1,129 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringRequest -import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringResponse -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.ml.HealthFeatureGetter -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.params.PushMLModel -import com.twitter.util.Future -import com.twitter.frigate.pushservice.util.CandidateUtil -import com.twitter.frigate.thriftscala.UserMediaRepresentation -import com.twitter.hss.api.thriftscala.UserHealthSignalResponse -import com.twitter.storehaus.ReadableStore - -object BqmlHealthModelPredicates { - - def healthModelOonPredicate( - bqmlHealthModelScorer: PushMLModelScorer, - producerMediaRepresentationStore: ReadableStore[Long, UserMediaRepresentation], - userHealthScoreStore: ReadableStore[Long, UserHealthSignalResponse], - tweetHealthScoreStore: ReadableStore[TweetScoringRequest, TweetScoringResponse] - )( - implicit stats: StatsReceiver - ): NamedPredicate[ - PushCandidate with TweetCandidate with RecommendationType with TweetAuthor - ] = { - val name = "bqml_health_model_based_predicate" - val scopedStatsReceiver = stats.scope(name) - - val allCandidatesCounter = scopedStatsReceiver.counter("all_candidates") - val oonCandidatesCounter = scopedStatsReceiver.counter("oon_candidates") - val filteredOonCandidatesCounter = - scopedStatsReceiver.counter("filtered_oon_candidates") - val emptyScoreCandidatesCounter = scopedStatsReceiver.counter("empty_score_candidates") - val healthScoreStat = scopedStatsReceiver.stat("health_model_dist") - - Predicate - .fromAsync { candidate: PushCandidate with TweetCandidate with RecommendationType => - val target = candidate.target - val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(candidate.commonRecType) || - RecTypes.outOfNetworkTopicTweetTypes.contains(candidate.commonRecType) - - lazy val enableBqmlHealthModelPredicateParam = - target.params(PushFeatureSwitchParams.EnableBqmlHealthModelPredicateParam) - lazy val enableBqmlHealthModelPredictionForInNetworkCandidates = - target.params( - PushFeatureSwitchParams.EnableBqmlHealthModelPredictionForInNetworkCandidatesParam) - lazy val bqmlHealthModelPredicateFilterThresholdParam = - target.params(PushFeatureSwitchParams.BqmlHealthModelPredicateFilterThresholdParam) - lazy val healthModelId = target.params(PushFeatureSwitchParams.BqmlHealthModelTypeParam) - lazy val enableBqmlHealthModelScoreHistogramParam = - target.params(PushFeatureSwitchParams.EnableBqmlHealthModelScoreHistogramParam) - val healthModelScoreFeature = "bqml_health_model_score" - - val histogramBinSize = 0.05 - lazy val healthCandidateScoreHistogramCounters = - bqmlHealthModelScorer.getScoreHistogramCounters( - scopedStatsReceiver, - "health_score_histogram", - histogramBinSize) - - candidate match { - case candidate: PushCandidate with TweetAuthor with TweetAuthorDetails - if enableBqmlHealthModelPredicateParam && (isOonCandidate || enableBqmlHealthModelPredictionForInNetworkCandidates) => - HealthFeatureGetter - .getFeatures( - candidate, - producerMediaRepresentationStore, - userHealthScoreStore, - Some(tweetHealthScoreStore)) - .flatMap { healthFeatures => - allCandidatesCounter.incr() - candidate.mergeFeatures(healthFeatures) - - val healthModelScoreFutOpt = - if (candidate.numericFeatures.contains(healthModelScoreFeature)) { - Future.value(candidate.numericFeatures.get(healthModelScoreFeature)) - } else - bqmlHealthModelScorer.singlePredicationForModelVersion( - healthModelId, - candidate - ) - - candidate.populateQualityModelScore( - PushMLModel.HealthNsfwProbability, - healthModelId, - healthModelScoreFutOpt - ) - - healthModelScoreFutOpt.map { - case Some(healthModelScore) => - healthScoreStat.add((healthModelScore * 10000).toFloat) - if (enableBqmlHealthModelScoreHistogramParam) { - healthCandidateScoreHistogramCounters( - math.ceil(healthModelScore / histogramBinSize).toInt).incr() - } - - if (CandidateUtil.shouldApplyHealthQualityFilters( - candidate) && isOonCandidate) { - oonCandidatesCounter.incr() - val threshold = bqmlHealthModelPredicateFilterThresholdParam - candidate.cachePredicateInfo( - name, - healthModelScore, - threshold, - healthModelScore > threshold) - if (healthModelScore > threshold) { - filteredOonCandidatesCounter.incr() - false - } else true - } else true - case _ => - emptyScoreCandidatesCounter.incr() - true - } - } - case _ => Future.True - } - } - .withStats(stats.scope(name)) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BqmlQualityModelPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BqmlQualityModelPredicates.scala deleted file mode 100644 index 76d52992b..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BqmlQualityModelPredicates.scala +++ /dev/null @@ -1,141 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.params.PushConstants.TweetMediaEmbeddingBQKeyIds -import com.twitter.frigate.pushservice.params.PushMLModel -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.util.CandidateUtil -import com.twitter.util.Future -import com.twitter.frigate.pushservice.util.CandidateUtil._ - -object BqmlQualityModelPredicates { - - def ingestExtraFeatures(cand: PushCandidate): Unit = { - val tagsCRCountFeature = "tagsCR_count" - val hasPushOpenOrNtabClickFeature = "has_PushOpenOrNtabClick" - val onlyPushOpenOrNtabClickFeature = "only_PushOpenOrNtabClick" - val firstTweetMediaEmbeddingFeature = "media_embedding_0" - val tweetMediaEmbeddingFeature = - "media.mediaunderstanding.media_embeddings.twitter_clip_as_sparse_continuous_feature" - - if (!cand.numericFeatures.contains(tagsCRCountFeature)) { - cand.numericFeatures(tagsCRCountFeature) = getTagsCRCount(cand) - } - if (!cand.booleanFeatures.contains(hasPushOpenOrNtabClickFeature)) { - cand.booleanFeatures(hasPushOpenOrNtabClickFeature) = isRelatedToMrTwistlyCandidate(cand) - } - if (!cand.booleanFeatures.contains(onlyPushOpenOrNtabClickFeature)) { - cand.booleanFeatures(onlyPushOpenOrNtabClickFeature) = isMrTwistlyCandidate(cand) - } - if (!cand.numericFeatures.contains(firstTweetMediaEmbeddingFeature)) { - val tweetMediaEmbedding = cand.sparseContinuousFeatures - .getOrElse(tweetMediaEmbeddingFeature, Map.empty[String, Double]) - Seq.range(0, TweetMediaEmbeddingBQKeyIds.size).foreach { i => - cand.numericFeatures(s"media_embedding_$i") = - tweetMediaEmbedding.getOrElse(TweetMediaEmbeddingBQKeyIds(i).toString, 0.0) - } - } - } - - def BqmlQualityModelOonPredicate( - bqmlQualityModelScorer: PushMLModelScorer - )( - implicit stats: StatsReceiver - ): NamedPredicate[ - PushCandidate with TweetCandidate with RecommendationType - ] = { - - val name = "bqml_quality_model_based_predicate" - val scopedStatsReceiver = stats.scope(name) - val oonCandidatesCounter = scopedStatsReceiver.counter("oon_candidates") - val inCandidatesCounter = scopedStatsReceiver.counter("in_candidates") - val filteredOonCandidatesCounter = - scopedStatsReceiver.counter("filtered_oon_candidates") - val bucketedCandidatesCounter = scopedStatsReceiver.counter("bucketed_oon_candidates") - val emptyScoreCandidatesCounter = scopedStatsReceiver.counter("empty_score_candidates") - val histogramBinSize = 0.05 - - Predicate - .fromAsync { candidate: PushCandidate with TweetCandidate with RecommendationType => - val target = candidate.target - val crt = candidate.commonRecType - val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(crt) || - RecTypes.outOfNetworkTopicTweetTypes.contains(crt) - - lazy val enableBqmlQualityModelScoreHistogramParam = - target.params(PushFeatureSwitchParams.EnableBqmlQualityModelScoreHistogramParam) - - lazy val qualityCandidateScoreHistogramCounters = - bqmlQualityModelScorer.getScoreHistogramCounters( - scopedStatsReceiver, - "quality_score_histogram", - histogramBinSize) - - if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && (isOonCandidate || target - .params(PushParams.EnableBqmlReportModelPredictionForF1Tweets)) - && target.params(PushFeatureSwitchParams.EnableBqmlQualityModelPredicateParam)) { - ingestExtraFeatures(candidate) - - lazy val shouldFilterFutSeq = - target - .params(PushFeatureSwitchParams.BqmlQualityModelBucketModelIdListParam) - .zip(target.params(PushFeatureSwitchParams.BqmlQualityModelBucketThresholdListParam)) - .map { - case (modelId, bucketThreshold) => - val scoreFutOpt = - bqmlQualityModelScorer.singlePredicationForModelVersion(modelId, candidate) - - candidate.populateQualityModelScore( - PushMLModel.FilteringProbability, - modelId, - scoreFutOpt - ) - - if (isOonCandidate) { - oonCandidatesCounter.incr() - scoreFutOpt.map { - case Some(score) => - if (score >= bucketThreshold) { - bucketedCandidatesCounter.incr() - if (modelId == target.params( - PushFeatureSwitchParams.BqmlQualityModelTypeParam)) { - if (enableBqmlQualityModelScoreHistogramParam) { - val scoreHistogramBinId = - math.ceil(score / histogramBinSize).toInt - qualityCandidateScoreHistogramCounters(scoreHistogramBinId).incr() - } - if (score >= target.params( - PushFeatureSwitchParams.BqmlQualityModelPredicateThresholdParam)) { - filteredOonCandidatesCounter.incr() - true - } else false - } else false - } else false - case _ => - emptyScoreCandidatesCounter.incr() - false - } - } else { - inCandidatesCounter.incr() - Future.False - } - } - - Future.collect(shouldFilterFutSeq).flatMap { shouldFilterSeq => - if (shouldFilterSeq.contains(true)) { - Future.False - } else Future.True - } - } else Future.True - } - .withStats(stats.scope(name)) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CaretFeedbackHistoryFilter.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CaretFeedbackHistoryFilter.scala deleted file mode 100644 index 8ccccd14d..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CaretFeedbackHistoryFilter.scala +++ /dev/null @@ -1,99 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.frigate.common.base.TargetUser -import com.twitter.frigate.common.candidate.CaretFeedbackHistory -import com.twitter.frigate.common.candidate.TargetABDecider -import com.twitter.frigate.common.util.MrNtabCopyObjects -import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails -import com.twitter.notificationservice.thriftscala.GenericNotificationMetadata -import com.twitter.notificationservice.thriftscala.GenericType - -object CaretFeedbackHistoryFilter { - - def caretFeedbackHistoryFilter( - categories: Seq[String] - ): TargetUser with TargetABDecider with CaretFeedbackHistory => Seq[CaretFeedbackDetails] => Seq[ - CaretFeedbackDetails - ] = { target => caretFeedbackDetailsSeq => - caretFeedbackDetailsSeq.filter { caretFeedbackDetails => - caretFeedbackDetails.genericNotificationMetadata match { - case Some(genericNotificationMetadata) => - isFeedbackSupportedGenericType(genericNotificationMetadata) - case None => false - } - } - } - - private def filterCriteria( - caretFeedbackDetails: CaretFeedbackDetails, - genericTypes: Seq[GenericType] - ): Boolean = { - caretFeedbackDetails.genericNotificationMetadata match { - case Some(genericNotificationMetadata) => - genericTypes.contains(genericNotificationMetadata.genericType) - case None => false - } - } - - def caretFeedbackHistoryFilterByGenericType( - genericTypes: Seq[GenericType] - ): TargetUser with TargetABDecider with CaretFeedbackHistory => Seq[CaretFeedbackDetails] => Seq[ - CaretFeedbackDetails - ] = { target => caretFeedbackDetailsSeq => - caretFeedbackDetailsSeq.filter { caretFeedbackDetails => - filterCriteria(caretFeedbackDetails, genericTypes) - } - } - - def caretFeedbackHistoryFilterByGenericTypeDenyList( - genericTypes: Seq[GenericType] - ): TargetUser with TargetABDecider with CaretFeedbackHistory => Seq[CaretFeedbackDetails] => Seq[ - CaretFeedbackDetails - ] = { target => caretFeedbackDetailsSeq => - caretFeedbackDetailsSeq.filterNot { caretFeedbackDetails => - filterCriteria(caretFeedbackDetails, genericTypes) - } - } - - def caretFeedbackHistoryFilterByRefreshableType( - refreshableTypes: Set[Option[String]] - ): TargetUser with TargetABDecider with CaretFeedbackHistory => Seq[CaretFeedbackDetails] => Seq[ - CaretFeedbackDetails - ] = { target => caretFeedbackDetailsSeq => - caretFeedbackDetailsSeq.filter { caretFeedbackDetails => - caretFeedbackDetails.genericNotificationMetadata match { - case Some(genericNotificationMetadata) => - refreshableTypes.contains(genericNotificationMetadata.refreshableType) - case None => false - } - } - } - - def caretFeedbackHistoryFilterByRefreshableTypeDenyList( - refreshableTypes: Set[Option[String]] - ): TargetUser with TargetABDecider with CaretFeedbackHistory => Seq[CaretFeedbackDetails] => Seq[ - CaretFeedbackDetails - ] = { target => caretFeedbackDetailsSeq => - caretFeedbackDetailsSeq.filter { caretFeedbackDetails => - caretFeedbackDetails.genericNotificationMetadata match { - case Some(genericNotificationMetadata) => - !refreshableTypes.contains(genericNotificationMetadata.refreshableType) - case None => true - } - } - } - - private def isFeedbackSupportedGenericType( - notificationMetadata: GenericNotificationMetadata - ): Boolean = { - val genericNotificationTypeName = - (notificationMetadata.genericType, notificationMetadata.refreshableType) match { - case (GenericType.RefreshableNotification, Some(refreshableType)) => refreshableType - case _ => notificationMetadata.genericType.name - } - - MrNtabCopyObjects.AllNtabCopyTypes - .flatMap(_.refreshableType) - .contains(genericNotificationTypeName) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CasLockPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CasLockPredicate.scala deleted file mode 100644 index 22067405a..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CasLockPredicate.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.CasLock -import com.twitter.frigate.common.util.CasSuccess -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.util.Duration -import com.twitter.util.Future - -object CasLockPredicate { - def apply( - casLock: CasLock, - expiryDuration: Duration - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val stats = statsReceiver.scope("predicate_addcaslock_for_candidate") - Predicate - .fromAsync { candidate: PushCandidate => - if (candidate.target.pushContext.exists(_.darkWrite.exists(_ == true))) { - Future.True - } else if (candidate.commonRecType == CommonRecommendationType.MagicFanoutSportsEvent) { - Future.True - } else { - candidate.target.history flatMap { h => - val now = candidate.createdAt - val expiry = now + expiryDuration - val oldTimestamp = h.lastNotificationTime map { - _.inSeconds - } getOrElse 0 - casLock.cas(candidate.target.targetId, oldTimestamp, now.inSeconds, expiry) map { - casResult => - stats.counter(s"cas_$casResult").incr() - casResult == CasSuccess - } - } - } - } - .withStats(stats) - .withName("add_cas_lock") - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CrtDeciderPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CrtDeciderPredicate.scala deleted file mode 100644 index 4b1abf221..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CrtDeciderPredicate.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.decider.Decider -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate - -object CrtDeciderPredicate { - val name = "crt_decider" - def apply( - decider: Decider - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate] = { - Predicate - .from { (candidate: PushCandidate) => - val prefix = "frigate_pushservice_" - val deciderKey = prefix + candidate.commonRecType - decider.feature(deciderKey).isAvailable - } - .withStats(statsReceiver.scope(s"predicate_$name")) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/DiscoverTwitterPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/DiscoverTwitterPredicate.scala deleted file mode 100644 index cb55be356..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/DiscoverTwitterPredicate.scala +++ /dev/null @@ -1,47 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.TargetUser -import com.twitter.frigate.common.candidate.FrigateHistory -import com.twitter.frigate.common.history.History -import com.twitter.frigate.common.predicate.FrigateHistoryFatiguePredicate -import com.twitter.frigate.common.predicate.{FatiguePredicate => TargetFatiguePredicate} -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.hermit.predicate.Predicate -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration - -object DiscoverTwitterPredicate { - - /** - * Predicate used to determine if a minimum duration has elapsed since the last MR push - * for a CRT to be valid. - * @param name Identifier of the caller (used for stats) - * @param intervalParam The minimum duration interval - * @param stats StatsReceiver - * @return Target Predicate - */ - def minDurationElapsedSinceLastMrPushPredicate( - name: String, - intervalParam: Param[Duration], - stats: StatsReceiver - ): Predicate[Target] = - Predicate - .fromAsync { target: Target => - val interval = - target.params(intervalParam) - FrigateHistoryFatiguePredicate( - minInterval = interval, - getSortedHistory = { h: History => - val magicRecsOnlyHistory = - TargetFatiguePredicate.magicRecsPushOnlyFilter(h.sortedPushDmHistory) - TargetFatiguePredicate.magicRecsNewUserPlaybookPushFilter(magicRecsOnlyHistory) - } - ).flatContraMap { target: TargetUser with FrigateHistory => - target.history - }.apply(Seq(target)).map { - _.head - } - }.withStats(stats.scope(s"${name}_predicate_mr_push_min_interval")) - .withName(s"${name}_predicate_mr_push_min_interval") -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/FatiguePredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/FatiguePredicate.scala deleted file mode 100644 index 457dc879c..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/FatiguePredicate.scala +++ /dev/null @@ -1,74 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.predicate.FatiguePredicate._ -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.{NotificationDisplayLocation => DisplayLocation} -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.util.Duration - -object FatiguePredicate { - - /** - * Predicate that operates on a candidate, and applies custom fatigue rules for the slice of history only - * corresponding to a given rec type. - * - * @param interval - * @param maxInInterval - * @param minInterval - * @param recommendationType - * @param statsReceiver - * @return - */ - def recTypeOnly( - interval: Duration, - maxInInterval: Int, - minInterval: Duration, - recommendationType: CommonRecommendationType, - notificationDisplayLocation: DisplayLocation = DisplayLocation.PushToMobileDevice - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate] = { - build( - interval = interval, - maxInInterval = maxInInterval, - minInterval = minInterval, - filterHistory = recOnlyFilter(recommendationType), - notificationDisplayLocation = notificationDisplayLocation - ).flatContraMap { candidate: PushCandidate => candidate.target.history } - .withStats(statsReceiver.scope(s"predicate_${recTypeOnlyFatigue}")) - .withName(recTypeOnlyFatigue) - } - - /** - * Predicate that operates on a candidate, and applies custom fatigue rules for the slice of history only - * corresponding to specified rec types - * - * @param interval - * @param maxInInterval - * @param minInterval - * @param statsReceiver - * @return - */ - def recTypeSetOnly( - interval: Duration, - maxInInterval: Int, - minInterval: Duration, - recTypes: Set[CommonRecommendationType], - notificationDisplayLocation: DisplayLocation = DisplayLocation.PushToMobileDevice - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "rec_type_set_fatigue" - build( - interval = interval, - maxInInterval = maxInInterval, - minInterval = minInterval, - filterHistory = recTypesOnlyFilter(recTypes), - notificationDisplayLocation = notificationDisplayLocation - ).flatContraMap { candidate: PushCandidate => candidate.target.history } - .withStats(statsReceiver.scope(s"${name}_predicate")) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/HealthPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/HealthPredicates.scala deleted file mode 100644 index f11ed1400..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/HealthPredicates.scala +++ /dev/null @@ -1,740 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringRequest -import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringResponse -import com.twitter.abuse.detection.scoring.thriftscala.{Model => TweetHealthModel} -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.NsfwTextDetectionModel -import com.twitter.frigate.pushservice.params.PushConstants -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.util.CandidateHydrationUtil -import com.twitter.frigate.pushservice.util.CandidateUtil -import com.twitter.frigate.pushservice.util.MediaAnnotationsUtil -import com.twitter.frigate.thriftscala.UserMediaRepresentation -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.hss.api.thriftscala.UserHealthSignal._ -import com.twitter.hss.api.thriftscala.SignalValue -import com.twitter.hss.api.thriftscala.UserHealthSignalResponse -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future -import com.twitter.util.Time - -object HealthPredicates { - - private val NsfwTextDetectionModelMap: Map[NsfwTextDetectionModel.Value, TweetHealthModel] = - Map( - NsfwTextDetectionModel.ProdModel -> TweetHealthModel.PnsfwTweetText, - NsfwTextDetectionModel.RetrainedModel -> TweetHealthModel.ExperimentalHealthModelScore1, - ) - - private def tweetIsSupportedLanguage( - candidate: PushCandidate, - supportedLanguages: Set[String] - ): Boolean = { - val tweetLanguage = - candidate.categoricalFeatures.getOrElse("RecTweet.TweetyPieResult.Language", "") - supportedLanguages.contains(tweetLanguage) - } - - def tweetHealthSignalScorePredicate( - tweetHealthScoreStore: ReadableStore[TweetScoringRequest, TweetScoringResponse], - applyToQuoteTweet: Boolean = false - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with TweetCandidate with TweetDetails] = { - val name = "tweet_health_signal_store_applyToQuoteTweet_" + applyToQuoteTweet.toString - val scopedStatsReceiver = stats.scope(name) - val numCandidatesStats = scopedStatsReceiver.scope("num_candidates") - val numCandidatesMediaNsfwScoreStats = numCandidatesStats.scope("media_nsfw_score") - - Predicate - .fromAsync { candidate: PushCandidate with TweetCandidate with TweetDetails => - numCandidatesStats.counter("all").incr() - val target = candidate.target - val tweetIdOpt = if (!applyToQuoteTweet) { - Some(candidate.tweetId) - } else candidate.tweetyPieResult.flatMap(_.quotedTweet.map(_.id)) - - tweetIdOpt match { - case Some(tweetId) => - val pMediaNsfwRequest = - TweetScoringRequest(tweetId, TweetHealthModel.ExperimentalHealthModelScore4) - tweetHealthScoreStore.get(pMediaNsfwRequest).map { - case Some(tweetScoringResponse) => - numCandidatesMediaNsfwScoreStats.counter("non_empty").incr() - val pMediaNsfwScore = tweetScoringResponse.score - - if (!applyToQuoteTweet) { - candidate - .cacheExternalScore("NsfwMediaProbability", Future.value(Some(pMediaNsfwScore))) - } - - val pMediaNsfwShouldBucket = - pMediaNsfwScore > target.params( - PushFeatureSwitchParams.PnsfwTweetMediaBucketingThreshold) - if (CandidateUtil.shouldApplyHealthQualityFilters( - candidate) && pMediaNsfwShouldBucket) { - numCandidatesMediaNsfwScoreStats.counter("bucketed").incr() - if (target.params(PushFeatureSwitchParams.PnsfwTweetMediaFilterOonOnly) - && !RecTypes.isOutOfNetworkTweetRecType(candidate.commonRecType)) { - true - } else { - val pMediaNsfwScoreThreshold = - if (applyToQuoteTweet) - target.params(PushFeatureSwitchParams.PnsfwQuoteTweetThreshold) - else if (candidate.hasPhoto) - target.params(PushFeatureSwitchParams.PnsfwTweetImageThreshold) - else target.params(PushFeatureSwitchParams.PnsfwTweetMediaThreshold) - candidate.cachePredicateInfo( - name + "_nsfwMedia", - pMediaNsfwScore, - pMediaNsfwScoreThreshold, - pMediaNsfwScore > pMediaNsfwScoreThreshold) - if (pMediaNsfwScore > pMediaNsfwScoreThreshold) { - numCandidatesMediaNsfwScoreStats.counter("filtered").incr() - false - } else true - } - } else true - case _ => - numCandidatesMediaNsfwScoreStats.counter("empty").incr() - if (candidate.hasPhoto || candidate.hasVideo) { - numCandidatesMediaNsfwScoreStats.counter("media_tweet_with_empty_score").incr() - } - true - } - case _ => Future.True - } - } - .withStats(stats.scope(s"predicate_$name")) - .withName(name) - } - - def healthSignalScoreSpammyTweetPredicate( - tweetHealthScoreStore: ReadableStore[TweetScoringRequest, TweetScoringResponse] - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with TweetCandidate with TweetDetails] = { - val name = "health_signal_store_spammy_tweet" - val statsScope = stats.scope(name) - val allCandidatesCounter = statsScope.counter("all_candidates") - val eligibleCandidatesCounter = statsScope.counter("eligible_candidates") - val oonCandidatesCounter = statsScope.counter("oon_candidates") - val inCandidatesCounter = statsScope.counter("in_candidates") - val bucketedCandidatesCounter = statsScope.counter("num_bucketed") - val nonEmptySpamScoreCounter = statsScope.counter("non_empty_spam_score") - val filteredOonCandidatesCounter = statsScope.counter("num_filtered_oon") - val filteredInCandidatesCounter = statsScope.counter("num_filtered_in") - - Predicate - .fromAsync { candidate: PushCandidate with TweetCandidate with TweetDetails => - allCandidatesCounter.incr() - val crt = candidate.commonRecType - val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(crt) || - RecTypes.outOfNetworkTopicTweetTypes.contains(crt) - if (isOonCandidate) { - oonCandidatesCounter.incr() - } - val target = candidate.target - if (target.params(PushFeatureSwitchParams.EnableSpammyTweetFilter)) { - eligibleCandidatesCounter.incr() - val tweetSpamScore = - TweetScoringRequest(candidate.tweetId, TweetHealthModel.SpammyTweetContent) - tweetHealthScoreStore.get(tweetSpamScore).map { - case (Some(tweetScoringResponse)) => - nonEmptySpamScoreCounter.incr() - val candidateSpamScore = tweetScoringResponse.score - - candidate - .cacheExternalScore("SpammyTweetScore", Future.value(Some(candidateSpamScore))) - - val tweetSpamShouldBucket = - candidateSpamScore > target.params( - PushFeatureSwitchParams.SpammyTweetBucketingThreshold) - if (CandidateUtil.shouldApplyHealthQualityFilters( - candidate) && tweetSpamShouldBucket) { - bucketedCandidatesCounter.incr() - if (isOonCandidate) { - val spamScoreThreshold = - target.params(PushFeatureSwitchParams.SpammyTweetOonThreshold) - if (candidateSpamScore > spamScoreThreshold) { - filteredOonCandidatesCounter.incr() - false - } else true - } else { - inCandidatesCounter.incr() - val spamScoreThreshold = - target.params(PushFeatureSwitchParams.SpammyTweetInThreshold) - if (candidateSpamScore > spamScoreThreshold) { - filteredInCandidatesCounter.incr() - false - } else true - } - } else true - case _ => true - } - } else Future.True - } - .withStats(stats.scope(s"predicate_$name")) - .withName(name) - } - - def healthSignalScorePnsfwTweetTextPredicate( - tweetHealthScoreStore: ReadableStore[TweetScoringRequest, TweetScoringResponse] - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with TweetCandidate] = { - val name = "health_signal_store_pnsfw_tweet_text" - val statsScope = stats.scope(name) - val allCandidatesCounter = statsScope.counter("all_candidates") - val nonEmptyNsfwTextScoreNum = statsScope.counter("non_empty_nsfw_text_score") - val filteredCounter = statsScope.counter("num_filtered") - val lowScoreCounter = statsScope.counter("low_score_count") - - Predicate - .fromAsync { candidate: PushCandidate with TweetCandidate => - val target = candidate.target - val predEnabled = - target.params(PushFeatureSwitchParams.EnableHealthSignalStorePnsfwTweetTextPredicate) - if (CandidateUtil.shouldApplyHealthQualityFilters( - candidate) && predEnabled && tweetIsSupportedLanguage(candidate, Set(""))) { - allCandidatesCounter.incr() - val pnsfwTextRequest = - TweetScoringRequest(candidate.tweetId, TweetHealthModel.PnsfwTweetText) - tweetHealthScoreStore.get(pnsfwTextRequest).flatMap { - case Some(tweetScoringResponse) => { - nonEmptyNsfwTextScoreNum.incr() - if (tweetScoringResponse.score < 1e-8) { - lowScoreCounter.incr() - } - - candidate - .cacheExternalScore( - "NsfwTextProbability-en", - Future.value(Some(tweetScoringResponse.score))) - val threshold = target.params(PushFeatureSwitchParams.PnsfwTweetTextThreshold) - candidate.cachePredicateInfo( - name, - tweetScoringResponse.score, - threshold, - tweetScoringResponse.score > threshold) - if (tweetScoringResponse.score > threshold) { - filteredCounter.incr() - Future.False - } else Future.True - } - case _ => Future.True - } - } else Future.True - } - .withStats(stats.scope(s"predicate_$name")) - .withName(name) - } - - def healthSignalScoreMultilingualPnsfwTweetTextPredicate( - tweetHealthScoreStore: ReadableStore[TweetScoringRequest, TweetScoringResponse] - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with TweetCandidate] = { - val name = "health_signal_store_multilingual_pnsfw_tweet_text" - val statsScope = stats.scope(name) - - val allLanguagesIdentifier = "all" - val languagesSelectedForStats = - Set("") + allLanguagesIdentifier - - val candidatesCounterMap: Map[String, Counter] = languagesSelectedForStats.map { lang => - lang -> statsScope.counter(f"candidates_$lang") - }.toMap - val nonEmptyHealthScoreMap: Map[String, Counter] = languagesSelectedForStats.map { lang => - lang -> statsScope.counter(f"non_empty_health_score_$lang") - }.toMap - val emptyHealthScoreMap: Map[String, Counter] = languagesSelectedForStats.map { lang => - lang -> statsScope.counter(f"empty_health_score_$lang") - }.toMap - val bucketedCounterMap: Map[String, Counter] = languagesSelectedForStats.map { lang => - lang -> statsScope.counter(f"num_candidates_bucketed_$lang") - }.toMap - val filteredCounterMap: Map[String, Counter] = languagesSelectedForStats.map { lang => - lang -> statsScope.counter(f"num_filtered_$lang") - }.toMap - val lowScoreCounterMap: Map[String, Counter] = languagesSelectedForStats.map { lang => - lang -> statsScope.counter(f"low_score_count_$lang") - }.toMap - - val wrongBucketingModelCounter = statsScope.counter("wrong_bucketing_model_count") - val wrongDetectionModelCounter = statsScope.counter("wrong_detection_model_count") - - def increaseCounterForLanguage(counterMap: Map[String, Counter], language: String): Unit = { - counterMap.get(allLanguagesIdentifier) match { - case Some(counter) => counter.incr() - case _ => - } - counterMap.get(language) match { - case Some(counter) => counter.incr() - case _ => - } - } - - Predicate - .fromAsync { candidate: PushCandidate with TweetCandidate => - val target = candidate.target - - val languageFeatureName = "RecTweet.TweetyPieResult.Language" - - lazy val isPredicateEnabledForTarget = target.params( - PushFeatureSwitchParams.EnableHealthSignalStoreMultilingualPnsfwTweetTextPredicate) - - lazy val targetNsfwTextDetectionModel: NsfwTextDetectionModel.Value = - target.params(PushFeatureSwitchParams.MultilingualPnsfwTweetTextModel) - - lazy val targetPredicateSupportedLanguageSeq: Seq[String] = - target.params(PushFeatureSwitchParams.MultilingualPnsfwTweetTextSupportedLanguages) - - lazy val bucketingModelSeq: Seq[NsfwTextDetectionModel.Value] = - target.params(PushFeatureSwitchParams.MultilingualPnsfwTweetTextBucketingModelList) - - lazy val bucketingThresholdPerLanguageSeq: Seq[Double] = - target.params(PushFeatureSwitchParams.MultilingualPnsfwTweetTextBucketingThreshold) - - lazy val filteringThresholdPerLanguageSeq: Seq[Double] = - target.params(PushFeatureSwitchParams.MultilingualPnsfwTweetTextFilteringThreshold) - - if (CandidateUtil.shouldApplyHealthQualityFilters( - candidate) && isPredicateEnabledForTarget) { - val candidateLanguage = - candidate.categoricalFeatures.getOrElse(languageFeatureName, "") - - val indexOfCandidateLanguage = - targetPredicateSupportedLanguageSeq.indexOf(candidateLanguage) - - val isCandidateLanguageSupported = indexOfCandidateLanguage >= 0 - - if (isCandidateLanguageSupported) { - increaseCounterForLanguage(candidatesCounterMap, candidateLanguage) - - val bucketingModelScoreMap: Map[NsfwTextDetectionModel.Value, Future[Option[Double]]] = - bucketingModelSeq.map { modelName => - NsfwTextDetectionModelMap.get(modelName) match { - case Some(targetNsfwTextDetectionModel) => - val pnsfwTweetTextRequest: TweetScoringRequest = - TweetScoringRequest(candidate.tweetId, targetNsfwTextDetectionModel) - - val scoreOptFut: Future[Option[Double]] = - tweetHealthScoreStore.get(pnsfwTweetTextRequest).map(_.map(_.score)) - - candidate - .cacheExternalScore("NsfwTextProbability", scoreOptFut) - - modelName -> scoreOptFut - case _ => - wrongBucketingModelCounter.incr() - modelName -> Future.None - } - }.toMap - - val candidateLanguageBucketingThreshold = - bucketingThresholdPerLanguageSeq(indexOfCandidateLanguage) - - val userShouldBeBucketedFut: Future[Boolean] = - Future - .collect(bucketingModelScoreMap.map { - case (_, modelScoreOptFut) => - modelScoreOptFut.map { - case Some(score) => - increaseCounterForLanguage(nonEmptyHealthScoreMap, candidateLanguage) - score > candidateLanguageBucketingThreshold - case _ => - increaseCounterForLanguage(emptyHealthScoreMap, candidateLanguage) - false - } - }.toSeq).map(_.contains(true)) - - val candidateShouldBeFilteredFut: Future[Boolean] = userShouldBeBucketedFut.flatMap { - userShouldBeBucketed => - if (userShouldBeBucketed) { - increaseCounterForLanguage(bucketedCounterMap, candidateLanguage) - - val candidateLanguageFilteringThreshold = - filteringThresholdPerLanguageSeq(indexOfCandidateLanguage) - - bucketingModelScoreMap.get(targetNsfwTextDetectionModel) match { - case Some(scoreOptFut) => - scoreOptFut.map { - case Some(score) => - val candidateShouldBeFiltered = - score > candidateLanguageFilteringThreshold - if (candidateShouldBeFiltered) { - increaseCounterForLanguage(filteredCounterMap, candidateLanguage) - } - candidateShouldBeFiltered - case _ => false - } - case _ => - wrongDetectionModelCounter.incr() - Future.False - } - } else { - increaseCounterForLanguage(lowScoreCounterMap, candidateLanguage) - Future.False - } - } - candidateShouldBeFilteredFut.map(result => !result) - } else Future.True - } else Future.True - } - .withStats(stats.scope(s"predicate_$name")) - .withName(name) - } - - def authorProfileBasedPredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with TweetCandidate] = { - val name = "author_profile" - val statsScope = stats.scope(name) - val filterByNsfwToken = statsScope.counter("filter_by_nsfw_token") - val filterByAccountAge = statsScope.counter("filter_by_account_age") - - Predicate - .fromAsync { candidate: PushCandidate with TweetCandidate => - val target = candidate.target - candidate match { - case cand: PushCandidate with TweetAuthorDetails => - cand.tweetAuthor.map { - case Some(author) => - val nsfwTokens = target.params(PushFeatureSwitchParams.NsfwTokensParam) - val accountAgeInHours = - (Time.now - Time.fromMilliseconds(author.createdAtMsec)).inHours - val isNsfwAccount = CandidateHydrationUtil.isNsfwAccount(author, nsfwTokens) - val isVerified = author.safety.map(_.verified).getOrElse(false) - - if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && !isVerified) { - val enableNsfwTokenCheck = - target.params(PushFeatureSwitchParams.EnableNsfwTokenBasedFiltering) - val minimumAllowedAge = - target.params(PushFeatureSwitchParams.MinimumAllowedAuthorAccountAgeInHours) - cand.cachePredicateInfo( - name + "_nsfwToken", - if (isNsfwAccount) 1.0 else 0.0, - 0.0, - enableNsfwTokenCheck && isNsfwAccount) - cand.cachePredicateInfo( - name + "_authorAge", - accountAgeInHours, - minimumAllowedAge, - accountAgeInHours < minimumAllowedAge) - - if (enableNsfwTokenCheck && isNsfwAccount) { - filterByNsfwToken.incr() - false - } else if (accountAgeInHours < minimumAllowedAge) { - filterByAccountAge.incr() - false - } else true - } else true - case _ => true - } - case _ => Future.value(true) - } - } - .withStats(stats.scope(s"predicate_$name")) - .withName(name) - } - - def authorSensitiveMediaPredicate( - producerMediaRepresentationStore: ReadableStore[Long, UserMediaRepresentation] - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with TweetAuthor] = { - val name = "author_sensitive_media_mrtwistly" - val statsScope = stats.scope(name) - val enableQueryNum = statsScope.counter("enable_query") - val nonEmptyMediaRepresentationNum = statsScope.counter("non_empty_media_representation") - val filteredOON = statsScope.counter("filtered_oon") - - Predicate - .fromAsync { candidate: PushCandidate with TweetAuthor => - val target = candidate.target - val useAggressiveThresholds = CandidateUtil.useAggressiveHealthThresholds(candidate) - - if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && - RecTypes.isOutOfNetworkTweetRecType(candidate.commonRecType) && - target.params(PushFeatureSwitchParams.EnableQueryAuthorMediaRepresentationStore)) { - enableQueryNum.incr() - - candidate.authorId match { - case Some(authorId) => - producerMediaRepresentationStore.get(authorId).map { - case Some(mediaRepresentation) => - nonEmptyMediaRepresentationNum.incr() - val sumScore: Double = mediaRepresentation.mediaRepresentation.values.sum - val nudityScore: Double = mediaRepresentation.mediaRepresentation - .getOrElse(MediaAnnotationsUtil.nudityCategoryId, 0.0) - val nudityRate = if (sumScore > 0) nudityScore / sumScore else 0.0 - - candidate - .cacheExternalScore("AuthorNudityScore", Future.value(Some(nudityScore))) - candidate.cacheExternalScore("AuthorNudityRate", Future.value(Some(nudityRate))) - - val threshold = if (useAggressiveThresholds) { - target.params( - PushFeatureSwitchParams.AuthorSensitiveMediaFilteringThresholdForMrTwistly) - } else { - target.params(PushFeatureSwitchParams.AuthorSensitiveMediaFilteringThreshold) - } - candidate.cachePredicateInfo( - name, - nudityRate, - threshold, - nudityRate > threshold, - Some(Map[String, Double]("sumScore" -> sumScore, "nudityScore" -> nudityScore))) - - if (nudityRate > threshold) { - filteredOON.incr() - false - } else true - case _ => true - } - case _ => Future.True - } - } else { - Future.True - } - } - .withStats(stats.scope(s"predicate_$name")) - .withName(name) - } - - def sensitiveMediaCategoryPredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with TweetCandidate] = { - val name = "sensitive_media_category" - val tweetMediaAnnotationFeature = - "tweet.mediaunderstanding.tweet_annotations.sensitive_category_probabilities" - val scopedStatsReceiver = stats.scope(name) - val allCandidatesCounter = scopedStatsReceiver.counter("all_candidates") - val nonZeroNudityCandidatesCounter = scopedStatsReceiver.counter("non_zero_nudity_candidates") - val nudityScoreStats = scopedStatsReceiver.stat("nudity_scores") - - Predicate - .fromAsync { candidate: PushCandidate => - allCandidatesCounter.incr() - val target = candidate.target - val nudityScore = candidate.sparseContinuousFeatures - .getOrElse(tweetMediaAnnotationFeature, Map.empty[String, Double]).getOrElse( - MediaAnnotationsUtil.nudityCategoryId, - 0.0) - if (nudityScore > 0) nonZeroNudityCandidatesCounter.incr() - nudityScoreStats.add(nudityScore.toFloat) - val threshold = - target.params(PushFeatureSwitchParams.TweetMediaSensitiveCategoryThresholdParam) - candidate.cachePredicateInfo(name, nudityScore, threshold, nudityScore > threshold) - if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && nudityScore > threshold) { - Future.False - } else { - Future.True - } - } - .withStats(stats.scope(s"predicate_$name")) - .withName(name) - } - - def profanityPredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with TweetCandidate] = { - val name = "profanity_filter" - val scopedStatsReceiver = stats.scope(name) - val allCandidatesCounter = scopedStatsReceiver.counter("all_candidates") - - Predicate - .fromAsync { candidate: PushCandidate => - allCandidatesCounter.incr() - val target = candidate.target - - lazy val enableFilter = - target.params(PushFeatureSwitchParams.EnableProfanityFilterParam) - val tweetSemanticCoreIds = candidate.sparseBinaryFeatures - .getOrElse(PushConstants.TweetSemanticCoreIdFeature, Set.empty[String]) - - if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && - tweetSemanticCoreIds.contains(PushConstants.ProfanityFilter_Id) && enableFilter) { - Future.False - } else { - Future.True - } - } - .withStats(stats.scope(s"predicate_$name")) - .withName(name) - } - - def agathaAbusiveTweetAuthorPredicateMrTwistly( - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with OutOfNetworkTweetCandidate] = { - val name = "agatha_abusive_tweet_author_mr_twistly" - val scopedStatsReceiver = stats.scope(name) - val allCandidatesCounter = scopedStatsReceiver.counter("all_candidates") - val isMrBackfillCRCandidateCounter = scopedStatsReceiver.counter("isMrBackfillCR_candidates") - Predicate - .fromAsync { cand: PushCandidate with OutOfNetworkTweetCandidate => - allCandidatesCounter.incr() - val target = cand.target - val tweetSemanticCoreIds = cand.sparseBinaryFeatures - .getOrElse(PushConstants.TweetSemanticCoreIdFeature, Set.empty[String]) - - val hasAbuseStrikeTop2Percent = - tweetSemanticCoreIds.contains(PushConstants.AbuseStrike_Top2Percent_Id) - val hasAbuseStrikeTop1Percent = - tweetSemanticCoreIds.contains(PushConstants.AbuseStrike_Top1Percent_Id) - val hasAbuseStrikeTop05Percent = - tweetSemanticCoreIds.contains(PushConstants.AbuseStrike_Top05Percent_Id) - - if (hasAbuseStrikeTop2Percent) { - scopedStatsReceiver.counter("abuse_strike_top_2_percent_candidates").incr() - } - if (hasAbuseStrikeTop1Percent) { - scopedStatsReceiver.counter("abuse_strike_top_1_percent_candidates").incr() - } - if (hasAbuseStrikeTop05Percent) { - scopedStatsReceiver.counter("abuse_strike_top_05_percent_candidates").incr() - } - - if (CandidateUtil.shouldApplyHealthQualityFilters(cand) && cand.isMrBackfillCR.getOrElse( - false)) { - isMrBackfillCRCandidateCounter.incr() - if (hasAbuseStrikeTop2Percent) { - if (target.params( - PushFeatureSwitchParams.EnableAbuseStrikeTop2PercentFilterSimCluster) && hasAbuseStrikeTop2Percent || - target.params( - PushFeatureSwitchParams.EnableAbuseStrikeTop1PercentFilterSimCluster) && hasAbuseStrikeTop1Percent || - target.params( - PushFeatureSwitchParams.EnableAbuseStrikeTop05PercentFilterSimCluster) && hasAbuseStrikeTop05Percent) { - Future.False - } else { - Future.True - } - } else { - Future.True - } - } else Future.True - } - .withStats(stats.scope(s"predicate_$name")) - .withName(name) - } - - def userHealthSignalsPredicate( - userHealthSignalStore: ReadableStore[Long, UserHealthSignalResponse] - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with TweetDetails] = { - val name = "agatha_user_health_model_score" - val scopedStatsReceiver = stats.scope(name) - val allCandidatesCounter = scopedStatsReceiver.counter("all_candidates") - val bucketedUserCandidatesCounter = - scopedStatsReceiver.counter("bucketed_user_candidates") - val filteredOON = scopedStatsReceiver.counter("filtered_oon") - - Predicate - .fromAsync { candidate: PushCandidate with TweetDetails => - allCandidatesCounter.incr() - val target = candidate.target - val useAggressiveThresholds = CandidateUtil.useAggressiveHealthThresholds(candidate) - - if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && target.params( - PushFeatureSwitchParams.EnableAgathaUserHealthModelPredicate)) { - val healthSignalsResponseFutOpt: Future[Option[UserHealthSignalResponse]] = - candidate.authorId match { - case Some(authorId) => userHealthSignalStore.get(authorId) - case _ => Future.None - } - healthSignalsResponseFutOpt.map { - case Some(response) => - val agathaRecentAbuseStrikeScore: Double = userHealthSignalValueToDouble( - response.signalValues - .getOrElse(AgathaRecentAbuseStrikeDouble, SignalValue.DoubleValue(0.0))) - val agathaCalibratedNSFWScore: Double = userHealthSignalValueToDouble( - response.signalValues - .getOrElse(AgathaCalibratedNsfwDouble, SignalValue.DoubleValue(0.0))) - val agathaTextNSFWScore: Double = userHealthSignalValueToDouble(response.signalValues - .getOrElse(NsfwTextUserScoreDouble, SignalValue.DoubleValue(0.0))) - - candidate - .cacheExternalScore( - "agathaRecentAbuseStrikeScore", - Future.value(Some(agathaRecentAbuseStrikeScore))) - candidate - .cacheExternalScore( - "agathaCalibratedNSFWScore", - Future.value(Some(agathaCalibratedNSFWScore))) - candidate - .cacheExternalScore("agathaTextNSFWScore", Future.value(Some(agathaTextNSFWScore))) - - val NSFWShouldBucket = agathaCalibratedNSFWScore > target.params( - PushFeatureSwitchParams.AgathaCalibratedNSFWBucketThreshold) - val textNSFWShouldBucket = agathaTextNSFWScore > target.params( - PushFeatureSwitchParams.AgathaTextNSFWBucketThreshold) - - if (NSFWShouldBucket || textNSFWShouldBucket) { - bucketedUserCandidatesCounter.incr() - if (NSFWShouldBucket) { - scopedStatsReceiver.counter("calibrated_nsfw_bucketed_user_candidates").incr() - } - if (textNSFWShouldBucket) { - scopedStatsReceiver.counter("text_nsfw_bucketed_user_candidates").incr() - } - - val (thresholdAgathaNsfw, thresholdTextNsfw) = if (useAggressiveThresholds) { - ( - target.params( - PushFeatureSwitchParams.AgathaCalibratedNSFWThresholdForMrTwistly), - target - .params(PushFeatureSwitchParams.AgathaTextNSFWThresholdForMrTwistly)) - } else { - ( - target.params(PushFeatureSwitchParams.AgathaCalibratedNSFWThreshold), - target.params(PushFeatureSwitchParams.AgathaTextNSFWThreshold)) - } - candidate.cachePredicateInfo( - name + "_agathaNsfw", - agathaCalibratedNSFWScore, - thresholdAgathaNsfw, - agathaCalibratedNSFWScore > thresholdAgathaNsfw) - candidate.cachePredicateInfo( - name + "_authorTextNsfw", - agathaTextNSFWScore, - thresholdTextNsfw, - agathaTextNSFWScore > thresholdTextNsfw) - - if ((agathaCalibratedNSFWScore > thresholdAgathaNsfw) || - (agathaTextNSFWScore > thresholdTextNsfw)) { - filteredOON.incr() - false - } else true - } else { - true - } - case _ => true - } - } else { - Future.True - } - } - .withStats(stats.scope(s"predicate_$name")) - .withName(name) - } - - def userHealthSignalValueToDouble(signalValue: SignalValue): Double = { - signalValue match { - case SignalValue.DoubleValue(value) => value - case _ => throw new Exception(f"Could not convert signal value to double") - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/JointDauAndQualityModelPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/JointDauAndQualityModelPredicate.scala deleted file mode 100644 index 63095f4db..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/JointDauAndQualityModelPredicate.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams.QualityPredicateIdParam -import com.twitter.frigate.pushservice.predicate.quality_model_predicate._ -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.util.Future - -object JointDauAndQualityModelPredicate { - - val name = "JointDauAndQualityModelPredicate" - - def apply()(implicit statsReceiver: StatsReceiver): NamedPredicate[PushCandidate] = { - val stats = statsReceiver.scope(s"predicate_$name") - - val defaultPred = WeightedOpenOrNtabClickQualityPredicate() - val qualityPredicateMap = QualityPredicateMap() - - Predicate - .fromAsync { candidate: PushCandidate => - if (!candidate.target.skipModelPredicate) { - - val modelPredicate = - qualityPredicateMap.getOrElse( - candidate.target.params(QualityPredicateIdParam), - defaultPred) - - val modelPredicateResultFut = - modelPredicate.apply(Seq(candidate)).map(_.headOption.getOrElse(false)) - - modelPredicateResultFut - } else Future.True - } - .withStats(stats) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ListPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ListPredicates.scala deleted file mode 100644 index cbfb670d8..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ListPredicates.scala +++ /dev/null @@ -1,110 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.model.ListRecommendationPushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.hermit.predicate.socialgraph.Edge -import com.twitter.hermit.predicate.socialgraph.RelationEdge -import com.twitter.hermit.predicate.socialgraph.SocialGraphPredicate -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.socialgraph.thriftscala.RelationshipType -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -object ListPredicates { - - def listNameExistsPredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[ListRecommendationPushCandidate] = { - Predicate - .fromAsync { candidate: ListRecommendationPushCandidate => - candidate.listName.map(_.isDefined) - } - .withStats(stats) - .withName("list_name_exists") - } - - def listAuthorExistsPredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[ListRecommendationPushCandidate] = { - Predicate - .fromAsync { candidate: ListRecommendationPushCandidate => - candidate.listOwnerId.map(_.isDefined) - } - .withStats(stats) - .withName("list_owner_exists") - } - - def listAuthorAcceptableToTargetUser( - edgeStore: ReadableStore[RelationEdge, Boolean] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[ListRecommendationPushCandidate] = { - val name = "list_author_acceptable_to_target_user" - val sgsPredicate = SocialGraphPredicate - .anyRelationExists( - edgeStore, - Set( - RelationshipType.Blocking, - RelationshipType.BlockedBy, - RelationshipType.Muting - ) - ) - .withStats(statsReceiver.scope("list_sgs_any_relation_exists")) - .withName("list_sgs_any_relation_exists") - - Predicate - .fromAsync { candidate: ListRecommendationPushCandidate => - candidate.listOwnerId.flatMap { - case Some(ownerId) => - sgsPredicate.apply(Seq(Edge(candidate.target.targetId, ownerId))).map(_.head) - case _ => Future.True - } - } - .withStats(statsReceiver.scope(s"predicate_$name")) - .withName(name) - } - - /** - * Checks if the list is acceptable to Target user => - * - Is Target not following the list - * - Is Target not muted the list - */ - def listAcceptablePredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[ListRecommendationPushCandidate] = { - val name = "list_acceptable_to_target_user" - Predicate - .fromAsync { candidate: ListRecommendationPushCandidate => - candidate.apiList.map { - case Some(apiList) => - !(apiList.following.contains(true) || apiList.muting.contains(true)) - case _ => false - } - } - .withStats(stats.scope(name)) - .withName(name) - } - - def listSubscriberCountPredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[ListRecommendationPushCandidate] = { - val name = "list_subscribe_count" - Predicate - .fromAsync { candidate: ListRecommendationPushCandidate => - candidate.apiList.map { apiListOpt => - apiListOpt.exists { apiList => - apiList.subscriberCount >= candidate.target.params( - PushFeatureSwitchParams.ListRecommendationsSubscriberCount) - } - } - } - .withStats(stats.scope(name)) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/LoggedOutPreRankingPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/LoggedOutPreRankingPredicates.scala deleted file mode 100644 index 9ba1c9f6f..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/LoggedOutPreRankingPredicates.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.common.base.TweetDetails -import com.twitter.frigate.common.predicate.tweet._ -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.hermit.predicate.NamedPredicate - -class LoggedOutPreRankingPredicatesBuilder(implicit statsReceiver: StatsReceiver) { - - private val TweetPredicates = List[NamedPredicate[PushCandidate]]( - TweetObjectExistsPredicate[ - TweetCandidate with TweetDetails - ].applyOnlyToTweetCandidatesWithTweetDetails - .withName("tweet_object_exists"), - PredicatesForCandidate.oldTweetRecsPredicate.applyOnlyToTweetCandidateWithTargetAndABDeciderAndMaxTweetAge - .withName("old_tweet"), - PredicatesForCandidate.tweetIsNotAreply.applyOnlyToTweetCandidateWithoutSocialContextWithTweetDetails - .withName("tweet_candidate_not_a_reply"), - TweetAuthorPredicates - .recTweetAuthorUnsuitable[TweetCandidate with TweetAuthorDetails] - .applyOnlyToTweetCandidateWithTweetAuthorDetails - .withName("tweet_author_unsuitable") - ) - - final def build(): List[NamedPredicate[PushCandidate]] = { - TweetPredicates - } - -} - -object LoggedOutPreRankingPredicates { - def apply(statsReceiver: StatsReceiver): List[NamedPredicate[PushCandidate]] = - new LoggedOutPreRankingPredicatesBuilder()(statsReceiver).build() -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/LoggedOutTargetPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/LoggedOutTargetPredicates.scala deleted file mode 100644 index 085ad73e9..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/LoggedOutTargetPredicates.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.abdecider.GuestRecipient -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.common.predicate.{FatiguePredicate => CommonFatiguePredicate} -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.conversions.DurationOps._ -import com.twitter.frigate.common.util.Experiments.LoggedOutRecsHoldback -import com.twitter.hermit.predicate.Predicate - -object LoggedOutTargetPredicates { - - def targetFatiguePredicate[T <: Target]( - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[T] = { - val name = "logged_out_target_min_duration_since_push" - CommonFatiguePredicate - .magicRecsPushTargetFatiguePredicate( - minInterval = 24.hours, - maxInInterval = 1 - ).withStats(statsReceiver.scope(name)) - .withName(name) - } - - def loggedOutRecsHoldbackPredicate[T <: Target]( - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[T] = { - val name = "logged_out_recs_holdback" - val guestIdNotFoundCounter = statsReceiver.scope("logged_out").counter("guest_id_not_found") - val controlBucketCounter = statsReceiver.scope("logged_out").counter("holdback_control") - val allowTrafficCounter = statsReceiver.scope("logged_out").counter("allow_traffic") - Predicate.from { target: T => - val guestId = target.targetGuestId match { - case Some(guest) => guest - case _ => - guestIdNotFoundCounter.incr() - throw new IllegalStateException("guest_id_not_found") - } - target.abDecider - .bucket(LoggedOutRecsHoldback.exptName, GuestRecipient(guestId)).map(_.name) match { - case Some(LoggedOutRecsHoldback.control) => - controlBucketCounter.incr() - false - case _ => - allowTrafficCounter.incr() - true - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/MlModelsHoldbackExperimentPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/MlModelsHoldbackExperimentPredicate.scala deleted file mode 100644 index 014393870..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/MlModelsHoldbackExperimentPredicate.scala +++ /dev/null @@ -1,71 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.util.Future - -object MlModelsHoldbackExperimentPredicate { - - val name = "MlModelsHoldbackExperimentPredicate" - - private val alwaysTruePred = PredicatesForCandidate.alwaysTruePushCandidatePredicate - - def getPredicateBasedOnCandidate( - pc: PushCandidate, - treatmentPred: Predicate[PushCandidate] - )( - implicit statsReceiver: StatsReceiver - ): Future[Predicate[PushCandidate]] = { - - Future - .join(Future.value(pc.target.skipFilters), pc.target.isInModelExclusionList) - .map { - case (skipFilters, isInModelExclusionList) => - if (skipFilters || - isInModelExclusionList || - pc.target.params(PushParams.DisableMlInFilteringParam) || - pc.target.params(PushFeatureSwitchParams.DisableMlInFilteringFeatureSwitchParam) || - pc.target.params(PushParams.DisableAllRelevanceParam) || - pc.target.params(PushParams.DisableHeavyRankingParam)) { - alwaysTruePred - } else { - treatmentPred - } - } - } - - def apply()(implicit statsReceiver: StatsReceiver): NamedPredicate[PushCandidate] = { - val stats = statsReceiver.scope(s"predicate_$name") - val statsProd = stats.scope("prod") - val counterAcceptedByModel = statsProd.counter("accepted") - val counterRejectedByModel = statsProd.counter("rejected") - val counterHoldback = stats.scope("holdback").counter("all") - val jointDauQualityPredicate = JointDauAndQualityModelPredicate() - - new Predicate[PushCandidate] { - def apply(items: Seq[PushCandidate]): Future[Seq[Boolean]] = { - val boolFuts = items.map { item => - getPredicateBasedOnCandidate(item, jointDauQualityPredicate)(statsReceiver) - .flatMap { predicate => - val predictionFut = predicate.apply(Seq(item)).map(_.headOption.getOrElse(false)) - predictionFut.foreach { prediction => - if (item.target.params(PushParams.DisableMlInFilteringParam) || item.target.params( - PushFeatureSwitchParams.DisableMlInFilteringFeatureSwitchParam)) { - counterHoldback.incr() - } else { - if (prediction) counterAcceptedByModel.incr() else counterRejectedByModel.incr() - } - } - predictionFut - } - } - Future.collect(boolFuts) - } - }.withStats(stats) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OONSpreadControlPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OONSpreadControlPredicate.scala deleted file mode 100644 index bcd9e30d0..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OONSpreadControlPredicate.scala +++ /dev/null @@ -1,116 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushConstants._ -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.util.CandidateUtil -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.util.Future - -object OONSpreadControlPredicate { - - def oonTweetSpreadControlPredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[ - PushCandidate with TweetCandidate with RecommendationType - ] = { - val name = "oon_tweet_spread_control_predicate" - val scopedStatsReceiver = stats.scope(name) - val allOonCandidatesCounter = scopedStatsReceiver.counter("all_oon_candidates") - val filteredCandidatesCounter = - scopedStatsReceiver.counter("filtered_oon_candidates") - - Predicate - .fromAsync { candidate: PushCandidate with TweetCandidate with RecommendationType => - val target = candidate.target - val crt = candidate.commonRecType - val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(crt) || - RecTypes.outOfNetworkTopicTweetTypes.contains(crt) - - lazy val minTweetSendsThreshold = - target.params(PushFeatureSwitchParams.MinTweetSendsThresholdParam) - lazy val spreadControlRatio = - target.params(PushFeatureSwitchParams.SpreadControlRatioParam) - lazy val favOverSendThreshold = - target.params(PushFeatureSwitchParams.FavOverSendThresholdParam) - - lazy val sentCount = candidate.numericFeatures.getOrElse(sentFeatureName, 0.0) - lazy val followerCount = - candidate.numericFeatures.getOrElse(authorActiveFollowerFeatureName, 0.0) - lazy val favCount = candidate.numericFeatures.getOrElse(favFeatureName, 0.0) - lazy val favOverSends = favCount / (sentCount + 1.0) - - if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && isOonCandidate) { - allOonCandidatesCounter.incr() - if (sentCount > minTweetSendsThreshold && - sentCount > spreadControlRatio * followerCount && - favOverSends < favOverSendThreshold) { - filteredCandidatesCounter.incr() - Future.False - } else Future.True - } else Future.True - } - .withStats(stats.scope(name)) - .withName(name) - } - - def oonAuthorSpreadControlPredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[ - PushCandidate with TweetCandidate with RecommendationType - ] = { - val name = "oon_author_spread_control_predicate" - val scopedStatsReceiver = stats.scope(name) - val allOonCandidatesCounter = scopedStatsReceiver.counter("all_oon_candidates") - val filteredCandidatesCounter = - scopedStatsReceiver.counter("filtered_oon_candidates") - - Predicate - .fromAsync { candidate: PushCandidate with TweetCandidate with RecommendationType => - val target = candidate.target - val crt = candidate.commonRecType - val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(crt) || - RecTypes.outOfNetworkTopicTweetTypes.contains(crt) - - lazy val minAuthorSendsThreshold = - target.params(PushFeatureSwitchParams.MinAuthorSendsThresholdParam) - lazy val spreadControlRatio = - target.params(PushFeatureSwitchParams.SpreadControlRatioParam) - lazy val reportRateThreshold = - target.params(PushFeatureSwitchParams.AuthorReportRateThresholdParam) - lazy val dislikeRateThreshold = - target.params(PushFeatureSwitchParams.AuthorDislikeRateThresholdParam) - - lazy val authorSentCount = - candidate.numericFeatures.getOrElse(authorSendCountFeatureName, 0.0) - lazy val authorReportCount = - candidate.numericFeatures.getOrElse(authorReportCountFeatureName, 0.0) - lazy val authorDislikeCount = - candidate.numericFeatures.getOrElse(authorDislikeCountFeatureName, 0.0) - lazy val followerCount = candidate.numericFeatures - .getOrElse(authorActiveFollowerFeatureName, 0.0) - lazy val reportRate = - authorReportCount / (authorSentCount + 1.0) - lazy val dislikeRate = - authorDislikeCount / (authorSentCount + 1.0) - - if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && isOonCandidate) { - allOonCandidatesCounter.incr() - if (authorSentCount > minAuthorSendsThreshold && - authorSentCount > spreadControlRatio * followerCount && - (reportRate > reportRateThreshold || dislikeRate > dislikeRateThreshold)) { - filteredCandidatesCounter.incr() - Future.False - } else Future.True - } else Future.True - } - .withStats(stats.scope(name)) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OONTweetNegativeFeedbackBasedPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OONTweetNegativeFeedbackBasedPredicate.scala deleted file mode 100644 index 3efb23d88..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OONTweetNegativeFeedbackBasedPredicate.scala +++ /dev/null @@ -1,82 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.util.CandidateUtil -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.util.Future - -object OONTweetNegativeFeedbackBasedPredicate { - - def ntabDislikeBasedPredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[ - PushCandidate with TweetCandidate with RecommendationType - ] = { - val name = "oon_tweet_dislike_based_predicate" - val scopedStatsReceiver = stats.scope(name) - val allOonCandidatesCounter = scopedStatsReceiver.counter("all_oon_candidates") - val oonCandidatesImpressedCounter = - scopedStatsReceiver.counter("oon_candidates_impressed") - val filteredCandidatesCounter = - scopedStatsReceiver.counter("filtered_oon_candidates") - - val ntabDislikeCountFeature = - "tweet.magic_recs_tweet_real_time_aggregates_v2.pair.v2.magicrecs.realtime.is_ntab_disliked.any_feature.Duration.Top.count" - val sentFeature = - "tweet.magic_recs_tweet_real_time_aggregates_v2.pair.v2.magicrecs.realtime.is_sent.any_feature.Duration.Top.count" - - Predicate - .fromAsync { candidate: PushCandidate with TweetCandidate with RecommendationType => - val target = candidate.target - val crt = candidate.commonRecType - val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(crt) || - RecTypes.outOfNetworkTopicTweetTypes.contains(crt) - - lazy val ntabDislikeCountThreshold = - target.params(PushFeatureSwitchParams.TweetNtabDislikeCountThresholdParam) - lazy val ntabDislikeRateThreshold = - target.params(PushFeatureSwitchParams.TweetNtabDislikeRateThresholdParam) - lazy val ntabDislikeCountThresholdForMrTwistly = - target.params(PushFeatureSwitchParams.TweetNtabDislikeCountThresholdForMrTwistlyParam) - lazy val ntabDislikeRateThresholdForMrTwistly = - target.params(PushFeatureSwitchParams.TweetNtabDislikeRateThresholdForMrTwistlyParam) - - val isMrTwistly = CandidateUtil.isMrTwistlyCandidate(candidate) - - lazy val dislikeCount = candidate.numericFeatures.getOrElse(ntabDislikeCountFeature, 0.0) - lazy val sentCount = candidate.numericFeatures.getOrElse(sentFeature, 0.0) - lazy val dislikeRate = if (sentCount > 0) dislikeCount / sentCount else 0.0 - - if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && isOonCandidate) { - allOonCandidatesCounter.incr() - val (countThreshold, rateThreshold) = if (isMrTwistly) { - (ntabDislikeCountThresholdForMrTwistly, ntabDislikeRateThresholdForMrTwistly) - } else { - (ntabDislikeCountThreshold, ntabDislikeRateThreshold) - } - candidate.cachePredicateInfo( - name + "_count", - dislikeCount, - countThreshold, - dislikeCount > countThreshold) - candidate.cachePredicateInfo( - name + "_rate", - dislikeRate, - rateThreshold, - dislikeRate > rateThreshold) - if (dislikeCount > countThreshold && dislikeRate > rateThreshold) { - filteredCandidatesCounter.incr() - Future.False - } else Future.True - } else Future.True - } - .withStats(stats.scope(name)) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OutOfNetworkCandidatesQualityPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OutOfNetworkCandidatesQualityPredicates.scala deleted file mode 100644 index 6f09df0c7..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OutOfNetworkCandidatesQualityPredicates.scala +++ /dev/null @@ -1,221 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.candidate.TargetABDecider -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.data_pipeline.features_common.MrRequestContextForFeatureStore -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.ml.featurestore.lib.dynamic.DynamicFeatureStoreClient -import com.twitter.util.Future -import com.twitter.frigate.pushservice.predicate.PostRankingPredicateHelper._ -import com.twitter.frigate.pushservice.util.CandidateUtil - -object OutOfNetworkCandidatesQualityPredicates { - - def getTweetCharLengthThreshold( - target: TargetUser with TargetABDecider, - language: String, - useMediaThresholds: Boolean - ): Double = { - lazy val sautOonWithMediaTweetLengthThreshold = - target.params(PushFeatureSwitchParams.SautOonWithMediaTweetLengthThresholdParam) - lazy val nonSautOonWithMediaTweetLengthThreshold = - target.params(PushFeatureSwitchParams.NonSautOonWithMediaTweetLengthThresholdParam) - lazy val sautOonWithoutMediaTweetLengthThreshold = - target.params(PushFeatureSwitchParams.SautOonWithoutMediaTweetLengthThresholdParam) - lazy val nonSautOonWithoutMediaTweetLengthThreshold = - target.params(PushFeatureSwitchParams.NonSautOonWithoutMediaTweetLengthThresholdParam) - val moreStrictForUndefinedLanguages = - target.params(PushFeatureSwitchParams.OonTweetLengthPredicateMoreStrictForUndefinedLanguages) - val isSautLanguage = if (moreStrictForUndefinedLanguages) { - isTweetLanguageInSautOrUndefined(language) - } else isTweetLanguageInSaut(language) - - (useMediaThresholds, isSautLanguage) match { - case (true, true) => - sautOonWithMediaTweetLengthThreshold - case (true, false) => - nonSautOonWithMediaTweetLengthThreshold - case (false, true) => - sautOonWithoutMediaTweetLengthThreshold - case (false, false) => - nonSautOonWithoutMediaTweetLengthThreshold - case _ => -1 - } - } - - def getTweetWordLengthThreshold( - target: TargetUser with TargetABDecider, - language: String, - useMediaThresholds: Boolean - ): Double = { - lazy val argfOonWithMediaTweetWordLengthThresholdParam = - target.params(PushFeatureSwitchParams.ArgfOonWithMediaTweetWordLengthThresholdParam) - lazy val esfthOonWithMediaTweetWordLengthThresholdParam = - target.params(PushFeatureSwitchParams.EsfthOonWithMediaTweetWordLengthThresholdParam) - - lazy val argfOonCandidatesWithMediaCondition = - isTweetLanguageInArgf(language) && useMediaThresholds - lazy val esfthOonCandidatesWithMediaCondition = - isTweetLanguageInEsfth(language) && useMediaThresholds - lazy val afirfOonCandidatesWithoutMediaCondition = - isTweetLanguageInAfirf(language) && !useMediaThresholds - - val afirfOonCandidatesWithoutMediaTweetWordLengthThreshold = 5 - if (argfOonCandidatesWithMediaCondition) { - argfOonWithMediaTweetWordLengthThresholdParam - } else if (esfthOonCandidatesWithMediaCondition) { - esfthOonWithMediaTweetWordLengthThresholdParam - } else if (afirfOonCandidatesWithoutMediaCondition) { - afirfOonCandidatesWithoutMediaTweetWordLengthThreshold - } else -1 - } - - def oonTweetLengthBasedPrerankingPredicate( - characterBased: Boolean - )( - implicit stats: StatsReceiver - ): NamedPredicate[OutOfNetworkTweetCandidate with TargetInfo[ - TargetUser with TargetABDecider - ]] = { - val name = "oon_tweet_length_based_preranking_predicate" - val scopedStats = stats.scope(s"${name}_charBased_$characterBased") - - Predicate - .fromAsync { - cand: OutOfNetworkTweetCandidate with TargetInfo[TargetUser with TargetABDecider] => - cand match { - case candidate: TweetAuthorDetails => - val target = candidate.target - val crt = candidate.commonRecType - - val updatedMediaLogic = - target.params(PushFeatureSwitchParams.OonTweetLengthPredicateUpdatedMediaLogic) - val updatedQuoteTweetLogic = - target.params(PushFeatureSwitchParams.OonTweetLengthPredicateUpdatedQuoteTweetLogic) - val useMediaThresholds = if (updatedMediaLogic || updatedQuoteTweetLogic) { - val hasMedia = updatedMediaLogic && (candidate.hasPhoto || candidate.hasVideo) - val hasQuoteTweet = updatedQuoteTweetLogic && candidate.quotedTweet.nonEmpty - hasMedia || hasQuoteTweet - } else RecTypes.isMediaType(crt) - val enableFilter = - target.params(PushFeatureSwitchParams.EnablePrerankingTweetLengthPredicate) - - val language = candidate.tweet.flatMap(_.language.map(_.language)).getOrElse("") - val tweetTextOpt = candidate.tweet.flatMap(_.coreData.map(_.text)) - - val (length: Double, threshold: Double) = if (characterBased) { - ( - tweetTextOpt.map(_.size.toDouble).getOrElse(9999.0), - getTweetCharLengthThreshold(target, language, useMediaThresholds)) - } else { - ( - tweetTextOpt.map(getTweetWordLength).getOrElse(999.0), - getTweetWordLengthThreshold(target, language, useMediaThresholds)) - } - scopedStats.counter("threshold_" + threshold.toString).incr() - - CandidateUtil.shouldApplyHealthQualityFiltersForPrerankingPredicates(candidate).map { - case true if enableFilter => - length > threshold - case _ => true - } - case _ => - scopedStats.counter("author_is_not_hydrated").incr() - Future.True - } - }.withStats(scopedStats) - .withName(name) - } - - private def isTweetLanguageInAfirf(candidateLanguage: String): Boolean = { - val setAFIRF: Set[String] = Set("") - setAFIRF.contains(candidateLanguage) - } - private def isTweetLanguageInEsfth(candidateLanguage: String): Boolean = { - val setESFTH: Set[String] = Set("") - setESFTH.contains(candidateLanguage) - } - private def isTweetLanguageInArgf(candidateLanguage: String): Boolean = { - val setARGF: Set[String] = Set("") - setARGF.contains(candidateLanguage) - } - - private def isTweetLanguageInSaut(candidateLanguage: String): Boolean = { - val setSAUT = Set("") - setSAUT.contains(candidateLanguage) - } - - private def isTweetLanguageInSautOrUndefined(candidateLanguage: String): Boolean = { - val setSautOrUndefined = Set("") - setSautOrUndefined.contains(candidateLanguage) - } - - def containTargetNegativeKeywords(text: String, denylist: Seq[String]): Boolean = { - if (denylist.isEmpty) - false - else { - denylist - .map { negativeKeyword => - text.toLowerCase().contains(negativeKeyword) - }.reduce(_ || _) - } - } - - def NegativeKeywordsPredicate( - postRankingFeatureStoreClient: DynamicFeatureStoreClient[MrRequestContextForFeatureStore] - )( - implicit stats: StatsReceiver - ): NamedPredicate[ - PushCandidate with TweetCandidate with RecommendationType - ] = { - - val name = "negative_keywords_predicate" - val scopedStatsReceiver = stats.scope(name) - val allOonCandidatesCounter = scopedStatsReceiver.counter("all_oon_candidates") - val filteredOonCandidatesCounter = scopedStatsReceiver.counter("filtered_oon_candidates") - val tweetLanguageFeature = "RecTweet.TweetyPieResult.Language" - - Predicate - .fromAsync { candidate: PushCandidate with TweetCandidate with RecommendationType => - val target = candidate.target - val crt = candidate.commonRecType - val isTwistlyCandidate = RecTypes.twistlyTweets.contains(crt) - - lazy val enableNegativeKeywordsPredicateParam = - target.params(PushFeatureSwitchParams.EnableNegativeKeywordsPredicateParam) - lazy val negativeKeywordsPredicateDenylist = - target.params(PushFeatureSwitchParams.NegativeKeywordsPredicateDenylist) - lazy val candidateLanguage = - candidate.categoricalFeatures.getOrElse(tweetLanguageFeature, "") - - if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && candidateLanguage.equals( - "en") && isTwistlyCandidate && enableNegativeKeywordsPredicateParam) { - allOonCandidatesCounter.incr() - - val tweetTextFuture: Future[String] = - getTweetText(candidate, postRankingFeatureStoreClient) - - tweetTextFuture.map { tweetText => - val containsNegativeWords = - containTargetNegativeKeywords(tweetText, negativeKeywordsPredicateDenylist) - candidate.cachePredicateInfo( - name, - if (containsNegativeWords) 1.0 else 0.0, - 0.0, - containsNegativeWords) - if (containsNegativeWords) { - filteredOonCandidatesCounter.incr() - false - } else true - } - } else Future.True - } - .withStats(stats.scope(name)) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PNegMultimodalPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PNegMultimodalPredicates.scala deleted file mode 100644 index f838d7ae6..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PNegMultimodalPredicates.scala +++ /dev/null @@ -1,83 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.abuse.detection.scoring.thriftscala.Model -import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringRequest -import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringResponse -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.util.CandidateUtil -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -object PNegMultimodalPredicates { - - def healthSignalScorePNegMultimodalPredicate( - tweetHealthScoreStore: ReadableStore[TweetScoringRequest, TweetScoringResponse] - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with TweetCandidate] = { - val name = "pneg_multimodal_predicate" - val statsScope = stats.scope(name) - val oonCandidatesCounter = statsScope.counter("oon_candidates") - val nonEmptyModelScoreCounter = statsScope.counter("non_empty_model_score") - val bucketedCounter = statsScope.counter("bucketed_oon_candidates") - val filteredCounter = statsScope.counter("filtered_oon_candidates") - - Predicate - .fromAsync { candidate: PushCandidate with TweetCandidate => - val target = candidate.target - val crt = candidate.commonRecType - val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(crt) || - RecTypes.outOfNetworkTopicTweetTypes.contains(crt) - - lazy val enablePNegMultimodalPredicateParam = - target.params(PushFeatureSwitchParams.EnablePNegMultimodalPredicateParam) - lazy val pNegMultimodalPredicateModelThresholdParam = - target.params(PushFeatureSwitchParams.PNegMultimodalPredicateModelThresholdParam) - lazy val pNegMultimodalPredicateBucketThresholdParam = - target.params(PushFeatureSwitchParams.PNegMultimodalPredicateBucketThresholdParam) - val pNegMultimodalEnabledForF1Tweets = - target.params(PushParams.EnablePnegMultimodalPredictionForF1Tweets) - - if (CandidateUtil.shouldApplyHealthQualityFilters( - candidate) && (isOonCandidate || pNegMultimodalEnabledForF1Tweets) && enablePNegMultimodalPredicateParam) { - - val pNegMultimodalRequest = TweetScoringRequest(candidate.tweetId, Model.PNegMultimodal) - tweetHealthScoreStore.get(pNegMultimodalRequest).map { - case Some(tweetScoringResponse) => - nonEmptyModelScoreCounter.incr() - - val pNegMultimodalScore = 1.0 - tweetScoringResponse.score - - candidate - .cacheExternalScore("PNegMultimodalScore", Future.value(Some(pNegMultimodalScore))) - - if (isOonCandidate) { - oonCandidatesCounter.incr() - - if (pNegMultimodalScore > pNegMultimodalPredicateBucketThresholdParam) { - bucketedCounter.incr() - if (pNegMultimodalScore > pNegMultimodalPredicateModelThresholdParam) { - filteredCounter.incr() - false - } else true - } else true - } else { - true - } - case _ => true - } - } else { - Future.True - } - } - .withStats(stats.scope(name)) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PostRankingPredicateHelper.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PostRankingPredicateHelper.scala deleted file mode 100644 index 604f7b07c..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PostRankingPredicateHelper.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.frigate.common.base._ -import com.twitter.frigate.data_pipeline.features_common.MrRequestContextForFeatureStore -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.ml.featurestore.catalog.entities.core.Tweet -import com.twitter.ml.featurestore.catalog.features.core.Tweet.Text -import com.twitter.ml.featurestore.lib.TweetId -import com.twitter.ml.featurestore.lib.dynamic.DynamicFeatureStoreClient -import com.twitter.ml.featurestore.lib.online.FeatureStoreRequest -import com.twitter.util.Future - -object PostRankingPredicateHelper { - - val tweetTextFeature = "tweet.core.tweet.text" - - def getTweetText( - candidate: PushCandidate with TweetCandidate, - dynamicClient: DynamicFeatureStoreClient[MrRequestContextForFeatureStore] - ): Future[String] = { - if (candidate.categoricalFeatures.contains(tweetTextFeature)) { - Future.value(candidate.categoricalFeatures.getOrElse(tweetTextFeature, "")) - } else { - val candidateTweetEntity = Tweet.withId(TweetId(candidate.tweetId)) - val featureStoreRequests = Seq( - FeatureStoreRequest( - entityIds = Seq(candidateTweetEntity) - )) - val predictionRecords = dynamicClient( - featureStoreRequests, - requestContext = candidate.target.mrRequestContextForFeatureStore) - - predictionRecords.map { records => - val tweetText = records.head - .getFeatureValue(candidateTweetEntity, Text).getOrElse( - "" - ) - candidate.categoricalFeatures(tweetTextFeature) = tweetText - tweetText - } - } - } - - def getTweetWordLength(tweetText: String): Double = { - val tweetTextWithoutUrl: String = - tweetText.replaceAll("https?://\\S+\\s?", "").replaceAll("[\\s]+", " ") - tweetTextWithoutUrl.trim().split(" ").length.toDouble - } - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PreRankingPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PreRankingPredicates.scala deleted file mode 100644 index 4b61b23e3..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PreRankingPredicates.scala +++ /dev/null @@ -1,158 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.SocialContextActions -import com.twitter.frigate.common.base.SocialContextUserDetails -import com.twitter.frigate.common.base.TargetInfo -import com.twitter.frigate.common.base.TargetUser -import com.twitter.frigate.common.base.TweetAuthor -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.common.base.TweetDetails -import com.twitter.frigate.common.candidate.FrigateHistory -import com.twitter.frigate.common.candidate.TargetABDecider -import com.twitter.frigate.common.candidate.TweetImpressionHistory -import com.twitter.frigate.common.predicate.socialcontext.{Predicates => SocialContextPredicates, _} -import com.twitter.frigate.common.predicate.tweet._ -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue.NtabCaretClickContFnFatiguePredicate -import com.twitter.hermit.predicate.NamedPredicate - -class PreRankingPredicatesBuilder( -)( - implicit statsReceiver: StatsReceiver) { - - private val SocialProofPredicates = List[NamedPredicate[PushCandidate]]( - SocialContextPredicates - .authorInSocialContext() - .applyOnlyToTweetAuthorWithSocialContextActions - .withName("author_social_context"), - SocialContextPredicates - .selfInSocialContext[TargetUser, SocialContextActions with TargetInfo[TargetUser]]() - .applyOnlyToSocialContextActionsWithTargetUser - .withName("self_social_context"), - SocialContextPredicates - .duplicateSocialContext[SocialContextActions]() - .applyOnlyToSocialContextActions - .withName("duplicate_social_context"), - SocialContextPredicates - .socialContextProtected[SocialContextUserDetails]() - .applyOnlyToSocialContextUserDetails - .withName("social_context_protected"), - SocialContextPredicates - .socialContextUnsuitable[SocialContextUserDetails]() - .applyOnlyToSocialContextUserDetails - .withName("social_context_unsuitable"), - SocialContextPredicates - .socialContextBlink[SocialContextUserDetails]() - .applyOnlyToSocialContextUserDetails - .withName("social_context_blink") - ) - - private val CommonPredicates = List[NamedPredicate[PushCandidate]]( - PredicatesForCandidate.candidateEnabledForEmailPredicate(), - PredicatesForCandidate.openAppExperimentUserCandidateAllowList(statsReceiver) - ) - - private val TweetPredicates = List[NamedPredicate[PushCandidate]]( - PredicatesForCandidate.tweetCandidateWithLessThan2SocialContextsIsAReply.applyOnlyToTweetCandidatesWithSocialContextActions - .withName("tweet_candidate_with_less_than_2_social_contexts_is_not_a_reply"), - PredicatesForCandidate.filterOONCandidatePredicate(), - PredicatesForCandidate.oldTweetRecsPredicate.applyOnlyToTweetCandidateWithTargetAndABDeciderAndMaxTweetAge - .withName("old_tweet"), - DuplicatePushTweetPredicate - .apply[ - TargetUser with FrigateHistory, - TweetCandidate with TargetInfo[TargetUser with FrigateHistory] - ] - .applyOnlyToTweetCandidateWithTargetAndFrigateHistory - .withName("duplicate_push_tweet"), - DuplicateEmailTweetPredicate - .apply[ - TargetUser with FrigateHistory, - TweetCandidate with TargetInfo[TargetUser with FrigateHistory] - ] - .applyOnlyToTweetCandidateWithTargetAndFrigateHistory - .withName("duplicate_email_tweet"), - TweetAuthorPredicates - .recTweetAuthorUnsuitable[TweetCandidate with TweetAuthorDetails] - .applyOnlyToTweetCandidateWithTweetAuthorDetails - .withName("tweet_author_unsuitable"), - TweetObjectExistsPredicate[ - TweetCandidate with TweetDetails - ].applyOnlyToTweetCandidatesWithTweetDetails - .withName("tweet_object_exists"), - TweetImpressionPredicate[ - TargetUser with TweetImpressionHistory, - TweetCandidate with TargetInfo[TargetUser with TweetImpressionHistory] - ].applyOnlyToTweetCandidateWithTargetAndTweetImpressionHistory - .withStats(statsReceiver.scope("tweet_impression")) - .withName("tweet_impression"), - SelfTweetPredicate[ - TargetUser, - TweetAuthor with TargetInfo[TargetUser]]().applyOnlyToTweetAuthorWithTargetInfo - .withName("self_author"), - PredicatesForCandidate.tweetIsNotAreply.applyOnlyToTweetCandidateWithoutSocialContextWithTweetDetails - .withName("tweet_candidate_not_a_reply"), - PredicatesForCandidate.f1CandidateIsNotAReply.applyOnlyToF1CandidateWithTargetAndABDecider - .withName("f1_candidate_is_not_a_reply"), - PredicatesForCandidate.outOfNetworkTweetCandidateIsNotAReply.applyOnlyToOutOfNetworkTweetCandidateWithTargetAndABDecider - .withName("out_of_network_tweet_candidate_is_not_a_reply"), - PredicatesForCandidate.outOfNetworkTweetCandidateEnabledCrTag.applyOnlyToOutOfNetworkTweetCandidateWithTargetAndABDecider - .withName("out_of_network_tweet_candidate_enabled_crtag"), - PredicatesForCandidate.outOfNetworkTweetCandidateEnabledCrtGroup.applyOnlyToOutOfNetworkTweetCandidateWithTargetAndABDecider - .withName("out_of_network_tweet_candidate_enabled_crt_group"), - OutOfNetworkCandidatesQualityPredicates - .oonTweetLengthBasedPrerankingPredicate(characterBased = true) - .applyOnlyToOutOfNetworkTweetCandidateWithTargetAndABDecider - .withName("oon_tweet_char_length_too_short"), - OutOfNetworkCandidatesQualityPredicates - .oonTweetLengthBasedPrerankingPredicate(characterBased = false) - .applyOnlyToOutOfNetworkTweetCandidateWithTargetAndABDecider - .withName("oon_tweet_word_length_too_short"), - PredicatesForCandidate - .protectedTweetF1ExemptPredicate[ - TargetUser with TargetABDecider, - TweetCandidate with TweetAuthorDetails with TargetInfo[ - TargetUser with TargetABDecider - ] - ] - .applyOnlyToTweetCandidateWithAuthorDetailsWithTargetABDecider - .withName("f1_exempt_tweet_author_protected"), - ) - - private val SgsPreRankingPredicates = List[NamedPredicate[PushCandidate]]( - SGSPredicatesForCandidate.authorBeingFollowed.applyOnlyToAuthorBeingFollowPredicates - .withName("author_not_being_followed"), - SGSPredicatesForCandidate.authorNotBeingDeviceFollowed.applyOnlyToBasicTweetPredicates - .withName("author_being_device_followed"), - SGSPredicatesForCandidate.recommendedTweetAuthorAcceptableToTargetUser.applyOnlyToBasicTweetPredicates - .withName("recommended_tweet_author_not_acceptable_to_target_user"), - SGSPredicatesForCandidate.disableInNetworkTweetPredicate.applyOnlyToBasicTweetPredicates - .withName("enable_in_network_tweet"), - SGSPredicatesForCandidate.disableOutNetworkTweetPredicate.applyOnlyToBasicTweetPredicates - .withName("enable_out_network_tweet") - ) - - private val SeeLessOftenPredicates = List[NamedPredicate[PushCandidate]]( - NtabCaretClickContFnFatiguePredicate - .ntabCaretClickContFnFatiguePredicates( - ) - .withName("seelessoften_cont_fn_fatigue") - ) - - final def build(): List[NamedPredicate[PushCandidate]] = { - TweetPredicates ++ - CommonPredicates ++ - SocialProofPredicates ++ - SgsPreRankingPredicates ++ - SeeLessOftenPredicates - } -} - -object PreRankingPredicates { - def apply( - statsReceiver: StatsReceiver - ): List[NamedPredicate[PushCandidate]] = - new PreRankingPredicatesBuilder()(statsReceiver).build() -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PredicatesForCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PredicatesForCandidate.scala deleted file mode 100644 index e18667b51..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PredicatesForCandidate.scala +++ /dev/null @@ -1,874 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.candidate.MaxTweetAge -import com.twitter.frigate.common.candidate.TargetABDecider -import com.twitter.frigate.common.predicate.tweet.TweetAuthorPredicates -import com.twitter.frigate.common.predicate._ -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.common.util.SnowflakeUtils -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.util.CandidateUtil -import com.twitter.frigate.thriftscala.ChannelName -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.gizmoduck.thriftscala.UserType -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.hermit.predicate.gizmoduck._ -import com.twitter.hermit.predicate.socialgraph.Edge -import com.twitter.hermit.predicate.socialgraph.MultiEdge -import com.twitter.hermit.predicate.socialgraph.RelationEdge -import com.twitter.hermit.predicate.socialgraph.SocialGraphPredicate -import com.twitter.service.metastore.gen.thriftscala.Location -import com.twitter.socialgraph.thriftscala.RelationshipType -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration -import com.twitter.util.Future - -object PredicatesForCandidate { - - def oldTweetRecsPredicate(implicit stats: StatsReceiver): Predicate[ - TweetCandidate with RecommendationType with TargetInfo[ - TargetUser with TargetABDecider with MaxTweetAge - ] - ] = { - val name = "old_tweet" - Predicate - .from[TweetCandidate with RecommendationType with TargetInfo[ - TargetUser with TargetABDecider with MaxTweetAge - ]] { candidate => - { - val crt = candidate.commonRecType - val defaultAge = if (RecTypes.mrModelingBasedTypes.contains(crt)) { - candidate.target.params(PushFeatureSwitchParams.ModelingBasedCandidateMaxTweetAgeParam) - } else if (RecTypes.GeoPopTweetTypes.contains(crt)) { - candidate.target.params(PushFeatureSwitchParams.GeoPopTweetMaxAgeInHours) - } else if (RecTypes.simclusterBasedTweets.contains(crt)) { - candidate.target.params( - PushFeatureSwitchParams.SimclusterBasedCandidateMaxTweetAgeParam) - } else if (RecTypes.detopicTypes.contains(crt)) { - candidate.target.params(PushFeatureSwitchParams.DetopicBasedCandidateMaxTweetAgeParam) - } else if (RecTypes.f1FirstDegreeTypes.contains(crt)) { - candidate.target.params(PushFeatureSwitchParams.F1CandidateMaxTweetAgeParam) - } else if (crt == CommonRecommendationType.ExploreVideoTweet) { - candidate.target.params(PushFeatureSwitchParams.ExploreVideoTweetAgeParam) - } else - candidate.target.params(PushFeatureSwitchParams.MaxTweetAgeParam) - SnowflakeUtils.isRecent(candidate.tweetId, defaultAge) - } - } - .withStats(stats.scope(name)) - .withName(name) - } - - def tweetIsNotAreply( - implicit stats: StatsReceiver - ): NamedPredicate[TweetCandidate with TweetDetails] = { - val name = "tweet_candidate_not_a_reply" - Predicate - .from[TweetCandidate with TweetDetails] { c => - c.isReply match { - case Some(true) => false - case _ => true - } - } - .withStats(stats.scope(name)) - .withName(name) - } - - /** - * Check if tweet contains any optouted free form interests. - * Currently, we use it for media categories and semantic core - * @param stats - * @return - */ - def noOptoutFreeFormInterestPredicate( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "free_form_interest_opt_out" - val tweetMediaAnnotationFeature = - "tweet.mediaunderstanding.tweet_annotations.safe_category_probabilities" - val tweetSemanticCoreFeature = - "tweet.core.tweet.semantic_core_annotations" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - val withOptOutFreeFormInterestsCounter = stats.counter("with_optout_interests") - val withoutOptOutInterestsCounter = stats.counter("without_optout_interests") - val withOptOutFreeFormInterestsFromMediaAnnotationCounter = - stats.counter("with_optout_interests_from_media_annotation") - val withOptOutFreeFormInterestsFromSemanticCoreCounter = - stats.counter("with_optout_interests_from_semantic_core") - Predicate - .fromAsync { candidate: PushCandidate => - val tweetSemanticCoreEntityIds = candidate.sparseBinaryFeatures - .getOrElse(tweetSemanticCoreFeature, Set.empty[String]).map { id => - id.split('.')(2) - }.toSet - val tweetMediaAnnotationIds = candidate.sparseContinuousFeatures - .getOrElse(tweetMediaAnnotationFeature, Map.empty[String, Double]).keys.toSet - - candidate.target.optOutFreeFormUserInterests.map { - case optOutUserInterests: Seq[String] => - withOptOutFreeFormInterestsCounter.incr() - val optOutUserInterestsSet = optOutUserInterests.toSet - val mediaAnnoIntersect = optOutUserInterestsSet.intersect(tweetMediaAnnotationIds) - val semanticCoreIntersect = optOutUserInterestsSet.intersect(tweetSemanticCoreEntityIds) - if (!mediaAnnoIntersect.isEmpty) { - withOptOutFreeFormInterestsFromMediaAnnotationCounter.incr() - } - if (!semanticCoreIntersect.isEmpty) { - withOptOutFreeFormInterestsFromSemanticCoreCounter.incr() - } - semanticCoreIntersect.isEmpty && mediaAnnoIntersect.isEmpty - case _ => - withoutOptOutInterestsCounter.incr() - true - } - } - .withStats(scopedStatsReceiver) - .withName(name) - } - - def tweetCandidateWithLessThan2SocialContextsIsAReply( - implicit stats: StatsReceiver - ): NamedPredicate[TweetCandidate with TweetDetails with SocialContextActions] = { - val name = "tweet_candidate_with_less_than_2_social_contexts_is_not_a_reply" - Predicate - .from[TweetCandidate with TweetDetails with SocialContextActions] { cand => - cand.isReply match { - case Some(true) if cand.socialContextTweetIds.size < 2 => false - case _ => true - } - } - .withStats(stats.scope(name)) - .withName(name) - } - - def f1CandidateIsNotAReply(implicit stats: StatsReceiver): NamedPredicate[F1Candidate] = { - val name = "f1_candidate_is_not_a_reply" - Predicate - .from[F1Candidate] { candidate => - candidate.isReply match { - case Some(true) => false - case _ => true - } - } - .withStats(stats.scope(name)) - .withName(name) - } - - def outOfNetworkTweetCandidateEnabledCrTag( - implicit stats: StatsReceiver - ): NamedPredicate[OutOfNetworkTweetCandidate with TargetInfo[TargetUser with TargetABDecider]] = { - val name = "out_of_network_tweet_candidate_enabled_crtag" - val scopedStats = stats.scope(name) - Predicate - .from[OutOfNetworkTweetCandidate with TargetInfo[TargetUser with TargetABDecider]] { cand => - val disabledCrTag = cand.target - .params(PushFeatureSwitchParams.OONCandidatesDisabledCrTagParam) - val candGeneratedByDisabledSignal = cand.tagsCR.exists { tagsCR => - val tagsCRSet = tagsCR.map(_.toString).toSet - tagsCRSet.nonEmpty && tagsCRSet.subsetOf(disabledCrTag.toSet) - } - if (candGeneratedByDisabledSignal) { - cand.tagsCR.getOrElse(Nil).foreach(tag => scopedStats.counter(tag.toString).incr()) - false - } else true - } - .withStats(scopedStats) - .withName(name) - } - - def outOfNetworkTweetCandidateEnabledCrtGroup( - implicit stats: StatsReceiver - ): NamedPredicate[OutOfNetworkTweetCandidate with TargetInfo[TargetUser with TargetABDecider]] = { - val name = "out_of_network_tweet_candidate_enabled_crt_group" - val scopedStats = stats.scope(name) - Predicate - .from[OutOfNetworkTweetCandidate with TargetInfo[TargetUser with TargetABDecider]] { cand => - val disabledCrtGroup = cand.target - .params(PushFeatureSwitchParams.OONCandidatesDisabledCrtGroupParam) - val crtGroup = CandidateUtil.getCrtGroup(cand.commonRecType) - val candGeneratedByDisabledCrt = disabledCrtGroup.contains(crtGroup) - if (candGeneratedByDisabledCrt) { - scopedStats.counter("filter_" + crtGroup.toString).incr() - false - } else true - } - .withStats(scopedStats) - .withName(name) - } - - def outOfNetworkTweetCandidateIsNotAReply( - implicit stats: StatsReceiver - ): NamedPredicate[OutOfNetworkTweetCandidate] = { - val name = "out_of_network_tweet_candidate_is_not_a_reply" - Predicate - .from[OutOfNetworkTweetCandidate] { cand => - cand.isReply match { - case Some(true) => false - case _ => true - } - } - .withStats(stats.scope(name)) - .withName(name) - } - - def recommendedTweetIsAuthoredBySelf( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate] = - Predicate - .from[PushCandidate] { - case tweetCandidate: PushCandidate with TweetDetails => - tweetCandidate.authorId match { - case Some(authorId) => authorId != tweetCandidate.target.targetId - case None => true - } - case _ => - true - } - .withStats(statsReceiver.scope("predicate_self_author")) - .withName("self_author") - - def authorInSocialContext(implicit statsReceiver: StatsReceiver): NamedPredicate[PushCandidate] = - Predicate - .from[PushCandidate] { - case tweetCandidate: PushCandidate with TweetDetails with SocialContextActions => - tweetCandidate.authorId match { - case Some(authorId) => - !tweetCandidate.socialContextUserIds.contains(authorId) - case None => true - } - case _ => true - } - .withStats(statsReceiver.scope("predicate_author_social_context")) - .withName("author_social_context") - - def selfInSocialContext(implicit statsReceiver: StatsReceiver): NamedPredicate[PushCandidate] = { - val name = "self_social_context" - Predicate - .from[PushCandidate] { - case candidate: PushCandidate with SocialContextActions => - !candidate.socialContextUserIds.contains(candidate.target.targetId) - case _ => - true - } - .withStats(statsReceiver.scope(s"${name}_predicate")) - .withName(name) - } - - def minSocialContext( - threshold: Int - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with SocialContextActions] = { - Predicate - .from { candidate: PushCandidate with SocialContextActions => - candidate.socialContextUserIds.size >= threshold - } - .withStats(statsReceiver.scope("predicate_min_social_context")) - .withName("min_social_context") - } - - private def anyWithheldContent( - userStore: ReadableStore[Long, User], - userCountryStore: ReadableStore[Long, Location] - )( - implicit statsReceiver: StatsReceiver - ): Predicate[TargetRecUser] = - GizmoduckUserPredicate.withheldContentPredicate( - userStore = userStore, - userCountryStore = userCountryStore, - statsReceiver = statsReceiver, - checkAllCountries = true - ) - - def targetUserExists(implicit statsReceiver: StatsReceiver): NamedPredicate[PushCandidate] = { - TargetUserPredicates - .targetUserExists()(statsReceiver) - .flatContraMap { candidate: PushCandidate => Future.value(candidate.target) } - .withName("target_user_exists") - } - - def secondaryDormantAccountPredicate( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "secondary_dormant_account" - TargetUserPredicates - .secondaryDormantAccountPredicate()(statsReceiver) - .on { candidate: PushCandidate => candidate.target } - .withStats(statsReceiver.scope(s"predicate_$name")) - .withName(name) - } - - def socialContextBeingFollowed( - edgeStore: ReadableStore[RelationEdge, Boolean] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with SocialContextActions] = - SocialGraphPredicate - .allRelationEdgesExist(edgeStore, RelationshipType.Following) - .on { candidate: PushCandidate with SocialContextActions => - candidate.socialContextUserIds.map { u => Edge(candidate.target.targetId, u) } - } - .withStats(statsReceiver.scope("predicate_social_context_being_followed")) - .withName("social_context_being_followed") - - private def edgeFromCandidate(candidate: PushCandidate with TweetAuthor): Option[Edge] = { - candidate.authorId map { authorId => Edge(candidate.target.targetId, authorId) } - } - - def authorNotBeingDeviceFollowed( - edgeStore: ReadableStore[RelationEdge, Boolean] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetAuthor] = { - SocialGraphPredicate - .relationExists(edgeStore, RelationshipType.DeviceFollowing) - .optionalOn( - edgeFromCandidate, - missingResult = false - ) - .flip - .withStats(statsReceiver.scope("predicate_author_not_device_followed")) - .withName("author_not_device_followed") - } - - def authorBeingFollowed( - edgeStore: ReadableStore[RelationEdge, Boolean] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetAuthor] = { - SocialGraphPredicate - .relationExists(edgeStore, RelationshipType.Following) - .optionalOn( - edgeFromCandidate, - missingResult = false - ) - .withStats(statsReceiver.scope("predicate_author_being_followed")) - .withName("author_being_followed") - } - - def authorNotBeingFollowed( - edgeStore: ReadableStore[RelationEdge, Boolean] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetAuthor] = { - SocialGraphPredicate - .relationExists(edgeStore, RelationshipType.Following) - .optionalOn( - edgeFromCandidate, - missingResult = false - ) - .flip - .withStats(statsReceiver.scope("predicate_author_not_being_followed")) - .withName("author_not_being_followed") - } - - def recommendedTweetAuthorAcceptableToTargetUser( - edgeStore: ReadableStore[RelationEdge, Boolean] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetAuthor] = { - val name = "recommended_tweet_author_acceptable_to_target_user" - SocialGraphPredicate - .anyRelationExists( - edgeStore, - Set( - RelationshipType.Blocking, - RelationshipType.BlockedBy, - RelationshipType.HideRecommendations, - RelationshipType.Muting - ) - ) - .flip - .optionalOn( - edgeFromCandidate, - missingResult = false - ) - .withStats(statsReceiver.scope(s"predicate_$name")) - .withName(name) - } - - def relationNotExistsPredicate( - edgeStore: ReadableStore[RelationEdge, Boolean], - relations: Set[RelationshipType] - ): Predicate[(Long, Iterable[Long])] = - SocialGraphPredicate - .anyRelationExistsForMultiEdge( - edgeStore, - relations - ) - .flip - .on { - case (targetUserId, userIds) => - MultiEdge(targetUserId, userIds.toSet) - } - - def blocking(edgeStore: ReadableStore[RelationEdge, Boolean]): Predicate[(Long, Iterable[Long])] = - relationNotExistsPredicate( - edgeStore, - Set(RelationshipType.BlockedBy, RelationshipType.Blocking) - ) - - def blockingOrMuting( - edgeStore: ReadableStore[RelationEdge, Boolean] - ): Predicate[(Long, Iterable[Long])] = - relationNotExistsPredicate( - edgeStore, - Set(RelationshipType.BlockedBy, RelationshipType.Blocking, RelationshipType.Muting) - ) - - def socialContextNotRetweetFollowing( - edgeStore: ReadableStore[RelationEdge, Boolean] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with SocialContextActions] = { - val name = "social_context_not_retweet_following" - relationNotExistsPredicate(edgeStore, Set(RelationshipType.NotRetweetFollowing)) - .optionalOn[PushCandidate with SocialContextActions]( - { - case candidate: PushCandidate with SocialContextActions - if RecTypes.isTweetRetweetType(candidate.commonRecType) => - Some((candidate.target.targetId, candidate.socialContextUserIds)) - case _ => - None - }, - missingResult = true - ) - .withStats(statsReceiver.scope(s"predicate_$name")) - .withName(name) - } - - def socialContextBlockingOrMuting( - edgeStore: ReadableStore[RelationEdge, Boolean] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with SocialContextActions] = - blockingOrMuting(edgeStore) - .on { candidate: PushCandidate with SocialContextActions => - (candidate.target.targetId, candidate.socialContextUserIds) - } - .withStats(statsReceiver.scope("predicate_social_context_blocking_or_muting")) - .withName("social_context_blocking_or_muting") - - /** - * Use hyrated Tweet object for F1 Protected experiment for checking null cast as Tweetypie hydration - * fails for protected Authors without passing in Target id. We do this specifically for - * F1 Protected Tweet Experiment in Earlybird Adaptor. - * For rest of the traffic refer to existing Nullcast Predicate - */ - def nullCastF1ProtectedExperientPredicate( - tweetypieStore: ReadableStore[Long, TweetyPieResult] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetCandidate with TweetDetails] = { - val name = "f1_exempted_null_cast_tweet" - val f1NullCastCheckCounter = statsReceiver.scope(name).counter("f1_null_cast_check") - Predicate - .fromAsync { tweetCandidate: PushCandidate with TweetCandidate with TweetDetails => - if (RecTypes.f1FirstDegreeTypes(tweetCandidate.commonRecType) && tweetCandidate.target - .params(PushFeatureSwitchParams.EnableF1FromProtectedTweetAuthors)) { - f1NullCastCheckCounter.incr() - tweetCandidate.tweet match { - case Some(tweetObj) => - baseNullCastTweet().apply(Seq(TweetyPieResult(tweetObj, None, None))).map(_.head) - case _ => Future.False - } - } else { - nullCastTweet(tweetypieStore).apply(Seq(tweetCandidate)).map(_.head) - } - } - .withStats(statsReceiver.scope(s"predicate_$name")) - .withName(name) - } - - private def baseNullCastTweet(): Predicate[TweetyPieResult] = - Predicate.from { t: TweetyPieResult => !t.tweet.coreData.exists { cd => cd.nullcast } } - - def nullCastTweet( - tweetyPieStore: ReadableStore[Long, TweetyPieResult] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetCandidate] = { - val name = "null_cast_tweet" - baseNullCastTweet() - .flatOptionContraMap[PushCandidate with TweetCandidate]( - f = (tweetCandidate: PushCandidate - with TweetCandidate) => tweetyPieStore.get(tweetCandidate.tweetId), - missingResult = false - ) - .withStats(statsReceiver.scope(s"predicate_$name")) - .withName(name) - } - - /** - * Use the predicate except fn is true. - */ - def exceptedPredicate[T <: PushCandidate]( - name: String, - fn: T => Future[Boolean], - predicate: Predicate[T] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[T] = { - Predicate - .fromAsync { e: T => fn(e) } - .or(predicate) - .withStats(statsReceiver.scope(name)) - .withName(name) - } - - /** - * - * @param edgeStore [[ReadableStore[RelationEdge, Boolean]]] - * @return - allow only out-network tweets if in-network tweets are disabled - */ - def disableInNetworkTweetPredicate( - edgeStore: ReadableStore[RelationEdge, Boolean] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetAuthor] = { - val name = "disable_in_network_tweet" - Predicate - .fromAsync { candidate: PushCandidate with TweetAuthor => - if (candidate.target.params(PushParams.DisableInNetworkTweetCandidatesParam)) { - authorNotBeingFollowed(edgeStore) - .apply(Seq(candidate)) - .map(_.head) - } else Future.True - }.withStats(statsReceiver.scope(name)) - .withName(name) - } - - /** - * - * @param edgeStore [[ReadableStore[RelationEdge, Boolean]]] - * @return - allow only in-network tweets if out-network tweets are disabled - */ - def disableOutNetworkTweetPredicate( - edgeStore: ReadableStore[RelationEdge, Boolean] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetAuthor] = { - val name = "disable_out_network_tweet" - Predicate - .fromAsync { candidate: PushCandidate with TweetAuthor => - if (candidate.target.params(PushFeatureSwitchParams.DisableOutNetworkTweetCandidatesFS)) { - authorBeingFollowed(edgeStore) - .apply(Seq(candidate)) - .map(_.head) - } else Future.True - }.withStats(statsReceiver.scope(name)) - .withName(name) - } - - def alwaysTruePredicate: NamedPredicate[PushCandidate] = { - Predicate - .all[PushCandidate] - .withName("predicate_AlwaysTrue") - } - - def alwaysTruePushCandidatePredicate: NamedPredicate[PushCandidate] = { - Predicate - .all[PushCandidate] - .withName("predicate_AlwaysTrue") - } - - def alwaysFalsePredicate(implicit statsReceiver: StatsReceiver): NamedPredicate[PushCandidate] = { - val name = "predicate_AlwaysFalse" - val scopedStatsReceiver = statsReceiver.scope(name) - Predicate - .from { candidate: PushCandidate => false } - .withStats(scopedStatsReceiver) - .withName(name) - } - - def accountCountryPredicate( - allowedCountries: Set[String] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "AccountCountryPredicate" - val stats = statsReceiver.scope(name) - AccountCountryPredicate(allowedCountries) - .on { candidate: PushCandidate => candidate.target } - .withStats(stats) - .withName(name) - } - - def paramPredicate[T <: PushCandidate]( - param: Param[Boolean] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[T] = { - val name = param.getClass.getSimpleName.stripSuffix("$") - TargetPredicates - .paramPredicate(param) - .on { candidate: PushCandidate => candidate.target } - .withStats(statsReceiver.scope(s"param_${name}_controlled_predicate")) - .withName(s"param_${name}_controlled_predicate") - } - - def isDeviceEligibleForNewsOrSports( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "is_device_eligible_for_news_or_sports" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - Predicate - .fromAsync { candidate: PushCandidate => - candidate.target.deviceInfo.map(_.exists(_.isNewsEligible)) - } - .withStats(scopedStatsReceiver) - .withName(name) - } - - def isDeviceEligibleForCreatorPush( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "is_device_eligible_for_creator_push" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - Predicate - .fromAsync { candidate: PushCandidate => - candidate.target.deviceInfo.map(_.exists(settings => - settings.isNewsEligible || settings.isRecommendationsEligible)) - } - .withStats(scopedStatsReceiver) - .withName(name) - } - - /** - * Like [[TargetUserPredicates.homeTimelineFatigue()]] but for candidate. - */ - def htlFatiguePredicate( - fatigueDuration: Param[Duration] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "htl_fatigue" - Predicate - .fromAsync { candidate: PushCandidate => - val _fatigueDuration = candidate.target.params(fatigueDuration) - TargetUserPredicates - .homeTimelineFatigue( - fatigueDuration = _fatigueDuration - ).apply(Seq(candidate.target)).map(_.head) - } - .withStats(statsReceiver.scope(name)) - .withName(name) - } - - def mrWebHoldbackPredicate( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "mr_web_holdback_for_candidate" - val scopedStats = stats.scope(name) - PredicatesForCandidate.exludeCrtFromPushHoldback - .or( - TargetPredicates - .webNotifsHoldback() - .on { candidate: PushCandidate => candidate.target } - ) - .withStats(scopedStats) - .withName(name) - } - - def candidateEnabledForEmailPredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "candidates_enabled_for_email" - Predicate - .from { candidate: PushCandidate => - if (candidate.target.isEmailUser) - candidate.isInstanceOf[TweetCandidate with TweetAuthor with RecommendationType] - else true - } - .withStats(stats.scope(name)) - .withName(name) - } - - def protectedTweetF1ExemptPredicate[ - T <: TargetUser with TargetABDecider, - Cand <: TweetCandidate with TweetAuthorDetails with TargetInfo[T] - ]( - implicit stats: StatsReceiver - ): NamedPredicate[ - TweetCandidate with TweetAuthorDetails with TargetInfo[ - TargetUser with TargetABDecider - ] - ] = { - val name = "f1_exempt_tweet_author_protected" - val skipForProtectedAuthorScope = stats.scope(name).scope("skip_protected_author_for_f1") - val authorIsProtectedCounter = skipForProtectedAuthorScope.counter("author_protected_true") - val authorIsNotProtectedCounter = skipForProtectedAuthorScope.counter("author_protected_false") - val authorNotFoundCounter = stats.scope(name).counter("author_not_found") - Predicate - .fromAsync[TweetCandidate with TweetAuthorDetails with TargetInfo[ - TargetUser with TargetABDecider - ]] { - case candidate: F1Candidate - if candidate.target.params(PushFeatureSwitchParams.EnableF1FromProtectedTweetAuthors) => - candidate.tweetAuthor.foreach { - case Some(author) => - if (GizmoduckUserPredicate.isProtected(author)) { - authorIsProtectedCounter.incr() - } else authorIsNotProtectedCounter.incr() - case _ => authorNotFoundCounter.incr() - } - Future.True - case cand => - TweetAuthorPredicates.recTweetAuthorProtected.apply(Seq(cand)).map(_.head) - } - .withStats(stats.scope(name)) - .withName(name) - } - - /** - * filter a notification if user has already received ANY prior notification about the space id - * @param stats - * @return - */ - def duplicateSpacesPredicate( - implicit stats: StatsReceiver - ): NamedPredicate[Space with PushCandidate] = { - val name = "duplicate_spaces_predicate" - Predicate - .fromAsync { c: Space with PushCandidate => - c.target.pushRecItems.map { pushRecItems => - !pushRecItems.spaceIds.contains(c.spaceId) - } - } - .withStats(stats.scope(name)) - .withName(name) - } - - def filterOONCandidatePredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "filter_oon_candidate" - - Predicate - .fromAsync[PushCandidate] { cand => - val crt = cand.commonRecType - val isOONCandidate = - RecTypes.isOutOfNetworkTweetRecType(crt) || RecTypes.outOfNetworkTopicTweetTypes - .contains(crt) || RecTypes.isOutOfNetworkSpaceType(crt) || RecTypes.userTypes.contains( - crt) - if (isOONCandidate) { - cand.target.notificationsFromOnlyPeopleIFollow.map { inNetworkOnly => - if (inNetworkOnly) { - stats.scope(name, crt.toString).counter("inNetworkOnlyOn").incr() - } else { - stats.scope(name, crt.toString).counter("inNetworkOnlyOff").incr() - } - !(inNetworkOnly && cand.target.params( - PushFeatureSwitchParams.EnableOONFilteringBasedOnUserSettings)) - } - } else Future.True - } - .withStats(stats.scope(name)) - .withName(name) - } - - def exludeCrtFromPushHoldback( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate] = Predicate - .from { candidate: PushCandidate => - val crtName = candidate.commonRecType.name - val target = candidate.target - target - .params(PushFeatureSwitchParams.CommonRecommendationTypeDenyListPushHoldbacks) - .exists(crtName.equalsIgnoreCase) - } - .withStats(stats.scope("exclude_crt_from_push_holdbacks")) - - def enableSendHandlerCandidates(implicit stats: StatsReceiver): NamedPredicate[PushCandidate] = { - val name = "sendhandler_enable_push_recommendations" - PredicatesForCandidate.exludeCrtFromPushHoldback - .or(PredicatesForCandidate.paramPredicate( - PushFeatureSwitchParams.EnablePushRecommendationsParam)) - .withStats(stats.scope(name)) - .withName(name) - } - - def openAppExperimentUserCandidateAllowList( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "open_app_experiment_user_candidate_allow_list" - Predicate - .fromAsync { candidate: PushCandidate => - val target = candidate.target - Future.join(target.isOpenAppExperimentUser, target.targetUser).map { - case (isOpenAppUser, targetUser) => - val shouldLimitOpenAppCrts = - isOpenAppUser || targetUser.exists(_.userType == UserType.Soft) - - if (shouldLimitOpenAppCrts) { - val listOfAllowedCrt = target - .params(PushFeatureSwitchParams.ListOfCrtsForOpenApp) - .flatMap(CommonRecommendationType.valueOf) - listOfAllowedCrt.contains(candidate.commonRecType) - } else true - } - }.withStats(stats.scope(name)) - .withName(name) - } - - def isTargetBlueVerified( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "is_target_already_blue_verified" - Predicate - .fromAsync { candidate: PushCandidate => - val target = candidate.target - target.isBlueVerified.map(_.getOrElse(false)) - }.withStats(stats.scope(name)) - .withName(name) - } - - def isTargetLegacyVerified( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "is_target_already_legacy_verified" - Predicate - .fromAsync { candidate: PushCandidate => - val target = candidate.target - target.isVerified.map(_.getOrElse(false)) - }.withStats(stats.scope(name)) - .withName(name) - } - - def isTargetSuperFollowCreator(implicit stats: StatsReceiver): NamedPredicate[PushCandidate] = { - val name = "is_target_already_super_follow_creator" - Predicate - .fromAsync { candidate: PushCandidate => - val target = candidate.target - target.isSuperFollowCreator.map( - _.getOrElse(false) - ) - }.withStats(stats.scope(name)) - .withName(name) - } - - def isChannelValidPredicate( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "is_channel_valid" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - Predicate - .fromAsync { candidate: PushCandidate => - candidate - .getChannels().map(channels => - !(channels.toSet.size == 1 && channels.head == ChannelName.None)) - } - .withStats(scopedStatsReceiver) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/SGSPredicatesForCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/SGSPredicatesForCandidate.scala deleted file mode 100644 index e335c8d9c..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/SGSPredicatesForCandidate.scala +++ /dev/null @@ -1,174 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.SocialGraphServiceRelationshipMap -import com.twitter.frigate.common.base.TweetAuthor -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.gizmoduck.thriftscala.UserType -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.hermit.predicate.socialgraph.Edge -import com.twitter.hermit.predicate.socialgraph.RelationEdge -import com.twitter.socialgraph.thriftscala.RelationshipType -import com.twitter.util.Future - -/** - * Refactor SGS predicates so that predicates can use relationshipMap we generate in hydrate step - */ -object SGSPredicatesForCandidate { - - case class RelationshipMapEdge(edge: Edge, relationshipMap: Map[RelationEdge, Boolean]) - - private def relationshipMapEdgeFromCandidate( - candidate: PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap - ): Option[RelationshipMapEdge] = { - candidate.authorId map { authorId => - RelationshipMapEdge(Edge(candidate.target.targetId, authorId), candidate.relationshipMap) - } - } - - def authorBeingFollowed( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap] = { - val name = "author_not_being_followed" - val stats = statsReceiver.scope(name) - val softUserCounter = stats.counter("soft_user") - - val sgsAuthorBeingFollowedPredicate = Predicate - .from { relationshipMapEdge: RelationshipMapEdge => - anyRelationExist(relationshipMapEdge, Set(RelationshipType.Following)) - } - - Predicate - .fromAsync { - candidate: PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap => - val target = candidate.target - target.targetUser.flatMap { - case Some(gizmoduckUser) if gizmoduckUser.userType == UserType.Soft => - softUserCounter.incr() - target.seedsWithWeight.map { followedUsersWithWeightOpt => - candidate.authorId match { - case Some(authorId) => - val followedUsers = followedUsersWithWeightOpt.getOrElse(Map.empty).keys - followedUsers.toSet.contains(authorId) - - case None => false - } - } - - case _ => - sgsAuthorBeingFollowedPredicate - .optionalOn(relationshipMapEdgeFromCandidate, missingResult = false) - .apply(Seq(candidate)) - .map(_.head) - } - }.withStats(stats) - .withName(name) - } - - def authorNotBeingDeviceFollowed( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap] = { - val name = "author_being_device_followed" - Predicate - .from { relationshipMapEdge: RelationshipMapEdge => - { - anyRelationExist(relationshipMapEdge, Set(RelationshipType.DeviceFollowing)) - } - } - .optionalOn(relationshipMapEdgeFromCandidate, missingResult = false) - .flip - .withStats(statsReceiver.scope(name)) - .withName(name) - } - - def recommendedTweetAuthorAcceptableToTargetUser( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap] = { - val name = "recommended_tweet_author_not_acceptable_to_target_user" - Predicate - .from { relationshipMapEdge: RelationshipMapEdge => - { - anyRelationExist( - relationshipMapEdge, - Set( - RelationshipType.Blocking, - RelationshipType.BlockedBy, - RelationshipType.HideRecommendations, - RelationshipType.Muting - )) - } - } - .flip - .optionalOn(relationshipMapEdgeFromCandidate, missingResult = false) - .withStats(statsReceiver.scope(name)) - .withName(name) - } - - def authorNotBeingFollowed( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap] = { - Predicate - .from { relationshipMapEdge: RelationshipMapEdge => - { - anyRelationExist(relationshipMapEdge, Set(RelationshipType.Following)) - } - } - .optionalOn(relationshipMapEdgeFromCandidate, missingResult = false) - .flip - .withStats(statsReceiver.scope("predicate_author_not_being_followed_pre_ranking")) - .withName("author_not_being_followed") - } - - def disableInNetworkTweetPredicate( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap] = { - val name = "enable_in_network_tweet" - Predicate - .fromAsync { - candidate: PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap => - if (candidate.target.params(PushParams.DisableInNetworkTweetCandidatesParam)) { - authorNotBeingFollowed - .apply(Seq(candidate)) - .map(_.head) - } else Future.True - }.withStats(statsReceiver.scope(name)) - .withName(name) - } - - def disableOutNetworkTweetPredicate( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap] = { - val name = "enable_out_network_tweet" - Predicate - .fromAsync { - candidate: PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap => - if (candidate.target.params(PushFeatureSwitchParams.DisableOutNetworkTweetCandidatesFS)) { - authorBeingFollowed - .apply(Seq(candidate)) - .map(_.head) - } else Future.True - }.withStats(statsReceiver.scope(name)) - .withName(name) - } - - /** - * Returns true if the provided relationshipEdge exists among - * @param candidate candidate - * @param relationships relaionships - * @return Boolean result - */ - private def anyRelationExist( - relationshipMapEdge: RelationshipMapEdge, - relationships: Set[RelationshipType] - ): Boolean = { - val resultSeq = relationships.map { relationship => - relationshipMapEdge.relationshipMap.getOrElse( - RelationEdge(relationshipMapEdge.edge, relationship), - false) - }.toSeq - resultSeq.contains(true) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ScarecrowPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ScarecrowPredicate.scala deleted file mode 100644 index a4728eba2..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ScarecrowPredicate.scala +++ /dev/null @@ -1,138 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.thriftscala._ -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.scarecrow.{ScarecrowPredicate => HermitScarecrowPredicate} -import com.twitter.relevance.feature_store.thriftscala.FeatureData -import com.twitter.relevance.feature_store.thriftscala.FeatureValue -import com.twitter.service.gen.scarecrow.thriftscala.Event -import com.twitter.service.gen.scarecrow.thriftscala.TieredActionResult -import com.twitter.storehaus.ReadableStore - -object ScarecrowPredicate { - val name = "" - - def candidateToEvent(candidate: PushCandidate): Event = { - val recommendedUserIdOpt = candidate match { - case tweetCandidate: TweetCandidate with TweetAuthor => - tweetCandidate.authorId - case userCandidate: UserCandidate => - Some(userCandidate.userId) - case _ => None - } - val hashtagsInTweet = candidate match { - case tweetCandidate: TweetCandidate with TweetDetails => - tweetCandidate.tweetyPieResult - .flatMap { tweetPieResult => - tweetPieResult.tweet.hashtags.map(_.map(_.text)) - }.getOrElse(Nil) - case _ => - Nil - } - val urlsInTweet = candidate match { - case tweetCandidate: TweetCandidate with TweetDetails => - tweetCandidate.tweetyPieResult - .flatMap { tweetPieResult => - tweetPieResult.tweet.urls.map(_.flatMap(_.expanded)) - } - case _ => None - } - val tweetIdOpt = candidate match { - case tweetCandidate: TweetCandidate => - Some(tweetCandidate.tweetId) - case _ => - None - } - val urlOpt = candidate match { - case candidate: UrlCandidate => - Some(candidate.url) - case _ => - None - } - val scUserIds = candidate match { - case hasSocialContext: SocialContextActions => Some(hasSocialContext.socialContextUserIds) - case _ => None - } - - val eventTitleOpt = candidate match { - case eventCandidate: EventCandidate with EventDetails => - Some(eventCandidate.eventTitle) - case _ => - None - } - - val urlTitleOpt = candidate match { - case candidate: UrlCandidate => - candidate.title - case _ => - None - } - - val urlDescriptionOpt = candidate match { - case candidate: UrlCandidate with UrlCandidateWithDetails => - candidate.description - case _ => - None - } - - Event( - "magicrecs_recommendation_write", - Map( - "targetUserId" -> FeatureData(Some(FeatureValue.LongValue(candidate.target.targetId))), - "type" -> FeatureData( - Some( - FeatureValue.StrValue(candidate.commonRecType.name) - ) - ), - "recommendedUserId" -> FeatureData(recommendedUserIdOpt map { id => - FeatureValue.LongValue(id) - }), - "tweetId" -> FeatureData(tweetIdOpt map { id => - FeatureValue.LongValue(id) - }), - "url" -> FeatureData(urlOpt map { url => - FeatureValue.StrValue(url) - }), - "hashtagsInTweet" -> FeatureData(Some(FeatureValue.StrListValue(hashtagsInTweet))), - "urlsInTweet" -> FeatureData(urlsInTweet.map(FeatureValue.StrListValue)), - "socialContexts" -> FeatureData(scUserIds.map { sc => - FeatureValue.LongListValue(sc) - }), - "eventTitle" -> FeatureData(eventTitleOpt.map { eventTitle => - FeatureValue.StrValue(eventTitle) - }), - "urlTitle" -> FeatureData(urlTitleOpt map { title => - FeatureValue.StrValue(title) - }), - "urlDescription" -> FeatureData(urlDescriptionOpt map { des => - FeatureValue.StrValue(des) - }) - ) - ) - } - - def candidateToPossibleEvent(c: PushCandidate): Option[Event] = { - if (c.frigateNotification.notificationDisplayLocation == NotificationDisplayLocation.PushToMobileDevice) { - Some(candidateToEvent(c)) - } else { - None - } - } - - def apply( - scarecrowCheckEventStore: ReadableStore[Event, TieredActionResult] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate] = { - HermitScarecrowPredicate(scarecrowCheckEventStore) - .optionalOn( - candidateToPossibleEvent, - missingResult = true - ) - .withStats(statsReceiver.scope(s"predicate_$name")) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/SpacePredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/SpacePredicate.scala deleted file mode 100644 index 044c0afdb..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/SpacePredicate.scala +++ /dev/null @@ -1,153 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.SpaceCandidate -import com.twitter.frigate.common.base.SpaceCandidateDetails -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.hermit.predicate.socialgraph.Edge -import com.twitter.hermit.predicate.socialgraph.RelationEdge -import com.twitter.hermit.predicate.socialgraph.SocialGraphPredicate -import com.twitter.socialgraph.thriftscala.RelationshipType -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.response.Err -import com.twitter.ubs.thriftscala.AudioSpace -import com.twitter.ubs.thriftscala.BroadcastState -import com.twitter.ubs.thriftscala.ParticipantUser -import com.twitter.ubs.thriftscala.Participants -import com.twitter.util.Future - -object SpacePredicate { - - /** Filters the request if the target is present in the space as a listener, speakeTestConfigr, or admin */ - def targetInSpace( - audioSpaceParticipantsStore: ReadableStore[String, Participants] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[SpaceCandidateDetails with RawCandidate] = { - val name = "target_in_space" - Predicate - .fromAsync[SpaceCandidateDetails with RawCandidate] { spaceCandidate => - audioSpaceParticipantsStore.get(spaceCandidate.spaceId).map { - case Some(participants) => - val allParticipants: Seq[ParticipantUser] = - (participants.admins ++ participants.speakers ++ participants.listeners).flatten.toSeq - val isInSpace = allParticipants.exists { participant => - participant.twitterUserId.contains(spaceCandidate.target.targetId) - } - !isInSpace - case None => false - } - }.withStats(statsReceiver.scope(name)) - .withName(name) - } - - /** - * - * @param audioSpaceStore: space metadata store - * @param statsReceiver: record stats - * @return: true if the space not started ELSE false to filter out notification - */ - def scheduledSpaceStarted( - audioSpaceStore: ReadableStore[String, AudioSpace] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[SpaceCandidate with RawCandidate] = { - val name = "scheduled_space_started" - Predicate - .fromAsync[SpaceCandidate with RawCandidate] { spaceCandidate => - audioSpaceStore - .get(spaceCandidate.spaceId) - .map(_.exists(_.state.contains(BroadcastState.NotStarted))) - .rescue { - case Err(Err.Authorization, _, _) => - Future.False - } - } - .withStats(statsReceiver.scope(name)) - .withName(name) - } - - private def relationshipMapEdgeFromSpaceCandidate( - candidate: RawCandidate with SpaceCandidate - ): Option[(Long, Seq[Long])] = { - candidate.hostId.map { spaceHostId => - (candidate.target.targetId, Seq(spaceHostId)) - } - } - - /** - * Check only host block for scheduled space reminders - * @return: True if no blocking relation between host and target user, else False - */ - def spaceHostTargetUserBlocking( - edgeStore: ReadableStore[RelationEdge, Boolean] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[SpaceCandidate with RawCandidate] = { - val name = "space_host_target_user_blocking" - PredicatesForCandidate - .blocking(edgeStore) - .optionalOn(relationshipMapEdgeFromSpaceCandidate, false) - .withStats(statsReceiver.scope(name)) - .withName(name) - } - - private def edgeFromCandidate( - candidate: PushCandidate with TweetAuthorDetails - ): Future[Option[Edge]] = { - candidate.tweetAuthor.map(_.map { author => Edge(candidate.target.targetId, author.id) }) - } - - def recommendedTweetAuthorAcceptableToTargetUser( - edgeStore: ReadableStore[RelationEdge, Boolean] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetAuthorDetails] = { - val name = "recommended_tweet_author_acceptable_to_target_user" - SocialGraphPredicate - .anyRelationExists( - edgeStore, - Set( - RelationshipType.Blocking, - RelationshipType.BlockedBy, - RelationshipType.HideRecommendations, - RelationshipType.Muting - ) - ) - .flip - .flatOptionContraMap( - edgeFromCandidate, - missingResult = false - ) - .withStats(statsReceiver.scope(s"predicate_$name")) - .withName(name) - } - - def narrowCastSpace( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[SpaceCandidateDetails with RawCandidate] = { - val name = "narrow_cast_space" - val narrowCastSpaceScope = statsReceiver.scope(name) - val employeeSpaceCounter = narrowCastSpaceScope.counter("employees") - val superFollowerSpaceCounter = narrowCastSpaceScope.counter("super_followers") - - Predicate - .fromAsync[SpaceCandidateDetails with RawCandidate] { candidate => - candidate.audioSpaceFut.map { - case Some(audioSpace) if audioSpace.narrowCastSpaceType.contains(1L) => - employeeSpaceCounter.incr() - candidate.target.params(PushFeatureSwitchParams.EnableEmployeeOnlySpaceNotifications) - case Some(audioSpace) if audioSpace.narrowCastSpaceType.contains(2L) => - superFollowerSpaceCounter.incr() - false - case _ => true - } - }.withStats(narrowCastSpaceScope) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetEngagementPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetEngagementPredicate.scala deleted file mode 100644 index d02e5b89a..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetEngagementPredicate.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.tweetypie.EngagementsPredicate -import com.twitter.hermit.predicate.tweetypie.Perspective -import com.twitter.hermit.predicate.tweetypie.UserTweet -import com.twitter.storehaus.ReadableStore - -object TargetEngagementPredicate { - val name = "target_engagement" - def apply( - perspectiveStore: ReadableStore[UserTweet, Perspective], - defaultForMissing: Boolean - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetCandidate] = { - EngagementsPredicate(perspectiveStore, defaultForMissing) - .on { candidate: PushCandidate with TweetCandidate => - UserTweet(candidate.target.targetId, candidate.tweetId) - } - .withStats(statsReceiver.scope(s"predicate_$name")) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetNtabCaretClickFatiguePredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetNtabCaretClickFatiguePredicate.scala deleted file mode 100644 index c5042bbc8..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetNtabCaretClickFatiguePredicate.scala +++ /dev/null @@ -1,91 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.TargetUser -import com.twitter.frigate.common.candidate.CaretFeedbackHistory -import com.twitter.frigate.common.candidate.FrigateHistory -import com.twitter.frigate.common.candidate.HTLVisitHistory -import com.twitter.frigate.common.candidate.TargetABDecider -import com.twitter.frigate.common.history.History -import com.twitter.frigate.common.predicate.FrigateHistoryFatiguePredicate.TimeSeries -import com.twitter.frigate.common.predicate.ntab_caret_fatigue.NtabCaretClickFatiguePredicateHelper -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.common.util.FeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.frigate.common.predicate.{FatiguePredicate => CommonFatiguePredicate} - -object TargetNtabCaretClickFatiguePredicate { - import NtabCaretClickFatiguePredicateHelper._ - - private val MagicRecsCategory = "MagicRecs" - - def apply[ - T <: TargetUser with TargetABDecider with CaretFeedbackHistory with FrigateHistory with HTLVisitHistory - ]( - filterHistory: TimeSeries => TimeSeries = - CommonFatiguePredicate.recTypesOnlyFilter(RecTypes.sharedNTabCaretFatigueTypes), - filterCaretFeedbackHistory: TargetUser with TargetABDecider with CaretFeedbackHistory => Seq[ - CaretFeedbackDetails - ] => Seq[CaretFeedbackDetails] = - CaretFeedbackHistoryFilter.caretFeedbackHistoryFilter(Seq(MagicRecsCategory)), - calculateFatiguePeriod: Seq[CaretFeedbackDetails] => Duration = calculateFatiguePeriodMagicRecs, - useMostRecentDislikeTime: Boolean = false, - name: String = "NtabCaretClickFatiguePredicate" - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[T] = { - - val scopedStats = statsReceiver.scope(name) - val crtStats = scopedStats.scope("crt") - Predicate - .fromAsync { target: T => - Future.join(target.history, target.caretFeedbacks).map { - case (history, Some(feedbackDetails)) => { - val feedbackDetailsDeduped = dedupFeedbackDetails( - filterCaretFeedbackHistory(target)(feedbackDetails), - scopedStats - ) - - val fatiguePeriod = - if (hasUserDislikeInLast30Days(feedbackDetailsDeduped) && target.params( - PushFeatureSwitchParams.EnableReducedFatigueRulesForSeeLessOften)) { - durationToFilterMRForSeeLessOftenExpt( - feedbackDetailsDeduped, - target.params(FeatureSwitchParams.NumberOfDaysToFilterMRForSeeLessOften), - target.params(FeatureSwitchParams.NumberOfDaysToReducePushCapForSeeLessOften), - scopedStats - ) - } else { - calculateFatiguePeriod(feedbackDetailsDeduped) - } - - val crtlist = feedbackDetailsDeduped - .flatMap { fd => - fd.genericNotificationMetadata.map { gm => - gm.genericType.name - } - }.distinct.sorted.mkString("-") - - if (fatiguePeriod > 0.days) { - crtStats.scope(crtlist).counter("fatigued").incr() - } else { - crtStats.scope(crtlist).counter("non_fatigued").incr() - } - - val hasRecentSent = - hasRecentSend(History(filterHistory(history.history.toSeq).toMap), fatiguePeriod) - !hasRecentSent - } - case _ => true - } - } - .withStats(scopedStats) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetPredicates.scala deleted file mode 100644 index 45d0b7578..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetPredicates.scala +++ /dev/null @@ -1,292 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.TargetUser -import com.twitter.frigate.common.candidate.FrigateHistory -import com.twitter.frigate.common.candidate.HTLVisitHistory -import com.twitter.frigate.common.candidate.TargetABDecider -import com.twitter.frigate.common.candidate.UserDetails -import com.twitter.frigate.common.predicate.TargetUserPredicates -import com.twitter.frigate.common.predicate.{FatiguePredicate => CommonFatiguePredicate} -import com.twitter.frigate.common.store.deviceinfo.MobileClientType -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.target.TargetScoringDetails -import com.twitter.frigate.pushservice.util.PushCapUtil -import com.twitter.frigate.thriftscala.NotificationDisplayLocation -import com.twitter.frigate.thriftscala.{CommonRecommendationType => CRT} -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration -import com.twitter.util.Future - -object TargetPredicates { - - def paramPredicate[T <: Target]( - param: Param[Boolean] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[T] = { - val name = param.getClass.getSimpleName.stripSuffix("$") - Predicate - .from { target: T => target.params(param) } - .withStats(statsReceiver.scope(s"param_${name}_controlled_predicate")) - .withName(s"param_${name}_controlled_predicate") - } - - /** - * Use the predicate except fn is true., Same as the candidate version but for Target - */ - def exceptedPredicate[T <: TargetUser]( - name: String, - fn: T => Future[Boolean], - predicate: Predicate[T] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[T] = { - Predicate - .fromAsync { e: T => fn(e) } - .or(predicate) - .withStats(statsReceiver.scope(name)) - .withName(name) - } - - /** - * Refresh For push handler target user predicate to fatigue on visiting Home timeline - */ - def targetHTLVisitPredicate[ - T <: TargetUser with UserDetails with TargetABDecider with HTLVisitHistory - ]( - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[T] = { - val name = "target_htl_visit_predicate" - Predicate - .fromAsync { target: T => - val hoursToFatigue = target.params(PushFeatureSwitchParams.HTLVisitFatigueTime) - TargetUserPredicates - .homeTimelineFatigue(hoursToFatigue.hours) - .apply(Seq(target)) - .map(_.head) - } - .withStats(statsReceiver.scope(name)) - .withName(name) - } - - def targetPushBitEnabledPredicate[T <: Target]( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[T] = { - val name = "push_bit_enabled" - val scopedStats = statsReceiver.scope(s"targetpredicate_$name") - - Predicate - .fromAsync { target: T => - target.deviceInfo - .map { info => - info.exists { deviceInfo => - deviceInfo.isRecommendationsEligible || - deviceInfo.isNewsEligible || - deviceInfo.isTopicsEligible || - deviceInfo.isSpacesEligible - } - } - }.withStats(scopedStats) - .withName(name) - } - - def targetFatiguePredicate[T <: Target]( - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[T] = { - val name = "target_fatigue_predicate" - val predicateStatScope = statsReceiver.scope(name) - Predicate - .fromAsync { target: T => - PushCapUtil - .getPushCapFatigue(target, predicateStatScope) - .flatMap { pushCapInfo => - CommonFatiguePredicate - .magicRecsPushTargetFatiguePredicate( - interval = pushCapInfo.fatigueInterval, - maxInInterval = pushCapInfo.pushcap - ) - .apply(Seq(target)) - .map(_.headOption.getOrElse(false)) - } - } - .withStats(predicateStatScope) - .withName(name) - } - - def teamExceptedPredicate[T <: TargetUser]( - predicate: NamedPredicate[T] - )( - implicit stats: StatsReceiver - ): NamedPredicate[T] = { - Predicate - .fromAsync { t: T => t.isTeamMember } - .or(predicate) - .withStats(stats.scope(predicate.name)) - .withName(predicate.name) - } - - def targetValidMobileSDKPredicate[T <: Target]( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[T] = { - val name = "valid_mobile_sdk" - val scopedStats = statsReceiver.scope(s"targetpredicate_$name") - - Predicate - .fromAsync { target: T => - TargetUserPredicates.validMobileSDKPredicate - .apply(Seq(target)).map(_.headOption.getOrElse(false)) - }.withStats(scopedStats) - .withName(name) - } - - def magicRecsMinDurationSinceSent[T <: Target]( - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[T] = { - val name = "target_min_duration_since_push" - Predicate - .fromAsync { target: T => - PushCapUtil.getMinDurationSincePush(target, statsReceiver).flatMap { minDurationSincePush => - CommonFatiguePredicate - .magicRecsMinDurationSincePush(interval = minDurationSincePush) - .apply(Seq(target)).map(_.head) - } - } - .withStats(statsReceiver.scope(name)) - .withName(name) - } - - def optoutProbPredicate[ - T <: TargetUser with TargetABDecider with TargetScoringDetails with FrigateHistory - ]( - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[T] = { - val name = "target_has_high_optout_probability" - Predicate - .fromAsync { target: T => - val isNewUser = target.is30DayNewUserFromSnowflakeIdTime - if (isNewUser) { - statsReceiver.scope(name).counter("all_new_users").incr() - } - target.bucketOptoutProbability - .flatMap { - case Some(optoutProb) => - if (optoutProb >= target.params(PushFeatureSwitchParams.BucketOptoutThresholdParam)) { - CommonFatiguePredicate - .magicRecsPushTargetFatiguePredicate( - interval = 24.hours, - maxInInterval = target.params(PushFeatureSwitchParams.OptoutExptPushCapParam) - ) - .apply(Seq(target)) - .map { values => - val isValid = values.headOption.getOrElse(false) - if (!isValid && isNewUser) { - statsReceiver.scope(name).counter("filtered_new_users").incr() - } - isValid - } - } else Future.True - case _ => Future.True - } - } - .withStats(statsReceiver.scope(name)) - .withName(name) - } - - /** - * Predicate used to specify CRT fatigue given interval and max number of candidates within interval. - * @param crt The specific CRT that this predicate is being applied to - * @param intervalParam The fatigue interval - * @param maxInIntervalParam The max number of the given CRT's candidates that are acceptable - * in the interval - * @param stats StatsReceiver - * @return Target Predicate - */ - def pushRecTypeFatiguePredicate( - crt: CRT, - intervalParam: Param[Duration], - maxInIntervalParam: FSBoundedParam[Int], - stats: StatsReceiver - ): Predicate[Target] = - Predicate.fromAsync { target: Target => - val interval = target.params(intervalParam) - val maxIninterval = target.params(maxInIntervalParam) - CommonFatiguePredicate - .recTypeTargetFatiguePredicate( - interval = interval, - maxInInterval = maxIninterval, - recommendationType = crt, - notificationDisplayLocation = NotificationDisplayLocation.PushToMobileDevice, - minInterval = 30.minutes - )(stats.scope(s"${crt}_push_candidate_fatigue")).apply(Seq(target)).map(_.head) - } - - def inlineActionFatiguePredicate( - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[Target] = { - val name = "inline_action_fatigue" - val predicateRequests = statsReceiver.scope(name).counter("requests") - val targetIsInExpt = statsReceiver.scope(name).counter("target_in_expt") - val predicateEnabled = statsReceiver.scope(name).counter("enabled") - val predicateDisabled = statsReceiver.scope(name).counter("disabled") - val inlineFatigueDisabled = statsReceiver.scope(name).counter("inline_fatigue_disabled") - - Predicate - .fromAsync { target: Target => - predicateRequests.incr() - if (target.params(PushFeatureSwitchParams.TargetInInlineActionAppVisitFatigue)) { - targetIsInExpt.incr() - target.inlineActionHistory.map { inlineHistory => - if (inlineHistory.nonEmpty && target.params( - PushFeatureSwitchParams.EnableInlineActionAppVisitFatigue)) { - predicateEnabled.incr() - val inlineFatigue = target.params(PushFeatureSwitchParams.InlineActionAppVisitFatigue) - val lookbackInMs = inlineFatigue.ago.inMilliseconds - val filteredHistory = inlineHistory.filter { - case (time, _) => time > lookbackInMs - } - filteredHistory.isEmpty - } else { - inlineFatigueDisabled.incr() - true - } - } - } else { - predicateDisabled.incr() - Future.True - } - } - .withStats(statsReceiver.scope(name)) - .withName(name) - } - - def webNotifsHoldback[T <: TargetUser with UserDetails with TargetABDecider]( - )( - implicit stats: StatsReceiver - ): NamedPredicate[T] = { - val name = "mr_web_notifs_holdback" - Predicate - .fromAsync { targetUserContext: T => - targetUserContext.deviceInfo.map { deviceInfoOpt => - val isPrimaryWeb = deviceInfoOpt.exists { - _.guessedPrimaryClient.exists { clientType => - clientType == MobileClientType.Web - } - } - !(isPrimaryWeb && targetUserContext.params(PushFeatureSwitchParams.MRWebHoldbackParam)) - } - } - .withStats(stats.scope(s"predicate_$name")) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TopTweetImpressionsPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TopTweetImpressionsPredicates.scala deleted file mode 100644 index be5993b71..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TopTweetImpressionsPredicates.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.model.TopTweetImpressionsPushCandidate -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate - -object TopTweetImpressionsPredicates { - - def topTweetImpressionsFatiguePredicate( - implicit stats: StatsReceiver - ): NamedPredicate[TopTweetImpressionsPushCandidate] = { - val name = "top_tweet_impressions_fatigue" - val scopedStats = stats.scope(name) - val bucketImpressionCounter = scopedStats.counter("bucket_impression_count") - Predicate - .fromAsync { candidate: TopTweetImpressionsPushCandidate => - val interval = candidate.target.params(FS.TopTweetImpressionsNotificationInterval) - val maxInInterval = candidate.target.params(FS.MaxTopTweetImpressionsNotifications) - val minInterval = candidate.target.params(FS.TopTweetImpressionsFatigueMinIntervalDuration) - bucketImpressionCounter.incr() - - val fatiguePredicate = FatiguePredicate.recTypeOnly( - interval = interval, - maxInInterval = maxInInterval, - minInterval = minInterval, - recommendationType = CommonRecommendationType.TweetImpressions - ) - fatiguePredicate.apply(Seq(candidate)).map(_.head) - } - .withStats(stats.scope(s"predicate_${name}")) - .withName(name) - } - - def topTweetImpressionsThreshold( - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[TopTweetImpressionsPushCandidate] = { - val name = "top_tweet_impressions_threshold" - val scopedStats = statsReceiver.scope(name) - val meetsImpressionsCounter = scopedStats.counter("meets_impressions_count") - val bucketImpressionCounter = scopedStats.counter("bucket_impression_count") - Predicate - .from[TopTweetImpressionsPushCandidate] { candidate => - val meetsImpressionsThreshold = - candidate.impressionsCount >= candidate.target.params(FS.TopTweetImpressionsThreshold) - if (meetsImpressionsThreshold) meetsImpressionsCounter.incr() - bucketImpressionCounter.incr() - meetsImpressionsThreshold - } - .withStats(statsReceiver.scope(s"predicate_${name}")) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetEngagementRatioPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetEngagementRatioPredicate.scala deleted file mode 100644 index d1a3a1c64..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetEngagementRatioPredicate.scala +++ /dev/null @@ -1,112 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushConstants -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.util.CandidateUtil -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.util.Future - -object TweetEngagementRatioPredicate { - - def QTtoNtabClickBasedPredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[ - PushCandidate with TweetCandidate with RecommendationType - ] = { - val name = "oon_tweet_engagement_filter_qt_to_ntabclick_ratio_based_predicate" - val scopedStatsReceiver = stats.scope(name) - val allOonCandidatesCounter = scopedStatsReceiver.counter("all_oon_candidates") - val filteredCandidatesCounter = - scopedStatsReceiver.counter("filtered_oon_candidates") - - val quoteCountFeature = - "tweet.core.tweet_counts.quote_count" - val ntabClickCountFeature = - "tweet.magic_recs_tweet_real_time_aggregates_v2.pair.v2.magicrecs.realtime.is_ntab_clicked.any_feature.Duration.Top.count" - - Predicate - .fromAsync { candidate: PushCandidate with TweetCandidate with RecommendationType => - val target = candidate.target - val crt = candidate.commonRecType - val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(crt) || - RecTypes.outOfNetworkTopicTweetTypes.contains(crt) - - lazy val QTtoNtabClickRatioThreshold = - target.params(PushFeatureSwitchParams.TweetQTtoNtabClickRatioThresholdParam) - lazy val quoteCount = candidate.numericFeatures.getOrElse(quoteCountFeature, 0.0) - lazy val ntabClickCount = candidate.numericFeatures.getOrElse(ntabClickCountFeature, 0.0) - lazy val quoteRate = if (ntabClickCount > 0) quoteCount / ntabClickCount else 1.0 - - if (isOonCandidate) allOonCandidatesCounter.incr() - if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && isOonCandidate) { - val ntabClickThreshold = 1000 - candidate.cachePredicateInfo( - name + "_count", - ntabClickCount, - ntabClickThreshold, - ntabClickCount >= ntabClickThreshold) - candidate.cachePredicateInfo( - name + "_ratio", - quoteRate, - QTtoNtabClickRatioThreshold, - quoteRate < QTtoNtabClickRatioThreshold) - if (ntabClickCount >= ntabClickThreshold && quoteRate < QTtoNtabClickRatioThreshold) { - filteredCandidatesCounter.incr() - Future.False - } else Future.True - } else Future.True - } - .withStats(stats.scope(name)) - .withName(name) - } - - def TweetReplyLikeRatioPredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with TweetCandidate] = { - val name = "tweet_reply_like_ratio" - val scopedStatsReceiver = stats.scope(name) - val allCandidatesCounter = scopedStatsReceiver.counter("all_candidates") - val filteredCandidatesCounter = scopedStatsReceiver.counter("filtered_candidates") - val bucketedCandidatesCounter = scopedStatsReceiver.counter("bucketed_candidates") - - Predicate - .fromAsync { candidate: PushCandidate => - allCandidatesCounter.incr() - val target = candidate.target - val likeCount = candidate.numericFeatures - .getOrElse(PushConstants.TweetLikesFeatureName, 0.0) - val replyCount = candidate.numericFeatures - .getOrElse(PushConstants.TweetRepliesFeatureName, 0.0) - val ratio = replyCount / likeCount.max(1) - val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(candidate.commonRecType) || - RecTypes.outOfNetworkTopicTweetTypes.contains(candidate.commonRecType) - - if (isOonCandidate - && CandidateUtil.shouldApplyHealthQualityFilters(candidate) - && replyCount > target.params( - PushFeatureSwitchParams.TweetReplytoLikeRatioReplyCountThreshold)) { - bucketedCandidatesCounter.incr() - if (ratio > target.params( - PushFeatureSwitchParams.TweetReplytoLikeRatioThresholdLowerBound) - && ratio < target.params( - PushFeatureSwitchParams.TweetReplytoLikeRatioThresholdUpperBound)) { - filteredCandidatesCounter.incr() - Future.False - } else { - Future.True - } - } else { - Future.True - } - } - .withStats(stats.scope(s"predicate_$name")) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetLanguagePredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetLanguagePredicate.scala deleted file mode 100644 index 4ff24ae77..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetLanguagePredicate.scala +++ /dev/null @@ -1,109 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.util.CandidateUtil -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.language.normalization.UserDisplayLanguage -import com.twitter.util.Future - -object TweetLanguagePredicate { - - def oonTweeetLanguageMatch( - )( - implicit stats: StatsReceiver - ): NamedPredicate[ - PushCandidate with RecommendationType with TweetDetails - ] = { - val name = "oon_tweet_language_predicate" - val scopedStatsReceiver = stats.scope(name) - val oonCandidatesCounter = - scopedStatsReceiver.counter("oon_candidates") - val enableFilterCounter = - scopedStatsReceiver.counter("enabled_filter") - val skipMediaTweetsCounter = - scopedStatsReceiver.counter("skip_media_tweets") - - Predicate - .fromAsync { candidate: PushCandidate with RecommendationType with TweetDetails => - val target = candidate.target - val crt = candidate.commonRecType - val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(crt) || - RecTypes.outOfNetworkTopicTweetTypes.contains(crt) - - if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && isOonCandidate) { - oonCandidatesCounter.incr() - - target.featureMap.map { featureMap => - val userPreferredLanguages = featureMap.sparseBinaryFeatures - .getOrElse("user.language.user.preferred_contents", Set.empty[String]) - val userEngagementLanguages = featureMap.sparseContinuousFeatures.getOrElse( - "user.language.user.engagements", - Map.empty[String, Double]) - val userFollowLanguages = featureMap.sparseContinuousFeatures.getOrElse( - "user.language.user.following_accounts", - Map.empty[String, Double]) - val userProducedTweetLanguages = featureMap.sparseContinuousFeatures - .getOrElse("user.language.user.produced_tweets", Map.empty) - val userDeviceLanguages = featureMap.sparseContinuousFeatures.getOrElse( - "user.language.user.recent_devices", - Map.empty[String, Double]) - val tweetLanguageOpt = candidate.categoricalFeatures - .get(target.params(PushFeatureSwitchParams.TweetLanguageFeatureNameParam)) - - if (userPreferredLanguages.isEmpty) - scopedStatsReceiver.counter("userPreferredLanguages_empty").incr() - if (userEngagementLanguages.isEmpty) - scopedStatsReceiver.counter("userEngagementLanguages_empty").incr() - if (userFollowLanguages.isEmpty) - scopedStatsReceiver.counter("userFollowLanguages_empty").incr() - if (userProducedTweetLanguages.isEmpty) - scopedStatsReceiver - .counter("userProducedTweetLanguages_empty") - .incr() - if (userDeviceLanguages.isEmpty) - scopedStatsReceiver.counter("userDeviceLanguages_empty").incr() - if (tweetLanguageOpt.isEmpty) scopedStatsReceiver.counter("tweetLanguage_empty").incr() - - val tweetLanguage = tweetLanguageOpt.getOrElse("und") - val undefinedTweetLanguages = Set("") - - if (!undefinedTweetLanguages.contains(tweetLanguage)) { - lazy val userInferredLanguageThreshold = - target.params(PushFeatureSwitchParams.UserInferredLanguageThresholdParam) - lazy val userDeviceLanguageThreshold = - target.params(PushFeatureSwitchParams.UserDeviceLanguageThresholdParam) - lazy val enableTweetLanguageFilter = - target.params(PushFeatureSwitchParams.EnableTweetLanguageFilter) - lazy val skipLanguageFilterForMediaTweets = - target.params(PushFeatureSwitchParams.SkipLanguageFilterForMediaTweets) - - lazy val allLanguages = userPreferredLanguages ++ - userEngagementLanguages.filter(_._2 > userInferredLanguageThreshold).keySet ++ - userFollowLanguages.filter(_._2 > userInferredLanguageThreshold).keySet ++ - userProducedTweetLanguages.filter(_._2 > userInferredLanguageThreshold).keySet ++ - userDeviceLanguages.filter(_._2 > userDeviceLanguageThreshold).keySet - - if (enableTweetLanguageFilter && allLanguages.nonEmpty) { - enableFilterCounter.incr() - val hasMedia = candidate.hasPhoto || candidate.hasVideo - - if (hasMedia && skipLanguageFilterForMediaTweets) { - skipMediaTweetsCounter.incr() - true - } else { - allLanguages.map(UserDisplayLanguage.toTweetLanguage).contains(tweetLanguage) - } - } else true - } else true - } - } else Future.True - } - .withStats(stats.scope(name)) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetWithheldContentPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetWithheldContentPredicate.scala deleted file mode 100644 index c05536909..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetWithheldContentPredicate.scala +++ /dev/null @@ -1,35 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.TweetDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.hermit.predicate.tweetypie.UserLocationAndTweet -import com.twitter.hermit.predicate.tweetypie.WithheldTweetPredicate -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.service.metastore.gen.thriftscala.Location -import com.twitter.util.Future - -object TweetWithheldContentPredicate { - val name = "withheld_content" - val defaultLocation = Location(city = "", region = "", countryCode = "", confidence = 0.0) - - def apply( - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetDetails] = { - Predicate - .fromAsync { candidate: PushCandidate with TweetDetails => - candidate.tweet match { - case Some(tweet) => - WithheldTweetPredicate(checkAllCountries = true) - .apply(Seq(UserLocationAndTweet(defaultLocation, tweet))) - .map(_.head) - case None => - Future.value(false) - } - } - .withStats(statsReceiver.scope(s"predicate_$name")) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/event/EventPredicatesForCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/event/EventPredicatesForCandidate.scala deleted file mode 100644 index 86c1f8abd..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/event/EventPredicatesForCandidate.scala +++ /dev/null @@ -1,155 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.event - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.EventCandidate -import com.twitter.frigate.common.base.TargetInfo -import com.twitter.frigate.common.base.TargetUser -import com.twitter.frigate.common.candidate.FrigateHistory -import com.twitter.frigate.common.history.RecItems -import com.twitter.frigate.magic_events.thriftscala.Locale -import com.twitter.frigate.pushservice.model.MagicFanoutEventHydratedCandidate -import com.twitter.frigate.pushservice.model.MagicFanoutEventPushCandidate -import com.twitter.frigate.pushservice.model.MagicFanoutNewsEventPushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutPredicatesUtil._ -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.util.Future - -object EventPredicatesForCandidate { - def hasTitle( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[MagicFanoutEventHydratedCandidate] = { - val name = "event_title_available" - val scopedStatsReceiver = statsReceiver.scope(s"predicate_$name") - Predicate - .fromAsync { candidate: MagicFanoutEventHydratedCandidate => - candidate.eventTitleFut.map(_.nonEmpty) - } - .withStats(scopedStatsReceiver) - .withName(name) - } - - def isNotDuplicateWithEventId( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[MagicFanoutEventHydratedCandidate] = { - val name = "duplicate_event_id" - Predicate - .fromAsync { candidate: MagicFanoutEventHydratedCandidate => - val useRelaxedFatigueLengthFut: Future[Boolean] = - candidate match { - case mfNewsEvent: MagicFanoutNewsEventPushCandidate => - mfNewsEvent.isHighPriorityEvent - case _ => Future.value(false) - } - Future.join(candidate.target.history, useRelaxedFatigueLengthFut).map { - case (history, useRelaxedFatigueLength) => - val filteredNotifications = if (useRelaxedFatigueLength) { - val relaxedFatigueInterval = - candidate.target - .params( - PushFeatureSwitchParams.MagicFanoutRelaxedEventIdFatigueIntervalInHours).hours - history.notificationMap.filterKeys { time => - time.untilNow <= relaxedFatigueInterval - }.values - } else history.notificationMap.values - !RecItems(filteredNotifications.toSeq).events.exists(_.eventId == candidate.eventId) - } - } - .withStats(statsReceiver.scope(s"predicate_$name")) - .withName(name) - } - - def isNotDuplicateWithEventIdForCandidate[ - T <: TargetUser with FrigateHistory, - Cand <: EventCandidate with TargetInfo[T] - ]( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[Cand] = { - val name = "is_not_duplicate_event" - Predicate - .fromAsync { candidate: Cand => - candidate.target.pushRecItems.map { - !_.events.map(_.eventId).contains(candidate.eventId) - } - } - .withStats(statsReceiver.scope(name)) - .withName(name) - } - - def accountCountryPredicateWithAllowlist( - implicit stats: StatsReceiver - ): NamedPredicate[MagicFanoutEventPushCandidate] = { - val name = "account_country_predicate_with_allowlist" - val scopedStats = stats.scope(name) - - val skipPredicate = Predicate - .from { candidate: MagicFanoutEventPushCandidate => - candidate.target.params(PushFeatureSwitchParams.MagicFanoutSkipAccountCountryPredicate) - } - .withStats(stats.scope("skip_account_country_predicate_mf")) - .withName("skip_account_country_predicate_mf") - - val excludeEventFromAccountCountryPredicateFiltering = Predicate - .from { candidate: MagicFanoutEventPushCandidate => - val eventId = candidate.eventId - val target = candidate.target - target - .params(PushFeatureSwitchParams.MagicFanoutEventAllowlistToSkipAccountCountryPredicate) - .exists(eventId.equals) - } - .withStats(stats.scope("exclude_event_from_account_country_predicate_filtering")) - .withName("exclude_event_from_account_country_predicate_filtering") - - skipPredicate - .or(excludeEventFromAccountCountryPredicateFiltering) - .or(accountCountryPredicate) - .withStats(scopedStats) - .withName(name) - } - - /** - * Check if user's country is targeted - * @param stats - */ - def accountCountryPredicate( - implicit stats: StatsReceiver - ): NamedPredicate[MagicFanoutEventPushCandidate] = { - val name = "account_country_predicate" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - val internationalLocalePassedCounter = - scopedStatsReceiver.counter("international_locale_passed") - val internationalLocaleFilteredCounter = - scopedStatsReceiver.counter("international_locale_filtered") - Predicate - .fromAsync { candidate: MagicFanoutEventPushCandidate => - candidate.target.countryCode.map { - case Some(countryCode) => - val denyListedCountryCodes: Seq[String] = - if (candidate.commonRecType == CommonRecommendationType.MagicFanoutNewsEvent) { - candidate.target - .params(PushFeatureSwitchParams.MagicFanoutDenyListedCountries) - } else if (candidate.commonRecType == CommonRecommendationType.MagicFanoutSportsEvent) { - candidate.target - .params(PushFeatureSwitchParams.MagicFanoutSportsEventDenyListedCountries) - } else Seq() - val eventCountries = - candidate.newsForYouMetadata - .flatMap(_.locales).getOrElse(Seq.empty[Locale]).flatMap(_.country) - if (isInCountryList(countryCode, eventCountries) - && !isInCountryList(countryCode, denyListedCountryCodes)) { - internationalLocalePassedCounter.incr() - true - } else { - internationalLocaleFilteredCounter.incr() - false - } - case _ => false - } - } - .withStats(scopedStatsReceiver) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutPredicatesForCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutPredicatesForCandidate.scala deleted file mode 100644 index 52371b488..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutPredicatesForCandidate.scala +++ /dev/null @@ -1,525 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.magic_fanout - -import com.twitter.audience_rewards.thriftscala.HasSuperFollowingRelationshipRequest -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.MagicFanoutCandidate -import com.twitter.frigate.common.base.MagicFanoutCreatorEventCandidate -import com.twitter.frigate.common.base.MagicFanoutProductLaunchCandidate -import com.twitter.frigate.common.history.RecItems -import com.twitter.frigate.common.predicate.FatiguePredicate.build -import com.twitter.frigate.common.predicate.FatiguePredicate.productLaunchTypeRecTypesOnlyFilter -import com.twitter.frigate.common.predicate.FatiguePredicate.recOnlyFilter -import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext -import com.twitter.frigate.common.store.interests.SemanticCoreEntityId -import com.twitter.frigate.common.util.IbisAppPushDeviceSettingsUtil -import com.twitter.frigate.magic_events.thriftscala.CreatorFanoutType -import com.twitter.frigate.magic_events.thriftscala.ProductType -import com.twitter.frigate.magic_events.thriftscala.TargetID -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.MagicFanoutEventHydratedCandidate -import com.twitter.frigate.pushservice.model.MagicFanoutEventPushCandidate -import com.twitter.frigate.pushservice.model.MagicFanoutNewsEventPushCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.predicate.FatiguePredicate -import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.NotificationDisplayLocation -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.interests.thriftscala.UserInterests -import com.twitter.simclusters_v2.thriftscala.EmbeddingType -import com.twitter.simclusters_v2.thriftscala.ModelVersion -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration -import com.twitter.util.Future - -object MagicFanoutPredicatesForCandidate { - - /** - * Check if Semantic Core reasons satisfy rank threshold ( for heavy users a non broad entity should satisfy the threshold) - */ - def magicFanoutErgInterestRankThresholdPredicate( - implicit stats: StatsReceiver - ): NamedPredicate[MagicFanoutEventHydratedCandidate] = { - val name = "magicfanout_interest_erg_rank_threshold" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - Predicate - .fromAsync { candidate: MagicFanoutEventHydratedCandidate => - candidate.target.isHeavyUserState.map { isHeavyUser => - lazy val rankThreshold = - if (isHeavyUser) { - candidate.target.params(PushFeatureSwitchParams.MagicFanoutRankErgThresholdHeavy) - } else { - candidate.target.params(PushFeatureSwitchParams.MagicFanoutRankErgThresholdNonHeavy) - } - MagicFanoutPredicatesUtil - .checkIfValidErgScEntityReasonExists( - candidate.effectiveMagicEventsReasons, - rankThreshold - ) - } - } - .withStats(scopedStatsReceiver) - .withName(name) - } - - def newsNotificationFatigue( - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "news_notification_fatigue" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - Predicate - .fromAsync { candidate: PushCandidate => - FatiguePredicate - .recTypeSetOnly( - notificationDisplayLocation = NotificationDisplayLocation.PushToMobileDevice, - recTypes = Set(CommonRecommendationType.MagicFanoutNewsEvent), - maxInInterval = - candidate.target.params(PushFeatureSwitchParams.MFMaxNumberOfPushesInInterval), - interval = candidate.target.params(PushFeatureSwitchParams.MFPushIntervalInHours), - minInterval = candidate.target.params(PushFeatureSwitchParams.MFMinIntervalFatigue) - ) - .apply(Seq(candidate)) - .map(_.headOption.getOrElse(false)) - - } - .withStats(scopedStatsReceiver) - .withName(name) - } - - /** - * Check if reason contains any optouted semantic core entity interests. - * - * @param stats - * - * @return - */ - def magicFanoutNoOptoutInterestPredicate( - implicit stats: StatsReceiver - ): NamedPredicate[MagicFanoutEventPushCandidate] = { - val name = "magicfanout_optout_interest_predicate" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - val withOptOutInterestsCounter = stats.counter("with_optout_interests") - val withoutOptOutInterestsCounter = stats.counter("without_optout_interests") - Predicate - .fromAsync { candidate: MagicFanoutEventPushCandidate => - candidate.target.optOutSemanticCoreInterests.map { - case ( - optOutUserInterests: Seq[SemanticCoreEntityId] - ) => - withOptOutInterestsCounter.incr() - optOutUserInterests - .intersect(candidate.annotatedAndInferredSemanticCoreEntities).isEmpty - case _ => - withoutOptOutInterestsCounter.incr() - true - } - } - .withStats(scopedStatsReceiver) - .withName(name) - } - - /** - * Checks if the target has only one device language language, - * and that language is targeted for that event - * - * @param statsReceiver - * - * @return - */ - def inferredUserDeviceLanguagePredicate( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[MagicFanoutEventPushCandidate] = { - val name = "inferred_device_language" - val scopedStats = statsReceiver.scope(s"predicate_$name") - Predicate - .fromAsync { candidate: MagicFanoutEventPushCandidate => - val target = candidate.target - target.deviceInfo.map { - _.flatMap { deviceInfo => - val languages = deviceInfo.deviceLanguages.getOrElse(Seq.empty[String]) - val distinctDeviceLanguages = - IbisAppPushDeviceSettingsUtil.distinctDeviceLanguages(languages) - - candidate.newsForYouMetadata.map { newsForYouMetadata => - val eventLocales = newsForYouMetadata.locales.getOrElse(Seq.empty) - val eventLanguages = eventLocales.flatMap(_.language).map(_.toLowerCase).distinct - - eventLanguages.intersect(distinctDeviceLanguages).nonEmpty - } - }.getOrElse(false) - } - } - .withStats(scopedStats) - .withName(name) - } - - /** - * Bypass predicate if high priority push - */ - def highPriorityNewsEventExceptedPredicate( - predicate: NamedPredicate[MagicFanoutNewsEventPushCandidate] - )( - implicit config: Config - ): NamedPredicate[MagicFanoutNewsEventPushCandidate] = { - PredicatesForCandidate.exceptedPredicate( - name = "high_priority_excepted_" + predicate.name, - fn = MagicFanoutPredicatesUtil.checkIfHighPriorityNewsEventForCandidate, - predicate - )(config.statsReceiver) - } - - /** - * Bypass predicate if high priority push - */ - def highPriorityEventExceptedPredicate( - predicate: NamedPredicate[MagicFanoutEventPushCandidate] - )( - implicit config: Config - ): NamedPredicate[MagicFanoutEventPushCandidate] = { - PredicatesForCandidate.exceptedPredicate( - name = "high_priority_excepted_" + predicate.name, - fn = MagicFanoutPredicatesUtil.checkIfHighPriorityEventForCandidate, - predicate - )(config.statsReceiver) - } - - def magicFanoutSimClusterTargetingPredicate( - implicit stats: StatsReceiver - ): NamedPredicate[MagicFanoutEventPushCandidate] = { - val name = "simcluster_targeting" - val scopedStats = stats.scope(s"predicate_$name") - val userStateCounters = scopedStats.scope("user_state") - Predicate - .fromAsync { candidate: MagicFanoutEventPushCandidate => - candidate.target.isHeavyUserState.map { isHeavyUser => - val simClusterEmbeddings = candidate.newsForYouMetadata.flatMap( - _.eventContextScribe.flatMap(_.simClustersEmbeddings)) - val TopKSimClustersCount = 50 - val eventSimClusterVectorOpt: Option[MagicFanoutPredicatesUtil.SimClusterScores] = - MagicFanoutPredicatesUtil.getEventSimClusterVector( - simClusterEmbeddings.map(_.toMap), - (ModelVersion.Model20m145kUpdated, EmbeddingType.FollowBasedTweet), - TopKSimClustersCount - ) - val userSimClusterVectorOpt: Option[MagicFanoutPredicatesUtil.SimClusterScores] = - MagicFanoutPredicatesUtil.getUserSimClusterVector(candidate.effectiveMagicEventsReasons) - (eventSimClusterVectorOpt, userSimClusterVectorOpt) match { - case ( - Some(eventSimClusterVector: MagicFanoutPredicatesUtil.SimClusterScores), - Some(userSimClusterVector)) => - val score = eventSimClusterVector - .normedDotProduct(userSimClusterVector, eventSimClusterVector) - val threshold = if (isHeavyUser) { - candidate.target.params( - PushFeatureSwitchParams.MagicFanoutSimClusterDotProductHeavyUserThreshold) - } else { - candidate.target.params( - PushFeatureSwitchParams.MagicFanoutSimClusterDotProductNonHeavyUserThreshold) - } - val isPassed = score >= threshold - userStateCounters.scope(isHeavyUser.toString).counter(s"$isPassed").incr() - isPassed - - case (None, Some(userSimClusterVector)) => - candidate.commonRecType == CommonRecommendationType.MagicFanoutSportsEvent - - case _ => false - } - } - } - .withStats(scopedStats) - .withName(name) - } - - def geoTargetingHoldback( - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with MagicFanoutCandidate] = { - Predicate - .from[PushCandidate with MagicFanoutCandidate] { candidate => - if (MagicFanoutPredicatesUtil.reasonsContainGeoTarget( - candidate.candidateMagicEventsReasons)) { - candidate.target.params(PushFeatureSwitchParams.EnableMfGeoTargeting) - } else true - } - .withStats(stats.scope("geo_targeting_holdback")) - .withName("geo_targeting_holdback") - } - - def geoOptOutPredicate( - userStore: ReadableStore[Long, User] - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with MagicFanoutCandidate] = { - Predicate - .fromAsync[PushCandidate with MagicFanoutCandidate] { candidate => - if (MagicFanoutPredicatesUtil.reasonsContainGeoTarget( - candidate.candidateMagicEventsReasons)) { - userStore.get(candidate.target.targetId).map { userOpt => - val isGeoAllowed = userOpt - .flatMap(_.account) - .exists(_.allowLocationHistoryPersonalization) - isGeoAllowed - } - } else { - Future.True - } - } - .withStats(stats.scope("geo_opt_out_predicate")) - .withName("geo_opt_out_predicate") - } - - /** - * Check if Semantic Core reasons contains valid utt reason & reason is within top k topics followed by user - */ - def magicFanoutTopicFollowsTargetingPredicate( - implicit stats: StatsReceiver, - interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests] - ): NamedPredicate[MagicFanoutEventHydratedCandidate] = { - val name = "magicfanout_topic_follows_targeting" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - Predicate - .fromAsync[PushCandidate with MagicFanoutEventHydratedCandidate] { candidate => - candidate.followedTopicLocalizedEntities.map(_.nonEmpty) - } - .withStats(scopedStatsReceiver) - .withName(name) - } - - /** Requires the magicfanout candidate to have a UserID reason which ranks below the follow - * rank threshold. If no UserID target exists the candidate is dropped. */ - def followRankThreshold( - threshold: Param[Int] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with MagicFanoutCandidate] = { - val name = "follow_rank_threshold" - Predicate - .from[PushCandidate with MagicFanoutCandidate] { c => - c.candidateMagicEventsReasons.exists { fanoutReason => - fanoutReason.reason match { - case TargetID.UserID(_) => - fanoutReason.rank.exists { rank => - rank <= c.target.params(threshold) - } - case _ => false - } - } - } - .withStats(statsReceiver.scope(name)) - .withName(name) - } - - def userGeneratedEventsPredicate( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with MagicFanoutEventHydratedCandidate] = { - val name = "user_generated_moments" - val stats = statsReceiver.scope(name) - - Predicate - .from { candidate: PushCandidate with MagicFanoutEventHydratedCandidate => - val isUgmMoment = candidate.semanticCoreEntityTags.values.flatten.toSet - .contains(MagicFanoutPredicatesUtil.UgmMomentTag) - if (isUgmMoment) { - candidate.target.params(PushFeatureSwitchParams.MagicFanoutNewsUserGeneratedEventsEnable) - } else true - }.withStats(stats) - .withName(name) - } - def escherbirdMagicfanoutEventParam( - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with MagicFanoutEventPushCandidate] = { - val name = "magicfanout_escherbird_fs" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - - Predicate - .fromAsync[PushCandidate with MagicFanoutEventPushCandidate] { candidate => - val candidateFrigateNotif = candidate.frigateNotification.magicFanoutEventNotification - val isEscherbirdEvent = candidateFrigateNotif.exists(_.isEscherbirdEvent.contains(true)) - scopedStatsReceiver.counter(s"with_escherbird_flag_$isEscherbirdEvent").incr() - - if (isEscherbirdEvent) { - - val listOfEventsSemanticCoreDomainIds = - candidate.target.params(PushFeatureSwitchParams.ListOfEventSemanticCoreDomainIds) - - val candScDomainEvent = - if (listOfEventsSemanticCoreDomainIds.nonEmpty) { - candidate.eventSemanticCoreDomainIds - .intersect(listOfEventsSemanticCoreDomainIds).nonEmpty - } else { - false - } - scopedStatsReceiver - .counter( - s"with_escherbird_fs_in_list_of_event_semantic_core_domains_$candScDomainEvent").incr() - Future.value(candScDomainEvent) - } else { - Future.True - } - } - .withStats(scopedStatsReceiver) - .withName(name) - } - - /** - * Checks if the user has custom targeting enabled.If so, bucket the user in experiment. This custom targeting refers to adding - * tweet authors as targets in the eventfanout service. - * @param stats [StatsReceiver] - * @return NamedPredicate[PushCandidate with MagicFanoutEventPushCandidate] - */ - def hasCustomTargetingForNewsEventsParam( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with MagicFanoutEventPushCandidate] = { - val name = "magicfanout_hascustomtargeting" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - - Predicate - .from[PushCandidate with MagicFanoutEventPushCandidate] { candidate => - candidate.candidateMagicEventsReasons.exists { fanoutReason => - fanoutReason.reason match { - case userIdReason: TargetID.UserID => - if (userIdReason.userID.hasCustomTargeting.contains(true)) { - candidate.target.params( - PushFeatureSwitchParams.MagicFanoutEnableCustomTargetingNewsEvent) - } else true - case _ => true - } - } - } - .withStats(scopedStatsReceiver) - .withName(name) - - } - - def magicFanoutProductLaunchFatigue( - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with MagicFanoutProductLaunchCandidate] = { - val name = "magic_fanout_product_launch_fatigue" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - Predicate - .fromAsync { candidate: PushCandidate with MagicFanoutProductLaunchCandidate => - val target = candidate.target - val (interval, maxInInterval, minInterval) = { - candidate.productLaunchType match { - case ProductType.BlueVerified => - ( - target.params(PushFeatureSwitchParams.ProductLaunchPushIntervalInHours), - target.params(PushFeatureSwitchParams.ProductLaunchMaxNumberOfPushesInInterval), - target.params(PushFeatureSwitchParams.ProductLaunchMinIntervalFatigue)) - case _ => - (Duration.fromDays(1), 0, Duration.Zero) - } - } - build( - interval = interval, - maxInInterval = maxInInterval, - minInterval = minInterval, - filterHistory = productLaunchTypeRecTypesOnlyFilter( - Set(CommonRecommendationType.MagicFanoutProductLaunch), - candidate.productLaunchType.toString), - notificationDisplayLocation = NotificationDisplayLocation.PushToMobileDevice - ).flatContraMap { candidate: PushCandidate => candidate.target.history } - .apply(Seq(candidate)) - .map(_.headOption.getOrElse(false)) - } - .withStats(scopedStatsReceiver) - .withName(name) - } - - def creatorPushTargetIsNotCreator( - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with MagicFanoutCreatorEventCandidate] = { - val name = "magic_fanout_creator_is_self" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - Predicate - .from { candidate: PushCandidate with MagicFanoutCreatorEventCandidate => - candidate.target.targetId != candidate.creatorId - } - .withStats(scopedStatsReceiver) - .withName(name) - } - - def duplicateCreatorPredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with MagicFanoutCreatorEventCandidate] = { - val name = "magic_fanout_creator_duplicate_creator_id" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - Predicate - .fromAsync { cand: PushCandidate with MagicFanoutCreatorEventCandidate => - cand.target.pushRecItems.map { recItems: RecItems => - !recItems.creatorIds.contains(cand.creatorId) - } - } - .withStats(scopedStatsReceiver) - .withName(name) - } - - def isSuperFollowingCreator( - )( - implicit config: Config, - stats: StatsReceiver - ): NamedPredicate[PushCandidate with MagicFanoutCreatorEventCandidate] = { - val name = "magic_fanout_is_already_superfollowing_creator" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - Predicate - .fromAsync { cand: PushCandidate with MagicFanoutCreatorEventCandidate => - config.hasSuperFollowingRelationshipStore - .get( - HasSuperFollowingRelationshipRequest( - sourceUserId = cand.target.targetId, - targetUserId = cand.creatorId)).map(_.getOrElse(false)) - } - .withStats(scopedStatsReceiver) - .withName(name) - } - - def magicFanoutCreatorPushFatiguePredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with MagicFanoutCreatorEventCandidate] = { - val name = "magic_fanout_creator_fatigue" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - Predicate - .fromAsync { candidate: PushCandidate with MagicFanoutCreatorEventCandidate => - val target = candidate.target - val (interval, maxInInterval, minInterval) = { - candidate.creatorFanoutType match { - case CreatorFanoutType.UserSubscription => - ( - target.params(PushFeatureSwitchParams.CreatorSubscriptionPushIntervalInHours), - target.params( - PushFeatureSwitchParams.CreatorSubscriptionPushMaxNumberOfPushesInInterval), - target.params(PushFeatureSwitchParams.CreatorSubscriptionPushhMinIntervalFatigue)) - case CreatorFanoutType.NewCreator => - ( - target.params(PushFeatureSwitchParams.NewCreatorPushIntervalInHours), - target.params(PushFeatureSwitchParams.NewCreatorPushMaxNumberOfPushesInInterval), - target.params(PushFeatureSwitchParams.NewCreatorPushMinIntervalFatigue)) - case _ => - (Duration.fromDays(1), 0, Duration.Zero) - } - } - build( - interval = interval, - maxInInterval = maxInInterval, - minInterval = minInterval, - filterHistory = recOnlyFilter(candidate.commonRecType), - notificationDisplayLocation = NotificationDisplayLocation.PushToMobileDevice - ).flatContraMap { candidate: PushCandidate => candidate.target.history } - .apply(Seq(candidate)) - .map(_.headOption.getOrElse(false)) - } - .withStats(scopedStatsReceiver) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutPredicatesUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutPredicatesUtil.scala deleted file mode 100644 index 306f2b3b6..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutPredicatesUtil.scala +++ /dev/null @@ -1,218 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.magic_fanout - -import com.twitter.eventdetection.event_context.util.SimClustersUtil -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.magic_events.thriftscala._ -import com.twitter.frigate.pushservice.model.MagicFanoutEventPushCandidate -import com.twitter.frigate.pushservice.model.MagicFanoutNewsEventPushCandidate -import com.twitter.frigate.pushservice.model.MagicFanoutProductLaunchPushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.simclusters_v2.common.SimClustersEmbedding -import com.twitter.simclusters_v2.thriftscala.EmbeddingType -import com.twitter.simclusters_v2.thriftscala.ModelVersion -import com.twitter.simclusters_v2.thriftscala.SimClustersEmbeddingId -import com.twitter.simclusters_v2.thriftscala.{SimClustersEmbedding => ThriftSimClustersEmbedding} -import com.twitter.util.Future - -object MagicFanoutPredicatesUtil { - - val UttDomain: Long = 0L - type DomainId = Long - type EntityId = Long - val BroadCategoryTag = "utt:broad_category" - val UgmMomentTag = "MMTS.isUGMMoment" - val TopKSimClustersCount = 50 - - case class SimClusterScores(simClusterScoreVector: Map[Int, Double]) { - def dotProduct(other: SimClusterScores): Double = { - simClusterScoreVector - .map { - case (clusterId, score) => other.simClusterScoreVector.getOrElse(clusterId, 0.0) * score - }.foldLeft(0.0) { _ + _ } - } - - def norm(): Double = { - val sumOfSquares: Double = simClusterScoreVector - .map { - case (clusterId, score) => score * score - }.foldLeft(0.0)(_ + _) - scala.math.sqrt(sumOfSquares) - } - - def normedDotProduct(other: SimClusterScores, normalizer: SimClusterScores): Double = { - val denominator = normalizer.norm() - val score = dotProduct(other) - if (denominator != 0.0) { - score / denominator - } else { - score - } - } - } - - private def isSemanticCoreEntityBroad( - semanticCoreEntityTags: Map[(DomainId, EntityId), Set[String]], - scEntityId: SemanticCoreID - ): Boolean = { - semanticCoreEntityTags - .getOrElse((scEntityId.domainId, scEntityId.entityId), Set.empty).contains(BroadCategoryTag) - } - - def isInCountryList(accountCountryCode: String, locales: Seq[String]): Boolean = { - locales.map(_.toLowerCase).contains(accountCountryCode.toLowerCase) - } - - /** - * Boolean check of if a MagicFanout is high priority push - */ - def checkIfHighPriorityNewsEventForCandidate( - candidate: MagicFanoutNewsEventPushCandidate - ): Future[Boolean] = { - candidate.isHighPriorityEvent.map { isHighPriority => - isHighPriority && (candidate.target.params(PushFeatureSwitchParams.EnableHighPriorityPush)) - } - } - - /** - * Boolean check of if a MagicFanout event is high priority push - */ - def checkIfHighPriorityEventForCandidate( - candidate: MagicFanoutEventPushCandidate - ): Future[Boolean] = { - candidate.isHighPriorityEvent.map { isHighPriority => - candidate.commonRecType match { - case CommonRecommendationType.MagicFanoutSportsEvent => - isHighPriority && (candidate.target.params( - PushFeatureSwitchParams.EnableHighPrioritySportsPush)) - case _ => false - } - } - } - - /** - * Boolean check if to skip target blue verified - */ - def shouldSkipBlueVerifiedCheckForCandidate( - candidate: MagicFanoutProductLaunchPushCandidate - ): Future[Boolean] = - Future.value( - candidate.target.params(PushFeatureSwitchParams.DisableIsTargetBlueVerifiedPredicate)) - - /** - * Boolean check if to skip target is legacy verified - */ - def shouldSkipLegacyVerifiedCheckForCandidate( - candidate: MagicFanoutProductLaunchPushCandidate - ): Future[Boolean] = - Future.value( - candidate.target.params(PushFeatureSwitchParams.DisableIsTargetLegacyVerifiedPredicate)) - - def shouldSkipSuperFollowCreatorCheckForCandidate( - candidate: MagicFanoutProductLaunchPushCandidate - ): Future[Boolean] = - Future.value( - !candidate.target.params(PushFeatureSwitchParams.EnableIsTargetSuperFollowCreatorPredicate)) - - /** - * Boolean check of if a reason of a MagicFanout is higher than the rank threshold of an event - */ - def checkIfErgScEntityReasonMeetsThreshold( - rankThreshold: Int, - reason: MagicEventsReason, - ): Boolean = { - reason.reason match { - case TargetID.SemanticCoreID(scEntityId: SemanticCoreID) => - reason.rank match { - case Some(rank) => rank < rankThreshold - case _ => false - } - case _ => false - } - } - - /** - * Check if MagicEventsReasons contains a reason that matches the thresholdw - */ - def checkIfValidErgScEntityReasonExists( - magicEventsReasons: Option[Seq[MagicEventsReason]], - rankThreshold: Int - )( - implicit stats: StatsReceiver - ): Boolean = { - magicEventsReasons match { - case Some(reasons) if reasons.exists(_.isNewUser.contains(true)) => true - case Some(reasons) => - reasons.exists { reason => - reason.source.contains(ReasonSource.ErgShortTermInterestSemanticCore) && - checkIfErgScEntityReasonMeetsThreshold( - rankThreshold, - reason - ) - } - - case _ => false - } - } - - /** - * Get event simcluster vector from event context - */ - def getEventSimClusterVector( - simClustersEmbeddingOption: Option[Map[SimClustersEmbeddingId, ThriftSimClustersEmbedding]], - embeddingMapKey: (ModelVersion, EmbeddingType), - topKSimClustersCount: Int - ): Option[SimClusterScores] = { - simClustersEmbeddingOption.map { thriftSimClustersEmbeddings => - val simClustersEmbeddings: Map[SimClustersEmbeddingId, SimClustersEmbedding] = - thriftSimClustersEmbeddings.map { - case (simClustersEmbeddingId, simClustersEmbeddingValue) => - (simClustersEmbeddingId, SimClustersEmbedding(simClustersEmbeddingValue)) - }.toMap - val emptySeq = Seq[(Int, Double)]() - val simClusterScoreTuple: Map[(ModelVersion, EmbeddingType), Seq[(Int, Double)]] = - SimClustersUtil - .getMaxTopKTweetSimClusters(simClustersEmbeddings, topKSimClustersCount) - SimClusterScores(simClusterScoreTuple.getOrElse(embeddingMapKey, emptySeq).toMap) - } - } - - /** - * Get user simcluster vector magic events reasons - */ - def getUserSimClusterVector( - magicEventsReasonsOpt: Option[Seq[MagicEventsReason]] - ): Option[SimClusterScores] = { - magicEventsReasonsOpt.map { magicEventsReasons: Seq[MagicEventsReason] => - val reasons: Seq[(Int, Double)] = magicEventsReasons.flatMap { reason => - reason.reason match { - case TargetID.SimClusterID(simClusterId: SimClusterID) => - Some((simClusterId.clusterId, reason.score.getOrElse(0.0))) - case _ => - None - } - } - SimClusterScores(reasons.toMap) - } - } - - def reasonsContainGeoTarget(reasons: Seq[MagicEventsReason]): Boolean = { - reasons.exists { reason => - val isGeoGraphSource = reason.source.contains(ReasonSource.GeoGraph) - reason.reason match { - case TargetID.PlaceID(_) if isGeoGraphSource => true - case _ => false - } - } - } - - def geoPlaceIdsFromReasons(reasons: Seq[MagicEventsReason]): Set[Long] = { - reasons.flatMap { reason => - val isGeoGraphSource = reason.source.contains(ReasonSource.GeoGraph) - reason.reason match { - case TargetID.PlaceID(PlaceID(id)) if isGeoGraphSource => Some(id) - case _ => None - } - }.toSet - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutSportsUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutSportsUtil.scala deleted file mode 100644 index 224be3ad5..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutSportsUtil.scala +++ /dev/null @@ -1,231 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.magic_fanout - -import com.twitter.datatools.entityservice.entities.sports.thriftscala.NflFootballGameLiveUpdate -import com.twitter.datatools.entityservice.entities.sports.thriftscala.SoccerMatchLiveUpdate -import com.twitter.datatools.entityservice.entities.sports.thriftscala.SoccerPeriod -import com.twitter.datatools.entityservice.entities.sports.thriftscala.SportsEventHomeAwayTeamScore -import com.twitter.datatools.entityservice.entities.sports.thriftscala.SportsEventStatus -import com.twitter.datatools.entityservice.entities.sports.thriftscala.SportsEventTeamAlignment.Away -import com.twitter.datatools.entityservice.entities.sports.thriftscala.SportsEventTeamAlignment.Home -import com.twitter.escherbird.metadata.thriftscala.EntityMegadata -import com.twitter.frigate.pushservice.params.SportGameEnum -import com.twitter.frigate.common.base.GenericGameScore -import com.twitter.frigate.common.base.NflGameScore -import com.twitter.frigate.common.base.SoccerGameScore -import com.twitter.frigate.common.base.TeamInfo -import com.twitter.frigate.common.base.TeamScore -import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -object MagicFanoutSportsUtil { - - def transformSoccerGameScore(game: SoccerMatchLiveUpdate): Option[SoccerGameScore] = { - require(game.status.isDefined) - val gameScore = transformToGameScore(game.score, game.status.get) - val _penaltyKicks = transformToGameScore(game.penaltyScore, game.status.get) - gameScore.map { score => - val _isGameEnd = game.status.get match { - case SportsEventStatus.Completed(_) => true - case _ => false - } - - val _isHalfTime = game.period.exists { period => - period match { - case SoccerPeriod.Halftime(_) => true - case _ => false - } - } - - val _isOvertime = game.period.exists { period => - period match { - case SoccerPeriod.PreOvertime(_) => true - case _ => false - } - } - - val _isPenaltyKicks = game.period.exists { period => - period match { - case SoccerPeriod.PrePenalty(_) => true - case SoccerPeriod.Penalty(_) => true - case _ => false - } - } - - val _gameMinute = game.gameMinute.map { soccerGameMinute => - game.minutesInInjuryTime match { - case Some(injuryTime) => s"($soccerGameMinute+$injuryTime′)" - case None => s"($soccerGameMinute′)" - } - } - - SoccerGameScore( - score.home, - score.away, - isGameOngoing = score.isGameOngoing, - penaltyKicks = _penaltyKicks, - gameMinute = _gameMinute, - isHalfTime = _isHalfTime, - isOvertime = _isOvertime, - isPenaltyKicks = _isPenaltyKicks, - isGameEnd = _isGameEnd - ) - } - } - - def transformNFLGameScore(game: NflFootballGameLiveUpdate): Option[NflGameScore] = { - require(game.status.isDefined) - - val gameScore = transformToGameScore(game.score, game.status.get) - gameScore.map { score => - val _isGameEnd = game.status.get match { - case SportsEventStatus.Completed(_) => true - case _ => false - } - - val _matchTime = (game.quarter, game.remainingSecondsInQuarter) match { - case (Some(quarter), Some(remainingSeconds)) if remainingSeconds != 0L => - val m = (remainingSeconds / 60) % 60 - val s = remainingSeconds % 60 - val formattedSeconds = "%02d:%02d".format(m, s) - s"(Q$quarter - $formattedSeconds)" - case (Some(quarter), None) => s"(Q$quarter)" - case _ => "" - } - - NflGameScore( - score.home, - score.away, - isGameOngoing = score.isGameOngoing, - isGameEnd = _isGameEnd, - matchTime = _matchTime - ) - } - } - - /** - Takes a score from Strato columns and turns it into an easier to handle structure (GameScore class) - We do this to easily access the home/away scenario for copy setting - */ - def transformToGameScore( - scoreOpt: Option[SportsEventHomeAwayTeamScore], - status: SportsEventStatus - ): Option[GenericGameScore] = { - val isGameOngoing = status match { - case SportsEventStatus.InProgress(_) => true - case SportsEventStatus.Completed(_) => false - case _ => false - } - - val scoresWithTeam = scoreOpt - .map { score => - score.scores.map { score => (score.score, score.participantAlignment, score.participantId) } - }.getOrElse(Seq()) - - val tuple = scoresWithTeam match { - case Seq(teamOne, teamTwo, _*) => Some((teamOne, teamTwo)) - case _ => None - } - tuple.flatMap { - case ((Some(teamOneScore), teamOneAlignment, teamOne), (Some(teamTwoScore), _, teamTwo)) => - teamOneAlignment.flatMap { - case Home(_) => - val home = TeamScore(teamOneScore, teamOne.entityId, teamOne.domainId) - val away = TeamScore(teamTwoScore, teamTwo.entityId, teamTwo.domainId) - Some(GenericGameScore(home, away, isGameOngoing)) - case Away(_) => - val away = TeamScore(teamOneScore, teamOne.entityId, teamOne.domainId) - val home = TeamScore(teamTwoScore, teamTwo.entityId, teamTwo.domainId) - Some(GenericGameScore(home, away, isGameOngoing)) - case _ => None - } - case _ => None - } - } - - def getTeamInfo( - team: TeamScore, - semanticCoreMegadataStore: ReadableStore[SemanticEntityForQuery, EntityMegadata] - ): Future[Option[TeamInfo]] = { - semanticCoreMegadataStore - .get(SemanticEntityForQuery(team.teamDomainId, team.teamEntityId)).map { - _.flatMap { - _.basicMetadata.map { metadata => - TeamInfo( - name = metadata.name, - twitterUserId = metadata.twitter.flatMap(_.preferredTwitterUserId)) - } - } - } - } - - def getNFLReadableName(name: String): String = { - val teamNames = - Seq("") - teamNames.find(teamName => name.contains(teamName)).getOrElse(name) - } - - def getSoccerIbisMap(game: SoccerGameScore): Map[String, String] = { - val gameMinuteMap = game.gameMinute - .map { gameMinute => Map("match_time" -> gameMinute) } - .getOrElse(Map.empty) - - val updateTypeMap = { - if (game.isGameEnd) Map("is_game_end" -> "true") - else if (game.isHalfTime) Map("is_half_time" -> "true") - else if (game.isOvertime) Map("is_overtime" -> "true") - else if (game.isPenaltyKicks) Map("is_penalty_kicks" -> "true") - else Map("is_score_update" -> "true") - } - - val awayScore = game match { - case SoccerGameScore(_, away, _, None, _, _, _, _, _) => - away.score.toString - case SoccerGameScore(_, away, _, Some(penaltyKick), _, _, _, _, _) => - s"${away.score} (${penaltyKick.away.score}) " - case _ => "" - } - - val homeScore = game match { - case SoccerGameScore(home, _, _, None, _, _, _, _, _) => - home.score.toString - case SoccerGameScore(home, _, _, Some(penaltyKick), _, _, _, _, _) => - s"${home.score} (${penaltyKick.home.score}) " - case _ => "" - } - - val scoresMap = Map( - "away_score" -> awayScore, - "home_score" -> homeScore, - ) - - gameType(SportGameEnum.Soccer) ++ updateTypeMap ++ gameMinuteMap ++ scoresMap - } - - def getNflIbisMap(game: NflGameScore): Map[String, String] = { - val gameMinuteMap = Map("match_time" -> game.matchTime) - - val updateTypeMap = { - if (game.isGameEnd) Map("is_game_end" -> "true") - else Map("is_score_update" -> "true") - } - - val awayScore = game.away.score - val homeScore = game.home.score - - val scoresMap = Map( - "away_score" -> awayScore.toString, - "home_score" -> homeScore.toString, - ) - - gameType(SportGameEnum.Nfl) ++ updateTypeMap ++ gameMinuteMap ++ scoresMap - } - - private def gameType(game: SportGameEnum.Value): Map[String, String] = { - game match { - case SportGameEnum.Soccer => Map("is_soccer_game" -> "true") - case SportGameEnum.Nfl => Map("is_nfl_game" -> "true") - case _ => Map.empty - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutTargetingPredicateWrappersForCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutTargetingPredicateWrappersForCandidate.scala deleted file mode 100644 index 758c9ef34..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutTargetingPredicateWrappersForCandidate.scala +++ /dev/null @@ -1,133 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.magic_fanout - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext -import com.twitter.frigate.common.util.FeatureSwitchParams -import com.twitter.frigate.common.util.MagicFanoutTargetingPredicatesEnum -import com.twitter.frigate.common.util.MagicFanoutTargetingPredicatesEnum.MagicFanoutTargetingPredicatesEnum -import com.twitter.frigate.pushservice.model.MagicFanoutEventPushCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.interests.thriftscala.UserInterests -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi.FSEnumParam - -object MagicFanoutTargetingPredicateWrappersForCandidate { - - /** - * Combine Prod and Experimental Targeting predicate logic - * @return: NamedPredicate[MagicFanoutNewsEventPushCandidate] - */ - def magicFanoutTargetingPredicate( - stats: StatsReceiver, - config: Config - ): NamedPredicate[MagicFanoutEventPushCandidate] = { - val name = "magic_fanout_targeting_predicate" - Predicate - .fromAsync { candidate: MagicFanoutEventPushCandidate => - val mfTargetingPredicateParam = getTargetingPredicateParams(candidate) - val mfTargetingPredicate = MagicFanoutTargetingPredicateMapForCandidate - .apply(config) - .get(candidate.target.params(mfTargetingPredicateParam)) - mfTargetingPredicate match { - case Some(predicate) => - predicate.apply(Seq(candidate)).map(_.head) - case None => - throw new Exception( - s"MFTargetingPredicateMap doesnt contain value for TargetingParam: ${FeatureSwitchParams.MFTargetingPredicate}") - } - } - .withStats(stats.scope(name)) - .withName(name) - } - - private def getTargetingPredicateParams( - candidate: MagicFanoutEventPushCandidate - ): FSEnumParam[MagicFanoutTargetingPredicatesEnum.type] = { - if (candidate.commonRecType == CommonRecommendationType.MagicFanoutSportsEvent) { - FeatureSwitchParams.MFCricketTargetingPredicate - } else FeatureSwitchParams.MFTargetingPredicate - } - - /** - * SimCluster and ERG and Topic Follows Targeting Predicate - */ - def simClusterErgTopicFollowsTargetingPredicate( - implicit stats: StatsReceiver, - interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests] - ): NamedPredicate[MagicFanoutEventPushCandidate] = { - simClusterErgTargetingPredicate - .or(MagicFanoutPredicatesForCandidate.magicFanoutTopicFollowsTargetingPredicate) - .withName("sim_cluster_erg_topic_follows_targeting") - } - - /** - * SimCluster and ERG and Topic Follows Targeting Predicate - */ - def simClusterErgTopicFollowsUserFollowsTargetingPredicate( - implicit stats: StatsReceiver, - interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests] - ): NamedPredicate[MagicFanoutEventPushCandidate] = { - simClusterErgTopicFollowsTargetingPredicate - .or( - MagicFanoutPredicatesForCandidate.followRankThreshold( - PushFeatureSwitchParams.MagicFanoutRealgraphRankThreshold)) - .withName("sim_cluster_erg_topic_follows_user_follows_targeting") - } - - /** - * SimCluster and ERG Targeting Predicate - */ - def simClusterErgTargetingPredicate( - implicit stats: StatsReceiver - ): NamedPredicate[MagicFanoutEventPushCandidate] = { - MagicFanoutPredicatesForCandidate.magicFanoutSimClusterTargetingPredicate - .or(MagicFanoutPredicatesForCandidate.magicFanoutErgInterestRankThresholdPredicate) - .withName("sim_cluster_erg_targeting") - } -} - -/** - * Object to initalze and get predicate map - */ -object MagicFanoutTargetingPredicateMapForCandidate { - - /** - * Called from the Config.scala at the time of server initialization - * @param statsReceiver: implict stats receiver - * @return Map[MagicFanoutTargetingPredicatesEnum, NamedPredicate[MagicFanoutNewsEventPushCandidate]] - */ - def apply( - config: Config - ): Map[MagicFanoutTargetingPredicatesEnum, NamedPredicate[MagicFanoutEventPushCandidate]] = { - Map( - MagicFanoutTargetingPredicatesEnum.SimClusterAndERGAndTopicFollows -> MagicFanoutTargetingPredicateWrappersForCandidate - .simClusterErgTopicFollowsTargetingPredicate( - config.statsReceiver, - config.interestsWithLookupContextStore), - MagicFanoutTargetingPredicatesEnum.SimClusterAndERG -> MagicFanoutTargetingPredicateWrappersForCandidate - .simClusterErgTargetingPredicate(config.statsReceiver), - MagicFanoutTargetingPredicatesEnum.SimCluster -> MagicFanoutPredicatesForCandidate - .magicFanoutSimClusterTargetingPredicate(config.statsReceiver), - MagicFanoutTargetingPredicatesEnum.ERG -> MagicFanoutPredicatesForCandidate - .magicFanoutErgInterestRankThresholdPredicate(config.statsReceiver), - MagicFanoutTargetingPredicatesEnum.TopicFollows -> MagicFanoutPredicatesForCandidate - .magicFanoutTopicFollowsTargetingPredicate( - config.statsReceiver, - config.interestsWithLookupContextStore), - MagicFanoutTargetingPredicatesEnum.UserFollows -> MagicFanoutPredicatesForCandidate - .followRankThreshold( - PushFeatureSwitchParams.MagicFanoutRealgraphRankThreshold - )(config.statsReceiver), - MagicFanoutTargetingPredicatesEnum.SimClusterAndERGAndTopicFollowsAndUserFollows -> - MagicFanoutTargetingPredicateWrappersForCandidate - .simClusterErgTopicFollowsUserFollowsTargetingPredicate( - config.statsReceiver, - config.interestsWithLookupContextStore - ) - ) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/CRTBasedNtabCaretClickFatiguePredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/CRTBasedNtabCaretClickFatiguePredicates.scala deleted file mode 100644 index 704f300a5..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/CRTBasedNtabCaretClickFatiguePredicates.scala +++ /dev/null @@ -1,973 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.notificationservice.thriftscala.GenericType -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.notificationservice.genericfeedbackstore.FeedbackPromptValue -import com.twitter.hermit.predicate.Predicate -import com.twitter.frigate.common.base.Candidate -import com.twitter.frigate.common.base.RecommendationType -import com.twitter.frigate.common.base.TargetInfo -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.SeeLessOftenType -import com.twitter.frigate.common.history.History -import com.twitter.frigate.common.predicate.FrigateHistoryFatiguePredicate.TimeSeries -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.common.predicate.ntab_caret_fatigue.NtabCaretClickFatiguePredicateHelper -import com.twitter.frigate.pushservice.predicate.CaretFeedbackHistoryFilter -import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.frigate.common.predicate.FatiguePredicate -import com.twitter.frigate.pushservice.util.PushCapUtil -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.util.PushDeviceUtil - -object CRTBasedNtabCaretClickFatiguePredicates { - - private val MagicRecsCategory = "MagicRecs" - - private val HighQualityRefreshableTypes: Set[Option[String]] = Set( - Some("MagicRecHighQualityTweet"), - ) - - private def getUserStateWeight(target: Target): Future[Double] = { - PushDeviceUtil.isNtabOnlyEligible.map { - case true => - target.params(PushFeatureSwitchParams.SeeLessOftenNtabOnlyNotifUserPushCapWeight) - case _ => 1.0 - } - } - - def crtToSeeLessOftenType( - crt: CommonRecommendationType, - candidate: Candidate - with RecommendationType - with TargetInfo[ - Target - ], - ): SeeLessOftenType = { - val crtToSeeLessOftenTypeMap: Map[CommonRecommendationType, SeeLessOftenType] = { - RecTypes.f1FirstDegreeTypes.map((_, SeeLessOftenType.F1Type)).toMap - } - - crtToSeeLessOftenTypeMap.getOrElse(crt, SeeLessOftenType.OtherTypes) - } - - def genericTypeToSeeLessOftenType( - genericType: GenericType, - candidate: Candidate - with RecommendationType - with TargetInfo[ - Target - ] - ): SeeLessOftenType = { - val genericTypeToSeeLessOftenTypeMap: Map[GenericType, SeeLessOftenType] = { - Map(GenericType.MagicRecFirstDegreeTweetRecent -> SeeLessOftenType.F1Type) - } - - genericTypeToSeeLessOftenTypeMap.getOrElse(genericType, SeeLessOftenType.OtherTypes) - } - - def getWeightForCaretFeedback( - dislikedType: SeeLessOftenType, - candidate: Candidate - with RecommendationType - with TargetInfo[ - Target - ] - ): Double = { - def getWeightFromDislikedAndCurrentType( - dislikedType: SeeLessOftenType, - currentType: SeeLessOftenType - ): Double = { - val weightMap: Map[(SeeLessOftenType, SeeLessOftenType), Double] = { - - Map( - (SeeLessOftenType.F1Type, SeeLessOftenType.F1Type) -> candidate.target.params( - PushFeatureSwitchParams.SeeLessOftenF1TriggerF1PushCapWeight), - (SeeLessOftenType.OtherTypes, SeeLessOftenType.OtherTypes) -> candidate.target.params( - PushFeatureSwitchParams.SeeLessOftenNonF1TriggerNonF1PushCapWeight), - (SeeLessOftenType.F1Type, SeeLessOftenType.OtherTypes) -> candidate.target.params( - PushFeatureSwitchParams.SeeLessOftenF1TriggerNonF1PushCapWeight), - (SeeLessOftenType.OtherTypes, SeeLessOftenType.F1Type) -> candidate.target.params( - PushFeatureSwitchParams.SeeLessOftenNonF1TriggerF1PushCapWeight) - ) - } - - weightMap - .getOrElse( - (dislikedType, currentType), - candidate.target.params(PushFeatureSwitchParams.SeeLessOftenDefaultPushCapWeight)) - } - - getWeightFromDislikedAndCurrentType( - dislikedType, - crtToSeeLessOftenType(candidate.commonRecType, candidate)) - } - - private def isOutsideCrtBasedNtabCaretClickFatiguePeriodContFn( - candidate: Candidate - with RecommendationType - with TargetInfo[ - Target - ], - history: History, - feedbackDetails: Seq[CaretFeedbackDetails], - filterHistory: TimeSeries => TimeSeries = - FatiguePredicate.recTypesOnlyFilter(RecTypes.sharedNTabCaretFatigueTypes), - filterCaretFeedbackHistory: Target => Seq[ - CaretFeedbackDetails - ] => Seq[CaretFeedbackDetails] = - CaretFeedbackHistoryFilter.caretFeedbackHistoryFilter(Seq(MagicRecsCategory)), - knobs: Seq[Double], - pushCapKnobs: Seq[Double], - powerKnobs: Seq[Double], - f1Weight: Double, - nonF1Weight: Double, - defaultPushCap: Int, - stats: StatsReceiver, - tripHqTweetWeight: Double = 0.0, - ): Boolean = { - val filteredFeedbackDetails = filterCaretFeedbackHistory(candidate.target)(feedbackDetails) - val weight = { - if (RecTypes.HighQualityTweetTypes.contains( - candidate.commonRecType) && (tripHqTweetWeight != 0)) { - tripHqTweetWeight - } else if (RecTypes.isF1Type(candidate.commonRecType)) { - f1Weight - } else { - nonF1Weight - } - } - val filteredHistory = History(filterHistory(history.history.toSeq).toMap) - isOutsideFatiguePeriod( - filteredHistory, - filteredFeedbackDetails, - Seq(), - ContinuousFunctionParam( - knobs, - pushCapKnobs, - powerKnobs, - weight, - defaultPushCap - ), - stats.scope( - if (RecTypes.isF1Type(candidate.commonRecType)) "mr_ntab_dislike_f1_candidate_fn" - else if (RecTypes.HighQualityTweetTypes.contains(candidate.commonRecType)) - "mr_ntab_dislike_high_quality_candidate_fn" - else "mr_ntab_dislike_nonf1_candidate_fn") - ) - } - - private def isOutsideFatiguePeriod( - history: History, - feedbackDetails: Seq[CaretFeedbackDetails], - feedbacks: Seq[FeedbackModel], - param: ContinuousFunctionParam, - stats: StatsReceiver - ): Boolean = { - val fatiguePeriod: Duration = - NtabCaretClickFatigueUtils.durationToFilterForFeedback( - feedbackDetails, - feedbacks, - param, - param.defaultValue, - stats - ) - - val hasRecentSent = - NtabCaretClickFatiguePredicateHelper.hasRecentSend(history, fatiguePeriod) - !hasRecentSent - - } - - def genericCRTBasedNtabCaretClickFnFatiguePredicate[ - Cand <: Candidate with RecommendationType with TargetInfo[ - Target - ] - ]( - filterHistory: TimeSeries => TimeSeries = - FatiguePredicate.recTypesOnlyFilter(RecTypes.sharedNTabCaretFatigueTypes), - filterCaretFeedbackHistory: Target => Seq[ - CaretFeedbackDetails - ] => Seq[CaretFeedbackDetails] = CaretFeedbackHistoryFilter - .caretFeedbackHistoryFilter(Seq(MagicRecsCategory)), - filterInlineFeedbackHistory: Seq[FeedbackModel] => Seq[FeedbackModel] = - NtabCaretClickFatigueUtils.feedbackModelFilterByCRT(RecTypes.sharedNTabCaretFatigueTypes) - )( - implicit stats: StatsReceiver - ): NamedPredicate[Cand] = { - val predicateName = "generic_crt_based_ntab_dislike_fatigue_fn" - Predicate - .fromAsync[Cand] { cand: Cand => - { - if (!cand.target.params(PushFeatureSwitchParams.EnableGenericCRTBasedFatiguePredicate)) { - Future.True - } else { - val scopedStats = stats.scope(predicateName) - val totalRequests = scopedStats.counter("mr_ntab_dislike_total") - val total90Day = - scopedStats.counter("mr_ntab_dislike_90day_dislike") - val totalDisabled = - scopedStats.counter("mr_ntab_dislike_not_90day_dislike") - val totalSuccess = scopedStats.counter("mr_ntab_dislike_success") - val totalFiltered = scopedStats.counter("mr_ntab_dislike_filtered") - val totalWithHistory = - scopedStats.counter("mr_ntab_dislike_with_history") - val totalWithoutHistory = - scopedStats.counter("mr_ntab_dislike_without_history") - totalRequests.incr() - - Future - .join( - cand.target.history, - cand.target.caretFeedbacks, - cand.target.dynamicPushcap, - cand.target.optoutAdjustedPushcap, - PushCapUtil.getDefaultPushCap(cand.target), - getUserStateWeight(cand.target) - ).map { - case ( - history, - Some(feedbackDetails), - dynamicPushcapOpt, - optoutAdjustedPushcapOpt, - defaultPushCap, - userStateWeight) => { - totalWithHistory.incr() - - val feedbackDetailsDeduped = - NtabCaretClickFatiguePredicateHelper.dedupFeedbackDetails( - filterCaretFeedbackHistory(cand.target)(feedbackDetails), - stats - ) - - val pushCap: Int = (dynamicPushcapOpt, optoutAdjustedPushcapOpt) match { - case (_, Some(optoutAdjustedPushcap)) => optoutAdjustedPushcap - case (Some(pushcapInfo), _) => pushcapInfo.pushcap - case _ => defaultPushCap - } - val filteredHistory = History(filterHistory(history.history.toSeq).toMap) - - val hasUserDislikeInLast90Days = - NtabCaretClickFatigueUtils.hasUserDislikeInLast90Days(feedbackDetailsDeduped) - val isF1TriggerFatigueEnabled = cand.target - .params(PushFeatureSwitchParams.EnableContFnF1TriggerSeeLessOftenFatigue) - val isNonF1TriggerFatigueEnabled = cand.target.params( - PushFeatureSwitchParams.EnableContFnNonF1TriggerSeeLessOftenFatigue) - - val isOutisdeSeeLessOftenFatigue = - if (hasUserDislikeInLast90Days && (isF1TriggerFatigueEnabled || isNonF1TriggerFatigueEnabled)) { - total90Day.incr() - - val feedbackDetailsGroupedBySeeLessOftenType: Map[Option[ - SeeLessOftenType - ], Seq[ - CaretFeedbackDetails - ]] = feedbackDetails.groupBy(feedbackDetail => - feedbackDetail.genericNotificationMetadata.map(x => - genericTypeToSeeLessOftenType(x.genericType, cand))) - - val isOutsideFatiguePeriodSeq = - for (elem <- feedbackDetailsGroupedBySeeLessOftenType if elem._1.isDefined) - yield { - val dislikedSeeLessOftenType: SeeLessOftenType = elem._1.get - val seqCaretFeedbackDetails: Seq[CaretFeedbackDetails] = elem._2 - - val weight = getWeightForCaretFeedback( - dislikedSeeLessOftenType, - cand) * userStateWeight - - if (isOutsideFatiguePeriod( - history = filteredHistory, - feedbackDetails = seqCaretFeedbackDetails, - feedbacks = Seq(), - param = ContinuousFunctionParam( - knobs = cand.target - .params(PushFeatureSwitchParams.SeeLessOftenListOfDayKnobs), - knobValues = cand.target - .params( - PushFeatureSwitchParams.SeeLessOftenListOfPushCapWeightKnobs).map( - _ * pushCap), - powers = cand.target - .params(PushFeatureSwitchParams.SeeLessOftenListOfPowerKnobs), - weight = weight, - defaultValue = pushCap - ), - scopedStats - )) { - true - } else { - false - } - } - - isOutsideFatiguePeriodSeq.forall(identity) - } else { - totalDisabled.incr() - true - } - - if (isOutisdeSeeLessOftenFatigue) { - totalSuccess.incr() - } else totalFiltered.incr() - - isOutisdeSeeLessOftenFatigue - } - - case _ => - totalSuccess.incr() - totalWithoutHistory.incr() - true - } - } - } - }.withStats(stats.scope(predicateName)) - .withName(predicateName) - } - - def f1TriggeredCRTBasedNtabCaretClickFnFatiguePredicate[ - Cand <: Candidate with RecommendationType with TargetInfo[ - Target - ] - ]( - filterHistory: TimeSeries => TimeSeries = - FatiguePredicate.recTypesOnlyFilter(RecTypes.sharedNTabCaretFatigueTypes), - filterCaretFeedbackHistory: Target => Seq[ - CaretFeedbackDetails - ] => Seq[CaretFeedbackDetails] = CaretFeedbackHistoryFilter - .caretFeedbackHistoryFilter(Seq(MagicRecsCategory)), - filterInlineFeedbackHistory: Seq[FeedbackModel] => Seq[FeedbackModel] = - NtabCaretClickFatigueUtils.feedbackModelFilterByCRT(RecTypes.sharedNTabCaretFatigueTypes) - )( - implicit stats: StatsReceiver - ): NamedPredicate[Cand] = { - val predicateName = "f1_triggered_crt_based_ntab_dislike_fatigue_fn" - Predicate - .fromAsync[Cand] { cand: Cand => - { - val scopedStats = stats.scope(predicateName) - val totalRequests = scopedStats.counter("mr_ntab_dislike_total") - val total90Day = - scopedStats.counter("mr_ntab_dislike_90day_dislike") - val totalDisabled = - scopedStats.counter("mr_ntab_dislike_not_90day_dislike") - val totalSuccess = scopedStats.counter("mr_ntab_dislike_success") - val totalFiltered = scopedStats.counter("mr_ntab_dislike_filtered") - val totalWithHistory = - scopedStats.counter("mr_ntab_dislike_with_history") - val totalWithoutHistory = - scopedStats.counter("mr_ntab_dislike_without_history") - totalRequests.incr() - - Future - .join( - cand.target.history, - cand.target.caretFeedbacks, - cand.target.dynamicPushcap, - cand.target.optoutAdjustedPushcap, - cand.target.notificationFeedbacks, - PushCapUtil.getDefaultPushCap(cand.target), - getUserStateWeight(cand.target) - ).map { - case ( - history, - Some(feedbackDetails), - dynamicPushcapOpt, - optoutAdjustedPushcapOpt, - Some(feedbacks), - defaultPushCap, - userStateWeight) => - totalWithHistory.incr() - - val feedbackDetailsDeduped = - NtabCaretClickFatiguePredicateHelper.dedupFeedbackDetails( - filterCaretFeedbackHistory(cand.target)(feedbackDetails), - stats - ) - - val pushCap: Int = (dynamicPushcapOpt, optoutAdjustedPushcapOpt) match { - case (_, Some(optoutAdjustedPushcap)) => optoutAdjustedPushcap - case (Some(pushcapInfo), _) => pushcapInfo.pushcap - case _ => defaultPushCap - } - val filteredHistory = History(filterHistory(history.history.toSeq).toMap) - - val isOutsideInlineDislikeFatigue = - if (cand.target - .params(PushFeatureSwitchParams.EnableContFnF1TriggerInlineFeedbackFatigue)) { - val weight = - if (RecTypes.isF1Type(cand.commonRecType)) { - cand.target - .params(PushFeatureSwitchParams.InlineFeedbackF1TriggerF1PushCapWeight) - } else { - cand.target - .params(PushFeatureSwitchParams.InlineFeedbackF1TriggerNonF1PushCapWeight) - } - - val inlineFeedbackFatigueParam = ContinuousFunctionParam( - cand.target - .params(PushFeatureSwitchParams.InlineFeedbackListOfDayKnobs), - cand.target - .params(PushFeatureSwitchParams.InlineFeedbackListOfPushCapWeightKnobs) - .map(_ * pushCap), - cand.target - .params(PushFeatureSwitchParams.InlineFeedbackListOfPowerKnobs), - weight, - pushCap - ) - - isInlineDislikeOutsideFatiguePeriod( - cand, - feedbacks - .collect { - case feedbackPromptValue: FeedbackPromptValue => - InlineFeedbackModel(feedbackPromptValue, None) - }, - filteredHistory, - Seq( - filterInlineFeedbackHistory, - NtabCaretClickFatigueUtils.feedbackModelFilterByCRT( - RecTypes.f1FirstDegreeTypes)), - inlineFeedbackFatigueParam, - scopedStats - ) - } else true - - lazy val isOutsidePromptDislikeFatigue = - if (cand.target - .params(PushFeatureSwitchParams.EnableContFnF1TriggerPromptFeedbackFatigue)) { - val weight = - if (RecTypes.isF1Type(cand.commonRecType)) { - cand.target - .params(PushFeatureSwitchParams.PromptFeedbackF1TriggerF1PushCapWeight) - } else { - cand.target - .params(PushFeatureSwitchParams.PromptFeedbackF1TriggerNonF1PushCapWeight) - } - - val promptFeedbackFatigueParam = ContinuousFunctionParam( - cand.target - .params(PushFeatureSwitchParams.PromptFeedbackListOfDayKnobs), - cand.target - .params(PushFeatureSwitchParams.PromptFeedbackListOfPushCapWeightKnobs) - .map(_ * pushCap), - cand.target - .params(PushFeatureSwitchParams.PromptFeedbackListOfPowerKnobs), - weight, - pushCap - ) - - isPromptDislikeOutsideFatiguePeriod( - feedbacks - .collect { - case feedbackPromptValue: FeedbackPromptValue => - PromptFeedbackModel(feedbackPromptValue, None) - }, - filteredHistory, - Seq( - filterInlineFeedbackHistory, - NtabCaretClickFatigueUtils.feedbackModelFilterByCRT( - RecTypes.f1FirstDegreeTypes)), - promptFeedbackFatigueParam, - scopedStats - ) - } else true - - isOutsideInlineDislikeFatigue && isOutsidePromptDislikeFatigue - - case _ => - totalSuccess.incr() - totalWithoutHistory.incr() - true - } - } - }.withStats(stats.scope(predicateName)) - .withName(predicateName) - } - - def nonF1TriggeredCRTBasedNtabCaretClickFnFatiguePredicate[ - Cand <: Candidate with RecommendationType with TargetInfo[ - Target - ] - ]( - filterHistory: TimeSeries => TimeSeries = - FatiguePredicate.recTypesOnlyFilter(RecTypes.sharedNTabCaretFatigueTypes), - filterCaretFeedbackHistory: Target => Seq[ - CaretFeedbackDetails - ] => Seq[CaretFeedbackDetails] = CaretFeedbackHistoryFilter - .caretFeedbackHistoryFilter(Seq(MagicRecsCategory)), - filterInlineFeedbackHistory: Seq[FeedbackModel] => Seq[FeedbackModel] = - NtabCaretClickFatigueUtils.feedbackModelFilterByCRT(RecTypes.sharedNTabCaretFatigueTypes) - )( - implicit stats: StatsReceiver - ): NamedPredicate[Cand] = { - val predicateName = "non_f1_triggered_crt_based_ntab_dislike_fatigue_fn" - Predicate - .fromAsync[Cand] { cand: Cand => - { - val scopedStats = stats.scope(predicateName) - val totalRequests = scopedStats.counter("mr_ntab_dislike_total") - val total90Day = - scopedStats.counter("mr_ntab_dislike_90day_dislike") - val totalDisabled = - scopedStats.counter("mr_ntab_dislike_not_90day_dislike") - val totalSuccess = scopedStats.counter("mr_ntab_dislike_success") - val totalFiltered = scopedStats.counter("mr_ntab_dislike_filtered") - val totalWithHistory = - scopedStats.counter("mr_ntab_dislike_with_history") - val totalWithoutHistory = - scopedStats.counter("mr_ntab_dislike_without_history") - val totalFeedbackSuccess = scopedStats.counter("mr_total_feedback_success") - totalRequests.incr() - - Future - .join( - cand.target.history, - cand.target.caretFeedbacks, - cand.target.dynamicPushcap, - cand.target.optoutAdjustedPushcap, - cand.target.notificationFeedbacks, - PushCapUtil.getDefaultPushCap(cand.target), - getUserStateWeight(cand.target), - ).map { - case ( - history, - Some(feedbackDetails), - dynamicPushcapOpt, - optoutAdjustedPushcapOpt, - Some(feedbacks), - defaultPushCap, - userStateWeight) => - totalWithHistory.incr() - - val filteredfeedbackDetails = - if (cand.target.params( - PushFeatureSwitchParams.AdjustTripHqTweetTriggeredNtabCaretClickFatigue)) { - val refreshableTypeFilter = CaretFeedbackHistoryFilter - .caretFeedbackHistoryFilterByRefreshableTypeDenyList( - HighQualityRefreshableTypes) - refreshableTypeFilter(cand.target)(feedbackDetails) - } else { - feedbackDetails - } - - val feedbackDetailsDeduped = - NtabCaretClickFatiguePredicateHelper.dedupFeedbackDetails( - filterCaretFeedbackHistory(cand.target)(filteredfeedbackDetails), - stats - ) - - val pushCap: Int = (dynamicPushcapOpt, optoutAdjustedPushcapOpt) match { - case (_, Some(optoutAdjustedPushcap)) => optoutAdjustedPushcap - case (Some(pushcapInfo), _) => pushcapInfo.pushcap - case _ => defaultPushCap - } - val filteredHistory = History(filterHistory(history.history.toSeq).toMap) - - val isOutsideInlineDislikeFatigue = - if (cand.target - .params( - PushFeatureSwitchParams.EnableContFnNonF1TriggerInlineFeedbackFatigue)) { - val weight = - if (RecTypes.isF1Type(cand.commonRecType)) - cand.target - .params(PushFeatureSwitchParams.InlineFeedbackNonF1TriggerF1PushCapWeight) - else - cand.target - .params( - PushFeatureSwitchParams.InlineFeedbackNonF1TriggerNonF1PushCapWeight) - - val inlineFeedbackFatigueParam = ContinuousFunctionParam( - cand.target - .params(PushFeatureSwitchParams.InlineFeedbackListOfDayKnobs), - cand.target - .params(PushFeatureSwitchParams.InlineFeedbackListOfPushCapWeightKnobs) - .map(_ * pushCap), - cand.target - .params(PushFeatureSwitchParams.InlineFeedbackListOfPowerKnobs), - weight, - pushCap - ) - - val excludedCRTs: Set[CommonRecommendationType] = - if (cand.target.params( - PushFeatureSwitchParams.AdjustTripHqTweetTriggeredNtabCaretClickFatigue)) { - RecTypes.f1FirstDegreeTypes ++ RecTypes.HighQualityTweetTypes - } else { - RecTypes.f1FirstDegreeTypes - } - - isInlineDislikeOutsideFatiguePeriod( - cand, - feedbacks - .collect { - case feedbackPromptValue: FeedbackPromptValue => - InlineFeedbackModel(feedbackPromptValue, None) - }, - filteredHistory, - Seq( - filterInlineFeedbackHistory, - NtabCaretClickFatigueUtils.feedbackModelExcludeCRT(excludedCRTs)), - inlineFeedbackFatigueParam, - scopedStats - ) - } else true - - lazy val isOutsidePromptDislikeFatigue = - if (cand.target - .params( - PushFeatureSwitchParams.EnableContFnNonF1TriggerPromptFeedbackFatigue)) { - val weight = - if (RecTypes.isF1Type(cand.commonRecType)) - cand.target - .params(PushFeatureSwitchParams.PromptFeedbackNonF1TriggerF1PushCapWeight) - else - cand.target - .params( - PushFeatureSwitchParams.PromptFeedbackNonF1TriggerNonF1PushCapWeight) - - val promptFeedbackFatigueParam = ContinuousFunctionParam( - cand.target - .params(PushFeatureSwitchParams.PromptFeedbackListOfDayKnobs), - cand.target - .params(PushFeatureSwitchParams.PromptFeedbackListOfPushCapWeightKnobs) - .map(_ * pushCap), - cand.target - .params(PushFeatureSwitchParams.PromptFeedbackListOfPowerKnobs), - weight, - pushCap - ) - - isPromptDislikeOutsideFatiguePeriod( - feedbacks - .collect { - case feedbackPromptValue: FeedbackPromptValue => - PromptFeedbackModel(feedbackPromptValue, None) - }, - filteredHistory, - Seq( - filterInlineFeedbackHistory, - NtabCaretClickFatigueUtils.feedbackModelExcludeCRT( - RecTypes.f1FirstDegreeTypes)), - promptFeedbackFatigueParam, - scopedStats - ) - } else true - - isOutsideInlineDislikeFatigue && isOutsidePromptDislikeFatigue - case _ => - totalFeedbackSuccess.incr() - totalWithoutHistory.incr() - true - } - } - }.withStats(stats.scope(predicateName)) - .withName(predicateName) - } - - def tripHqTweetTriggeredCRTBasedNtabCaretClickFnFatiguePredicate[ - Cand <: Candidate with RecommendationType with TargetInfo[ - Target - ] - ]( - filterHistory: TimeSeries => TimeSeries = - FatiguePredicate.recTypesOnlyFilter(RecTypes.sharedNTabCaretFatigueTypes), - filterCaretFeedbackHistory: Target => Seq[ - CaretFeedbackDetails - ] => Seq[CaretFeedbackDetails] = CaretFeedbackHistoryFilter - .caretFeedbackHistoryFilter(Seq(MagicRecsCategory)), - filterInlineFeedbackHistory: Seq[FeedbackModel] => Seq[FeedbackModel] = - NtabCaretClickFatigueUtils.feedbackModelFilterByCRT(RecTypes.sharedNTabCaretFatigueTypes) - )( - implicit stats: StatsReceiver - ): NamedPredicate[Cand] = { - val predicateName = "trip_hq_tweet_triggered_crt_based_ntab_dislike_fatigue_fn" - Predicate - .fromAsync[Cand] { cand: Cand => - { - val scopedStats = stats.scope(predicateName) - val totalRequests = scopedStats.counter("mr_ntab_dislike_total") - val total90Day = - scopedStats.counter("mr_ntab_dislike_90day_dislike") - val totalDisabled = - scopedStats.counter("mr_ntab_dislike_not_90day_dislike") - val totalSuccess = scopedStats.counter("mr_ntab_dislike_success") - val totalFiltered = scopedStats.counter("mr_ntab_dislike_filtered") - val totalWithHistory = - scopedStats.counter("mr_ntab_dislike_with_history") - val totalWithoutHistory = - scopedStats.counter("mr_ntab_dislike_without_history") - val totalFeedbackSuccess = scopedStats.counter("mr_total_feedback_success") - totalRequests.incr() - - Future - .join( - cand.target.history, - cand.target.caretFeedbacks, - cand.target.dynamicPushcap, - cand.target.optoutAdjustedPushcap, - cand.target.notificationFeedbacks, - PushCapUtil.getDefaultPushCap(cand.target), - getUserStateWeight(cand.target), - ).map { - case ( - history, - Some(feedbackDetails), - dynamicPushcapOpt, - optoutAdjustedPushcapOpt, - Some(feedbacks), - defaultPushCap, - userStateWeight) => - totalWithHistory.incr() - if (cand.target.params( - PushFeatureSwitchParams.AdjustTripHqTweetTriggeredNtabCaretClickFatigue)) { - - val refreshableTypeFilter = CaretFeedbackHistoryFilter - .caretFeedbackHistoryFilterByRefreshableType(HighQualityRefreshableTypes) - val filteredfeedbackDetails = refreshableTypeFilter(cand.target)(feedbackDetails) - - val feedbackDetailsDeduped = - NtabCaretClickFatiguePredicateHelper.dedupFeedbackDetails( - filterCaretFeedbackHistory(cand.target)(filteredfeedbackDetails), - stats - ) - - val pushCap: Int = (dynamicPushcapOpt, optoutAdjustedPushcapOpt) match { - case (_, Some(optoutAdjustedPushcap)) => optoutAdjustedPushcap - case (Some(pushcapInfo), _) => pushcapInfo.pushcap - case _ => defaultPushCap - } - val filteredHistory = History(filterHistory(history.history.toSeq).toMap) - - val isOutsideInlineDislikeFatigue = - if (cand.target - .params( - PushFeatureSwitchParams.EnableContFnNonF1TriggerInlineFeedbackFatigue)) { - val weight = { - if (RecTypes.HighQualityTweetTypes.contains(cand.commonRecType)) { - cand.target - .params( - PushFeatureSwitchParams.InlineFeedbackNonF1TriggerNonF1PushCapWeight) - } else { - cand.target - .params( - PushFeatureSwitchParams.InlineFeedbackNonF1TriggerF1PushCapWeight) - } - } - - val inlineFeedbackFatigueParam = ContinuousFunctionParam( - cand.target - .params(PushFeatureSwitchParams.InlineFeedbackListOfDayKnobs), - cand.target - .params(PushFeatureSwitchParams.InlineFeedbackListOfPushCapWeightKnobs) - .map(_ * pushCap), - cand.target - .params(PushFeatureSwitchParams.InlineFeedbackListOfPowerKnobs), - weight, - pushCap - ) - - val includedCRTs: Set[CommonRecommendationType] = - RecTypes.HighQualityTweetTypes - - isInlineDislikeOutsideFatiguePeriod( - cand, - feedbacks - .collect { - case feedbackPromptValue: FeedbackPromptValue => - InlineFeedbackModel(feedbackPromptValue, None) - }, - filteredHistory, - Seq( - filterInlineFeedbackHistory, - NtabCaretClickFatigueUtils.feedbackModelFilterByCRT(includedCRTs)), - inlineFeedbackFatigueParam, - scopedStats - ) - } else true - - lazy val isOutsidePromptDislikeFatigue = - if (cand.target - .params( - PushFeatureSwitchParams.EnableContFnNonF1TriggerPromptFeedbackFatigue)) { - val weight = - if (RecTypes.isF1Type(cand.commonRecType)) - cand.target - .params( - PushFeatureSwitchParams.PromptFeedbackNonF1TriggerF1PushCapWeight) - else - cand.target - .params( - PushFeatureSwitchParams.PromptFeedbackNonF1TriggerNonF1PushCapWeight) - - val promptFeedbackFatigueParam = ContinuousFunctionParam( - cand.target - .params(PushFeatureSwitchParams.PromptFeedbackListOfDayKnobs), - cand.target - .params(PushFeatureSwitchParams.PromptFeedbackListOfPushCapWeightKnobs) - .map(_ * pushCap), - cand.target - .params(PushFeatureSwitchParams.PromptFeedbackListOfPowerKnobs), - weight, - pushCap - ) - - isPromptDislikeOutsideFatiguePeriod( - feedbacks - .collect { - case feedbackPromptValue: FeedbackPromptValue => - PromptFeedbackModel(feedbackPromptValue, None) - }, - filteredHistory, - Seq( - filterInlineFeedbackHistory, - NtabCaretClickFatigueUtils.feedbackModelExcludeCRT( - RecTypes.f1FirstDegreeTypes)), - promptFeedbackFatigueParam, - scopedStats - ) - } else true - - isOutsideInlineDislikeFatigue && isOutsidePromptDislikeFatigue - } else { - true - } - case _ => - totalFeedbackSuccess.incr() - totalWithoutHistory.incr() - true - } - } - }.withStats(stats.scope(predicateName)) - .withName(predicateName) - } - - private def getDedupedInlineFeedbackByType( - inlineFeedbacks: Seq[FeedbackModel], - feedbackType: FeedbackTypeEnum.Value, - revertedFeedbackType: FeedbackTypeEnum.Value - ): Seq[FeedbackModel] = { - inlineFeedbacks - .filter(feedback => - feedback.feedbackTypeEnum == feedbackType || - feedback.feedbackTypeEnum == revertedFeedbackType) - .groupBy(feedback => feedback.notificationImpressionId.getOrElse("")) - .toSeq - .collect { - case (impressionId, feedbacks: Seq[FeedbackModel]) if (feedbacks.nonEmpty) => - val latestFeedback = feedbacks.maxBy(feedback => feedback.timestampMs) - if (latestFeedback.feedbackTypeEnum == feedbackType) - Some(latestFeedback) - else None - case _ => None - } - .flatten - } - - private def getDedupedInlineFeedback( - inlineFeedbacks: Seq[FeedbackModel], - target: Target - ): Seq[FeedbackModel] = { - val inlineDislikeFeedback = - if (target.params(PushFeatureSwitchParams.UseInlineDislikeForFatigue)) { - getDedupedInlineFeedbackByType( - inlineFeedbacks, - FeedbackTypeEnum.InlineDislike, - FeedbackTypeEnum.InlineRevertedDislike) - } else Seq() - val inlineDismissFeedback = - if (target.params(PushFeatureSwitchParams.UseInlineDismissForFatigue)) { - getDedupedInlineFeedbackByType( - inlineFeedbacks, - FeedbackTypeEnum.InlineDismiss, - FeedbackTypeEnum.InlineRevertedDismiss) - } else Seq() - val inlineSeeLessFeedback = - if (target.params(PushFeatureSwitchParams.UseInlineSeeLessForFatigue)) { - getDedupedInlineFeedbackByType( - inlineFeedbacks, - FeedbackTypeEnum.InlineSeeLess, - FeedbackTypeEnum.InlineRevertedSeeLess) - } else Seq() - val inlineNotRelevantFeedback = - if (target.params(PushFeatureSwitchParams.UseInlineNotRelevantForFatigue)) { - getDedupedInlineFeedbackByType( - inlineFeedbacks, - FeedbackTypeEnum.InlineNotRelevant, - FeedbackTypeEnum.InlineRevertedNotRelevant) - } else Seq() - - inlineDislikeFeedback ++ inlineDismissFeedback ++ inlineSeeLessFeedback ++ inlineNotRelevantFeedback - } - - private def isInlineDislikeOutsideFatiguePeriod( - candidate: Candidate - with RecommendationType - with TargetInfo[ - Target - ], - inlineFeedbacks: Seq[FeedbackModel], - filteredHistory: History, - feedbackFilters: Seq[Seq[FeedbackModel] => Seq[FeedbackModel]], - inlineFeedbackFatigueParam: ContinuousFunctionParam, - stats: StatsReceiver - ): Boolean = { - val scopedStats = stats.scope("inline_dislike_fatigue") - - val inlineNegativeFeedback = - getDedupedInlineFeedback(inlineFeedbacks, candidate.target) - - val hydratedInlineNegativeFeedback = FeedbackModelHydrator.HydrateNotification( - inlineNegativeFeedback, - filteredHistory.history.toSeq.map(_._2)) - - if (isOutsideFatiguePeriod( - filteredHistory, - Seq(), - feedbackFilters.foldLeft(hydratedInlineNegativeFeedback)((feedbacks, feedbackFilter) => - feedbackFilter(feedbacks)), - inlineFeedbackFatigueParam, - scopedStats - )) { - scopedStats.counter("feedback_inline_dislike_success").incr() - true - } else { - scopedStats.counter("feedback_inline_dislike_filtered").incr() - false - } - } - - private def isPromptDislikeOutsideFatiguePeriod( - feedbacks: Seq[FeedbackModel], - filteredHistory: History, - feedbackFilters: Seq[Seq[FeedbackModel] => Seq[FeedbackModel]], - inlineFeedbackFatigueParam: ContinuousFunctionParam, - stats: StatsReceiver - ): Boolean = { - val scopedStats = stats.scope("prompt_dislike_fatigue") - - val promptDislikeFeedback = feedbacks - .filter(feedback => feedback.feedbackTypeEnum == FeedbackTypeEnum.PromptIrrelevant) - val hydratedPromptDislikeFeedback = FeedbackModelHydrator.HydrateNotification( - promptDislikeFeedback, - filteredHistory.history.toSeq.map(_._2)) - - if (isOutsideFatiguePeriod( - filteredHistory, - Seq(), - feedbackFilters.foldLeft(hydratedPromptDislikeFeedback)((feedbacks, feedbackFilter) => - feedbackFilter(feedbacks)), - inlineFeedbackFatigueParam, - scopedStats - )) { - scopedStats.counter("feedback_prompt_dislike_success").incr() - true - } else { - scopedStats.counter("feedback_prompt_dislike_filtered").incr() - false - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/ContinuousFunction.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/ContinuousFunction.scala deleted file mode 100644 index 862541b63..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/ContinuousFunction.scala +++ /dev/null @@ -1,148 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue - -import com.twitter.finagle.stats.StatsReceiver - -case class ContinuousFunctionParam( - knobs: Seq[Double], - knobValues: Seq[Double], - powers: Seq[Double], - weight: Double, - defaultValue: Double) { - - def validateParams(): Boolean = { - knobs.size > 0 && knobs.size - 1 == powers.size && knobs.size == knobValues.size - } -} - -object ContinuousFunction { - - /** - * Evalutate the value for function f(x) = w(x - b)^power - * where w and b are decided by the start, startVal, end, endVal - * such that - * w(start - b) ^ power = startVal - * w(end - b) ^ power = endVal - * - * @param value the value at which we will evaluate the param - * @return weight * f(value) - */ - def evaluateFn( - value: Double, - start: Double, - startVal: Double, - end: Double, - endVal: Double, - power: Double, - weight: Double - ): Double = { - val b = - (math.pow(startVal / endVal, 1 / power) * end - start) / (math.pow( - startVal / endVal, - 1 / power) - 1) - val w = startVal / math.pow(start - b, power) - weight * w * math.pow(value - b, power) - } - - /** - * Evaluate value for function f(x), and return weight * f(x) - * - * f(x) is a piecewise function - * f(x) = w_i * (x - b_i)^powers[i] for knobs[i] <= x < knobs[i+1] - * such that - * w(knobs[i] - b) ^ power = knobVals[i] - * w(knobs[i+1] - b) ^ power = knobVals[i+1] - * - * @return Evaluate value for weight * f(x), for the function described above. If the any of the input is invalid, returns defaultVal - */ - def safeEvaluateFn( - value: Double, - knobs: Seq[Double], - knobVals: Seq[Double], - powers: Seq[Double], - weight: Double, - defaultVal: Double, - statsReceiver: StatsReceiver - ): Double = { - val totalStats = statsReceiver.counter("safe_evalfn_total") - val validStats = - statsReceiver.counter("safe_evalfn_valid") - val validEndCaseStats = - statsReceiver.counter("safe_evalfn_valid_endcase") - val invalidStats = statsReceiver.counter("safe_evalfn_invalid") - - totalStats.incr() - if (knobs.size <= 0 || knobs.size - 1 != powers.size || knobs.size != knobVals.size) { - invalidStats.incr() - defaultVal - } else { - val endIndex = knobs.indexWhere(knob => knob > value) - validStats.incr() - endIndex match { - case -1 => { - validEndCaseStats.incr() - knobVals(knobVals.size - 1) * weight - } - case 0 => { - validEndCaseStats.incr() - knobVals(0) * weight - } - case _ => { - val startIndex = endIndex - 1 - evaluateFn( - value, - knobs(startIndex), - knobVals(startIndex), - knobs(endIndex), - knobVals(endIndex), - powers(startIndex), - weight) - } - } - } - } - - def safeEvaluateFn( - value: Double, - fnParams: ContinuousFunctionParam, - statsReceiver: StatsReceiver - ): Double = { - val totalStats = statsReceiver.counter("safe_evalfn_total") - val validStats = - statsReceiver.counter("safe_evalfn_valid") - val validEndCaseStats = - statsReceiver.counter("safe_evalfn_valid_endcase") - val invalidStats = statsReceiver.counter("safe_evalfn_invalid") - - totalStats.incr() - - if (fnParams.validateParams()) { - val endIndex = fnParams.knobs.indexWhere(knob => knob > value) - validStats.incr() - endIndex match { - case -1 => { - validEndCaseStats.incr() - fnParams.knobValues(fnParams.knobValues.size - 1) * fnParams.weight - } - case 0 => { - validEndCaseStats.incr() - fnParams.knobValues(0) * fnParams.weight - } - case _ => { - val startIndex = endIndex - 1 - evaluateFn( - value, - fnParams.knobs(startIndex), - fnParams.knobValues(startIndex), - fnParams.knobs(endIndex), - fnParams.knobValues(endIndex), - fnParams.powers(startIndex), - fnParams.weight - ) - } - } - } else { - invalidStats.incr() - fnParams.defaultValue - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/FeedbackModel.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/FeedbackModel.scala deleted file mode 100644 index 654889901..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/FeedbackModel.scala +++ /dev/null @@ -1,136 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue - -import com.twitter.notificationservice.thriftscala.GenericType -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.notificationservice.genericfeedbackstore.FeedbackPromptValue -import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails -import com.twitter.notificationservice.feedback.thriftscala.FeedbackMetadata -import com.twitter.notificationservice.feedback.thriftscala.InlineFeedback -import com.twitter.notificationservice.feedback.thriftscala.FeedbackValue -import com.twitter.notificationservice.feedback.thriftscala.YesOrNoAnswer - -object FeedbackTypeEnum extends Enumeration { - val Unknown = Value - val CaretDislike = Value - val InlineDislike = Value - val InlineLike = Value - val InlineRevertedLike = Value - val InlineRevertedDislike = Value - val PromptRelevant = Value - val PromptIrrelevant = Value - val InlineDismiss = Value - val InlineRevertedDismiss = Value - val InlineSeeLess = Value - val InlineRevertedSeeLess = Value - val InlineNotRelevant = Value - val InlineRevertedNotRelevant = Value - - def safeFindByName(name: String): Value = - values.find(_.toString.toLowerCase() == name.toLowerCase()).getOrElse(Unknown) -} - -trait FeedbackModel { - - def timestampMs: Long - - def feedbackTypeEnum: FeedbackTypeEnum.Value - - def notificationImpressionId: Option[String] - - def notification: Option[FrigateNotification] = None -} - -case class CaretFeedbackModel( - caretFeedbackDetails: CaretFeedbackDetails, - notificationOpt: Option[FrigateNotification] = None) - extends FeedbackModel { - - override def timestampMs: Long = caretFeedbackDetails.eventTimestamp - - override def feedbackTypeEnum: FeedbackTypeEnum.Value = FeedbackTypeEnum.CaretDislike - - override def notificationImpressionId: Option[String] = caretFeedbackDetails.impressionId - - override def notification: Option[FrigateNotification] = notificationOpt - - def notificationGenericType: Option[GenericType] = { - caretFeedbackDetails.genericNotificationMetadata match { - case Some(genericNotificationMetadata) => - Some(genericNotificationMetadata.genericType) - case None => None - } - } -} - -case class InlineFeedbackModel( - feedback: FeedbackPromptValue, - notificationOpt: Option[FrigateNotification] = None) - extends FeedbackModel { - - override def timestampMs: Long = feedback.createdAt.inMilliseconds - - override def feedbackTypeEnum: FeedbackTypeEnum.Value = { - feedback.feedbackValue match { - case FeedbackValue( - _, - _, - _, - Some(FeedbackMetadata.InlineFeedback(InlineFeedback(Some(answer))))) => - FeedbackTypeEnum.safeFindByName("inline" + answer) - case _ => FeedbackTypeEnum.Unknown - } - } - - override def notificationImpressionId: Option[String] = Some(feedback.feedbackValue.impressionId) - - override def notification: Option[FrigateNotification] = notificationOpt -} - -case class PromptFeedbackModel( - feedback: FeedbackPromptValue, - notificationOpt: Option[FrigateNotification] = None) - extends FeedbackModel { - - override def timestampMs: Long = feedback.createdAt.inMilliseconds - - override def feedbackTypeEnum: FeedbackTypeEnum.Value = { - feedback.feedbackValue match { - case FeedbackValue(_, _, _, Some(FeedbackMetadata.YesOrNoAnswer(answer))) => - answer match { - case YesOrNoAnswer.Yes => FeedbackTypeEnum.PromptRelevant - case YesOrNoAnswer.No => FeedbackTypeEnum.PromptIrrelevant - case _ => FeedbackTypeEnum.Unknown - } - case _ => FeedbackTypeEnum.Unknown - } - } - - override def notificationImpressionId: Option[String] = Some(feedback.feedbackValue.impressionId) - - override def notification: Option[FrigateNotification] = notificationOpt -} - -object FeedbackModelHydrator { - - def HydrateNotification( - feedbacks: Seq[FeedbackModel], - history: Seq[FrigateNotification] - ): Seq[FeedbackModel] = { - feedbacks.map { - case feedback @ (inlineFeedback: InlineFeedbackModel) => - inlineFeedback.copy(notificationOpt = history.find( - _.impressionId - .equals(feedback.notificationImpressionId))) - case feedback @ (caretFeedback: CaretFeedbackModel) => - caretFeedback.copy(notificationOpt = history.find( - _.impressionId - .equals(feedback.notificationImpressionId))) - case feedback @ (promptFeedback: PromptFeedbackModel) => - promptFeedback.copy(notificationOpt = history.find( - _.impressionId - .equals(feedback.notificationImpressionId))) - case feedback => feedback - } - - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/MagicFanoutNtabCaretFatiguePredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/MagicFanoutNtabCaretFatiguePredicate.scala deleted file mode 100644 index 040543660..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/MagicFanoutNtabCaretFatiguePredicate.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.predicate.ntab_caret_fatigue.NtabCaretClickFatiguePredicateHelper -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.hermit.predicate.NamedPredicate - -object MagicFanoutNtabCaretFatiguePredicate { - val name = "MagicFanoutNtabCaretFatiguePredicateForCandidate" - - private val MomentsCategory = "Moments" - private val MomentsViaMagicRecsCategory = "MomentsViaMagicRecs" - - def apply()(implicit globalStats: StatsReceiver): NamedPredicate[PushCandidate] = { - val scopedStats = globalStats.scope(name) - val genericTypeCategories = Seq(MomentsCategory, MomentsViaMagicRecsCategory) - val crts = RecTypes.magicFanoutEventTypes - RecTypeNtabCaretClickFatiguePredicate - .apply( - genericTypeCategories, - crts, - NtabCaretClickFatiguePredicateHelper.calculateFatiguePeriodMagicRecs, - useMostRecentDislikeTime = true, - name = name - ).withStats(scopedStats).withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickCandidateFatiguePredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickCandidateFatiguePredicate.scala deleted file mode 100644 index 376d9b11f..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickCandidateFatiguePredicate.scala +++ /dev/null @@ -1,87 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.predicate.FatiguePredicate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.frigate.common.base.Candidate -import com.twitter.frigate.common.base.TargetInfo -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.common.base.{RecommendationType => BaseRecommendationType} -import com.twitter.frigate.common.predicate.CandidateWithRecommendationTypeAndTargetInfoWithCaretFeedbackHistory -import com.twitter.frigate.common.predicate.FrigateHistoryFatiguePredicate.TimeSeries -import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.predicate.CaretFeedbackHistoryFilter - -object NtabCaretClickContFnFatiguePredicate { - - private val MagicRecsCategory = "MagicRecs" - - def ntabCaretClickContFnFatiguePredicates( - filterHistory: TimeSeries => TimeSeries = - FatiguePredicate.recTypesOnlyFilter(RecTypes.sharedNTabCaretFatigueTypes), - filterCaretFeedbackHistory: Target => Seq[ - CaretFeedbackDetails - ] => Seq[CaretFeedbackDetails] = - CaretFeedbackHistoryFilter.caretFeedbackHistoryFilter(Seq(MagicRecsCategory)), - filterInlineFeedbackHistory: Seq[FeedbackModel] => Seq[FeedbackModel] = - NtabCaretClickFatigueUtils.feedbackModelFilterByCRT(RecTypes.sharedNTabCaretFatigueTypes), - name: String = "NTabCaretClickFnCandidatePredicates" - )( - implicit globalStats: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val scopedStats = globalStats.scope(name) - CRTBasedNtabCaretClickFatiguePredicates - .f1TriggeredCRTBasedNtabCaretClickFnFatiguePredicate[ - Candidate with BaseRecommendationType with TargetInfo[ - Target - ] - ]( - filterHistory = filterHistory, - filterCaretFeedbackHistory = filterCaretFeedbackHistory, - filterInlineFeedbackHistory = filterInlineFeedbackHistory - ) - .applyOnlyToCandidateWithRecommendationTypeAndTargetWithCaretFeedbackHistory - .withName("f1_triggered_fn_seelessoften_fatigue") - .andThen( - CRTBasedNtabCaretClickFatiguePredicates - .nonF1TriggeredCRTBasedNtabCaretClickFnFatiguePredicate[ - Candidate with BaseRecommendationType with TargetInfo[ - Target - ] - ]( - filterHistory = filterHistory, - filterCaretFeedbackHistory = filterCaretFeedbackHistory, - filterInlineFeedbackHistory = filterInlineFeedbackHistory - ) - .applyOnlyToCandidateWithRecommendationTypeAndTargetWithCaretFeedbackHistory) - .withName("nonf1_triggered_fn_seelessoften_fatigue") - .andThen( - CRTBasedNtabCaretClickFatiguePredicates - .tripHqTweetTriggeredCRTBasedNtabCaretClickFnFatiguePredicate[ - Candidate with BaseRecommendationType with TargetInfo[ - Target - ] - ]( - filterHistory = filterHistory, - filterCaretFeedbackHistory = filterCaretFeedbackHistory, - filterInlineFeedbackHistory = filterInlineFeedbackHistory - ) - .applyOnlyToCandidateWithRecommendationTypeAndTargetWithCaretFeedbackHistory) - .withName("trip_hq_tweet_triggered_fn_seelessoften_fatigue") - .andThen( - CRTBasedNtabCaretClickFatiguePredicates - .genericCRTBasedNtabCaretClickFnFatiguePredicate[ - Candidate with BaseRecommendationType with TargetInfo[ - Target - ] - ]( - filterHistory = filterHistory, - filterCaretFeedbackHistory = filterCaretFeedbackHistory, - filterInlineFeedbackHistory = filterInlineFeedbackHistory) - .applyOnlyToCandidateWithRecommendationTypeAndTargetWithCaretFeedbackHistory - .withName("generic_fn_seelessoften_fatigue") - ) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickFatiguePredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickFatiguePredicate.scala deleted file mode 100644 index 579f4b25f..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickFatiguePredicate.scala +++ /dev/null @@ -1,47 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.predicate.ntab_caret_fatigue.NtabCaretClickFatiguePredicateHelper -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.util.Future - -object NtabCaretClickFatiguePredicate { - val name = "NtabCaretClickFatiguePredicate" - - def isSpacesTypeAndTeamMember(candidate: PushCandidate): Future[Boolean] = { - candidate.target.isTeamMember.map { isTeamMember => - val isSpacesType = RecTypes.isRecommendedSpacesType(candidate.commonRecType) - isTeamMember && isSpacesType - } - } - - def apply()(implicit globalStats: StatsReceiver): NamedPredicate[PushCandidate] = { - val scopedStats = globalStats.scope(name) - val genericTypeCategories = Seq("MagicRecs") - val crts = RecTypes.sharedNTabCaretFatigueTypes - val recTypeNtabCaretClickFatiguePredicate = - RecTypeNtabCaretClickFatiguePredicate.apply( - genericTypeCategories, - crts, - NtabCaretClickFatiguePredicateHelper.calculateFatiguePeriodMagicRecs, - useMostRecentDislikeTime = false - ) - Predicate - .fromAsync { candidate: PushCandidate => - isSpacesTypeAndTeamMember(candidate).flatMap { isSpacesTypeAndTeamMember => - if (RecTypes.sharedNTabCaretFatigueTypes( - candidate.commonRecType) && !isSpacesTypeAndTeamMember) { - recTypeNtabCaretClickFatiguePredicate - .apply(Seq(candidate)).map(_.headOption.getOrElse(false)) - } else { - Future.True - } - } - } - .withStats(scopedStats) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickFatigueUtils.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickFatigueUtils.scala deleted file mode 100644 index cc2c0c072..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickFatigueUtils.scala +++ /dev/null @@ -1,108 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.predicate.ntab_caret_fatigue.NtabCaretClickFatiguePredicateHelper -import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails -import com.twitter.util.Duration -import com.twitter.conversions.DurationOps._ -import scala.math.min -import com.twitter.util.Time -import com.twitter.frigate.thriftscala.{CommonRecommendationType => CRT} - -object NtabCaretClickFatigueUtils { - - private def pushCapForFeedback( - feedbackDetails: Seq[CaretFeedbackDetails], - feedbacks: Seq[FeedbackModel], - param: ContinuousFunctionParam, - statsReceiver: StatsReceiver - ): Double = { - val stats = statsReceiver.scope("mr_seelessoften_contfn_pushcap") - val pushCapTotal = stats.counter("pushcap_total") - val pushCapInvalid = - stats.counter("pushcap_invalid") - - pushCapTotal.incr() - val timeSinceMostRecentDislikeMs = - NtabCaretClickFatiguePredicateHelper.getDurationSinceMostRecentDislike(feedbackDetails) - val mostRecentFeedbackTimestamp: Option[Long] = - feedbacks - .map { feedback => - feedback.timestampMs - }.reduceOption(_ max _) - val timeSinceMostRecentFeedback: Option[Duration] = - mostRecentFeedbackTimestamp.map(Time.now - Time.fromMilliseconds(_)) - - val nTabDislikePushCap = timeSinceMostRecentDislikeMs match { - case Some(lastDislikeTimeMs) => { - ContinuousFunction.safeEvaluateFn(lastDislikeTimeMs.inDays.toDouble, param, stats) - } - case _ => { - pushCapInvalid.incr() - param.defaultValue - } - } - val feedbackPushCap = timeSinceMostRecentFeedback match { - case Some(lastDislikeTimeVal) => { - ContinuousFunction.safeEvaluateFn(lastDislikeTimeVal.inDays.toDouble, param, stats) - } - case _ => { - pushCapInvalid.incr() - param.defaultValue - } - } - - min(nTabDislikePushCap, feedbackPushCap) - } - - def durationToFilterForFeedback( - feedbackDetails: Seq[CaretFeedbackDetails], - feedbacks: Seq[FeedbackModel], - param: ContinuousFunctionParam, - defaultPushCap: Double, - statsReceiver: StatsReceiver - ): Duration = { - val pushCap = min( - pushCapForFeedback(feedbackDetails, feedbacks, param, statsReceiver), - defaultPushCap - ) - if (pushCap <= 0) { - Duration.Top - } else { - 24.hours / pushCap - } - } - - def hasUserDislikeInLast90Days(feedbackDetails: Seq[CaretFeedbackDetails]): Boolean = { - val timeSinceMostRecentDislike = - NtabCaretClickFatiguePredicateHelper.getDurationSinceMostRecentDislike(feedbackDetails) - - timeSinceMostRecentDislike.exists(_ < 90.days) - } - - def feedbackModelFilterByCRT( - crts: Set[CRT] - ): Seq[FeedbackModel] => Seq[ - FeedbackModel - ] = { feedbacks => - feedbacks.filter { feedback => - feedback.notification match { - case Some(notification) => crts.contains(notification.commonRecommendationType) - case None => false - } - } - } - - def feedbackModelExcludeCRT( - crts: Set[CRT] - ): Seq[FeedbackModel] => Seq[ - FeedbackModel - ] = { feedbacks => - feedbacks.filter { feedback => - feedback.notification match { - case Some(notification) => !crts.contains(notification.commonRecommendationType) - case None => true - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/RecTypeNtabCaretFatiguePredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/RecTypeNtabCaretFatiguePredicate.scala deleted file mode 100644 index d83650de0..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/RecTypeNtabCaretFatiguePredicate.scala +++ /dev/null @@ -1,87 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.predicate.FatiguePredicate -import com.twitter.frigate.pushservice.predicate.CaretFeedbackHistoryFilter -import com.twitter.frigate.pushservice.predicate.{ - TargetNtabCaretClickFatiguePredicate => CommonNtabCaretClickFatiguePredicate -} -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.thriftscala.NotificationDisplayLocation -import com.twitter.frigate.thriftscala.{CommonRecommendationType => CRT} -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails -import com.twitter.util.Duration -import com.twitter.util.Future - -object RecTypeNtabCaretClickFatiguePredicate { - val defaultName = "RecTypeNtabCaretClickFatiguePredicateForCandidate" - - private def candidateFatiguePredicate( - genericTypeCategories: Seq[String], - crts: Set[CRT] - )( - implicit stats: StatsReceiver - ): NamedPredicate[ - PushCandidate - ] = { - val name = "f1TriggeredCRTBasedFatiguePredciate" - val scopedStats = stats.scope(s"predicate_$name") - Predicate - .fromAsync { candidate: PushCandidate => - if (candidate.frigateNotification.notificationDisplayLocation == NotificationDisplayLocation.PushToMobileDevice) { - if (candidate.target.params(PushParams.EnableFatigueNtabCaretClickingParam)) { - NtabCaretClickContFnFatiguePredicate - .ntabCaretClickContFnFatiguePredicates( - filterHistory = FatiguePredicate.recTypesOnlyFilter(crts), - filterCaretFeedbackHistory = - CaretFeedbackHistoryFilter.caretFeedbackHistoryFilter(genericTypeCategories), - filterInlineFeedbackHistory = - NtabCaretClickFatigueUtils.feedbackModelFilterByCRT(crts) - ).apply(Seq(candidate)) - .map(_.headOption.getOrElse(false)) - } else Future.True - } else { - Future.True - } - }.withStats(scopedStats) - .withName(name) - } - - def apply( - genericTypeCategories: Seq[String], - crts: Set[CRT], - calculateFatiguePeriod: Seq[CaretFeedbackDetails] => Duration, - useMostRecentDislikeTime: Boolean, - name: String = defaultName - )( - implicit globalStats: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val scopedStats = globalStats.scope(name) - val commonNtabCaretClickFatiguePredicate = CommonNtabCaretClickFatiguePredicate( - filterCaretFeedbackHistory = - CaretFeedbackHistoryFilter.caretFeedbackHistoryFilter(genericTypeCategories), - filterHistory = FatiguePredicate.recTypesOnlyFilter(crts), - calculateFatiguePeriod = calculateFatiguePeriod, - useMostRecentDislikeTime = useMostRecentDislikeTime, - name = name - )(globalStats) - - Predicate - .fromAsync { candidate: PushCandidate => - if (candidate.frigateNotification.notificationDisplayLocation == NotificationDisplayLocation.PushToMobileDevice) { - if (candidate.target.params(PushParams.EnableFatigueNtabCaretClickingParam)) { - commonNtabCaretClickFatiguePredicate - .apply(Seq(candidate.target)) - .map(_.headOption.getOrElse(false)) - } else Future.True - } else { - Future.True - } - }.andThen(candidateFatiguePredicate(genericTypeCategories, crts)) - .withStats(scopedStats) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/package.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/package.scala deleted file mode 100644 index 61c1f78cc..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/package.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.twitter.frigate.pushservice - -import com.twitter.frigate.common.base.Candidate -import com.twitter.frigate.common.base.SocialGraphServiceRelationshipMap -import com.twitter.frigate.common.base.TweetAuthor -import com.twitter.frigate.common.rec_types.RecTypes.isInNetworkTweetType -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.hermit.predicate.Predicate - -package object predicate { - implicit class CandidatesWithAuthorFollowPredicates( - predicate: Predicate[ - PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap - ]) { - def applyOnlyToAuthorBeingFollowPredicates: Predicate[Candidate] = - predicate.optionalOn[Candidate]( - { - case candidate: PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap - if isInNetworkTweetType(candidate.commonRecType) => - Some(candidate) - case _ => - None - }, - missingResult = true - ) - } - - implicit class TweetCandidateWithTweetAuthor( - predicate: Predicate[ - PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap - ]) { - def applyOnlyToBasicTweetPredicates: Predicate[Candidate] = - predicate.optionalOn[Candidate]( - { - case candidate: PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap - if isInNetworkTweetType(candidate.commonRecType) => - Some(candidate) - case _ => - None - }, - missingResult = true - ) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/OpenOrNtabClickQualityPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/OpenOrNtabClickQualityPredicate.scala deleted file mode 100644 index d7f38349b..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/OpenOrNtabClickQualityPredicate.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.quality_model_predicate - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.util.Future - -object ExplicitOONCFilterPredicate extends QualityPredicateBase { - override lazy val name = "open_or_ntab_click_explicit_threshold" - - override lazy val thresholdExtractor = (t: Target) => - Future.value(t.params(PushFeatureSwitchParams.QualityPredicateExplicitThresholdParam)) - - override def scoreExtractor = (candidate: PushCandidate) => - candidate.mrWeightedOpenOrNtabClickRankingProbability -} - -object WeightedOpenOrNtabClickQualityPredicate extends QualityPredicateBase { - override lazy val name = "weighted_open_or_ntab_click_model" - - override lazy val thresholdExtractor = (t: Target) => { - Future.value(0.0) - } - - override def scoreExtractor = - (candidate: PushCandidate) => candidate.mrWeightedOpenOrNtabClickFilteringProbability -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/QualityPredicateCommon.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/QualityPredicateCommon.scala deleted file mode 100644 index d22f8c68f..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/QualityPredicateCommon.scala +++ /dev/null @@ -1,165 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.quality_model_predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.model.PushTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.target.TargetScoringDetails -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.util.Future - -object PDauCohort extends Enumeration { - type PDauCohort = Value - - val cohort1 = Value - val cohort2 = Value - val cohort3 = Value - val cohort4 = Value - val cohort5 = Value - val cohort6 = Value -} - -object PDauCohortUtil { - - case class DauThreshold( - threshold1: Double, - threshold2: Double, - threshold3: Double, - threshold4: Double, - threshold5: Double) - - val defaultDAUProb = 0.0 - - val dauProbThresholds = DauThreshold( - threshold1 = 0.05, - threshold2 = 0.14, - threshold3 = 0.33, - threshold4 = 0.7, - threshold5 = 0.959 - ) - - val finerThresholdMap = - Map( - PDauCohort.cohort2 -> List(0.05, 0.0539, 0.0563, 0.0600, 0.0681, 0.0733, 0.0800, 0.0849, - 0.0912, 0.0975, 0.1032, 0.1092, 0.1134, 0.1191, 0.1252, 0.1324, 0.14), - PDauCohort.cohort3 -> List(0.14, 0.1489, 0.1544, 0.1625, 0.1704, 0.1797, 0.1905, 0.2001, - 0.2120, 0.2248, 0.2363, 0.2500, 0.2650, 0.2801, 0.2958, 0.3119, 0.33), - PDauCohort.cohort4 -> List(0.33, 0.3484, 0.3686, 0.3893, 0.4126, 0.4350, 0.4603, 0.4856, - 0.5092, 0.5348, 0.5602, 0.5850, 0.6087, 0.6319, 0.6548, 0.6779, 0.7), - PDauCohort.cohort5 -> List(0.7, 0.7295, 0.7581, 0.7831, 0.8049, 0.8251, 0.8444, 0.8612, - 0.8786, 0.8936, 0.9043, 0.9175, 0.9290, 0.9383, 0.9498, 0.9587, 0.959) - ) - - def getBucket(targetUser: PushTypes.Target, doImpression: Boolean) = { - implicit val stats = targetUser.stats.scope("PDauCohortUtil") - if (doImpression) targetUser.getBucket _ else targetUser.getBucketWithoutImpression _ - } - - def threshold1(targetUser: PushTypes.Target): Double = dauProbThresholds.threshold1 - - def threshold2(targetUser: PushTypes.Target): Double = dauProbThresholds.threshold2 - - def threshold3(targetUser: PushTypes.Target): Double = dauProbThresholds.threshold3 - - def threshold4(targetUser: PushTypes.Target): Double = dauProbThresholds.threshold4 - - def threshold5(targetUser: PushTypes.Target): Double = dauProbThresholds.threshold5 - - def thresholdForCohort(targetUser: PushTypes.Target, dauCohort: Int): Double = { - if (dauCohort == 0) 0.0 - else if (dauCohort == 1) threshold1(targetUser) - else if (dauCohort == 2) threshold2(targetUser) - else if (dauCohort == 3) threshold3(targetUser) - else if (dauCohort == 4) threshold4(targetUser) - else if (dauCohort == 5) threshold5(targetUser) - else 1.0 - } - - def getPDauCohort(dauProbability: Double, thresholds: DauThreshold): PDauCohort.Value = { - dauProbability match { - case dauProb if dauProb >= 0.0 && dauProb < thresholds.threshold1 => PDauCohort.cohort1 - case dauProb if dauProb >= thresholds.threshold1 && dauProb < thresholds.threshold2 => - PDauCohort.cohort2 - case dauProb if dauProb >= thresholds.threshold2 && dauProb < thresholds.threshold3 => - PDauCohort.cohort3 - case dauProb if dauProb >= thresholds.threshold3 && dauProb < thresholds.threshold4 => - PDauCohort.cohort4 - case dauProb if dauProb >= thresholds.threshold4 && dauProb < thresholds.threshold5 => - PDauCohort.cohort5 - case dauProb if dauProb >= thresholds.threshold5 && dauProb <= 1.0 => PDauCohort.cohort6 - } - } - - def getDauProb(target: TargetScoringDetails): Future[Double] = { - target.dauProbability.map { dauProb => - dauProb.map(_.probability).getOrElse(defaultDAUProb) - } - } - - def getPDauCohort(target: TargetScoringDetails): Future[PDauCohort.Value] = { - getDauProb(target).map { getPDauCohort(_, dauProbThresholds) } - } - - def getPDauCohortWithPDau(target: TargetScoringDetails): Future[(PDauCohort.Value, Double)] = { - getDauProb(target).map { prob => - (getPDauCohort(prob, dauProbThresholds), prob) - } - } - - def updateStats( - target: PushTypes.Target, - modelName: String, - predicateResult: Boolean - )( - implicit statsReceiver: StatsReceiver - ): Unit = { - val dauCohortOp = getPDauCohort(target) - dauCohortOp.map { dauCohort => - val cohortStats = statsReceiver.scope(modelName).scope(dauCohort.toString) - cohortStats.counter(s"filter_$predicateResult").incr() - } - if (target.isNewSignup) { - val newUserModelStats = statsReceiver.scope(modelName) - newUserModelStats.counter(s"new_user_filter_$predicateResult").incr() - } - } -} - -trait QualityPredicateBase { - def name: String - def thresholdExtractor: Target => Future[Double] - def scoreExtractor: PushCandidate => Future[Option[Double]] - def isPredicateEnabled: PushCandidate => Future[Boolean] = _ => Future.True - def comparator: (Double, Double) => Boolean = - (score: Double, threshold: Double) => score >= threshold - def updateCustomStats( - candidate: PushCandidate, - score: Double, - threshold: Double, - result: Boolean - )( - implicit statsReceiver: StatsReceiver - ): Unit = {} - - def apply()(implicit statsReceiver: StatsReceiver): NamedPredicate[PushCandidate] = { - Predicate - .fromAsync { candidate: PushCandidate => - isPredicateEnabled(candidate).flatMap { - case true => - scoreExtractor(candidate).flatMap { scoreOpt => - thresholdExtractor(candidate.target).map { threshold => - val score = scoreOpt.getOrElse(0.0) - val result = comparator(score, threshold) - PDauCohortUtil.updateStats(candidate.target, name, result) - updateCustomStats(candidate, score, threshold, result) - result - } - } - case _ => Future.True - } - } - .withStats(statsReceiver.scope(s"predicate_$name")) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/QualityPredicateMap.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/QualityPredicateMap.scala deleted file mode 100644 index 9ac360df0..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/QualityPredicateMap.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.quality_model_predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.QualityPredicateEnum -import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate -import com.twitter.hermit.predicate.NamedPredicate - -object QualityPredicateMap { - - def apply( - )( - implicit statsReceiver: StatsReceiver - ): Map[QualityPredicateEnum.Value, NamedPredicate[PushCandidate]] = { - Map( - QualityPredicateEnum.WeightedOpenOrNtabClick -> WeightedOpenOrNtabClickQualityPredicate(), - QualityPredicateEnum.ExplicitOpenOrNtabClickFilter -> ExplicitOONCFilterPredicate(), - QualityPredicateEnum.AlwaysTrue -> PredicatesForCandidate.alwaysTruePushCandidatePredicate, - ) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/CRTBoostRanker.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/CRTBoostRanker.scala deleted file mode 100644 index de95c0695..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/CRTBoostRanker.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.frigate.pushservice.rank - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.thriftscala.CommonRecommendationType - -/** - * This Ranker re-ranks MR candidates, boosting input CRTs. - * Relative ranking between input CRTs and rest of the candidates doesn't change - * - * Ex: T: Tweet candidate, F: input CRT candidatess - * - * T3, F2, T1, T2, F1 => F2, F1, T3, T1, T2 - */ -case class CRTBoostRanker(statsReceiver: StatsReceiver) { - - private val recsToBoostStat = statsReceiver.stat("recs_to_boost") - private val otherRecsStat = statsReceiver.stat("other_recs") - - private def boostCrtToTop( - inputCandidates: Seq[CandidateDetails[PushCandidate]], - crtToBoost: CommonRecommendationType - ): Seq[CandidateDetails[PushCandidate]] = { - val (upRankedCandidates, otherCandidates) = - inputCandidates.partition(_.candidate.commonRecType == crtToBoost) - recsToBoostStat.add(upRankedCandidates.size) - otherRecsStat.add(otherCandidates.size) - upRankedCandidates ++ otherCandidates - } - - final def boostCrtsToTop( - inputCandidates: Seq[CandidateDetails[PushCandidate]], - crtsToBoost: Seq[CommonRecommendationType] - ): Seq[CandidateDetails[PushCandidate]] = { - crtsToBoost.headOption match { - case Some(crt) => - val upRankedCandidates = boostCrtToTop(inputCandidates, crt) - boostCrtsToTop(upRankedCandidates, crtsToBoost.tail) - case None => inputCandidates - } - } - - final def boostCrtsToTopStableOrder( - inputCandidates: Seq[CandidateDetails[PushCandidate]], - crtsToBoost: Seq[CommonRecommendationType] - ): Seq[CandidateDetails[PushCandidate]] = { - val crtsToBoostSet = crtsToBoost.toSet - val (upRankedCandidates, otherCandidates) = inputCandidates.partition(candidateDetail => - crtsToBoostSet.contains(candidateDetail.candidate.commonRecType)) - - upRankedCandidates ++ otherCandidates - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/CRTDownRanker.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/CRTDownRanker.scala deleted file mode 100644 index 4a8d74504..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/CRTDownRanker.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.frigate.pushservice.rank - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.thriftscala.CommonRecommendationType - -/** - * This Ranker re-ranks MR candidates, down ranks input CRTs. - * Relative ranking between input CRTs and rest of the candidates doesn't change - * - * Ex: T: Tweet candidate, F: input CRT candidates - * - * T3, F2, T1, T2, F1 => T3, T1, T2, F2, F1 - */ -case class CRTDownRanker(statsReceiver: StatsReceiver) { - - private val recsToDownRankStat = statsReceiver.stat("recs_to_down_rank") - private val otherRecsStat = statsReceiver.stat("other_recs") - private val downRankerRequests = statsReceiver.counter("down_ranker_requests") - - private def downRank( - inputCandidates: Seq[CandidateDetails[PushCandidate]], - crtToDownRank: CommonRecommendationType - ): Seq[CandidateDetails[PushCandidate]] = { - downRankerRequests.incr() - val (downRankedCandidates, otherCandidates) = - inputCandidates.partition(_.candidate.commonRecType == crtToDownRank) - recsToDownRankStat.add(downRankedCandidates.size) - otherRecsStat.add(otherCandidates.size) - otherCandidates ++ downRankedCandidates - } - - final def downRank( - inputCandidates: Seq[CandidateDetails[PushCandidate]], - crtsToDownRank: Seq[CommonRecommendationType] - ): Seq[CandidateDetails[PushCandidate]] = { - crtsToDownRank.headOption match { - case Some(crt) => - val downRankedCandidates = downRank(inputCandidates, crt) - downRank(downRankedCandidates, crtsToDownRank.tail) - case None => inputCandidates - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/LoggedOutRanker.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/LoggedOutRanker.scala deleted file mode 100644 index 5ab0c240a..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/LoggedOutRanker.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.frigate.pushservice.rank - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -class LoggedOutRanker(tweetyPieStore: ReadableStore[Long, TweetyPieResult], stats: StatsReceiver) { - private val statsReceiver = stats.scope(this.getClass.getSimpleName) - private val rankedCandidates = statsReceiver.counter("ranked_candidates_count") - - def rank( - candidates: Seq[CandidateDetails[PushCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - val tweetIds = candidates.map { cand => cand.candidate.asInstanceOf[TweetCandidate].tweetId } - val results = tweetyPieStore.multiGet(tweetIds.toSet).values.toSeq - val futureOfResults = Future.traverseSequentially(results)(r => r) - val tweetsFut = futureOfResults.map { tweetyPieResults => - tweetyPieResults.map(_.map(_.tweet)) - } - val sortedTweetsFuture = tweetsFut.map { tweets => - tweets - .map { tweet => - if (tweet.isDefined && tweet.get.counts.isDefined) { - tweet.get.id -> tweet.get.counts.get.favoriteCount.getOrElse(0L) - } else { - 0 -> 0L - } - }.sortBy(_._2)(Ordering[Long].reverse) - } - val finalCandidates = sortedTweetsFuture.map { sortedTweets => - sortedTweets - .map { tweet => - candidates.find(_.candidate.asInstanceOf[TweetCandidate].tweetId == tweet._1).orNull - }.filter { cand => cand != null } - } - finalCandidates.map { fc => - rankedCandidates.incr(fc.size) - } - finalCandidates - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/ModelBasedRanker.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/ModelBasedRanker.scala deleted file mode 100644 index 372888a66..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/ModelBasedRanker.scala +++ /dev/null @@ -1,204 +0,0 @@ -package com.twitter.frigate.pushservice.rank - -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.util.Future - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.params.MrQualityUprankingPartialTypeEnum -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.params.PushConstants.OoncQualityCombinedScore - -object ModelBasedRanker { - - def rankBySpecifiedScore( - candidatesDetails: Seq[CandidateDetails[PushCandidate]], - scoreExtractor: PushCandidate => Future[Option[Double]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - - val scoredCandidatesFutures = candidatesDetails.map { cand => - scoreExtractor(cand.candidate).map { scoreOp => (cand, scoreOp.getOrElse(0.0)) } - } - - Future.collect(scoredCandidatesFutures).map { scores => - val sorted = scores.sortBy { candidateDetails => -1 * candidateDetails._2 } - sorted.map(_._1) - } - } - - def populatePredictionScoreStats( - candidatesDetails: Seq[CandidateDetails[PushCandidate]], - scoreExtractor: PushCandidate => Future[Option[Double]], - predictionScoreStats: StatsReceiver - ): Unit = { - val scoreScaleFactorForStat = 10000 - val statName = "prediction_scores" - candidatesDetails.map { - case CandidateDetails(candidate, source) => - val crt = candidate.commonRecType - scoreExtractor(candidate).map { scoreOp => - val scaledScore = (scoreOp.getOrElse(0.0) * scoreScaleFactorForStat).toFloat - predictionScoreStats.scope("all_candidates").stat(statName).add(scaledScore) - predictionScoreStats.scope(crt.toString()).stat(statName).add(scaledScore) - } - } - } - - def populateMrWeightedOpenOrNtabClickScoreStats( - candidatesDetails: Seq[CandidateDetails[PushCandidate]], - predictionScoreStats: StatsReceiver - ): Unit = { - populatePredictionScoreStats( - candidatesDetails, - candidate => candidate.mrWeightedOpenOrNtabClickRankingProbability, - predictionScoreStats - ) - } - - def populateMrQualityUprankingScoreStats( - candidatesDetails: Seq[CandidateDetails[PushCandidate]], - predictionScoreStats: StatsReceiver - ): Unit = { - populatePredictionScoreStats( - candidatesDetails, - candidate => candidate.mrQualityUprankingProbability, - predictionScoreStats - ) - } - - def rankByMrWeightedOpenOrNtabClickScore( - candidatesDetails: Seq[CandidateDetails[PushCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - - rankBySpecifiedScore( - candidatesDetails, - candidate => candidate.mrWeightedOpenOrNtabClickRankingProbability - ) - } - - def transformSigmoid( - score: Double, - weight: Double = 1.0, - bias: Double = 0.0 - ): Double = { - val base = -1.0 * (weight * score + bias) - val cappedBase = math.max(math.min(base, 100.0), -100.0) - 1.0 / (1.0 + math.exp(cappedBase)) - } - - def transformLinear( - score: Double, - bar: Double = 1.0 - ): Double = { - val positiveBar = math.abs(bar) - val cappedScore = math.max(math.min(score, positiveBar), -1.0 * positiveBar) - cappedScore / positiveBar - } - - def transformIdentity( - score: Double - ): Double = score - - def rankByQualityOoncCombinedScore( - candidatesDetails: Seq[CandidateDetails[PushCandidate]], - qualityScoreTransform: Double => Double, - qualityScoreBoost: Double = 1.0 - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - - rankBySpecifiedScore( - candidatesDetails, - candidate => { - val ooncScoreFutOpt: Future[Option[Double]] = - candidate.mrWeightedOpenOrNtabClickRankingProbability - val qualityScoreFutOpt: Future[Option[Double]] = - candidate.mrQualityUprankingProbability - Future - .join( - ooncScoreFutOpt, - qualityScoreFutOpt - ).map { - case (Some(ooncScore), Some(qualityScore)) => - val transformedQualityScore = qualityScoreTransform(qualityScore) - val combinedScore = ooncScore * (1.0 + qualityScoreBoost * transformedQualityScore) - candidate - .cacheExternalScore(OoncQualityCombinedScore, Future.value(Some(combinedScore))) - Some(combinedScore) - case _ => None - } - } - ) - } - - def rerankByProducerQualityOoncCombinedScore( - candidateDetails: Seq[CandidateDetails[PushCandidate]] - )( - implicit stat: StatsReceiver - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - val scopedStat = stat.scope("producer_quality_reranking") - val oonCandidates = candidateDetails.filter { - case CandidateDetails(pushCandidate: PushCandidate, _) => - tweetCandidateSelector(pushCandidate, MrQualityUprankingPartialTypeEnum.Oon) - } - - val rankedOonCandidatesFut = rankBySpecifiedScore( - oonCandidates, - candidate => { - val baseScoreFutureOpt: Future[Option[Double]] = { - val qualityCombinedScoreFutureOpt = - candidate.getExternalCachedScoreByName(OoncQualityCombinedScore) - val ooncScoreFutureOpt = candidate.mrWeightedOpenOrNtabClickRankingProbability - Future.join(qualityCombinedScoreFutureOpt, ooncScoreFutureOpt).map { - case (Some(qualityCombinedScore), _) => - scopedStat.counter("quality_combined_score").incr() - Some(qualityCombinedScore) - case (_, ooncScoreOpt) => - scopedStat.counter("oonc_score").incr() - ooncScoreOpt - } - } - baseScoreFutureOpt.map { - case Some(baseScore) => - val boostRatio = candidate.mrProducerQualityUprankingBoost.getOrElse(1.0) - if (boostRatio > 1.0) scopedStat.counter("author_uprank").incr() - else if (boostRatio < 1.0) scopedStat.counter("author_downrank").incr() - else scopedStat.counter("author_noboost").incr() - Some(baseScore * boostRatio) - case _ => - scopedStat.counter("empty_score").incr() - None - } - } - ) - - rankedOonCandidatesFut.map { rankedOonCandidates => - val sortedOonCandidateIterator = rankedOonCandidates.toIterator - candidateDetails.map { ooncRankedCandidate => - val isOon = tweetCandidateSelector( - ooncRankedCandidate.candidate, - MrQualityUprankingPartialTypeEnum.Oon) - - if (sortedOonCandidateIterator.hasNext && isOon) - sortedOonCandidateIterator.next() - else ooncRankedCandidate - } - } - } - - def tweetCandidateSelector( - pushCandidate: PushCandidate, - selectedCandidateType: MrQualityUprankingPartialTypeEnum.Value - ): Boolean = { - pushCandidate match { - case candidate: PushCandidate with TweetCandidate => - selectedCandidateType match { - case MrQualityUprankingPartialTypeEnum.Oon => - val crt = candidate.commonRecType - RecTypes.isOutOfNetworkTweetRecType(crt) || RecTypes.outOfNetworkTopicTweetTypes - .contains(crt) - case _ => true - } - case _ => false - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/PushserviceRanker.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/PushserviceRanker.scala deleted file mode 100644 index 26a3a4239..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/PushserviceRanker.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.frigate.pushservice.rank - -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.common.base.Ranker -import com.twitter.util.Future - -trait PushserviceRanker[T, C] extends Ranker[T, C] { - - /** - * Initial Ranking of input candidates - */ - def initialRank(target: T, candidates: Seq[CandidateDetails[C]]): Future[Seq[CandidateDetails[C]]] - - /** - * Re-ranks input ranked candidates. Useful when a subset of candidates are ranked - * by a different logic, while preserving the initial ranking for the rest - */ - def reRank( - target: T, - rankedCandidates: Seq[CandidateDetails[C]] - ): Future[Seq[CandidateDetails[C]]] - - /** - * Final ranking that does Initial + Rerank - */ - override final def rank(target: T, candidates: Seq[CandidateDetails[C]]): ( - Future[Seq[CandidateDetails[C]]] - ) = { - initialRank(target, candidates).flatMap { rankedCandidates => reRank(target, rankedCandidates) } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/RFPHLightRanker.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/RFPHLightRanker.scala deleted file mode 100644 index 3fdae08c3..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/RFPHLightRanker.scala +++ /dev/null @@ -1,139 +0,0 @@ -package com.twitter.frigate.pushservice.rank -import com.twitter.contentrecommender.thriftscala.LightRankingCandidate -import com.twitter.contentrecommender.thriftscala.LightRankingFeatureHydrationContext -import com.twitter.contentrecommender.thriftscala.MagicRecsFeatureHydrationContext -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.common.base.RandomRanker -import com.twitter.frigate.common.base.Ranker -import com.twitter.frigate.common.base.TweetAuthor -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushConstants -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.ml.featurestore.lib.UserId -import com.twitter.nrel.lightranker.MagicRecsServeDataRecordLightRanker -import com.twitter.util.Future - -class RFPHLightRanker( - lightRanker: MagicRecsServeDataRecordLightRanker, - stats: StatsReceiver) - extends Ranker[Target, PushCandidate] { - - private val statsReceiver = stats.scope(this.getClass.getSimpleName) - - private val lightRankerCandidateCounter = statsReceiver.counter("light_ranker_candidate_count") - private val lightRankerRequestCounter = statsReceiver.counter("light_ranker_request_count") - private val lightRankingStats: StatsReceiver = statsReceiver.scope("light_ranking") - private val restrictLightRankingCounter: Counter = - lightRankingStats.counter("restrict_light_ranking") - private val selectedLightRankerScribedTargetCandidateCountStats: Stat = - lightRankingStats.stat("selected_light_ranker_scribed_target_candidate_count") - private val selectedLightRankerScribedCandidatesStats: Stat = - lightRankingStats.stat("selected_light_ranker_scribed_candidates") - private val lightRankingRandomBaselineStats: StatsReceiver = - statsReceiver.scope("light_ranking_random_baseline") - - override def rank( - target: Target, - candidates: Seq[CandidateDetails[PushCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - val enableLightRanker = target.params(PushFeatureSwitchParams.EnableLightRankingParam) - val restrictLightRanker = target.params(PushParams.RestrictLightRankingParam) - val lightRankerSelectionThreshold = - target.params(PushFeatureSwitchParams.LightRankingNumberOfCandidatesParam) - val randomRanker = RandomRanker[Target, PushCandidate]()(lightRankingRandomBaselineStats) - - if (enableLightRanker && candidates.length > lightRankerSelectionThreshold && !target.scribeFeatureForRequestScribe) { - val (tweetCandidates, nonTweetCandidates) = - candidates.partition { - case CandidateDetails(pushCandidate: PushCandidate with TweetCandidate, source) => true - case _ => false - } - val lightRankerSelectedTweetCandidatesFut = { - if (restrictLightRanker) { - restrictLightRankingCounter.incr() - lightRankThenTake( - target, - tweetCandidates - .asInstanceOf[Seq[CandidateDetails[PushCandidate with TweetCandidate]]], - PushConstants.RestrictLightRankingCandidatesThreshold - ) - } else if (target.params(PushFeatureSwitchParams.EnableRandomBaselineLightRankingParam)) { - randomRanker.rank(target, tweetCandidates).map { randomLightRankerCands => - randomLightRankerCands.take(lightRankerSelectionThreshold) - } - } else { - lightRankThenTake( - target, - tweetCandidates - .asInstanceOf[Seq[CandidateDetails[PushCandidate with TweetCandidate]]], - lightRankerSelectionThreshold - ) - } - } - lightRankerSelectedTweetCandidatesFut.map { returnedTweetCandidates => - nonTweetCandidates ++ returnedTweetCandidates - } - } else if (target.scribeFeatureForRequestScribe) { - val downSampleRate: Double = - if (target.params(PushParams.DownSampleLightRankingScribeCandidatesParam)) - PushConstants.DownSampleLightRankingScribeCandidatesRate - else target.params(PushFeatureSwitchParams.LightRankingScribeCandidatesDownSamplingParam) - val selectedCandidateCounter: Int = math.ceil(candidates.size * downSampleRate).toInt - selectedLightRankerScribedTargetCandidateCountStats.add(selectedCandidateCounter.toFloat) - - randomRanker.rank(target, candidates).map { randomLightRankerCands => - val selectedCandidates = randomLightRankerCands.take(selectedCandidateCounter) - selectedLightRankerScribedCandidatesStats.add(selectedCandidates.size.toFloat) - selectedCandidates - } - } else Future.value(candidates) - } - - private def lightRankThenTake( - target: Target, - candidates: Seq[CandidateDetails[PushCandidate with TweetCandidate]], - numOfCandidates: Int - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - lightRankerCandidateCounter.incr(candidates.length) - lightRankerRequestCounter.incr() - val lightRankerCandidates: Seq[LightRankingCandidate] = candidates.map { - case CandidateDetails(tweetCandidate, _) => - val tweetAuthor = tweetCandidate match { - case t: TweetCandidate with TweetAuthor => t.authorId - case _ => None - } - val hydrationContext: LightRankingFeatureHydrationContext = - LightRankingFeatureHydrationContext.MagicRecsHydrationContext( - MagicRecsFeatureHydrationContext( - tweetAuthor = tweetAuthor, - pushString = tweetCandidate.getPushCopy.flatMap(_.pushStringGroup).map(_.toString)) - ) - LightRankingCandidate( - tweetId = tweetCandidate.tweetId, - hydrationContext = Some(hydrationContext) - ) - } - val modelName = target.params(PushFeatureSwitchParams.LightRankingModelTypeParam) - val lightRankedCandidatesFut = { - lightRanker - .rank(UserId(target.targetId), lightRankerCandidates, modelName) - } - - lightRankedCandidatesFut.map { lightRankedCandidates => - val lrScoreMap = lightRankedCandidates.map { lrCand => - lrCand.tweetId -> lrCand.score - }.toMap - val candScoreMap: Seq[Option[Double]] = candidates.map { candidateDetails => - lrScoreMap.get(candidateDetails.candidate.tweetId) - } - sortCandidatesByScore(candidates, candScoreMap) - .take(numOfCandidates) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/RFPHRanker.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/RFPHRanker.scala deleted file mode 100644 index 83bdf3932..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/RFPHRanker.scala +++ /dev/null @@ -1,297 +0,0 @@ -package com.twitter.frigate.pushservice.rank -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.common.base.Ranker -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.ml.HealthFeatureGetter -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.params.MrQualityUprankingPartialTypeEnum -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushMLModel -import com.twitter.frigate.pushservice.params.PushModelName -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.util.MediaAnnotationsUtil.updateMediaCategoryStats -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.util.Future -import com.twitter.frigate.pushservice.params.MrQualityUprankingTransformTypeEnum -import com.twitter.storehaus.ReadableStore -import com.twitter.frigate.thriftscala.UserMediaRepresentation -import com.twitter.hss.api.thriftscala.UserHealthSignalResponse - -class RFPHRanker( - randomRanker: Ranker[Target, PushCandidate], - weightedOpenOrNtabClickModelScorer: PushMLModelScorer, - subscriptionCreatorRanker: SubscriptionCreatorRanker, - userHealthSignalStore: ReadableStore[Long, UserHealthSignalResponse], - producerMediaRepresentationStore: ReadableStore[Long, UserMediaRepresentation], - stats: StatsReceiver) - extends PushserviceRanker[Target, PushCandidate] { - - private val statsReceiver = stats.scope(this.getClass.getSimpleName) - - private val boostCRTsRanker = CRTBoostRanker(statsReceiver.scope("boost_desired_crts")) - private val crtDownRanker = CRTDownRanker(statsReceiver.scope("down_rank_desired_crts")) - - private val crtsToDownRank = statsReceiver.stat("crts_to_downrank") - private val crtsToUprank = statsReceiver.stat("crts_to_uprank") - - private val randomRankingCounter = stats.counter("randomRanking") - private val mlRankingCounter = stats.counter("mlRanking") - private val disableAllRelevanceCounter = stats.counter("disableAllRelevance") - private val disableHeavyRankingCounter = stats.counter("disableHeavyRanking") - - private val heavyRankerCandidateCounter = stats.counter("heavy_ranker_candidate_count") - private val heavyRankerScoreStats = statsReceiver.scope("heavy_ranker_prediction_scores") - - private val producerUprankingCounter = statsReceiver.counter("producer_quality_upranking") - private val producerBoostedCounter = statsReceiver.counter("producer_boosted_candidates") - private val producerDownboostedCounter = statsReceiver.counter("producer_downboosted_candidates") - - override def initialRank( - target: Target, - candidates: Seq[CandidateDetails[PushCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - - heavyRankerCandidateCounter.incr(candidates.size) - - updateMediaCategoryStats(candidates)(stats) - target.targetUserState - .flatMap { targetUserState => - val useRandomRanking = target.skipMlRanker || target.params( - PushParams.UseRandomRankingParam - ) - - if (useRandomRanking) { - randomRankingCounter.incr() - randomRanker.rank(target, candidates) - } else if (target.params(PushParams.DisableAllRelevanceParam)) { - disableAllRelevanceCounter.incr() - Future.value(candidates) - } else if (target.params(PushParams.DisableHeavyRankingParam) || target.params( - PushFeatureSwitchParams.DisableHeavyRankingModelFSParam)) { - disableHeavyRankingCounter.incr() - Future.value(candidates) - } else { - mlRankingCounter.incr() - - val scoredCandidatesFut = scoring(target, candidates) - - target.rankingModelParam.map { rankingModelParam => - val modelName = PushModelName( - PushMLModel.WeightedOpenOrNtabClickProbability, - target.params(rankingModelParam)).toString - ModelBasedRanker.populateMrWeightedOpenOrNtabClickScoreStats( - candidates, - heavyRankerScoreStats.scope(modelName) - ) - } - - if (target.params( - PushFeatureSwitchParams.EnableQualityUprankingCrtScoreStatsForHeavyRankingParam)) { - val modelName = PushModelName( - PushMLModel.FilteringProbability, - target.params(PushFeatureSwitchParams.QualityUprankingModelTypeParam) - ).toString - ModelBasedRanker.populateMrQualityUprankingScoreStats( - candidates, - heavyRankerScoreStats.scope(modelName) - ) - } - - val ooncRankedCandidatesFut = - scoredCandidatesFut.flatMap(ModelBasedRanker.rankByMrWeightedOpenOrNtabClickScore) - - val qualityUprankedCandidatesFut = - if (target.params(PushFeatureSwitchParams.EnableQualityUprankingForHeavyRankingParam)) { - ooncRankedCandidatesFut.flatMap { ooncRankedCandidates => - val transformFunc: Double => Double = - target.params(PushFeatureSwitchParams.QualityUprankingTransformTypeParam) match { - case MrQualityUprankingTransformTypeEnum.Linear => - ModelBasedRanker.transformLinear( - _, - bar = target.params( - PushFeatureSwitchParams.QualityUprankingLinearBarForHeavyRankingParam)) - case MrQualityUprankingTransformTypeEnum.Sigmoid => - ModelBasedRanker.transformSigmoid( - _, - weight = target.params( - PushFeatureSwitchParams.QualityUprankingSigmoidWeightForHeavyRankingParam), - bias = target.params( - PushFeatureSwitchParams.QualityUprankingSigmoidBiasForHeavyRankingParam) - ) - case _ => ModelBasedRanker.transformIdentity - } - - ModelBasedRanker.rankByQualityOoncCombinedScore( - ooncRankedCandidates, - transformFunc, - target.params(PushFeatureSwitchParams.QualityUprankingBoostForHeavyRankingParam) - ) - } - } else ooncRankedCandidatesFut - - if (target.params( - PushFeatureSwitchParams.EnableProducersQualityBoostingForHeavyRankingParam)) { - producerUprankingCounter.incr() - qualityUprankedCandidatesFut.flatMap(cands => - ModelBasedRanker.rerankByProducerQualityOoncCombinedScore(cands)(statsReceiver)) - } else qualityUprankedCandidatesFut - } - } - } - - private def scoring( - target: Target, - candidates: Seq[CandidateDetails[PushCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - - val ooncScoredCandidatesFut = target.rankingModelParam.map { rankingModelParam => - weightedOpenOrNtabClickModelScorer.scoreByBatchPredictionForModelVersion( - target, - candidates, - rankingModelParam - ) - } - - val scoredCandidatesFut = { - if (target.params(PushFeatureSwitchParams.EnableQualityUprankingForHeavyRankingParam)) { - ooncScoredCandidatesFut.map { candidates => - weightedOpenOrNtabClickModelScorer.scoreByBatchPredictionForModelVersion( - target = target, - candidatesDetails = candidates, - modelVersionParam = PushFeatureSwitchParams.QualityUprankingModelTypeParam, - overridePushMLModelOpt = Some(PushMLModel.FilteringProbability) - ) - } - } else ooncScoredCandidatesFut - } - - scoredCandidatesFut.foreach { candidates => - val oonCandidates = candidates.filter { - case CandidateDetails(pushCandidate: PushCandidate, _) => - ModelBasedRanker.tweetCandidateSelector( - pushCandidate, - MrQualityUprankingPartialTypeEnum.Oon) - } - setProducerQuality( - target, - oonCandidates, - userHealthSignalStore, - producerMediaRepresentationStore) - } - } - - private def setProducerQuality( - target: Target, - candidates: Seq[CandidateDetails[PushCandidate]], - userHealthSignalStore: ReadableStore[Long, UserHealthSignalResponse], - producerMediaRepresentationStore: ReadableStore[Long, UserMediaRepresentation] - ): Unit = { - lazy val boostRatio = - target.params(PushFeatureSwitchParams.QualityUprankingBoostForHighQualityProducersParam) - lazy val downboostRatio = - target.params(PushFeatureSwitchParams.QualityUprankingDownboostForLowQualityProducersParam) - candidates.foreach { - case CandidateDetails(pushCandidate, _) => - HealthFeatureGetter - .getFeatures(pushCandidate, producerMediaRepresentationStore, userHealthSignalStore).map { - featureMap => - val agathaNsfwScore = featureMap.numericFeatures.getOrElse("agathaNsfwScore", 0.5) - val textNsfwScore = featureMap.numericFeatures.getOrElse("textNsfwScore", 0.15) - val nudityRate = featureMap.numericFeatures.getOrElse("nudityRate", 0.0) - val activeFollowers = featureMap.numericFeatures.getOrElse("activeFollowers", 0.0) - val favorsRcvd28Days = featureMap.numericFeatures.getOrElse("favorsRcvd28Days", 0.0) - val tweets28Days = featureMap.numericFeatures.getOrElse("tweets28Days", 0.0) - val authorDislikeCount = featureMap.numericFeatures - .getOrElse("authorDislikeCount", 0.0) - val authorDislikeRate = featureMap.numericFeatures.getOrElse("authorDislikeRate", 0.0) - val authorReportRate = featureMap.numericFeatures.getOrElse("authorReportRate", 0.0) - val abuseStrikeTop2Percent = - featureMap.booleanFeatures.getOrElse("abuseStrikeTop2Percent", false) - val abuseStrikeTop1Percent = - featureMap.booleanFeatures.getOrElse("abuseStrikeTop1Percent", false) - val hasNsfwToken = featureMap.booleanFeatures.getOrElse("hasNsfwToken", false) - - if ((activeFollowers > 3000000) || - (activeFollowers > 1000000 && agathaNsfwScore < 0.7 && nudityRate < 0.01 && !hasNsfwToken && !abuseStrikeTop2Percent) || - (activeFollowers > 100000 && agathaNsfwScore < 0.7 && nudityRate < 0.01 && !hasNsfwToken && !abuseStrikeTop2Percent && - tweets28Days > 0 && favorsRcvd28Days / tweets28Days > 3000 && authorReportRate < 0.000001 && authorDislikeRate < 0.0005)) { - producerBoostedCounter.incr() - pushCandidate.setProducerQualityUprankingBoost(boostRatio) - } else if (activeFollowers < 5 || agathaNsfwScore > 0.9 || nudityRate > 0.03 || hasNsfwToken || abuseStrikeTop1Percent || - textNsfwScore > 0.4 || (authorDislikeRate > 0.005 && authorDislikeCount > 5) || - (tweets28Days > 56 && favorsRcvd28Days / tweets28Days < 100)) { - producerDownboostedCounter.incr() - pushCandidate.setProducerQualityUprankingBoost(downboostRatio) - } else pushCandidate.setProducerQualityUprankingBoost(1.0) - } - } - } - - private def rerankBySubscriptionCreatorRanker( - target: Target, - rankedCandidates: Future[Seq[CandidateDetails[PushCandidate]]], - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - if (target.params(PushFeatureSwitchParams.SoftRankCandidatesFromSubscriptionCreators)) { - val factor = target.params(PushFeatureSwitchParams.SoftRankFactorForSubscriptionCreators) - subscriptionCreatorRanker.boostByScoreFactor(rankedCandidates, factor) - } else - subscriptionCreatorRanker.boostSubscriptionCreator(rankedCandidates) - } - - override def reRank( - target: Target, - rankedCandidates: Seq[CandidateDetails[PushCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - val numberOfF1Candidates = - rankedCandidates.count(candidateDetails => - RecTypes.isF1Type(candidateDetails.candidate.commonRecType)) - lazy val threshold = - target.params(PushFeatureSwitchParams.NumberOfF1CandidatesThresholdForOONBackfill) - lazy val enableOONBackfillBasedOnF1 = - target.params(PushFeatureSwitchParams.EnableOONBackfillBasedOnF1Candidates) - - val f1BoostedCandidates = - if (enableOONBackfillBasedOnF1 && numberOfF1Candidates > threshold) { - boostCRTsRanker.boostCrtsToTopStableOrder( - rankedCandidates, - RecTypes.f1FirstDegreeTypes.toSeq) - } else rankedCandidates - - val topTweetsByGeoDownRankedCandidates = - if (target.params(PushFeatureSwitchParams.BackfillRankTopTweetsByGeoCandidates)) { - crtDownRanker.downRank( - f1BoostedCandidates, - Seq(CommonRecommendationType.GeoPopTweet) - ) - } else f1BoostedCandidates - - val reRankedCandidatesWithBoostedCrts = { - val listOfCrtsToUpRank = target - .params(PushFeatureSwitchParams.ListOfCrtsToUpRank) - .flatMap(CommonRecommendationType.valueOf) - crtsToUprank.add(listOfCrtsToUpRank.size) - boostCRTsRanker.boostCrtsToTop(topTweetsByGeoDownRankedCandidates, listOfCrtsToUpRank) - } - - val reRankedCandidatesWithDownRankedCrts = { - val listOfCrtsToDownRank = target - .params(PushFeatureSwitchParams.ListOfCrtsToDownRank) - .flatMap(CommonRecommendationType.valueOf) - crtsToDownRank.add(listOfCrtsToDownRank.size) - crtDownRanker.downRank(reRankedCandidatesWithBoostedCrts, listOfCrtsToDownRank) - } - - val rerankBySubscriptionCreatorFut = { - if (target.params(PushFeatureSwitchParams.BoostCandidatesFromSubscriptionCreators)) { - rerankBySubscriptionCreatorRanker( - target, - Future.value(reRankedCandidatesWithDownRankedCrts)) - } else Future.value(reRankedCandidatesWithDownRankedCrts) - } - - rerankBySubscriptionCreatorFut - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/SubscriptionCreatorRanker.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/SubscriptionCreatorRanker.scala deleted file mode 100644 index 3a2bff9a5..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/SubscriptionCreatorRanker.scala +++ /dev/null @@ -1,110 +0,0 @@ -package com.twitter.frigate.pushservice.rank - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.common.base.TweetAuthor -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.storehaus.FutureOps -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -class SubscriptionCreatorRanker( - superFollowEligibilityUserStore: ReadableStore[Long, Boolean], - statsReceiver: StatsReceiver) { - - private val scopedStats = statsReceiver.scope("SubscriptionCreatorRanker") - private val boostStats = scopedStats.scope("boostSubscriptionCreator") - private val softUprankStats = scopedStats.scope("boostByScoreFactor") - private val boostTotalCandidates = boostStats.stat("total_input_candidates") - private val softRankTotalCandidates = softUprankStats.stat("total_input_candidates") - private val softRankNumCandidatesCreators = softUprankStats.counter("candidates_from_creators") - private val softRankNumCandidatesNonCreators = - softUprankStats.counter("candidates_not_from_creators") - private val boostNumCandidatesCreators = boostStats.counter("candidates_from_creators") - private val boostNumCandidatesNonCreators = - boostStats.counter("candidates_not_from_creators") - - def boostSubscriptionCreator( - inputCandidatesFut: Future[Seq[CandidateDetails[PushCandidate]]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - - inputCandidatesFut.flatMap { inputCandidates => - boostTotalCandidates.add(inputCandidates.size) - val tweetAuthorIds = inputCandidates.flatMap { - case CandidateDetails(candidate: TweetCandidate with TweetAuthor, s) => - candidate.authorId - case _ => None - }.toSet - - FutureOps - .mapCollect(superFollowEligibilityUserStore.multiGet(tweetAuthorIds)) - .map { creatorAuthorMap => - val (upRankedCandidates, otherCandidates) = inputCandidates.partition { - case CandidateDetails(candidate: TweetCandidate with TweetAuthor, s) => - candidate.authorId match { - case Some(authorId) => - creatorAuthorMap(authorId).getOrElse(false) - case _ => false - } - case _ => false - } - boostNumCandidatesCreators.incr(upRankedCandidates.size) - boostNumCandidatesNonCreators.incr(otherCandidates.size) - upRankedCandidates ++ otherCandidates - } - } - } - - def boostByScoreFactor( - inputCandidatesFut: Future[Seq[CandidateDetails[PushCandidate]]], - factor: Double = 1.0, - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - - inputCandidatesFut.flatMap { inputCandidates => - softRankTotalCandidates.add(inputCandidates.size) - val tweetAuthorIds = inputCandidates.flatMap { - case CandidateDetails(candidate: TweetCandidate with TweetAuthor, s) => - candidate.authorId - case _ => None - }.toSet - - FutureOps - .mapCollect(superFollowEligibilityUserStore.multiGet(tweetAuthorIds)) - .flatMap { creatorAuthorMap => - val (upRankedCandidates, otherCandidates) = inputCandidates.partition { - case CandidateDetails(candidate: TweetCandidate with TweetAuthor, s) => - candidate.authorId match { - case Some(authorId) => - creatorAuthorMap(authorId).getOrElse(false) - case _ => false - } - case _ => false - } - softRankNumCandidatesCreators.incr(upRankedCandidates.size) - softRankNumCandidatesNonCreators.incr(otherCandidates.size) - - ModelBasedRanker.rankBySpecifiedScore( - inputCandidates, - candidate => { - val isFromCreator = candidate match { - case candidate: TweetCandidate with TweetAuthor => - candidate.authorId match { - case Some(authorId) => - creatorAuthorMap(authorId).getOrElse(false) - case _ => false - } - case _ => false - } - candidate.mrWeightedOpenOrNtabClickRankingProbability.map { - case Some(score) => - if (isFromCreator) Some(score * factor) - else Some(score) - case _ => None - } - } - ) - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/LoggedOutRefreshForPushHandler.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/LoggedOutRefreshForPushHandler.scala deleted file mode 100644 index f626d5b08..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/LoggedOutRefreshForPushHandler.scala +++ /dev/null @@ -1,259 +0,0 @@ -package com.twitter.frigate.pushservice.refresh_handler - -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.common.base.CandidateResult -import com.twitter.frigate.common.base.CandidateSource -import com.twitter.frigate.common.base.FetchRankFlowWithHydratedCandidates -import com.twitter.frigate.common.base.Invalid -import com.twitter.frigate.common.base.OK -import com.twitter.frigate.common.base.Response -import com.twitter.frigate.common.base.Result -import com.twitter.frigate.common.base.Stats.track -import com.twitter.frigate.common.base.Stats.trackSeq -import com.twitter.frigate.common.logger.MRLogger -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.adaptor.LoggedOutPushCandidateSourceGenerator -import com.twitter.frigate.pushservice.predicate.LoggedOutPreRankingPredicates -import com.twitter.frigate.pushservice.predicate.LoggedOutTargetPredicates -import com.twitter.frigate.pushservice.rank.LoggedOutRanker -import com.twitter.frigate.pushservice.take.LoggedOutRefreshForPushNotifier -import com.twitter.frigate.pushservice.scriber.MrRequestScribeHandler -import com.twitter.frigate.pushservice.target.LoggedOutPushTargetUserBuilder -import com.twitter.frigate.pushservice.thriftscala.LoggedOutRequest -import com.twitter.frigate.pushservice.thriftscala.LoggedOutResponse -import com.twitter.frigate.pushservice.thriftscala.PushContext -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.hermit.predicate.SequentialPredicate -import com.twitter.util.Future - -class LoggedOutRefreshForPushHandler( - val loPushTargetUserBuilder: LoggedOutPushTargetUserBuilder, - val loPushCandidateSourceGenerator: LoggedOutPushCandidateSourceGenerator, - candidateHydrator: PushCandidateHydrator, - val loRanker: LoggedOutRanker, - val loRfphNotifier: LoggedOutRefreshForPushNotifier, - loMrRequestScriberNode: String -)( - globalStats: StatsReceiver) - extends FetchRankFlowWithHydratedCandidates[Target, RawCandidate, PushCandidate] { - - val log = MRLogger("LORefreshForPushHandler") - implicit val statsReceiver: StatsReceiver = - globalStats.scope("LORefreshForPushHandler") - private val loggedOutBuildStats = statsReceiver.scope("logged_out_build_target") - private val loggedOutProcessStats = statsReceiver.scope("logged_out_process") - private val loggedOutNotifyStats = statsReceiver.scope("logged_out_notify") - private val loCandidateHydrationStats: StatsReceiver = - statsReceiver.scope("logged_out_candidate_hydration") - val mrLORequestCandidateScribeStats = - statsReceiver.scope("mr_logged_out_request_scribe_candidates") - - val mrRequestScribeHandler = - new MrRequestScribeHandler(loMrRequestScriberNode, statsReceiver.scope("lo_mr_request_scribe")) - val loMrRequestTargetScribeStats = statsReceiver.scope("lo_mr_request_scribe_target") - - lazy val loCandSourceEligibleCounter: Counter = - loCandidateStats.counter("logged_out_cand_source_eligible") - lazy val loCandSourceNotEligibleCounter: Counter = - loCandidateStats.counter("logged_out_cand_source_not_eligible") - lazy val allCandidatesCounter: Counter = statsReceiver.counter("all_logged_out_candidates") - val allCandidatesFilteredPreRank = filterStats.counter("all_logged_out_candidates_filtered") - - override def targetPredicates(target: Target): List[Predicate[Target]] = List( - LoggedOutTargetPredicates.targetFatiguePredicate(), - LoggedOutTargetPredicates.loggedOutRecsHoldbackPredicate() - ) - - override def isTargetValid(target: Target): Future[Result] = { - val resultFut = - if (target.skipFilters) { - Future.value(OK) - } else { - predicateSeq(target).track(Seq(target)).map { resultArr => - trackTargetPredStats(resultArr(0)) - } - } - track(targetStats)(resultFut) - } - - override def rank( - target: Target, - candidateDetails: Seq[CandidateDetails[PushCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - loRanker.rank(candidateDetails) - } - - override def validCandidates( - target: Target, - candidates: Seq[PushCandidate] - ): Future[Seq[Result]] = { - Future.value(candidates.map { c => OK }) - } - - override def desiredCandidateCount(target: Target): Int = 1 - - private val loggedOutPreRankingPredicates = - LoggedOutPreRankingPredicates(filterStats.scope("logged_out_predicates")) - - private val loggedOutPreRankingPredicateChain = - new SequentialPredicate[PushCandidate](loggedOutPreRankingPredicates) - - override def filter( - target: Target, - candidates: Seq[CandidateDetails[PushCandidate]] - ): Future[ - (Seq[CandidateDetails[PushCandidate]], Seq[CandidateResult[PushCandidate, Result]]) - ] = { - val predicateChain = loggedOutPreRankingPredicateChain - predicateChain - .track(candidates.map(_.candidate)) - .map { results => - val resultForPreRankingFiltering = - results - .zip(candidates) - .foldLeft( - ( - Seq.empty[CandidateDetails[PushCandidate]], - Seq.empty[CandidateResult[PushCandidate, Result]] - ) - ) { - case ((goodCandidates, filteredCandidates), (result, candidateDetails)) => - result match { - case None => - (goodCandidates :+ candidateDetails, filteredCandidates) - - case Some(pred: NamedPredicate[_]) => - val r = Invalid(Some(pred.name)) - ( - goodCandidates, - filteredCandidates :+ CandidateResult[PushCandidate, Result]( - candidateDetails.candidate, - candidateDetails.source, - r - ) - ) - case Some(_) => - val r = Invalid(Some("Filtered by un-named predicate")) - ( - goodCandidates, - filteredCandidates :+ CandidateResult[PushCandidate, Result]( - candidateDetails.candidate, - candidateDetails.source, - r - ) - ) - } - } - resultForPreRankingFiltering match { - case (validCandidates, _) if validCandidates.isEmpty && candidates.nonEmpty => - allCandidatesFilteredPreRank.incr() - case _ => () - - } - resultForPreRankingFiltering - - } - - } - - override def candidateSources( - target: Target - ): Future[Seq[CandidateSource[Target, RawCandidate]]] = { - Future - .collect(loPushCandidateSourceGenerator.sources.map { cs => - cs.isCandidateSourceAvailable(target).map { isEligible => - if (isEligible) { - loCandSourceEligibleCounter.incr() - Some(cs) - } else { - loCandSourceNotEligibleCounter.incr() - None - } - } - }).map(_.flatten) - } - - override def process( - target: Target, - externalCandidates: Seq[RawCandidate] = Nil - ): Future[Response[PushCandidate, Result]] = { - isTargetValid(target).flatMap { - case OK => - for { - candidatesFromSources <- trackSeq(fetchStats)(fetchCandidates(target)) - externalCandidateDetails = externalCandidates.map( - CandidateDetails(_, "logged_out_refresh_for_push_handler_external_candidates")) - allCandidates = candidatesFromSources ++ externalCandidateDetails - hydratedCandidatesWithCopy <- - trackSeq(loCandidateHydrationStats)(hydrateCandidates(allCandidates)) - (candidates, preRankingFilteredCandidates) <- - track(filterStats)(filter(target, hydratedCandidatesWithCopy)) - rankedCandidates <- trackSeq(rankingStats)(rank(target, candidates)) - allTakeCandidateResults <- track(takeStats)( - take(target, rankedCandidates, desiredCandidateCount(target)) - ) - _ <- track(mrLORequestCandidateScribeStats)( - mrRequestScribeHandler.scribeForCandidateFiltering( - target, - hydratedCandidatesWithCopy, - preRankingFilteredCandidates, - rankedCandidates, - rankedCandidates, - rankedCandidates, - allTakeCandidateResults - )) - - } yield { - val takeCandidateResults = allTakeCandidateResults.filterNot { candResult => - candResult.result == MoreThanDesiredCandidates - } - val allCandidateResults = takeCandidateResults ++ preRankingFilteredCandidates - allCandidatesCounter.incr(allCandidateResults.size) - Response(OK, allCandidateResults) - } - - case result: Result => - for (_ <- track(loMrRequestTargetScribeStats)( - mrRequestScribeHandler.scribeForTargetFiltering(target, result))) yield { - Response(result, Nil) - } - } - } - - def buildTarget( - guestId: Long, - inputPushContext: Option[PushContext] - ): Future[Target] = - loPushTargetUserBuilder.buildTarget(guestId, inputPushContext) - - /** - * Hydrate candidate by querying downstream services - * - * @param candidates - candidates - * - * @return - hydrated candidates - */ - override def hydrateCandidates( - candidates: Seq[CandidateDetails[RawCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = candidateHydrator(candidates) - - override def batchForCandidatesCheck(target: Target): Int = 1 - - def refreshAndSend(request: LoggedOutRequest): Future[LoggedOutResponse] = { - for { - target <- track(loggedOutBuildStats)( - loPushTargetUserBuilder.buildTarget(request.guestId, request.context)) - response <- track(loggedOutProcessStats)(process(target, externalCandidates = Seq.empty)) - loggedOutRefreshResponse <- - track(loggedOutNotifyStats)(loRfphNotifier.checkResponseAndNotify(response)) - } yield { - loggedOutRefreshResponse - } - } - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/PushCandidateHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/PushCandidateHydrator.scala deleted file mode 100644 index b8bf675bd..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/PushCandidateHydrator.scala +++ /dev/null @@ -1,239 +0,0 @@ -package com.twitter.frigate.pushservice.refresh_handler - -import com.twitter.channels.common.thriftscala.ApiList -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.rec_types.RecTypes.isInNetworkTweetType -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.TrendTweetPushCandidate -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.refresh_handler.cross.CandidateCopyExpansion -import com.twitter.frigate.pushservice.util.CandidateHydrationUtil._ -import com.twitter.frigate.pushservice.util.MrUserStateUtil -import com.twitter.frigate.pushservice.util.RelationshipUtil -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.hermit.predicate.socialgraph.RelationEdge -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -case class PushCandidateHydrator( - socialGraphServiceProcessStore: ReadableStore[RelationEdge, Boolean], - safeUserStore: ReadableStore[Long, User], - apiListStore: ReadableStore[Long, ApiList], - candidateCopyCross: CandidateCopyExpansion -)( - implicit statsReceiver: StatsReceiver, - implicit val weightedOpenOrNtabClickModelScorer: PushMLModelScorer) { - - lazy val candidateWithCopyNumStat = statsReceiver.stat("candidate_with_copy_num") - lazy val hydratedCandidateStat = statsReceiver.scope("hydrated_candidates") - lazy val mrUserStateStat = statsReceiver.scope("mr_user_state") - - lazy val queryStep = statsReceiver.scope("query_step") - lazy val relationEdgeWithoutDuplicateInQueryStep = - queryStep.counter("number_of_relationEdge_without_duplicate_in_query_step") - lazy val relationEdgeWithoutDuplicateInQueryStepDistribution = - queryStep.stat("number_of_relationEdge_without_duplicate_in_query_step_distribution") - - case class Entities( - users: Set[Long] = Set.empty[Long], - relationshipEdges: Set[RelationEdge] = Set.empty[RelationEdge]) { - def merge(otherEntities: Entities): Entities = { - this.copy( - users = this.users ++ otherEntities.users, - relationshipEdges = - this.relationshipEdges ++ otherEntities.relationshipEdges - ) - } - } - - case class EntitiesMap( - userMap: Map[Long, User] = Map.empty[Long, User], - relationshipMap: Map[RelationEdge, Boolean] = Map.empty[RelationEdge, Boolean]) - - private def updateCandidateAndCrtStats( - candidate: RawCandidate, - candidateType: String, - numEntities: Int = 1 - ): Unit = { - statsReceiver - .scope(candidateType).scope(candidate.commonRecType.name).stat( - "totalEntitiesPerCandidateTypePerCrt").add(numEntities) - statsReceiver.scope(candidateType).stat("totalEntitiesPerCandidateType").add(numEntities) - } - - private def collectEntities( - candidateDetailsSeq: Seq[CandidateDetails[RawCandidate]] - ): Entities = { - candidateDetailsSeq - .map { candidateDetails => - val pushCandidate = candidateDetails.candidate - - val userEntities = pushCandidate match { - case tweetWithSocialContext: RawCandidate with TweetWithSocialContextTraits => - val authorIdOpt = getAuthorIdFromTweetCandidate(tweetWithSocialContext) - val scUserIds = tweetWithSocialContext.socialContextUserIds.toSet - updateCandidateAndCrtStats(pushCandidate, "tweetWithSocialContext", scUserIds.size + 1) - Entities(users = scUserIds ++ authorIdOpt.toSet) - - case _ => Entities() - } - - val relationEntities = { - if (isInNetworkTweetType(pushCandidate.commonRecType)) { - Entities( - relationshipEdges = - RelationshipUtil.getPreCandidateRelationshipsForInNetworkTweets(pushCandidate).toSet - ) - } else Entities() - } - - userEntities.merge(relationEntities) - } - .foldLeft(Entities()) { (e1, e2) => e1.merge(e2) } - - } - - /** - * This method calls Gizmoduck and Social Graph Service, keep the results in EntitiesMap - * and passed onto the update candidate phase in the hydration step - * - * @param entities contains all userIds and relationEdges for all candidates - * @return EntitiesMap contains userMap and relationshipMap - */ - private def queryEntities(entities: Entities): Future[EntitiesMap] = { - - relationEdgeWithoutDuplicateInQueryStep.incr(entities.relationshipEdges.size) - relationEdgeWithoutDuplicateInQueryStepDistribution.add(entities.relationshipEdges.size) - - val relationshipMapFuture = Future - .collect(socialGraphServiceProcessStore.multiGet(entities.relationshipEdges)) - .map { resultMap => - resultMap.collect { - case (relationshipEdge, Some(res)) => relationshipEdge -> res - case (relationshipEdge, None) => relationshipEdge -> false - } - } - - val userMapFuture = Future - .collect(safeUserStore.multiGet(entities.users)) - .map { userMap => - userMap.collect { - case (userId, Some(user)) => - userId -> user - } - } - - Future.join(userMapFuture, relationshipMapFuture).map { - case (uMap, rMap) => EntitiesMap(userMap = uMap, relationshipMap = rMap) - } - } - - /** - * @param candidateDetails: recommendation candidates for a user - * @return sequence of candidates tagged with push and ntab copy id - */ - private def expandCandidatesWithCopy( - candidateDetails: Seq[CandidateDetails[RawCandidate]] - ): Future[Seq[(CandidateDetails[RawCandidate], CopyIds)]] = { - candidateCopyCross.expandCandidatesWithCopyId(candidateDetails) - } - - def updateCandidates( - candidateDetailsWithCopies: Seq[(CandidateDetails[RawCandidate], CopyIds)], - entitiesMaps: EntitiesMap - ): Seq[CandidateDetails[PushCandidate]] = { - candidateDetailsWithCopies.map { - case (candidateDetail, copyIds) => - val pushCandidate = candidateDetail.candidate - val userMap = entitiesMaps.userMap - val relationshipMap = entitiesMaps.relationshipMap - - val hydratedCandidate = pushCandidate match { - - case f1TweetCandidate: F1FirstDegree => - getHydratedCandidateForF1FirstDegreeTweet( - f1TweetCandidate, - userMap, - relationshipMap, - copyIds) - - case tweetRetweet: TweetRetweetCandidate => - getHydratedCandidateForTweetRetweet(tweetRetweet, userMap, copyIds) - - case tweetFavorite: TweetFavoriteCandidate => - getHydratedCandidateForTweetFavorite(tweetFavorite, userMap, copyIds) - - case tripTweetCandidate: OutOfNetworkTweetCandidate with TripCandidate => - getHydratedCandidateForTripTweetCandidate(tripTweetCandidate, userMap, copyIds) - - case outOfNetworkTweetCandidate: OutOfNetworkTweetCandidate with TopicCandidate => - getHydratedCandidateForOutOfNetworkTweetCandidate( - outOfNetworkTweetCandidate, - userMap, - copyIds) - - case topicProofTweetCandidate: TopicProofTweetCandidate => - getHydratedTopicProofTweetCandidate(topicProofTweetCandidate, userMap, copyIds) - - case subscribedSearchTweetCandidate: SubscribedSearchTweetCandidate => - getHydratedSubscribedSearchTweetCandidate( - subscribedSearchTweetCandidate, - userMap, - copyIds) - - case listRecommendation: ListPushCandidate => - getHydratedListCandidate(apiListStore, listRecommendation, copyIds) - - case discoverTwitterCandidate: DiscoverTwitterCandidate => - getHydratedCandidateForDiscoverTwitterCandidate(discoverTwitterCandidate, copyIds) - - case topTweetImpressionsCandidate: TopTweetImpressionsCandidate => - getHydratedCandidateForTopTweetImpressionsCandidate( - topTweetImpressionsCandidate, - copyIds) - - case trendTweetCandidate: TrendTweetCandidate => - new TrendTweetPushCandidate( - trendTweetCandidate, - trendTweetCandidate.authorId.flatMap(userMap.get), - copyIds) - - case unknownCandidate => - throw new IllegalArgumentException( - s"Incorrect candidate for hydration: ${unknownCandidate.commonRecType}") - } - - CandidateDetails( - hydratedCandidate, - source = candidateDetail.source - ) - } - } - - def apply( - candidateDetails: Seq[CandidateDetails[RawCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - val isLoggedOutRequest = - candidateDetails.headOption.exists(_.candidate.target.isLoggedOutUser) - if (!isLoggedOutRequest) { - candidateDetails.headOption.map { cd => - MrUserStateUtil.updateMrUserStateStats(cd.candidate.target)(mrUserStateStat) - } - } - - expandCandidatesWithCopy(candidateDetails).flatMap { candidateDetailsWithCopy => - candidateWithCopyNumStat.add(candidateDetailsWithCopy.size) - val entities = collectEntities(candidateDetailsWithCopy.map(_._1)) - queryEntities(entities).flatMap { entitiesMap => - val updatedCandidates = updateCandidates(candidateDetailsWithCopy, entitiesMap) - updatedCandidates.foreach { cand => - hydratedCandidateStat.counter(cand.candidate.commonRecType.name).incr() - } - Future.value(updatedCandidates) - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHFeatureHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHFeatureHydrator.scala deleted file mode 100644 index 6d1172cb9..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHFeatureHydrator.scala +++ /dev/null @@ -1,69 +0,0 @@ -package com.twitter.frigate.pushservice.refresh_handler - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.common.base.FeatureMap -import com.twitter.frigate.data_pipeline.features_common.MrRequestContextForFeatureStore -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.ml.HydrationContextBuilder -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.util.MrUserStateUtil -import com.twitter.nrel.heavyranker.FeatureHydrator -import com.twitter.util.Future - -class RFPHFeatureHydrator( - featureHydrator: FeatureHydrator -)( - implicit globalStats: StatsReceiver) { - - implicit val statsReceiver: StatsReceiver = - globalStats.scope("RefreshForPushHandler") - - //stat for feature hydration - private val featureHydrationEnabledCounter = statsReceiver.counter("featureHydrationEnabled") - private val mrUserStateStat = statsReceiver.scope("mr_user_state") - - private def hydrateFromRelevanceHydrator( - candidateDetails: Seq[CandidateDetails[PushCandidate]], - mrRequestContextForFeatureStore: MrRequestContextForFeatureStore - ): Future[Unit] = { - val pushCandidates = candidateDetails.map(_.candidate) - val candidatesAndContextsFut = Future.collect(pushCandidates.map { pc => - val contextFut = HydrationContextBuilder.build(pc) - contextFut.map { ctx => (pc, ctx) } - }) - candidatesAndContextsFut.flatMap { candidatesAndContexts => - val contexts = candidatesAndContexts.map(_._2) - val resultsFut = featureHydrator.hydrateCandidate(contexts, mrRequestContextForFeatureStore) - resultsFut.map { hydrationResult => - candidatesAndContexts.foreach { - case (pushCandidate, context) => - val resultFeatures = hydrationResult.getOrElse(context, FeatureMap()) - pushCandidate.mergeFeatures(resultFeatures) - } - } - } - } - - def candidateFeatureHydration( - candidateDetails: Seq[CandidateDetails[PushCandidate]], - mrRequestContextForFeatureStore: MrRequestContextForFeatureStore - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - candidateDetails.headOption match { - case Some(cand) => - val target = cand.candidate.target - MrUserStateUtil.updateMrUserStateStats(target)(mrUserStateStat) - if (target.params(PushParams.DisableAllRelevanceParam)) { - Future.value(candidateDetails) - } else { - featureHydrationEnabledCounter.incr() - for { - _ <- hydrateFromRelevanceHydrator(candidateDetails, mrRequestContextForFeatureStore) - } yield { - candidateDetails - } - } - case _ => Future.Nil - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHPrerankFilter.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHPrerankFilter.scala deleted file mode 100644 index fe52428b3..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHPrerankFilter.scala +++ /dev/null @@ -1,104 +0,0 @@ -package com.twitter.frigate.pushservice.refresh_handler - -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.predicate.PreRankingPredicates -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.SequentialPredicate -import com.twitter.util._ - -class RFPHPrerankFilter( -)( - globalStats: StatsReceiver) { - def filter( - target: Target, - hydratedCandidates: Seq[CandidateDetails[PushCandidate]] - ): Future[ - (Seq[CandidateDetails[PushCandidate]], Seq[CandidateResult[PushCandidate, Result]]) - ] = { - lazy val filterStats: StatsReceiver = globalStats.scope("RefreshForPushHandler/filter") - lazy val okFilterCounter: Counter = filterStats.counter("ok") - lazy val invalidFilterCounter: Counter = filterStats.counter("invalid") - lazy val invalidFilterStat: StatsReceiver = filterStats.scope("invalid") - lazy val invalidFilterReasonStat: StatsReceiver = invalidFilterStat.scope("reason") - val allCandidatesFilteredPreRank = filterStats.counter("all_candidates_filtered") - - lazy val preRankingPredicates = PreRankingPredicates( - filterStats.scope("predicates") - ) - - lazy val preRankingPredicateChain = - new SequentialPredicate[PushCandidate](preRankingPredicates) - - val predicateChain = if (target.pushContext.exists(_.predicatesToEnable.exists(_.nonEmpty))) { - val predicatesToEnable = target.pushContext.flatMap(_.predicatesToEnable).getOrElse(Nil) - new SequentialPredicate[PushCandidate](preRankingPredicates.filter { pred => - predicatesToEnable.contains(pred.name) - }) - } else preRankingPredicateChain - - predicateChain - .track(hydratedCandidates.map(_.candidate)) - .map { results => - val resultForPreRankFiltering = results - .zip(hydratedCandidates) - .foldLeft( - ( - Seq.empty[CandidateDetails[PushCandidate]], - Seq.empty[CandidateResult[PushCandidate, Result]] - ) - ) { - case ((goodCandidates, filteredCandidates), (result, candidateDetails)) => - result match { - case None => - okFilterCounter.incr() - (goodCandidates :+ candidateDetails, filteredCandidates) - - case Some(pred: NamedPredicate[_]) => - invalidFilterCounter.incr() - invalidFilterReasonStat.counter(pred.name).incr() - invalidFilterReasonStat - .scope(candidateDetails.candidate.commonRecType.toString).counter( - pred.name).incr() - - val r = Invalid(Some(pred.name)) - ( - goodCandidates, - filteredCandidates :+ CandidateResult[PushCandidate, Result]( - candidateDetails.candidate, - candidateDetails.source, - r - ) - ) - case Some(_) => - invalidFilterCounter.incr() - invalidFilterReasonStat.counter("unknown").incr() - invalidFilterReasonStat - .scope(candidateDetails.candidate.commonRecType.toString).counter( - "unknown").incr() - - val r = Invalid(Some("Filtered by un-named predicate")) - ( - goodCandidates, - filteredCandidates :+ CandidateResult[PushCandidate, Result]( - candidateDetails.candidate, - candidateDetails.source, - r - ) - ) - } - } - - resultForPreRankFiltering match { - case (validCandidates, _) if validCandidates.isEmpty && hydratedCandidates.nonEmpty => - allCandidatesFilteredPreRank.incr() - case _ => () - } - - resultForPreRankFiltering - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHRestrictStep.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHRestrictStep.scala deleted file mode 100644 index 037479111..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHRestrictStep.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.frigate.pushservice.refresh_handler - -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.common.base.TargetUser -import com.twitter.frigate.common.candidate.TargetABDecider -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.target.TargetScoringDetails - -class RFPHRestrictStep()(implicit stats: StatsReceiver) { - - private val statsReceiver: StatsReceiver = stats.scope("RefreshForPushHandler") - private val restrictStepStats: StatsReceiver = statsReceiver.scope("restrict") - private val restrictStepNumCandidatesDroppedStat: Stat = - restrictStepStats.stat("candidates_dropped") - - /** - * Limit the number of candidates that enter the Take step - */ - def restrict( - target: TargetUser with TargetABDecider with TargetScoringDetails, - candidates: Seq[CandidateDetails[PushCandidate]] - ): (Seq[CandidateDetails[PushCandidate]], Seq[CandidateDetails[PushCandidate]]) = { - if (target.params(PushFeatureSwitchParams.EnableRestrictStep)) { - val restrictSizeParam = PushFeatureSwitchParams.RestrictStepSize - val (newCandidates, filteredCandidates) = candidates.splitAt(target.params(restrictSizeParam)) - val numDropped = candidates.length - newCandidates.length - restrictStepNumCandidatesDroppedStat.add(numDropped) - (newCandidates, filteredCandidates) - } else (candidates, Seq.empty) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHStatsRecorder.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHStatsRecorder.scala deleted file mode 100644 index c09b4348a..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHStatsRecorder.scala +++ /dev/null @@ -1,77 +0,0 @@ -package com.twitter.frigate.pushservice.refresh_handler - -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.thriftscala.CommonRecommendationType - -class RFPHStatsRecorder(implicit statsReceiver: StatsReceiver) { - - private val selectedCandidateScoreStats: StatsReceiver = - statsReceiver.scope("score_of_sent_candidate_times_10000") - - private val emptyScoreStats: StatsReceiver = - statsReceiver.scope("score_of_sent_candidate_empty") - - def trackPredictionScoreStats(candidate: PushCandidate): Unit = { - candidate.mrWeightedOpenOrNtabClickRankingProbability.foreach { - case Some(s) => - selectedCandidateScoreStats - .stat("weighted_open_or_ntab_click_ranking") - .add((s * 10000).toFloat) - case None => - emptyScoreStats.counter("weighted_open_or_ntab_click_ranking").incr() - } - candidate.mrWeightedOpenOrNtabClickFilteringProbability.foreach { - case Some(s) => - selectedCandidateScoreStats - .stat("weighted_open_or_ntab_click_filtering") - .add((s * 10000).toFloat) - case None => - emptyScoreStats.counter("weighted_open_or_ntab_click_filtering").incr() - } - candidate.mrWeightedOpenOrNtabClickRankingProbability.foreach { - case Some(s) => - selectedCandidateScoreStats - .scope(candidate.commonRecType.toString) - .stat("weighted_open_or_ntab_click_ranking") - .add((s * 10000).toFloat) - case None => - emptyScoreStats - .scope(candidate.commonRecType.toString) - .counter("weighted_open_or_ntab_click_ranking") - .incr() - } - } - - def refreshRequestExceptionStats( - exception: Throwable, - bStats: StatsReceiver - ): Unit = { - bStats.counter("failures").incr() - bStats.scope("failures").counter(exception.getClass.getCanonicalName).incr() - } - - def loggedOutRequestExceptionStats( - exception: Throwable, - bStats: StatsReceiver - ): Unit = { - bStats.counter("logged_out_failures").incr() - bStats.scope("failures").counter(exception.getClass.getCanonicalName).incr() - } - - def rankDistributionStats( - candidatesDetails: Seq[CandidateDetails[PushCandidate]], - numRecsPerTypeStat: (CommonRecommendationType => Stat) - ): Unit = { - candidatesDetails - .groupBy { c => - c.candidate.commonRecType - } - .mapValues { s => - s.size - } - .foreach { case (crt, numRecs) => numRecsPerTypeStat(crt).add(numRecs) } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RefreshForPushHandler.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RefreshForPushHandler.scala deleted file mode 100644 index 17fb846cf..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RefreshForPushHandler.scala +++ /dev/null @@ -1,292 +0,0 @@ -package com.twitter.frigate.pushservice.refresh_handler - -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.Stats.track -import com.twitter.frigate.common.base.Stats.trackSeq -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.logger.MRLogger -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.adaptor._ -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.rank.RFPHLightRanker -import com.twitter.frigate.pushservice.rank.RFPHRanker -import com.twitter.frigate.pushservice.scriber.MrRequestScribeHandler -import com.twitter.frigate.pushservice.take.candidate_validator.RFPHCandidateValidator -import com.twitter.frigate.pushservice.target.PushTargetUserBuilder -import com.twitter.frigate.pushservice.target.RFPHTargetPredicates -import com.twitter.frigate.pushservice.util.RFPHTakeStepUtil -import com.twitter.frigate.pushservice.util.AdhocStatsUtil -import com.twitter.frigate.pushservice.thriftscala.PushContext -import com.twitter.frigate.pushservice.thriftscala.RefreshRequest -import com.twitter.frigate.pushservice.thriftscala.RefreshResponse -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.hermit.predicate.Predicate -import com.twitter.timelines.configapi.FeatureValue -import com.twitter.util._ - -case class ResultWithDebugInfo(result: Result, predicateResults: Seq[PredicateWithResult]) - -class RefreshForPushHandler( - val pushTargetUserBuilder: PushTargetUserBuilder, - val candSourceGenerator: PushCandidateSourceGenerator, - rfphRanker: RFPHRanker, - candidateHydrator: PushCandidateHydrator, - candidateValidator: RFPHCandidateValidator, - rfphTakeStepUtil: RFPHTakeStepUtil, - rfphRestrictStep: RFPHRestrictStep, - val rfphNotifier: RefreshForPushNotifier, - rfphStatsRecorder: RFPHStatsRecorder, - mrRequestScriberNode: String, - rfphFeatureHydrator: RFPHFeatureHydrator, - rfphPrerankFilter: RFPHPrerankFilter, - rfphLightRanker: RFPHLightRanker -)( - globalStats: StatsReceiver) - extends FetchRankFlowWithHydratedCandidates[Target, RawCandidate, PushCandidate] { - - val log = MRLogger("RefreshForPushHandler") - - implicit val statsReceiver: StatsReceiver = - globalStats.scope("RefreshForPushHandler") - private val maxCandidatesToBatchInTakeStat: Stat = - statsReceiver.stat("max_cands_to_batch_in_take") - - private val rfphRequestCounter = statsReceiver.counter("requests") - - private val buildTargetStats = statsReceiver.scope("build_target") - private val processStats = statsReceiver.scope("process") - private val notifyStats = statsReceiver.scope("notify") - - private val lightRankingStats: StatsReceiver = statsReceiver.scope("light_ranking") - private val reRankingStats: StatsReceiver = statsReceiver.scope("rerank") - private val featureHydrationLatency: StatsReceiver = - statsReceiver.scope("featureHydrationLatency") - private val candidateHydrationStats: StatsReceiver = statsReceiver.scope("candidate_hydration") - - lazy val candSourceEligibleCounter: Counter = - candidateStats.counter("cand_source_eligible") - lazy val candSourceNotEligibleCounter: Counter = - candidateStats.counter("cand_source_not_eligible") - - //pre-ranking stats - val allCandidatesFilteredPreRank = filterStats.counter("all_candidates_filtered") - - // total invalid candidates - val totalStats: StatsReceiver = statsReceiver.scope("total") - val totalInvalidCandidatesStat: Stat = totalStats.stat("candidates_invalid") - - val mrRequestScribeBuiltStats: Counter = statsReceiver.counter("mr_request_scribe_built") - - val mrRequestCandidateScribeStats = statsReceiver.scope("mr_request_scribe_candidates") - val mrRequestTargetScribeStats = statsReceiver.scope("mr_request_scribe_target") - - val mrRequestScribeHandler = - new MrRequestScribeHandler(mrRequestScriberNode, statsReceiver.scope("mr_request_scribe")) - - val adhocStatsUtil = new AdhocStatsUtil(statsReceiver.scope("adhoc_stats")) - - private def numRecsPerTypeStat(crt: CommonRecommendationType) = - fetchStats.scope(crt.toString).stat("dist") - - // static list of target predicates - private val targetPredicates = RFPHTargetPredicates(targetStats.scope("predicates")) - - def buildTarget( - userId: Long, - inputPushContext: Option[PushContext], - forcedFeatureValues: Option[Map[String, FeatureValue]] = None - ): Future[Target] = - pushTargetUserBuilder.buildTarget(userId, inputPushContext, forcedFeatureValues) - - override def targetPredicates(target: Target): List[Predicate[Target]] = targetPredicates - - override def isTargetValid(target: Target): Future[Result] = { - val resultFut = if (target.skipFilters) { - Future.value(trackTargetPredStats(None)) - } else { - predicateSeq(target).track(Seq(target)).map { resultArr => - trackTargetPredStats(resultArr(0)) - } - } - track(targetStats)(resultFut) - } - - override def candidateSources( - target: Target - ): Future[Seq[CandidateSource[Target, RawCandidate]]] = { - Future - .collect(candSourceGenerator.sources.map { cs => - cs.isCandidateSourceAvailable(target).map { isEligible => - if (isEligible) { - candSourceEligibleCounter.incr() - Some(cs) - } else { - candSourceNotEligibleCounter.incr() - None - } - } - }).map(_.flatten) - } - - override def updateCandidateCounter( - candidateResults: Seq[CandidateResult[PushCandidate, Result]] - ): Unit = { - candidateResults.foreach { - case candidateResult if candidateResult.result == OK => - okCandidateCounter.incr() - case candidateResult if candidateResult.result.isInstanceOf[Invalid] => - invalidCandidateCounter.incr() - case _ => - } - } - - override def hydrateCandidates( - candidates: Seq[CandidateDetails[RawCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = candidateHydrator(candidates) - - override def filter( - target: Target, - hydratedCandidates: Seq[CandidateDetails[PushCandidate]] - ): Future[ - (Seq[CandidateDetails[PushCandidate]], Seq[CandidateResult[PushCandidate, Result]]) - ] = rfphPrerankFilter.filter(target, hydratedCandidates) - - def lightRankAndTake( - target: Target, - candidates: Seq[CandidateDetails[PushCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - rfphLightRanker.rank(target, candidates) - } - - override def rank( - target: Target, - candidatesDetails: Seq[CandidateDetails[PushCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - val featureHydratedCandidatesFut = trackSeq(featureHydrationLatency)( - rfphFeatureHydrator - .candidateFeatureHydration(candidatesDetails, target.mrRequestContextForFeatureStore) - ) - featureHydratedCandidatesFut.flatMap { featureHydratedCandidates => - rfphStatsRecorder.rankDistributionStats(featureHydratedCandidates, numRecsPerTypeStat) - rfphRanker.initialRank(target, candidatesDetails) - } - } - - def reRank( - target: Target, - rankedCandidates: Seq[CandidateDetails[PushCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - rfphRanker.reRank(target, rankedCandidates) - } - - override def validCandidates( - target: Target, - candidates: Seq[PushCandidate] - ): Future[Seq[Result]] = { - Future.collect(candidates.map { candidate => - rfphTakeStepUtil.isCandidateValid(candidate, candidateValidator).map(res => res.result) - }) - } - - override def desiredCandidateCount(target: Target): Int = target.desiredCandidateCount - - override def batchForCandidatesCheck(target: Target): Int = { - val fsParam = PushFeatureSwitchParams.NumberOfMaxCandidatesToBatchInRFPHTakeStep - val maxToBatch = target.params(fsParam) - maxCandidatesToBatchInTakeStat.add(maxToBatch) - maxToBatch - } - - override def process( - target: Target, - externalCandidates: Seq[RawCandidate] = Nil - ): Future[Response[PushCandidate, Result]] = { - isTargetValid(target).flatMap { - case OK => - for { - candidatesFromSources <- trackSeq(fetchStats)(fetchCandidates(target)) - externalCandidateDetails = externalCandidates.map( - CandidateDetails(_, "refresh_for_push_handler_external_candidate")) - allCandidates = candidatesFromSources ++ externalCandidateDetails - hydratedCandidatesWithCopy <- - trackSeq(candidateHydrationStats)(hydrateCandidates(allCandidates)) - _ = adhocStatsUtil.getCandidateSourceStats(hydratedCandidatesWithCopy) - (candidates, preRankingFilteredCandidates) <- - track(filterStats)(filter(target, hydratedCandidatesWithCopy)) - _ = adhocStatsUtil.getPreRankingFilterStats(preRankingFilteredCandidates) - lightRankerFilteredCandidates <- - trackSeq(lightRankingStats)(lightRankAndTake(target, candidates)) - _ = adhocStatsUtil.getLightRankingStats(lightRankerFilteredCandidates) - rankedCandidates <- trackSeq(rankingStats)(rank(target, lightRankerFilteredCandidates)) - _ = adhocStatsUtil.getRankingStats(rankedCandidates) - rerankedCandidates <- trackSeq(reRankingStats)(reRank(target, rankedCandidates)) - _ = adhocStatsUtil.getReRankingStats(rerankedCandidates) - (restrictedCandidates, restrictFilteredCandidates) = - rfphRestrictStep.restrict(target, rerankedCandidates) - allTakeCandidateResults <- track(takeStats)( - take(target, restrictedCandidates, desiredCandidateCount(target)) - ) - _ = adhocStatsUtil.getTakeCandidateResultStats(allTakeCandidateResults) - _ <- track(mrRequestCandidateScribeStats)( - mrRequestScribeHandler.scribeForCandidateFiltering( - target, - hydratedCandidatesWithCopy, - preRankingFilteredCandidates, - rankedCandidates, - rerankedCandidates, - restrictFilteredCandidates, - allTakeCandidateResults - )) - } yield { - - /** - * Take processes post restrict step candidates and returns both: - * 1. valid + invalid candidates - * 2. Candidates that are not processed (more than desired) + restricted candidates - * We need #2 only for importance sampling - */ - val takeCandidateResults = - allTakeCandidateResults.filterNot { candResult => - candResult.result == MoreThanDesiredCandidates - } - - val totalInvalidCandidates = { - preRankingFilteredCandidates.size + //pre-ranking filtered candidates - (rerankedCandidates.length - restrictedCandidates.length) + //candidates reject in restrict step - takeCandidateResults.count(_.result != OK) //candidates reject in take step - } - takeInvalidCandidateDist.add( - takeCandidateResults - .count(_.result != OK) - ) // take step invalid candidates - totalInvalidCandidatesStat.add(totalInvalidCandidates) - val allCandidateResults = takeCandidateResults ++ preRankingFilteredCandidates - Response(OK, allCandidateResults) - } - - case result: Result => - for (_ <- track(mrRequestTargetScribeStats)( - mrRequestScribeHandler.scribeForTargetFiltering(target, result))) yield { - mrRequestScribeBuiltStats.incr() - Response(result, Nil) - } - } - } - - def refreshAndSend(request: RefreshRequest): Future[RefreshResponse] = { - rfphRequestCounter.incr() - for { - target <- track(buildTargetStats)( - pushTargetUserBuilder - .buildTarget(request.userId, request.context)) - response <- track(processStats)(process(target, externalCandidates = Seq.empty)) - refreshResponse <- track(notifyStats)(rfphNotifier.checkResponseAndNotify(response, target)) - } yield { - refreshResponse - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RefreshForPushNotifier.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RefreshForPushNotifier.scala deleted file mode 100644 index ae68d46ea..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RefreshForPushNotifier.scala +++ /dev/null @@ -1,128 +0,0 @@ -package com.twitter.frigate.pushservice.refresh_handler - -import com.twitter.finagle.stats.BroadcastStatsReceiver -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.Stats.track -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.config.CommonConstants -import com.twitter.frigate.common.util.PushServiceUtil.FilteredRefreshResponseFut -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.take.CandidateNotifier -import com.twitter.frigate.pushservice.util.ResponseStatsTrackUtils.trackStatsForResponseToRequest -import com.twitter.frigate.pushservice.thriftscala.PushStatus -import com.twitter.frigate.pushservice.thriftscala.RefreshResponse -import com.twitter.util.Future -import com.twitter.util.JavaTimer -import com.twitter.util.Timer - -class RefreshForPushNotifier( - rfphStatsRecorder: RFPHStatsRecorder, - candidateNotifier: CandidateNotifier -)( - globalStats: StatsReceiver) { - - private implicit val statsReceiver: StatsReceiver = - globalStats.scope("RefreshForPushHandler") - - private val pushStats: StatsReceiver = statsReceiver.scope("push") - private val sendLatency: StatsReceiver = statsReceiver.scope("send_handler") - implicit private val timer: Timer = new JavaTimer(true) - - private def notify( - candidatesResult: CandidateResult[PushCandidate, Result], - target: Target, - receivers: Seq[StatsReceiver] - ): Future[RefreshResponse] = { - - val candidate = candidatesResult.candidate - - val predsResult = candidatesResult.result - - if (predsResult != OK) { - val invalidResult = predsResult - invalidResult match { - case Invalid(Some(reason)) => - Future.value(RefreshResponse(PushStatus.Filtered, Some(reason))) - case _ => - Future.value(RefreshResponse(PushStatus.Filtered, None)) - } - } else { - rfphStatsRecorder.trackPredictionScoreStats(candidate) - - val isQualityUprankingCandidate = candidate.mrQualityUprankingBoost.isDefined - val commonRecTypeStats = Seq( - statsReceiver.scope(candidate.commonRecType.toString), - globalStats.scope(candidate.commonRecType.toString) - ) - val qualityUprankingStats = Seq( - statsReceiver.scope("QualityUprankingCandidates").scope(candidate.commonRecType.toString), - globalStats.scope("QualityUprankingCandidates").scope(candidate.commonRecType.toString) - ) - - val receiversWithRecTypeStats = { - if (isQualityUprankingCandidate) { - receivers ++ commonRecTypeStats ++ qualityUprankingStats - } else { - receivers ++ commonRecTypeStats - } - } - track(sendLatency)(candidateNotifier.notify(candidate).map { res => - trackStatsForResponseToRequest( - candidate.commonRecType, - candidate.target, - res, - receiversWithRecTypeStats - )(globalStats) - RefreshResponse(res.status) - }) - } - } - - def checkResponseAndNotify( - response: Response[PushCandidate, Result], - targetUserContext: Target - ): Future[RefreshResponse] = { - val receivers = Seq(statsReceiver) - val refreshResponse = response match { - case Response(OK, processedCandidates) => - // valid rec candidates - val validCandidates = processedCandidates.filter(_.result == OK) - - // top rec candidate - validCandidates.headOption match { - case Some(candidatesResult) => - candidatesResult.result match { - case OK => - notify(candidatesResult, targetUserContext, receivers) - .onSuccess { nr => - pushStats.scope("result").counter(nr.status.name).incr() - } - case _ => - targetUserContext.isTeamMember.flatMap { isTeamMember => - FilteredRefreshResponseFut - } - } - case _ => - FilteredRefreshResponseFut - } - case Response(Invalid(reason), _) => - // invalid target with known reason - FilteredRefreshResponseFut.map(_.copy(targetFilteredBy = reason)) - case _ => - // invalid target - FilteredRefreshResponseFut - } - - val bStats = BroadcastStatsReceiver(receivers) - Stat - .timeFuture(bStats.stat("latency"))( - refreshResponse - .raiseWithin(CommonConstants.maxPushRequestDuration) - ) - .onFailure { exception => - rfphStatsRecorder.refreshRequestExceptionStats(exception, bStats) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/BaseCopyFramework.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/BaseCopyFramework.scala deleted file mode 100644 index 47426a386..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/BaseCopyFramework.scala +++ /dev/null @@ -1,79 +0,0 @@ -package com.twitter.frigate.pushservice.refresh_handler.cross - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.MRNtabCopy -import com.twitter.frigate.common.util.MRPushCopy -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.util.Future - -abstract class BaseCopyFramework(statsReceiver: StatsReceiver) { - - private val NoAvailableCopyStat = statsReceiver.scope("no_copy_for_crt") - private val NoAvailableNtabCopyStat = statsReceiver.scope("no_ntab_copy") - - /** - * Instantiate push copy filters - */ - protected final val copyFilters = new CopyFilters(statsReceiver.scope("filters")) - - /** - * - * The following method fetches all the push copies for a [[com.twitter.frigate.thriftscala.CommonRecommendationType]] - * associated with a candidate and then filters the eligible copies based on - * [[PushTypes.PushCandidate]] features. These filters are defined in - * [[CopyFilters]] - * - * @param rawCandidate - [[RawCandidate]] object representing a recommendation candidate - * - * @return - set of eligible push copies for a given candidate - */ - protected[cross] final def getEligiblePushCopiesFromCandidate( - rawCandidate: RawCandidate - ): Future[Seq[MRPushCopy]] = { - val pushCopiesFromRectype = CandidateToCopy.getPushCopiesFromRectype(rawCandidate.commonRecType) - - if (pushCopiesFromRectype.isEmpty) { - NoAvailableCopyStat.counter(rawCandidate.commonRecType.name).incr() - throw new IllegalStateException(s"No Copy defined for CRT: " + rawCandidate.commonRecType) - } - pushCopiesFromRectype - .map(pushCopySet => copyFilters.execute(rawCandidate, pushCopySet.toSeq)) - .getOrElse(Future.value(Seq.empty)) - } - - /** - * - * This method essentially forms the base for cross-step for the MagicRecs Copy Framework. Given - * a recommendation type this returns a set of tuples wherein each tuple is a pair of push and - * ntab copy eligible for the said recommendation type - * - * @param rawCandidate - [[RawCandidate]] object representing a recommendation candidate - * @return - Set of eligible [[MRPushCopy]], Option[[MRNtabCopy]] for a given recommendation type - */ - protected[cross] final def getEligiblePushAndNtabCopiesFromCandidate( - rawCandidate: RawCandidate - ): Future[Seq[(MRPushCopy, Option[MRNtabCopy])]] = { - - val eligiblePushCopies = getEligiblePushCopiesFromCandidate(rawCandidate) - - eligiblePushCopies.map { pushCopies => - val setBuilder = Set.newBuilder[(MRPushCopy, Option[MRNtabCopy])] - pushCopies.foreach { pushCopy => - val ntabCopies = CandidateToCopy.getNtabcopiesFromPushcopy(pushCopy) - val pushNtabCopyPairs = ntabCopies match { - case Some(ntabCopySet) => - if (ntabCopySet.isEmpty) { - NoAvailableNtabCopyStat.counter(s"copy_id: ${pushCopy.copyId}").incr() - Set(pushCopy -> None) - } // push copy only - else ntabCopySet.map(pushCopy -> Some(_)) - - case None => - Set.empty[(MRPushCopy, Option[MRNtabCopy])] // no push or ntab copy - } - setBuilder ++= pushNtabCopyPairs - } - setBuilder.result().toSeq - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateCopyExpansion.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateCopyExpansion.scala deleted file mode 100644 index 9748c90ff..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateCopyExpansion.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.frigate.pushservice.refresh_handler.cross - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.common.util.MRNtabCopy -import com.twitter.frigate.common.util.MRPushCopy -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.util.Future - -/** - * @param statsReceiver - stats receiver object - */ -class CandidateCopyExpansion(statsReceiver: StatsReceiver) - extends BaseCopyFramework(statsReceiver) { - - /** - * - * Given a [[CandidateDetails]] object representing a push recommendation candidate this method - * expands it to multiple candidates, each tagged with a push copy id and ntab copy id to - * represent the eligible copies for the given recommendation candidate - * - * @param candidateDetails - [[CandidateDetails]] objects containing a recommendation candidate - * - * @return - list of tuples of [[PushTypes.RawCandidate]] and [[CopyIds]] - */ - private final def crossCandidateDetailsWithCopyId( - candidateDetails: CandidateDetails[RawCandidate] - ): Future[Seq[(CandidateDetails[RawCandidate], CopyIds)]] = { - val eligibleCopyPairs = getEligiblePushAndNtabCopiesFromCandidate(candidateDetails.candidate) - val copyPairs = eligibleCopyPairs.map(_.map { - case (pushCopy: MRPushCopy, ntabCopy: Option[MRNtabCopy]) => - CopyIds( - pushCopyId = Some(pushCopy.copyId), - ntabCopyId = ntabCopy.map(_.copyId) - ) - }) - - copyPairs.map(_.map((candidateDetails, _))) - } - - /** - * - * This method takes as input a list of [[CandidateDetails]] objects which contain the push - * recommendation candidates for a given target user. It expands each input candidate into - * multiple candidates, each tagged with a push copy id and ntab copy id to represent the eligible - * copies for the given recommendation candidate - * - * @param candidateDetailsSeq - list of fetched candidates for push recommendation - * @return - list of tuples of [[RawCandidate]] and [[CopyIds]] - */ - final def expandCandidatesWithCopyId( - candidateDetailsSeq: Seq[CandidateDetails[RawCandidate]] - ): Future[Seq[(CandidateDetails[RawCandidate], CopyIds)]] = - Future.collect(candidateDetailsSeq.map(crossCandidateDetailsWithCopyId)).map(_.flatten) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateCopyPair.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateCopyPair.scala deleted file mode 100644 index 4eca41730..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateCopyPair.scala +++ /dev/null @@ -1,11 +0,0 @@ -package com.twitter.frigate.pushservice.refresh_handler.cross - -import com.twitter.frigate.common.util.MRPushCopy -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate - -/** - * - * @param candidate: [[RawCandidate]] is a recommendation candidate - * @param pushCopy: [[MRPushCopy]] eligible for candidate - */ -case class CandidateCopyPair(candidate: RawCandidate, pushCopy: MRPushCopy) diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateToCopy.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateToCopy.scala deleted file mode 100644 index e7fbefe16..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateToCopy.scala +++ /dev/null @@ -1,263 +0,0 @@ -package com.twitter.frigate.pushservice.refresh_handler.cross - -import com.twitter.frigate.common.util.MrNtabCopyObjects -import com.twitter.frigate.common.util.MrPushCopyObjects -import com.twitter.frigate.common.util._ -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.CommonRecommendationType._ - -object CandidateToCopy { - - // Static map from a CommonRecommendationType to set of eligible push notification copies - private[cross] val rectypeToPushCopy: Map[CommonRecommendationType, Set[ - MRPushCopy - ]] = - Map[CommonRecommendationType, Set[MRPushCopy]]( - F1FirstdegreeTweet -> Set( - MrPushCopyObjects.FirstDegreeJustTweetedBoldTitle - ), - F1FirstdegreePhoto -> Set( - MrPushCopyObjects.FirstDegreePhotoJustTweetedBoldTitle - ), - F1FirstdegreeVideo -> Set( - MrPushCopyObjects.FirstDegreeVideoJustTweetedBoldTitle - ), - TweetRetweet -> Set( - MrPushCopyObjects.TweetRetweetWithOneDisplaySocialContextsWithText, - MrPushCopyObjects.TweetRetweetWithTwoDisplaySocialContextsWithText, - MrPushCopyObjects.TweetRetweetWithOneDisplayAndKOtherSocialContextsWithText - ), - TweetRetweetPhoto -> Set( - MrPushCopyObjects.TweetRetweetPhotoWithOneDisplaySocialContextWithText, - MrPushCopyObjects.TweetRetweetPhotoWithTwoDisplaySocialContextsWithText, - MrPushCopyObjects.TweetRetweetPhotoWithOneDisplayAndKOtherSocialContextsWithText - ), - TweetRetweetVideo -> Set( - MrPushCopyObjects.TweetRetweetVideoWithOneDisplaySocialContextWithText, - MrPushCopyObjects.TweetRetweetVideoWithTwoDisplaySocialContextsWithText, - MrPushCopyObjects.TweetRetweetVideoWithOneDisplayAndKOtherSocialContextsWithText - ), - TweetFavorite -> Set( - MrPushCopyObjects.TweetLikeOneSocialContextWithText, - MrPushCopyObjects.TweetLikeTwoSocialContextWithText, - MrPushCopyObjects.TweetLikeMultipleSocialContextWithText - ), - TweetFavoritePhoto -> Set( - MrPushCopyObjects.TweetLikePhotoOneSocialContextWithText, - MrPushCopyObjects.TweetLikePhotoTwoSocialContextWithText, - MrPushCopyObjects.TweetLikePhotoMultipleSocialContextWithText - ), - TweetFavoriteVideo -> Set( - MrPushCopyObjects.TweetLikeVideoOneSocialContextWithText, - MrPushCopyObjects.TweetLikeVideoTwoSocialContextWithText, - MrPushCopyObjects.TweetLikeVideoMultipleSocialContextWithText - ), - UnreadBadgeCount -> Set(MrPushCopyObjects.UnreadBadgeCount), - InterestBasedTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), - InterestBasedPhoto -> Set(MrPushCopyObjects.RecommendedForYouPhoto), - InterestBasedVideo -> Set(MrPushCopyObjects.RecommendedForYouVideo), - UserFollow -> Set( - MrPushCopyObjects.UserFollowWithOneSocialContext, - MrPushCopyObjects.UserFollowWithTwoSocialContext, - MrPushCopyObjects.UserFollowOneDisplayAndKOtherSocialContext - ), - HermitUser -> Set( - MrPushCopyObjects.HermitUserWithOneSocialContext, - MrPushCopyObjects.HermitUserWithTwoSocialContext, - MrPushCopyObjects.HermitUserWithOneDisplayAndKOtherSocialContexts - ), - TriangularLoopUser -> Set( - MrPushCopyObjects.TriangularLoopUserWithOneSocialContext, - MrPushCopyObjects.TriangularLoopUserWithTwoSocialContexts, - MrPushCopyObjects.TriangularLoopUserOneDisplayAndKotherSocialContext - ), - ForwardAddressbookUserFollow -> Set(MrPushCopyObjects.ForwardAddressBookUserFollow), - NewsArticleNewsLanding -> Set(MrPushCopyObjects.NewsArticleNewsLandingCopy), - TopicProofTweet -> Set(MrPushCopyObjects.TopicProofTweet), - UserInterestinTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), - UserInterestinPhoto -> Set(MrPushCopyObjects.RecommendedForYouPhoto), - UserInterestinVideo -> Set(MrPushCopyObjects.RecommendedForYouVideo), - TwistlyTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), - TwistlyPhoto -> Set(MrPushCopyObjects.RecommendedForYouPhoto), - TwistlyVideo -> Set(MrPushCopyObjects.RecommendedForYouVideo), - ElasticTimelineTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), - ElasticTimelinePhoto -> Set(MrPushCopyObjects.RecommendedForYouPhoto), - ElasticTimelineVideo -> Set(MrPushCopyObjects.RecommendedForYouVideo), - ExploreVideoTweet -> Set(MrPushCopyObjects.ExploreVideoTweet), - List -> Set(MrPushCopyObjects.ListRecommendation), - InterestBasedUserFollow -> Set(MrPushCopyObjects.UserFollowInterestBasedCopy), - PastEmailEngagementTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), - PastEmailEngagementPhoto -> Set(MrPushCopyObjects.RecommendedForYouPhoto), - PastEmailEngagementVideo -> Set(MrPushCopyObjects.RecommendedForYouVideo), - ExplorePush -> Set(MrPushCopyObjects.ExplorePush), - ConnectTabPush -> Set(MrPushCopyObjects.ConnectTabPush), - ConnectTabWithUserPush -> Set(MrPushCopyObjects.ConnectTabWithUserPush), - AddressBookUploadPush -> Set(MrPushCopyObjects.AddressBookPush), - InterestPickerPush -> Set(MrPushCopyObjects.InterestPickerPush), - CompleteOnboardingPush -> Set(MrPushCopyObjects.CompleteOnboardingPush), - GeoPopTweet -> Set(MrPushCopyObjects.GeoPopPushCopy), - TagSpaceTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), - FrsTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), - TwhinTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), - MrModelingBasedTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), - DetopicTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), - TweetImpressions -> Set(MrPushCopyObjects.TopTweetImpressions), - TrendTweet -> Set(MrPushCopyObjects.TrendTweet), - ReverseAddressbookTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), - ForwardAddressbookTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), - SpaceInNetwork -> Set(MrPushCopyObjects.SpaceHost), - SpaceOutOfNetwork -> Set(MrPushCopyObjects.SpaceHost), - SubscribedSearch -> Set(MrPushCopyObjects.SubscribedSearchTweet), - TripGeoTweet -> Set(MrPushCopyObjects.TripGeoTweetPushCopy), - CrowdSearchTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), - Digest -> Set(MrPushCopyObjects.Digest), - TripHqTweet -> Set(MrPushCopyObjects.TripHqTweetPushCopy) - ) - - // Static map from a push copy to set of eligible ntab copies - private[cross] val pushcopyToNtabcopy: Map[MRPushCopy, Set[MRNtabCopy]] = - Map[MRPushCopy, Set[MRNtabCopy]]( - MrPushCopyObjects.FirstDegreeJustTweetedBoldTitle -> Set( - MrNtabCopyObjects.FirstDegreeTweetRecent), - MrPushCopyObjects.FirstDegreePhotoJustTweetedBoldTitle -> Set( - MrNtabCopyObjects.FirstDegreeTweetRecent - ), - MrPushCopyObjects.FirstDegreeVideoJustTweetedBoldTitle -> Set( - MrNtabCopyObjects.FirstDegreeTweetRecent - ), - MrPushCopyObjects.TweetRetweetWithOneDisplaySocialContextsWithText -> Set( - MrNtabCopyObjects.TweetRetweetWithOneDisplaySocialContext - ), - MrPushCopyObjects.TweetRetweetWithTwoDisplaySocialContextsWithText -> Set( - MrNtabCopyObjects.TweetRetweetWithTwoDisplaySocialContexts - ), - MrPushCopyObjects.TweetRetweetWithOneDisplayAndKOtherSocialContextsWithText -> Set( - MrNtabCopyObjects.TweetRetweetWithOneDisplayAndKOtherSocialContexts - ), - MrPushCopyObjects.TweetRetweetPhotoWithOneDisplaySocialContextWithText -> Set( - MrNtabCopyObjects.TweetRetweetPhotoWithOneDisplaySocialContext - ), - MrPushCopyObjects.TweetRetweetPhotoWithTwoDisplaySocialContextsWithText -> Set( - MrNtabCopyObjects.TweetRetweetPhotoWithTwoDisplaySocialContexts - ), - MrPushCopyObjects.TweetRetweetPhotoWithOneDisplayAndKOtherSocialContextsWithText -> Set( - MrNtabCopyObjects.TweetRetweetPhotoWithOneDisplayAndKOtherSocialContexts - ), - MrPushCopyObjects.TweetRetweetVideoWithOneDisplaySocialContextWithText -> Set( - MrNtabCopyObjects.TweetRetweetVideoWithOneDisplaySocialContext - ), - MrPushCopyObjects.TweetRetweetVideoWithTwoDisplaySocialContextsWithText -> Set( - MrNtabCopyObjects.TweetRetweetVideoWithTwoDisplaySocialContexts - ), - MrPushCopyObjects.TweetRetweetVideoWithOneDisplayAndKOtherSocialContextsWithText -> Set( - MrNtabCopyObjects.TweetRetweetVideoWithOneDisplayAndKOtherSocialContexts - ), - MrPushCopyObjects.TweetLikeOneSocialContextWithText -> Set( - MrNtabCopyObjects.TweetLikeWithOneDisplaySocialContext - ), - MrPushCopyObjects.TweetLikeTwoSocialContextWithText -> Set( - MrNtabCopyObjects.TweetLikeWithTwoDisplaySocialContexts - ), - MrPushCopyObjects.TweetLikeMultipleSocialContextWithText -> Set( - MrNtabCopyObjects.TweetLikeWithOneDisplayAndKOtherSocialContexts - ), - MrPushCopyObjects.TweetLikePhotoOneSocialContextWithText -> Set( - MrNtabCopyObjects.TweetLikePhotoWithOneDisplaySocialContext - ), - MrPushCopyObjects.TweetLikePhotoTwoSocialContextWithText -> Set( - MrNtabCopyObjects.TweetLikePhotoWithTwoDisplaySocialContexts - ), - MrPushCopyObjects.TweetLikePhotoMultipleSocialContextWithText -> Set( - MrNtabCopyObjects.TweetLikePhotoWithOneDisplayAndKOtherSocialContexts - ), - MrPushCopyObjects.TweetLikeVideoOneSocialContextWithText -> Set( - MrNtabCopyObjects.TweetLikeVideoWithOneDisplaySocialContext - ), - MrPushCopyObjects.TweetLikeVideoTwoSocialContextWithText -> Set( - MrNtabCopyObjects.TweetLikeVideoWithTwoDisplaySocialContexts - ), - MrPushCopyObjects.TweetLikeVideoMultipleSocialContextWithText -> Set( - MrNtabCopyObjects.TweetLikeVideoWithOneDisplayAndKOtherSocialContexts - ), - MrPushCopyObjects.UnreadBadgeCount -> Set.empty[MRNtabCopy], - MrPushCopyObjects.RecommendedForYouTweet -> Set(MrNtabCopyObjects.RecommendedForYouCopy), - MrPushCopyObjects.RecommendedForYouPhoto -> Set(MrNtabCopyObjects.RecommendedForYouCopy), - MrPushCopyObjects.RecommendedForYouVideo -> Set(MrNtabCopyObjects.RecommendedForYouCopy), - MrPushCopyObjects.GeoPopPushCopy -> Set(MrNtabCopyObjects.RecommendedForYouCopy), - MrPushCopyObjects.UserFollowWithOneSocialContext -> Set( - MrNtabCopyObjects.UserFollowWithOneDisplaySocialContext - ), - MrPushCopyObjects.UserFollowWithTwoSocialContext -> Set( - MrNtabCopyObjects.UserFollowWithTwoDisplaySocialContexts - ), - MrPushCopyObjects.UserFollowOneDisplayAndKOtherSocialContext -> Set( - MrNtabCopyObjects.UserFollowWithOneDisplayAndKOtherSocialContexts - ), - MrPushCopyObjects.HermitUserWithOneSocialContext -> Set( - MrNtabCopyObjects.UserFollowWithOneDisplaySocialContext - ), - MrPushCopyObjects.HermitUserWithTwoSocialContext -> Set( - MrNtabCopyObjects.UserFollowWithTwoDisplaySocialContexts - ), - MrPushCopyObjects.HermitUserWithOneDisplayAndKOtherSocialContexts -> Set( - MrNtabCopyObjects.UserFollowWithOneDisplayAndKOtherSocialContexts - ), - MrPushCopyObjects.TriangularLoopUserWithOneSocialContext -> Set( - MrNtabCopyObjects.TriangularLoopUserWithOneSocialContext - ), - MrPushCopyObjects.TriangularLoopUserWithTwoSocialContexts -> Set( - MrNtabCopyObjects.TriangularLoopUserWithTwoSocialContexts - ), - MrPushCopyObjects.TriangularLoopUserOneDisplayAndKotherSocialContext -> Set( - MrNtabCopyObjects.TriangularLoopUserOneDisplayAndKOtherSocialContext - ), - MrPushCopyObjects.NewsArticleNewsLandingCopy -> Set( - MrNtabCopyObjects.NewsArticleNewsLandingCopy - ), - MrPushCopyObjects.UserFollowInterestBasedCopy -> Set( - MrNtabCopyObjects.UserFollowInterestBasedCopy - ), - MrPushCopyObjects.ForwardAddressBookUserFollow -> Set( - MrNtabCopyObjects.ForwardAddressBookUserFollow), - MrPushCopyObjects.ConnectTabPush -> Set( - MrNtabCopyObjects.ConnectTabPush - ), - MrPushCopyObjects.ExplorePush -> Set.empty[MRNtabCopy], - MrPushCopyObjects.ConnectTabWithUserPush -> Set( - MrNtabCopyObjects.UserFollowInterestBasedCopy), - MrPushCopyObjects.AddressBookPush -> Set(MrNtabCopyObjects.AddressBook), - MrPushCopyObjects.InterestPickerPush -> Set(MrNtabCopyObjects.InterestPicker), - MrPushCopyObjects.CompleteOnboardingPush -> Set(MrNtabCopyObjects.CompleteOnboarding), - MrPushCopyObjects.TopicProofTweet -> Set(MrNtabCopyObjects.TopicProofTweet), - MrPushCopyObjects.TopTweetImpressions -> Set(MrNtabCopyObjects.TopTweetImpressions), - MrPushCopyObjects.TrendTweet -> Set(MrNtabCopyObjects.TrendTweet), - MrPushCopyObjects.SpaceHost -> Set(MrNtabCopyObjects.SpaceHost), - MrPushCopyObjects.SubscribedSearchTweet -> Set(MrNtabCopyObjects.SubscribedSearchTweet), - MrPushCopyObjects.TripGeoTweetPushCopy -> Set(MrNtabCopyObjects.RecommendedForYouCopy), - MrPushCopyObjects.Digest -> Set(MrNtabCopyObjects.Digest), - MrPushCopyObjects.TripHqTweetPushCopy -> Set(MrNtabCopyObjects.HighQualityTweet), - MrPushCopyObjects.ExploreVideoTweet -> Set(MrNtabCopyObjects.ExploreVideoTweet), - MrPushCopyObjects.ListRecommendation -> Set(MrNtabCopyObjects.ListRecommendation), - MrPushCopyObjects.MagicFanoutCreatorSubscription -> Set( - MrNtabCopyObjects.MagicFanoutCreatorSubscription), - MrPushCopyObjects.MagicFanoutNewCreator -> Set(MrNtabCopyObjects.MagicFanoutNewCreator) - ) - - /** - * - * @param crt - [[CommonRecommendationType]] used for a frigate push notification - * - * @return - Set of [[MRPushCopy]] objects representing push copies eligibile for a - * [[CommonRecommendationType]] - */ - def getPushCopiesFromRectype(crt: CommonRecommendationType): Option[Set[MRPushCopy]] = - rectypeToPushCopy.get(crt) - - /** - * - * @param pushcopy - [[MRPushCopy]] object representing a push notification copy - * @return - Set of [[MRNtabCopy]] objects that can be paired with a given [[MRPushCopy]] - */ - def getNtabcopiesFromPushcopy(pushcopy: MRPushCopy): Option[Set[MRNtabCopy]] = - pushcopyToNtabcopy.get(pushcopy) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CopyFilters.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CopyFilters.scala deleted file mode 100644 index 0fe5f5cdd..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CopyFilters.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.frigate.pushservice.refresh_handler.cross - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.util.MRPushCopy -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.hermit.predicate.Predicate -import com.twitter.util.Future - -private[cross] class CopyFilters(statsReceiver: StatsReceiver) { - - private val copyPredicates = new CopyPredicates(statsReceiver.scope("copy_predicate")) - - def execute(rawCandidate: RawCandidate, pushCopies: Seq[MRPushCopy]): Future[Seq[MRPushCopy]] = { - val candidateCopyPairs: Seq[CandidateCopyPair] = - pushCopies.map(CandidateCopyPair(rawCandidate, _)) - - val compositePredicate: Predicate[CandidateCopyPair] = rawCandidate match { - case _: F1FirstDegree | _: OutOfNetworkTweetCandidate | _: EventCandidate | - _: TopicProofTweetCandidate | _: ListPushCandidate | _: HermitInterestBasedUserFollow | - _: UserFollowWithoutSocialContextCandidate | _: DiscoverTwitterCandidate | - _: TopTweetImpressionsCandidate | _: TrendTweetCandidate | - _: SubscribedSearchTweetCandidate | _: DigestCandidate => - copyPredicates.alwaysTruePredicate - - case _: SocialContextActions => copyPredicates.displaySocialContextPredicate - - case _ => copyPredicates.unrecognizedCandidatePredicate // block unrecognised candidates - } - - // apply predicate to all [[MRPushCopy]] objects - val filterResults: Future[Seq[Boolean]] = compositePredicate(candidateCopyPairs) - filterResults.map { results: Seq[Boolean] => - val seqBuilder = Seq.newBuilder[MRPushCopy] - results.zip(pushCopies).foreach { - case (result, pushCopy) => if (result) seqBuilder += pushCopy - } - seqBuilder.result() - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CopyPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CopyPredicates.scala deleted file mode 100644 index 980af1554..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CopyPredicates.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.frigate.pushservice.refresh_handler.cross - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.SocialContextActions -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.hermit.predicate.Predicate - -class CopyPredicates(statsReceiver: StatsReceiver) { - val alwaysTruePredicate = Predicate - .from { _: CandidateCopyPair => - true - }.withStats(statsReceiver.scope("always_true_copy_predicate")) - - val unrecognizedCandidatePredicate = alwaysTruePredicate.flip - .withStats(statsReceiver.scope("unrecognized_candidate")) - - val displaySocialContextPredicate = Predicate - .from { candidateCopyPair: CandidateCopyPair => - candidateCopyPair.candidate match { - case candidateWithScActions: RawCandidate with SocialContextActions => - val socialContextUserIds = candidateWithScActions.socialContextActions.map(_.userId) - val countSocialContext = socialContextUserIds.size - val pushCopy = candidateCopyPair.pushCopy - - countSocialContext match { - case 1 => pushCopy.hasOneDisplaySocialContext && !pushCopy.hasOtherSocialContext - case 2 => pushCopy.hasTwoDisplayContext && !pushCopy.hasOtherSocialContext - case c if c > 2 => - pushCopy.hasOneDisplaySocialContext && pushCopy.hasOtherSocialContext - case _ => false - } - - case _ => false - } - }.withStats(statsReceiver.scope("display_social_context_predicate")) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/scriber/MrRequestScribeHandler.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/scriber/MrRequestScribeHandler.scala deleted file mode 100644 index 90095056a..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/scriber/MrRequestScribeHandler.scala +++ /dev/null @@ -1,388 +0,0 @@ -package com.twitter.frigate.pushservice.scriber - -import com.twitter.bijection.Base64String -import com.twitter.bijection.Injection -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.core_workflows.user_model.thriftscala.{UserState => ThriftUserState} -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.tracing.Trace -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.common.base.CandidateResult -import com.twitter.frigate.common.base.Invalid -import com.twitter.frigate.common.base.OK -import com.twitter.frigate.common.base.Result -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.data_pipeline.features_common.PushQualityModelFeatureContext -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.scribe.thriftscala.CandidateFilteredOutStep -import com.twitter.frigate.scribe.thriftscala.CandidateRequestInfo -import com.twitter.frigate.scribe.thriftscala.MrRequestScribe -import com.twitter.frigate.scribe.thriftscala.TargetUserInfo -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.frigate.thriftscala.TweetNotification -import com.twitter.frigate.thriftscala.{SocialContextAction => TSocialContextAction} -import com.twitter.logging.Logger -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.Feature -import com.twitter.ml.api.FeatureType -import com.twitter.ml.api.util.SRichDataRecord -import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions -import com.twitter.nrel.heavyranker.PushPredictionHelper -import com.twitter.util.Future -import com.twitter.util.Time -import java.util.UUID -import scala.collection.mutable - -class MrRequestScribeHandler(mrRequestScriberNode: String, stats: StatsReceiver) { - - private val mrRequestScribeLogger = Logger(mrRequestScriberNode) - - private val mrRequestScribeTargetFilteringStats = - stats.counter("MrRequestScribeHandler_target_filtering") - private val mrRequestScribeCandidateFilteringStats = - stats.counter("MrRequestScribeHandler_candidate_filtering") - private val mrRequestScribeInvalidStats = - stats.counter("MrRequestScribeHandler_invalid_filtering") - private val mrRequestScribeUnsupportedFeatureTypeStats = - stats.counter("MrRequestScribeHandler_unsupported_feature_type") - private val mrRequestScribeNotIncludedFeatureStats = - stats.counter("MrRequestScribeHandler_not_included_features") - - private final val MrRequestScribeInjection: Injection[MrRequestScribe, String] = BinaryScalaCodec( - MrRequestScribe - ) andThen Injection.connect[Array[Byte], Base64String, String] - - /** - * - * @param target : Target user id - * @param result : Result for target filtering - * - * @return - */ - def scribeForTargetFiltering(target: Target, result: Result): Future[Option[MrRequestScribe]] = { - if (target.isLoggedOutUser || !enableTargetFilteringScribing(target)) { - Future.None - } else { - val predicate = result match { - case Invalid(reason) => reason - case _ => - mrRequestScribeInvalidStats.incr() - throw new IllegalStateException("Invalid reason for Target Filtering " + result) - } - buildScribeThrift(target, predicate, None).map { targetFilteredScribe => - writeAtTargetFilteringStep(target, targetFilteredScribe) - Some(targetFilteredScribe) - } - } - } - - /** - * - * @param target : Target user id - * @param hydratedCandidates : Candidates hydrated with details: impressionId, frigateNotification and source - * @param preRankingFilteredCandidates : Candidates result filtered out at preRanking filtering step - * @param rankedCandidates : Sorted candidates details ranked by ranking step - * @param rerankedCandidates : Sorted candidates details ranked by reranking step - * @param restrictFilteredCandidates : Candidates details filtered out at restrict step - * @param allTakeCandidateResults : Candidates results at take step, include the candidates we take and the candidates filtered out at take step [with different result] - * - * @return - */ - def scribeForCandidateFiltering( - target: Target, - hydratedCandidates: Seq[CandidateDetails[PushCandidate]], - preRankingFilteredCandidates: Seq[CandidateResult[PushCandidate, Result]], - rankedCandidates: Seq[CandidateDetails[PushCandidate]], - rerankedCandidates: Seq[CandidateDetails[PushCandidate]], - restrictFilteredCandidates: Seq[CandidateDetails[PushCandidate]], - allTakeCandidateResults: Seq[CandidateResult[PushCandidate, Result]] - ): Future[Seq[MrRequestScribe]] = { - if (target.isLoggedOutUser || target.isEmailUser) { - Future.Nil - } else if (enableCandidateFilteringScribing(target)) { - val hydrateFeature = - target.params(PushFeatureSwitchParams.EnableMrRequestScribingWithFeatureHydrating) || - target.scribeFeatureForRequestScribe - - val candidateRequestInfoSeq = generateCandidatesScribeInfo( - hydratedCandidates, - preRankingFilteredCandidates, - rankedCandidates, - rerankedCandidates, - restrictFilteredCandidates, - allTakeCandidateResults, - isFeatureHydratingEnabled = hydrateFeature - ) - val flattenStructure = - target.params(PushFeatureSwitchParams.EnableFlattenMrRequestScribing) || hydrateFeature - candidateRequestInfoSeq.flatMap { candidateRequestInfos => - if (flattenStructure) { - Future.collect { - candidateRequestInfos.map { candidateRequestInfo => - buildScribeThrift(target, None, Some(Seq(candidateRequestInfo))) - .map { mrRequestScribe => - writeAtCandidateFilteringStep(target, mrRequestScribe) - mrRequestScribe - } - } - } - } else { - buildScribeThrift(target, None, Some(candidateRequestInfos)) - .map { mrRequestScribe => - writeAtCandidateFilteringStep(target, mrRequestScribe) - Seq(mrRequestScribe) - } - } - } - } else Future.Nil - - } - - private def buildScribeThrift( - target: Target, - targetFilteredOutPredicate: Option[String], - candidatesRequestInfo: Option[Seq[CandidateRequestInfo]] - ): Future[MrRequestScribe] = { - Future - .join( - target.targetUserState, - generateTargetFeatureScribeInfo(target), - target.targetUser).map { - case (userStateOption, targetFeatureOption, gizmoduckUserOpt) => - val userState = userStateOption.map(userState => ThriftUserState(userState.id)) - val targetFeatures = - targetFeatureOption.map(ScalaToJavaDataRecordConversions.javaDataRecord2ScalaDataRecord) - val traceId = Trace.id.traceId.toLong - - MrRequestScribe( - requestId = UUID.randomUUID.toString.replaceAll("-", ""), - scribedTimeMs = Time.now.inMilliseconds, - targetUserId = target.targetId, - targetUserInfo = Some( - TargetUserInfo( - userState, - features = targetFeatures, - userType = gizmoduckUserOpt.map(_.userType)) - ), - targetFilteredOutPredicate = targetFilteredOutPredicate, - candidates = candidatesRequestInfo, - traceId = Some(traceId) - ) - } - } - - private def generateTargetFeatureScribeInfo( - target: Target - ): Future[Option[DataRecord]] = { - val featureList = - target.params(PushFeatureSwitchParams.TargetLevelFeatureListForMrRequestScribing) - if (featureList.nonEmpty) { - PushPredictionHelper - .getDataRecordFromTargetFeatureMap( - target.targetId, - target.featureMap, - stats - ).map { dataRecord => - val richRecord = - new SRichDataRecord(dataRecord, PushQualityModelFeatureContext.featureContext) - - val selectedRecord = - SRichDataRecord(new DataRecord(), PushQualityModelFeatureContext.featureContext) - featureList.map { featureName => - val feature: Feature[_] = { - try { - PushQualityModelFeatureContext.featureContext.getFeature(featureName) - } catch { - case _: Exception => - mrRequestScribeNotIncludedFeatureStats.incr() - throw new IllegalStateException( - "Scribing features not included in FeatureContext: " + featureName) - } - } - - richRecord.getFeatureValueOpt(feature).foreach { featureVal => - feature.getFeatureType() match { - case FeatureType.BINARY => - selectedRecord.setFeatureValue( - feature.asInstanceOf[Feature[Boolean]], - featureVal.asInstanceOf[Boolean]) - case FeatureType.CONTINUOUS => - selectedRecord.setFeatureValue( - feature.asInstanceOf[Feature[Double]], - featureVal.asInstanceOf[Double]) - case FeatureType.STRING => - selectedRecord.setFeatureValue( - feature.asInstanceOf[Feature[String]], - featureVal.asInstanceOf[String]) - case FeatureType.DISCRETE => - selectedRecord.setFeatureValue( - feature.asInstanceOf[Feature[Long]], - featureVal.asInstanceOf[Long]) - case _ => - mrRequestScribeUnsupportedFeatureTypeStats.incr() - } - } - } - Some(selectedRecord.getRecord) - } - } else Future.None - } - - private def generateCandidatesScribeInfo( - hydratedCandidates: Seq[CandidateDetails[PushCandidate]], - preRankingFilteredCandidates: Seq[CandidateResult[PushCandidate, Result]], - rankedCandidates: Seq[CandidateDetails[PushCandidate]], - rerankedCandidates: Seq[CandidateDetails[PushCandidate]], - restrictFilteredCandidates: Seq[CandidateDetails[PushCandidate]], - allTakeCandidateResults: Seq[CandidateResult[PushCandidate, Result]], - isFeatureHydratingEnabled: Boolean - ): Future[Seq[CandidateRequestInfo]] = { - val candidatesMap = new mutable.HashMap[String, CandidateRequestInfo] - - hydratedCandidates.foreach { hydratedCandidate => - val frgNotif = hydratedCandidate.candidate.frigateNotification - val simplifiedTweetNotificationOpt = frgNotif.tweetNotification.map { tweetNotification => - TweetNotification( - tweetNotification.tweetId, - Seq.empty[TSocialContextAction], - tweetNotification.tweetAuthorId) - } - val simplifiedFrigateNotification = FrigateNotification( - frgNotif.commonRecommendationType, - frgNotif.notificationDisplayLocation, - tweetNotification = simplifiedTweetNotificationOpt - ) - candidatesMap(hydratedCandidate.candidate.impressionId) = CandidateRequestInfo( - candidateId = "", - candidateSource = hydratedCandidate.source.substring( - 0, - Math.min(6, hydratedCandidate.source.length) - ), - frigateNotification = Some(simplifiedFrigateNotification), - modelScore = None, - rankPosition = None, - rerankPosition = None, - features = None, - isSent = Some(false) - ) - } - - preRankingFilteredCandidates.foreach { preRankingFilteredCandidateResult => - candidatesMap(preRankingFilteredCandidateResult.candidate.impressionId) = - candidatesMap(preRankingFilteredCandidateResult.candidate.impressionId) - .copy( - candidateFilteredOutPredicate = preRankingFilteredCandidateResult.result match { - case Invalid(reason) => reason - case _ => { - mrRequestScribeInvalidStats.incr() - throw new IllegalStateException( - "Invalid reason for Candidate Filtering " + preRankingFilteredCandidateResult.result) - } - }, - candidateFilteredOutStep = Some(CandidateFilteredOutStep.PreRankFiltering) - ) - } - - for { - _ <- Future.collectToTry { - rankedCandidates.zipWithIndex.map { - case (rankedCandidateDetail, index) => - val modelScoresFut = { - val crt = rankedCandidateDetail.candidate.commonRecType - if (RecTypes.notEligibleForModelScoreTracking.contains(crt)) Future.None - else rankedCandidateDetail.candidate.modelScores.map(Some(_)) - } - - modelScoresFut.map { modelScores => - candidatesMap(rankedCandidateDetail.candidate.impressionId) = - candidatesMap(rankedCandidateDetail.candidate.impressionId).copy( - rankPosition = Some(index), - modelScore = modelScores - ) - } - } - } - - _ = rerankedCandidates.zipWithIndex.foreach { - case (rerankedCandidateDetail, index) => { - candidatesMap(rerankedCandidateDetail.candidate.impressionId) = - candidatesMap(rerankedCandidateDetail.candidate.impressionId).copy( - rerankPosition = Some(index) - ) - } - } - - _ <- Future.collectToTry { - rerankedCandidates.map { rerankedCandidateDetail => - if (isFeatureHydratingEnabled) { - PushPredictionHelper - .getDataRecord( - rerankedCandidateDetail.candidate.target.targetHydrationContext, - rerankedCandidateDetail.candidate.target.featureMap, - rerankedCandidateDetail.candidate.candidateHydrationContext, - rerankedCandidateDetail.candidate.candidateFeatureMap(), - stats - ).map { features => - candidatesMap(rerankedCandidateDetail.candidate.impressionId) = - candidatesMap(rerankedCandidateDetail.candidate.impressionId).copy( - features = Some( - ScalaToJavaDataRecordConversions.javaDataRecord2ScalaDataRecord(features)) - ) - } - } else Future.Unit - } - } - - _ = restrictFilteredCandidates.foreach { restrictFilteredCandidateDetatil => - candidatesMap(restrictFilteredCandidateDetatil.candidate.impressionId) = - candidatesMap(restrictFilteredCandidateDetatil.candidate.impressionId) - .copy(candidateFilteredOutStep = Some(CandidateFilteredOutStep.Restrict)) - } - - _ = allTakeCandidateResults.foreach { allTakeCandidateResult => - allTakeCandidateResult.result match { - case OK => - candidatesMap(allTakeCandidateResult.candidate.impressionId) = - candidatesMap(allTakeCandidateResult.candidate.impressionId).copy(isSent = Some(true)) - case Invalid(reason) => - candidatesMap(allTakeCandidateResult.candidate.impressionId) = - candidatesMap(allTakeCandidateResult.candidate.impressionId).copy( - candidateFilteredOutPredicate = reason, - candidateFilteredOutStep = Some(CandidateFilteredOutStep.PostRankFiltering)) - case _ => - mrRequestScribeInvalidStats.incr() - throw new IllegalStateException( - "Invalid reason for Candidate Filtering " + allTakeCandidateResult.result) - } - } - } yield candidatesMap.values.toSeq - } - - private def enableTargetFilteringScribing(target: Target): Boolean = { - target.params(PushParams.EnableMrRequestScribing) && target.params( - PushFeatureSwitchParams.EnableMrRequestScribingForTargetFiltering) - } - - private def enableCandidateFilteringScribing(target: Target): Boolean = { - target.params(PushParams.EnableMrRequestScribing) && target.params( - PushFeatureSwitchParams.EnableMrRequestScribingForCandidateFiltering) - } - - private def writeAtTargetFilteringStep(target: Target, mrRequestScribe: MrRequestScribe) = { - logToScribe(mrRequestScribe) - mrRequestScribeTargetFilteringStats.incr() - } - - private def writeAtCandidateFilteringStep(target: Target, mrRequestScribe: MrRequestScribe) = { - logToScribe(mrRequestScribe) - mrRequestScribeCandidateFilteringStats.incr() - } - - private def logToScribe(mrRequestScribe: MrRequestScribe): Unit = { - val logEntry: String = MrRequestScribeInjection(mrRequestScribe) - mrRequestScribeLogger.info(logEntry) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/SendHandler.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/SendHandler.scala deleted file mode 100644 index e235f76cb..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/SendHandler.scala +++ /dev/null @@ -1,250 +0,0 @@ -package com.twitter.frigate.pushservice.send_handler - -import com.twitter.finagle.stats.BroadcastStatsReceiver -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.common.base.CandidateFilteringOnlyFlow -import com.twitter.frigate.common.base.CandidateResult -import com.twitter.frigate.common.base.FeatureMap -import com.twitter.frigate.common.base.OK -import com.twitter.frigate.common.base.Response -import com.twitter.frigate.common.base.Result -import com.twitter.frigate.common.base.Stats.track -import com.twitter.frigate.common.config.CommonConstants -import com.twitter.frigate.common.logger.MRLogger -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.common.util.InvalidRequestException -import com.twitter.frigate.common.util.MrNtabCopyObjects -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.HydrationContextBuilder -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams.EnableMagicFanoutNewsForYouNtabCopy -import com.twitter.frigate.pushservice.scriber.MrRequestScribeHandler -import com.twitter.frigate.pushservice.send_handler.generator.PushRequestToCandidate -import com.twitter.frigate.pushservice.take.SendHandlerNotifier -import com.twitter.frigate.pushservice.take.candidate_validator.SendHandlerPostCandidateValidator -import com.twitter.frigate.pushservice.take.candidate_validator.SendHandlerPreCandidateValidator -import com.twitter.frigate.pushservice.target.PushTargetUserBuilder -import com.twitter.frigate.pushservice.util.ResponseStatsTrackUtils.trackStatsForResponseToRequest -import com.twitter.frigate.pushservice.util.SendHandlerPredicateUtil -import com.twitter.frigate.pushservice.thriftscala.PushRequest -import com.twitter.frigate.pushservice.thriftscala.PushRequestScribe -import com.twitter.frigate.pushservice.thriftscala.PushResponse -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.nrel.heavyranker.FeatureHydrator -import com.twitter.util._ - -/** - * A handler for sending PushRequests - */ -class SendHandler( - pushTargetUserBuilder: PushTargetUserBuilder, - preCandidateValidator: SendHandlerPreCandidateValidator, - postCandidateValidator: SendHandlerPostCandidateValidator, - sendHandlerNotifier: SendHandlerNotifier, - candidateHydrator: SendHandlerPushCandidateHydrator, - featureHydrator: FeatureHydrator, - sendHandlerPredicateUtil: SendHandlerPredicateUtil, - mrRequestScriberNode: String -)( - implicit val statsReceiver: StatsReceiver, - implicit val config: Config) - extends CandidateFilteringOnlyFlow[Target, RawCandidate, PushCandidate] { - - implicit private val timer: Timer = new JavaTimer(true) - val stats = statsReceiver.scope("SendHandler") - val log = MRLogger("SendHandler") - - private val buildTargetStats = stats.scope("build_target") - - private val candidateHydrationLatency: Stat = - stats.stat("candidateHydrationLatency") - - private val candidatePreValidatorLatency: Stat = - stats.stat("candidatePreValidatorLatency") - - private val candidatePostValidatorLatency: Stat = - stats.stat("candidatePostValidatorLatency") - - private val featureHydrationLatency: StatsReceiver = - stats.scope("featureHydrationLatency") - - private val mrRequestScribeHandler = - new MrRequestScribeHandler(mrRequestScriberNode, stats.scope("mr_request_scribe")) - - def apply(request: PushRequest): Future[PushResponse] = { - val receivers = Seq( - stats, - stats.scope(request.notification.commonRecommendationType.toString) - ) - val bStats = BroadcastStatsReceiver(receivers) - bStats.counter("requests").incr() - Stat - .timeFuture(bStats.stat("latency"))( - process(request).raiseWithin(CommonConstants.maxPushRequestDuration)) - .onSuccess { - case (pushResp, rawCandidate) => - trackStatsForResponseToRequest( - rawCandidate.commonRecType, - rawCandidate.target, - pushResp, - receivers)(statsReceiver) - if (!request.context.exists(_.darkWrite.contains(true))) { - config.requestScribe(PushRequestScribe(request, pushResp)) - } - } - .onFailure { ex => - bStats.counter("failures").incr() - bStats.scope("failures").counter(ex.getClass.getCanonicalName).incr() - } - .map { - case (pushResp, _) => pushResp - } - } - - private def process(request: PushRequest): Future[(PushResponse, RawCandidate)] = { - val recType = request.notification.commonRecommendationType - - track(buildTargetStats)( - pushTargetUserBuilder - .buildTarget( - request.userId, - request.context - ) - ).flatMap { targetUser => - val responseWithScribedInfo = request.context.exists { context => - context.responseWithScribedInfo.contains(true) - } - val newRequest = - if (request.notification.commonRecommendationType == CommonRecommendationType.MagicFanoutNewsEvent && - targetUser.params(EnableMagicFanoutNewsForYouNtabCopy)) { - val newNotification = request.notification.copy(ntabCopyId = - Some(MrNtabCopyObjects.MagicFanoutNewsForYouCopy.copyId)) - request.copy(notification = newNotification) - } else request - - if (RecTypes.isSendHandlerType(recType) || newRequest.context.exists( - _.allowCRT.contains(true))) { - - val rawCandidateFut = PushRequestToCandidate.generatePushCandidate( - newRequest.notification, - targetUser - ) - - rawCandidateFut.flatMap { rawCandidate => - val pushResponse = process(targetUser, Seq(rawCandidate)).flatMap { - sendHandlerNotifier.checkResponseAndNotify(_, responseWithScribedInfo) - } - - pushResponse.map { pushResponse => - (pushResponse, rawCandidate) - } - } - } else { - Future.exception(InvalidRequestException(s"${recType.name} not supported in SendHandler")) - } - } - } - - private def hydrateFeatures( - candidateDetails: Seq[CandidateDetails[PushCandidate]], - target: Target, - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - - candidateDetails.headOption match { - case Some(candidateDetail) - if RecTypes.notEligibleForModelScoreTracking(candidateDetail.candidate.commonRecType) => - Future.value(candidateDetails) - - case Some(candidateDetail) => - val hydrationContextFut = HydrationContextBuilder.build(candidateDetail.candidate) - hydrationContextFut.flatMap { hc => - featureHydrator - .hydrateCandidate(Seq(hc), target.mrRequestContextForFeatureStore) - .map { hydrationResult => - val features = hydrationResult.getOrElse(hc, FeatureMap()) - candidateDetail.candidate.mergeFeatures(features) - candidateDetails - } - } - case _ => Future.Nil - } - } - - override def process( - target: Target, - externalCandidates: Seq[RawCandidate] - ): Future[Response[PushCandidate, Result]] = { - val candidate = externalCandidates.map(CandidateDetails(_, "realtime")) - - for { - hydratedCandidatesWithCopy <- hydrateCandidates(candidate) - - (candidates, preHydrationFilteredCandidates) <- track(filterStats)( - filter(target, hydratedCandidatesWithCopy) - ) - - featureHydratedCandidates <- - track(featureHydrationLatency)(hydrateFeatures(candidates, target)) - - allTakeCandidateResults <- track(takeStats)( - take(target, featureHydratedCandidates, desiredCandidateCount(target)) - ) - - _ <- mrRequestScribeHandler.scribeForCandidateFiltering( - target = target, - hydratedCandidates = hydratedCandidatesWithCopy, - preRankingFilteredCandidates = preHydrationFilteredCandidates, - rankedCandidates = featureHydratedCandidates, - rerankedCandidates = Seq.empty, - restrictFilteredCandidates = Seq.empty, // no restrict step - allTakeCandidateResults = allTakeCandidateResults - ) - } yield { - - /** - * We combine the results for all filtering steps and pass on in sequence to next step - * - * This is done to ensure the filtering reason for the candidate from multiple levels of - * filtering is carried all the way until [[PushResponse]] is built and returned from - * frigate-pushservice-send - */ - Response(OK, allTakeCandidateResults ++ preHydrationFilteredCandidates) - } - } - - override def hydrateCandidates( - candidates: Seq[CandidateDetails[RawCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - Stat.timeFuture(candidateHydrationLatency)(candidateHydrator(candidates)) - } - - // Filter Step - pre-predicates and app specific predicates - override def filter( - target: Target, - hydratedCandidatesDetails: Seq[CandidateDetails[PushCandidate]] - ): Future[ - (Seq[CandidateDetails[PushCandidate]], Seq[CandidateResult[PushCandidate, Result]]) - ] = { - Stat.timeFuture(candidatePreValidatorLatency)( - sendHandlerPredicateUtil.preValidationForCandidate( - hydratedCandidatesDetails, - preCandidateValidator - )) - } - - // Post Validation - Take step - override def validCandidates( - target: Target, - candidates: Seq[PushCandidate] - ): Future[Seq[Result]] = { - Stat.timeFuture(candidatePostValidatorLatency)(Future.collect(candidates.map { candidate => - sendHandlerPredicateUtil - .postValidationForCandidate(candidate, postCandidateValidator) - .map(res => res.result) - })) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/SendHandlerPushCandidateHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/SendHandlerPushCandidateHydrator.scala deleted file mode 100644 index f8f102790..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/SendHandlerPushCandidateHydrator.scala +++ /dev/null @@ -1,184 +0,0 @@ -package com.twitter.frigate.pushservice.send_handler - -import com.twitter.escherbird.metadata.thriftscala.EntityMegadata -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext -import com.twitter.frigate.common.util.MrNtabCopyObjects -import com.twitter.frigate.common.util.MrPushCopyObjects -import com.twitter.frigate.magic_events.thriftscala.FanoutEvent -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.store.EventRequest -import com.twitter.frigate.pushservice.store.UttEntityHydrationStore -import com.twitter.frigate.pushservice.util.CandidateHydrationUtil._ -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery -import com.twitter.interests.thriftscala.UserInterests -import com.twitter.livevideo.timeline.domain.v2.{Event => LiveEvent} -import com.twitter.simclusters_v2.thriftscala.SimClustersInferredEntities -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.client.UserId -import com.twitter.ubs.thriftscala.AudioSpace -import com.twitter.util.Future - -case class SendHandlerPushCandidateHydrator( - lexServiceStore: ReadableStore[EventRequest, LiveEvent], - fanoutMetadataStore: ReadableStore[(Long, Long), FanoutEvent], - semanticCoreMegadataStore: ReadableStore[SemanticEntityForQuery, EntityMegadata], - safeUserStore: ReadableStore[Long, User], - simClusterToEntityStore: ReadableStore[Int, SimClustersInferredEntities], - audioSpaceStore: ReadableStore[String, AudioSpace], - interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests], - uttEntityHydrationStore: UttEntityHydrationStore, - superFollowCreatorTweetCountStore: ReadableStore[UserId, Int] -)( - implicit statsReceiver: StatsReceiver, - implicit val weightedOpenOrNtabClickModelScorer: PushMLModelScorer) { - - lazy val candidateWithCopyNumStat = statsReceiver.stat("candidate_with_copy_num") - lazy val hydratedCandidateStat = statsReceiver.scope("hydrated_candidates") - - def updateCandidates( - candidateDetails: Seq[CandidateDetails[RawCandidate]], - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - - Future.collect { - candidateDetails.map { candidateDetail => - val pushCandidate = candidateDetail.candidate - - val copyIds = getCopyIdsByCRT(pushCandidate.commonRecType) - - val hydratedCandidateFut = pushCandidate match { - case magicFanoutNewsEventCandidate: MagicFanoutNewsEventCandidate => - getHydratedCandidateForMagicFanoutNewsEvent( - magicFanoutNewsEventCandidate, - copyIds, - lexServiceStore, - fanoutMetadataStore, - semanticCoreMegadataStore, - simClusterToEntityStore, - interestsLookupStore, - uttEntityHydrationStore - ) - - case scheduledSpaceSubscriberCandidate: ScheduledSpaceSubscriberCandidate => - getHydratedCandidateForScheduledSpaceSubscriber( - scheduledSpaceSubscriberCandidate, - safeUserStore, - copyIds, - audioSpaceStore - ) - case scheduledSpaceSpeakerCandidate: ScheduledSpaceSpeakerCandidate => - getHydratedCandidateForScheduledSpaceSpeaker( - scheduledSpaceSpeakerCandidate, - safeUserStore, - copyIds, - audioSpaceStore - ) - case magicFanoutSportsEventCandidate: MagicFanoutSportsEventCandidate with MagicFanoutSportsScoreInformation => - getHydratedCandidateForMagicFanoutSportsEvent( - magicFanoutSportsEventCandidate, - copyIds, - lexServiceStore, - fanoutMetadataStore, - semanticCoreMegadataStore, - interestsLookupStore, - uttEntityHydrationStore - ) - case magicFanoutProductLaunchCandidate: MagicFanoutProductLaunchCandidate => - getHydratedCandidateForMagicFanoutProductLaunch( - magicFanoutProductLaunchCandidate, - copyIds) - case creatorEventCandidate: MagicFanoutCreatorEventCandidate => - getHydratedCandidateForMagicFanoutCreatorEvent( - creatorEventCandidate, - safeUserStore, - copyIds, - superFollowCreatorTweetCountStore) - case _ => - throw new IllegalArgumentException("Incorrect candidate type when update candidates") - } - - hydratedCandidateFut.map { hydratedCandidate => - hydratedCandidateStat.counter(hydratedCandidate.commonRecType.name).incr() - CandidateDetails( - hydratedCandidate, - source = candidateDetail.source - ) - } - } - } - } - - private def getCopyIdsByCRT(crt: CommonRecommendationType): CopyIds = { - crt match { - case CommonRecommendationType.MagicFanoutNewsEvent => - CopyIds( - pushCopyId = Some(MrPushCopyObjects.MagicFanoutNewsPushCopy.copyId), - ntabCopyId = Some(MrNtabCopyObjects.MagicFanoutNewsForYouCopy.copyId), - aggregationId = None - ) - - case CommonRecommendationType.ScheduledSpaceSubscriber => - CopyIds( - pushCopyId = Some(MrPushCopyObjects.ScheduledSpaceSubscriber.copyId), - ntabCopyId = Some(MrNtabCopyObjects.ScheduledSpaceSubscriber.copyId), - aggregationId = None - ) - case CommonRecommendationType.ScheduledSpaceSpeaker => - CopyIds( - pushCopyId = Some(MrPushCopyObjects.ScheduledSpaceSpeaker.copyId), - ntabCopyId = Some(MrNtabCopyObjects.ScheduledSpaceSpeakerNow.copyId), - aggregationId = None - ) - case CommonRecommendationType.SpaceSpeaker => - CopyIds( - pushCopyId = Some(MrPushCopyObjects.SpaceSpeaker.copyId), - ntabCopyId = Some(MrNtabCopyObjects.SpaceSpeaker.copyId), - aggregationId = None - ) - case CommonRecommendationType.SpaceHost => - CopyIds( - pushCopyId = Some(MrPushCopyObjects.SpaceHost.copyId), - ntabCopyId = Some(MrNtabCopyObjects.SpaceHost.copyId), - aggregationId = None - ) - case CommonRecommendationType.MagicFanoutSportsEvent => - CopyIds( - pushCopyId = Some(MrPushCopyObjects.MagicFanoutSportsPushCopy.copyId), - ntabCopyId = Some(MrNtabCopyObjects.MagicFanoutSportsCopy.copyId), - aggregationId = None - ) - case CommonRecommendationType.MagicFanoutProductLaunch => - CopyIds( - pushCopyId = Some(MrPushCopyObjects.MagicFanoutProductLaunch.copyId), - ntabCopyId = Some(MrNtabCopyObjects.ProductLaunch.copyId), - aggregationId = None - ) - case CommonRecommendationType.CreatorSubscriber => - CopyIds( - pushCopyId = Some(MrPushCopyObjects.MagicFanoutCreatorSubscription.copyId), - ntabCopyId = Some(MrNtabCopyObjects.MagicFanoutCreatorSubscription.copyId), - aggregationId = None - ) - case CommonRecommendationType.NewCreator => - CopyIds( - pushCopyId = Some(MrPushCopyObjects.MagicFanoutNewCreator.copyId), - ntabCopyId = Some(MrNtabCopyObjects.MagicFanoutNewCreator.copyId), - aggregationId = None - ) - case _ => - throw new IllegalArgumentException("Incorrect candidate type when fetch copy ids") - } - } - - def apply( - candidateDetails: Seq[CandidateDetails[RawCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - updateCandidates(candidateDetails) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/CandidateGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/CandidateGenerator.scala deleted file mode 100644 index 45907fa8e..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/CandidateGenerator.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.twitter.frigate.pushservice.send_handler.generator - -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.util.Future - -trait CandidateGenerator { - - /** - * Build RawCandidate from FrigateNotification - * @param target - * @param frigateNotification - * @return RawCandidate - */ - def getCandidate(target: Target, frigateNotification: FrigateNotification): Future[RawCandidate] -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutCreatorEventCandidateGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutCreatorEventCandidateGenerator.scala deleted file mode 100644 index 10a7acb89..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutCreatorEventCandidateGenerator.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.twitter.frigate.pushservice.send_handler.generator - -import com.twitter.frigate.common.base.MagicFanoutCreatorEventCandidate -import com.twitter.frigate.magic_events.thriftscala.CreatorFanoutType -import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.model.PushTypes -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.util.Future - -object MagicFanoutCreatorEventCandidateGenerator extends CandidateGenerator { - override def getCandidate( - targetUser: PushTypes.Target, - notification: FrigateNotification - ): Future[PushTypes.RawCandidate] = { - - require( - notification.commonRecommendationType == CommonRecommendationType.CreatorSubscriber || notification.commonRecommendationType == CommonRecommendationType.NewCreator, - "MagicFanoutCreatorEvent: unexpected CRT " + notification.commonRecommendationType - ) - require( - notification.creatorSubscriptionNotification.isDefined, - "MagicFanoutCreatorEvent: creatorSubscriptionNotification is not defined") - require( - notification.creatorSubscriptionNotification.exists(_.magicFanoutPushId.isDefined), - "MagicFanoutCreatorEvent: magicFanoutPushId is not defined") - require( - notification.creatorSubscriptionNotification.exists(_.fanoutReasons.isDefined), - "MagicFanoutCreatorEvent: fanoutReasons is not defined") - require( - notification.creatorSubscriptionNotification.exists(_.creatorId.isDefined), - "MagicFanoutCreatorEvent: creatorId is not defined") - if (notification.commonRecommendationType == CommonRecommendationType.CreatorSubscriber) { - require( - notification.creatorSubscriptionNotification - .exists(_.subscriberId.isDefined), - "MagicFanoutCreatorEvent: subscriber id is not defined" - ) - } - - val creatorSubscriptionNotification = notification.creatorSubscriptionNotification.get - - val candidate = new RawCandidate with MagicFanoutCreatorEventCandidate { - - override val target: Target = targetUser - - override val pushId: Long = - creatorSubscriptionNotification.magicFanoutPushId.get - - override val candidateMagicEventsReasons: Seq[MagicEventsReason] = - creatorSubscriptionNotification.fanoutReasons.get - - override val creatorFanoutType: CreatorFanoutType = - creatorSubscriptionNotification.creatorFanoutType - - override val commonRecType: CommonRecommendationType = - notification.commonRecommendationType - - override val frigateNotification: FrigateNotification = notification - - override val subscriberId: Option[Long] = creatorSubscriptionNotification.subscriberId - - override val creatorId: Long = creatorSubscriptionNotification.creatorId.get - } - - Future.value(candidate) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutNewsEventCandidateGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutNewsEventCandidateGenerator.scala deleted file mode 100644 index 7b351c91a..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutNewsEventCandidateGenerator.scala +++ /dev/null @@ -1,57 +0,0 @@ -package com.twitter.frigate.pushservice.send_handler.generator - -import com.twitter.frigate.common.base.MagicFanoutNewsEventCandidate -import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.frigate.thriftscala.MagicFanoutEventNotificationDetails -import com.twitter.util.Future - -object MagicFanoutNewsEventCandidateGenerator extends CandidateGenerator { - - override def getCandidate( - targetUser: Target, - notification: FrigateNotification - ): Future[RawCandidate] = { - - /** - * frigateNotification recommendation type should be [[CommonRecommendationType.MagicFanoutNewsEvent]] - * AND pushId field should be set - **/ - require( - notification.commonRecommendationType == CommonRecommendationType.MagicFanoutNewsEvent, - "MagicFanoutNewsEvent: unexpected CRT " + notification.commonRecommendationType - ) - - require( - notification.magicFanoutEventNotification.exists(_.pushId.isDefined), - "MagicFanoutNewsEvent: pushId is not defined") - - val magicFanoutEventNotification = notification.magicFanoutEventNotification.get - - val candidate = new RawCandidate with MagicFanoutNewsEventCandidate { - - override val target: Target = targetUser - - override val eventId: Long = magicFanoutEventNotification.eventId - - override val pushId: Long = magicFanoutEventNotification.pushId.get - - override val candidateMagicEventsReasons: Seq[MagicEventsReason] = - magicFanoutEventNotification.eventReasons.getOrElse(Seq.empty) - - override val momentId: Option[Long] = magicFanoutEventNotification.momentId - - override val eventLanguage: Option[String] = magicFanoutEventNotification.eventLanguage - - override val details: Option[MagicFanoutEventNotificationDetails] = - magicFanoutEventNotification.details - - override val frigateNotification: FrigateNotification = notification - } - - Future.value(candidate) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutProductLaunchCandidateGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutProductLaunchCandidateGenerator.scala deleted file mode 100644 index 6844b1b06..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutProductLaunchCandidateGenerator.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.frigate.pushservice.send_handler.generator - -import com.twitter.frigate.common.base.MagicFanoutProductLaunchCandidate -import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason -import com.twitter.frigate.magic_events.thriftscala.ProductType -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.model.PushTypes -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.util.Future - -object MagicFanoutProductLaunchCandidateGenerator extends CandidateGenerator { - - override def getCandidate( - targetUser: PushTypes.Target, - notification: FrigateNotification - ): Future[PushTypes.RawCandidate] = { - - require( - notification.commonRecommendationType == CommonRecommendationType.MagicFanoutProductLaunch, - "MagicFanoutProductLaunch: unexpected CRT " + notification.commonRecommendationType - ) - require( - notification.magicFanoutProductLaunchNotification.isDefined, - "MagicFanoutProductLaunch: magicFanoutProductLaunchNotification is not defined") - require( - notification.magicFanoutProductLaunchNotification.exists(_.magicFanoutPushId.isDefined), - "MagicFanoutProductLaunch: magicFanoutPushId is not defined") - require( - notification.magicFanoutProductLaunchNotification.exists(_.fanoutReasons.isDefined), - "MagicFanoutProductLaunch: fanoutReasons is not defined") - - val magicFanoutProductLaunchNotification = notification.magicFanoutProductLaunchNotification.get - - val candidate = new RawCandidate with MagicFanoutProductLaunchCandidate { - - override val target: Target = targetUser - - override val pushId: Long = - magicFanoutProductLaunchNotification.magicFanoutPushId.get - - override val candidateMagicEventsReasons: Seq[MagicEventsReason] = - magicFanoutProductLaunchNotification.fanoutReasons.get - - override val productLaunchType: ProductType = - magicFanoutProductLaunchNotification.productLaunchType - - override val frigateNotification: FrigateNotification = notification - } - - Future.value(candidate) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutSportsEventCandidateGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutSportsEventCandidateGenerator.scala deleted file mode 100644 index cdd37833e..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutSportsEventCandidateGenerator.scala +++ /dev/null @@ -1,153 +0,0 @@ -package com.twitter.frigate.pushservice.send_handler.generator - -import com.twitter.datatools.entityservice.entities.sports.thriftscala.BaseballGameLiveUpdate -import com.twitter.datatools.entityservice.entities.sports.thriftscala.BasketballGameLiveUpdate -import com.twitter.datatools.entityservice.entities.sports.thriftscala.CricketMatchLiveUpdate -import com.twitter.datatools.entityservice.entities.sports.thriftscala.NflFootballGameLiveUpdate -import com.twitter.datatools.entityservice.entities.sports.thriftscala.SoccerMatchLiveUpdate -import com.twitter.escherbird.common.thriftscala.Domains -import com.twitter.escherbird.common.thriftscala.QualifiedId -import com.twitter.escherbird.metadata.thriftscala.EntityMegadata -import com.twitter.frigate.common.base.BaseGameScore -import com.twitter.frigate.common.base.MagicFanoutSportsEventCandidate -import com.twitter.frigate.common.base.MagicFanoutSportsScoreInformation -import com.twitter.frigate.common.base.TeamInfo -import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason -import com.twitter.frigate.pushservice.exception.InvalidSportDomainException -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushConstants -import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutSportsUtil -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.frigate.thriftscala.MagicFanoutEventNotificationDetails -import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -object MagicFanoutSportsEventCandidateGenerator { - - final def getCandidate( - targetUser: Target, - notification: FrigateNotification, - basketballGameScoreStore: ReadableStore[QualifiedId, BasketballGameLiveUpdate], - baseballGameScoreStore: ReadableStore[QualifiedId, BaseballGameLiveUpdate], - cricketMatchScoreStore: ReadableStore[QualifiedId, CricketMatchLiveUpdate], - soccerMatchScoreStore: ReadableStore[QualifiedId, SoccerMatchLiveUpdate], - nflGameScoreStore: ReadableStore[QualifiedId, NflFootballGameLiveUpdate], - semanticCoreMegadataStore: ReadableStore[SemanticEntityForQuery, EntityMegadata], - ): Future[RawCandidate] = { - - /** - * frigateNotification recommendation type should be [[CommonRecommendationType.MagicFanoutSportsEvent]] - * AND pushId field should be set - * - * */ - require( - notification.commonRecommendationType == CommonRecommendationType.MagicFanoutSportsEvent, - "MagicFanoutSports: unexpected CRT " + notification.commonRecommendationType - ) - - require( - notification.magicFanoutEventNotification.exists(_.pushId.isDefined), - "MagicFanoutSportsEvent: pushId is not defined") - - val magicFanoutEventNotification = notification.magicFanoutEventNotification.get - val eventId = magicFanoutEventNotification.eventId - val _isScoreUpdate = magicFanoutEventNotification.isScoreUpdate.getOrElse(false) - - val gameScoresFut: Future[Option[BaseGameScore]] = { - if (_isScoreUpdate) { - semanticCoreMegadataStore - .get(SemanticEntityForQuery(PushConstants.SportsEventDomainId, eventId)) - .flatMap { - case Some(megadata) => - if (megadata.domains.contains(Domains.BasketballGame)) { - basketballGameScoreStore - .get(QualifiedId(Domains.BasketballGame.value, eventId)).map { - case Some(game) if game.status.isDefined => - val status = game.status.get - MagicFanoutSportsUtil.transformToGameScore(game.score, status) - case _ => None - } - } else if (megadata.domains.contains(Domains.BaseballGame)) { - baseballGameScoreStore - .get(QualifiedId(Domains.BaseballGame.value, eventId)).map { - case Some(game) if game.status.isDefined => - val status = game.status.get - MagicFanoutSportsUtil.transformToGameScore(game.runs, status) - case _ => None - } - } else if (megadata.domains.contains(Domains.NflFootballGame)) { - nflGameScoreStore - .get(QualifiedId(Domains.NflFootballGame.value, eventId)).map { - case Some(game) if game.status.isDefined => - val nflScore = MagicFanoutSportsUtil.transformNFLGameScore(game) - nflScore - case _ => None - } - } else if (megadata.domains.contains(Domains.SoccerMatch)) { - soccerMatchScoreStore - .get(QualifiedId(Domains.SoccerMatch.value, eventId)).map { - case Some(game) if game.status.isDefined => - val soccerScore = MagicFanoutSportsUtil.transformSoccerGameScore(game) - soccerScore - case _ => None - } - } else { - // The domains are not in our list of supported sports - throw new InvalidSportDomainException( - s"Domain for entity ${eventId} is not supported") - } - case _ => Future.None - } - } else Future.None - } - - val homeTeamInfoFut: Future[Option[TeamInfo]] = gameScoresFut.flatMap { - case Some(gameScore) => - MagicFanoutSportsUtil.getTeamInfo(gameScore.home, semanticCoreMegadataStore) - case _ => Future.None - } - - val awayTeamInfoFut: Future[Option[TeamInfo]] = gameScoresFut.flatMap { - case Some(gameScore) => - MagicFanoutSportsUtil.getTeamInfo(gameScore.away, semanticCoreMegadataStore) - case _ => Future.None - } - - val candidate = new RawCandidate - with MagicFanoutSportsEventCandidate - with MagicFanoutSportsScoreInformation { - - override val target: Target = targetUser - - override val eventId: Long = magicFanoutEventNotification.eventId - - override val pushId: Long = magicFanoutEventNotification.pushId.get - - override val candidateMagicEventsReasons: Seq[MagicEventsReason] = - magicFanoutEventNotification.eventReasons.getOrElse(Seq.empty) - - override val momentId: Option[Long] = magicFanoutEventNotification.momentId - - override val eventLanguage: Option[String] = magicFanoutEventNotification.eventLanguage - - override val details: Option[MagicFanoutEventNotificationDetails] = - magicFanoutEventNotification.details - - override val frigateNotification: FrigateNotification = notification - - override val homeTeamInfo: Future[Option[TeamInfo]] = homeTeamInfoFut - - override val awayTeamInfo: Future[Option[TeamInfo]] = awayTeamInfoFut - - override val gameScores: Future[Option[BaseGameScore]] = gameScoresFut - - override val isScoreUpdate: Boolean = _isScoreUpdate - } - - Future.value(candidate) - - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/PushRequestToCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/PushRequestToCandidate.scala deleted file mode 100644 index 8d7e81d3f..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/PushRequestToCandidate.scala +++ /dev/null @@ -1,49 +0,0 @@ -package com.twitter.frigate.pushservice.send_handler.generator - -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.exception.UnsupportedCrtException -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.frigate.thriftscala.{CommonRecommendationType => CRT} -import com.twitter.util.Future - -object PushRequestToCandidate { - final def generatePushCandidate( - frigateNotification: FrigateNotification, - target: Target - )( - implicit config: Config - ): Future[RawCandidate] = { - - val candidateGenerator: (Target, FrigateNotification) => Future[RawCandidate] = { - frigateNotification.commonRecommendationType match { - case CRT.MagicFanoutNewsEvent => MagicFanoutNewsEventCandidateGenerator.getCandidate - case CRT.ScheduledSpaceSubscriber => ScheduledSpaceSubscriberCandidateGenerator.getCandidate - case CRT.ScheduledSpaceSpeaker => ScheduledSpaceSpeakerCandidateGenerator.getCandidate - case CRT.MagicFanoutSportsEvent => - MagicFanoutSportsEventCandidateGenerator.getCandidate( - _, - _, - config.basketballGameScoreStore, - config.baseballGameScoreStore, - config.cricketMatchScoreStore, - config.soccerMatchScoreStore, - config.nflGameScoreStore, - config.semanticCoreMegadataStore - ) - case CRT.MagicFanoutProductLaunch => - MagicFanoutProductLaunchCandidateGenerator.getCandidate - case CRT.NewCreator => - MagicFanoutCreatorEventCandidateGenerator.getCandidate - case CRT.CreatorSubscriber => - MagicFanoutCreatorEventCandidateGenerator.getCandidate - case _ => - throw new UnsupportedCrtException( - "UnsupportedCrtException for SendHandler: " + frigateNotification.commonRecommendationType) - } - } - - candidateGenerator(target, frigateNotification) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/ScheduledSpaceSpeakerCandidateGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/ScheduledSpaceSpeakerCandidateGenerator.scala deleted file mode 100644 index e7821db76..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/ScheduledSpaceSpeakerCandidateGenerator.scala +++ /dev/null @@ -1,55 +0,0 @@ -package com.twitter.frigate.pushservice.send_handler.generator - -import com.twitter.frigate.common.base.ScheduledSpaceSpeakerCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.util.Future - -object ScheduledSpaceSpeakerCandidateGenerator extends CandidateGenerator { - - override def getCandidate( - targetUser: Target, - notification: FrigateNotification - ): Future[RawCandidate] = { - - /** - * frigateNotification recommendation type should be [[CommonRecommendationType.ScheduledSpaceSpeaker]] - * - **/ - require( - notification.commonRecommendationType == CommonRecommendationType.ScheduledSpaceSpeaker, - "ScheduledSpaceSpeaker: unexpected CRT " + notification.commonRecommendationType - ) - - val spaceNotification = notification.spaceNotification.getOrElse( - throw new IllegalStateException("ScheduledSpaceSpeaker notification object not defined")) - - require( - spaceNotification.hostUserId.isDefined, - "ScheduledSpaceSpeaker notification - hostUserId not defined" - ) - - val spaceHostId = spaceNotification.hostUserId - - require( - spaceNotification.scheduledStartTime.isDefined, - "ScheduledSpaceSpeaker notification - scheduledStartTime not defined" - ) - - val scheduledStartTime = spaceNotification.scheduledStartTime.get - - val candidate = new RawCandidate with ScheduledSpaceSpeakerCandidate { - override val target: Target = targetUser - override val frigateNotification: FrigateNotification = notification - override val spaceId: String = spaceNotification.broadcastId - override val hostId: Option[Long] = spaceHostId - override val startTime: Long = scheduledStartTime - override val speakerIds: Option[Seq[Long]] = spaceNotification.speakers - override val listenerIds: Option[Seq[Long]] = spaceNotification.listeners - } - - Future.value(candidate) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/ScheduledSpaceSubscriberCandidateGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/ScheduledSpaceSubscriberCandidateGenerator.scala deleted file mode 100644 index 484f17b2a..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/ScheduledSpaceSubscriberCandidateGenerator.scala +++ /dev/null @@ -1,55 +0,0 @@ -package com.twitter.frigate.pushservice.send_handler.generator - -import com.twitter.frigate.common.base.ScheduledSpaceSubscriberCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.util.Future - -object ScheduledSpaceSubscriberCandidateGenerator extends CandidateGenerator { - - override def getCandidate( - targetUser: Target, - notification: FrigateNotification - ): Future[RawCandidate] = { - - /** - * frigateNotification recommendation type should be [[CommonRecommendationType.ScheduledSpaceSubscriber]] - * - **/ - require( - notification.commonRecommendationType == CommonRecommendationType.ScheduledSpaceSubscriber, - "ScheduledSpaceSubscriber: unexpected CRT " + notification.commonRecommendationType - ) - - val spaceNotification = notification.spaceNotification.getOrElse( - throw new IllegalStateException("ScheduledSpaceSubscriber notification object not defined")) - - require( - spaceNotification.hostUserId.isDefined, - "ScheduledSpaceSubscriber notification - hostUserId not defined" - ) - - val spaceHostId = spaceNotification.hostUserId - - require( - spaceNotification.scheduledStartTime.isDefined, - "ScheduledSpaceSubscriber notification - scheduledStartTime not defined" - ) - - val scheduledStartTime = spaceNotification.scheduledStartTime.get - - val candidate = new RawCandidate with ScheduledSpaceSubscriberCandidate { - override val target: Target = targetUser - override val frigateNotification: FrigateNotification = notification - override val spaceId: String = spaceNotification.broadcastId - override val hostId: Option[Long] = spaceHostId - override val startTime: Long = scheduledStartTime - override val speakerIds: Option[Seq[Long]] = spaceNotification.speakers - override val listenerIds: Option[Seq[Long]] = spaceNotification.listeners - } - - Future.value(candidate) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/ContentMixerStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/ContentMixerStore.scala deleted file mode 100644 index 1f4171030..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/ContentMixerStore.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.content_mixer.thriftscala.ContentMixer -import com.twitter.content_mixer.thriftscala.ContentMixerRequest -import com.twitter.content_mixer.thriftscala.ContentMixerResponse -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -case class ContentMixerStore(contentMixer: ContentMixer.MethodPerEndpoint) - extends ReadableStore[ContentMixerRequest, ContentMixerResponse] { - - override def get(request: ContentMixerRequest): Future[Option[ContentMixerResponse]] = { - contentMixer.getCandidates(request).map { response => - Some(response) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/CopySelectionServiceStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/CopySelectionServiceStore.scala deleted file mode 100644 index b793ade7e..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/CopySelectionServiceStore.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.copyselectionservice.thriftscala._ -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -class CopySelectionServiceStore(copySelectionServiceClient: CopySelectionService.FinagledClient) - extends ReadableStore[CopySelectionRequestV1, Copy] { - override def get(k: CopySelectionRequestV1): Future[Option[Copy]] = - copySelectionServiceClient.getSelectedCopy(CopySelectionRequest.V1(k)).map { - case CopySelectionResponse.V1(response) => - Some(response.selectedCopy) - case _ => throw CopyServiceException(CopyServiceErrorCode.VersionNotFound) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/CrMixerTweetStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/CrMixerTweetStore.scala deleted file mode 100644 index dba016e6c..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/CrMixerTweetStore.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.cr_mixer.thriftscala.CrMixer -import com.twitter.cr_mixer.thriftscala.CrMixerTweetRequest -import com.twitter.cr_mixer.thriftscala.CrMixerTweetResponse -import com.twitter.cr_mixer.thriftscala.FrsTweetRequest -import com.twitter.cr_mixer.thriftscala.FrsTweetResponse -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.util.Future - -/** - * Store to get content recs from content recommender. - */ -case class CrMixerTweetStore( - crMixer: CrMixer.MethodPerEndpoint -)( - implicit statsReceiver: StatsReceiver = NullStatsReceiver) { - - private val requestsCounter = statsReceiver.counter("requests") - private val successCounter = statsReceiver.counter("success") - private val failuresCounter = statsReceiver.counter("failures") - private val nonEmptyCounter = statsReceiver.counter("non_empty") - private val emptyCounter = statsReceiver.counter("empty") - private val failuresScope = statsReceiver.scope("failures") - private val latencyStat = statsReceiver.stat("latency") - - private def updateStats[T](f: => Future[Option[T]]): Future[Option[T]] = { - requestsCounter.incr() - Stat - .timeFuture(latencyStat)(f) - .onSuccess { r => - if (r.isDefined) nonEmptyCounter.incr() else emptyCounter.incr() - successCounter.incr() - } - .onFailure { e => - { - failuresCounter.incr() - failuresScope.counter(e.getClass.getName).incr() - } - } - } - - def getTweetRecommendations( - request: CrMixerTweetRequest - ): Future[Option[CrMixerTweetResponse]] = { - updateStats(crMixer.getTweetRecommendations(request).map { response => - Some(response) - }) - } - - def getFRSTweetCandidates(request: FrsTweetRequest): Future[Option[FrsTweetResponse]] = { - updateStats(crMixer.getFrsBasedTweetRecommendations(request).map { response => - Some(response) - }) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/ExploreRankerStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/ExploreRankerStore.scala deleted file mode 100644 index eeeb62d27..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/ExploreRankerStore.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.explore_ranker.thriftscala.ExploreRanker -import com.twitter.explore_ranker.thriftscala.ExploreRankerResponse -import com.twitter.explore_ranker.thriftscala.ExploreRankerRequest -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -/** A Store for Video Tweet Recommendations from Explore - * - * @param exploreRankerService - */ -case class ExploreRankerStore(exploreRankerService: ExploreRanker.MethodPerEndpoint) - extends ReadableStore[ExploreRankerRequest, ExploreRankerResponse] { - - /** Method to get video recommendations - * - * @param request explore ranker request object - * @return - */ - override def get( - request: ExploreRankerRequest - ): Future[Option[ExploreRankerResponse]] = { - exploreRankerService.getRankedResults(request).map { response => - Some(response) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/FollowRecommendationsStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/FollowRecommendationsStore.scala deleted file mode 100644 index 0ab722cd4..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/FollowRecommendationsStore.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService -import com.twitter.follow_recommendations.thriftscala.Recommendation -import com.twitter.follow_recommendations.thriftscala.RecommendationRequest -import com.twitter.follow_recommendations.thriftscala.RecommendationResponse -import com.twitter.follow_recommendations.thriftscala.UserRecommendation -import com.twitter.inject.Logging -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -case class FollowRecommendationsStore( - frsClient: FollowRecommendationsThriftService.MethodPerEndpoint, - statsReceiver: StatsReceiver) - extends ReadableStore[RecommendationRequest, RecommendationResponse] - with Logging { - - private val scopedStats = statsReceiver.scope(getClass.getSimpleName) - private val requests = scopedStats.counter("requests") - private val valid = scopedStats.counter("valid") - private val invalid = scopedStats.counter("invalid") - private val numTotalResults = scopedStats.stat("total_results") - private val numValidResults = scopedStats.stat("valid_results") - - override def get(request: RecommendationRequest): Future[Option[RecommendationResponse]] = { - requests.incr() - frsClient.getRecommendations(request).map { response => - numTotalResults.add(response.recommendations.size) - val validRecs = response.recommendations.filter { - case Recommendation.User(_: UserRecommendation) => - valid.incr() - true - case _ => - invalid.incr() - false - } - - numValidResults.add(validRecs.size) - Some( - RecommendationResponse( - recommendations = validRecs - )) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/IbisStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/IbisStore.scala deleted file mode 100644 index 6c355c505..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/IbisStore.scala +++ /dev/null @@ -1,190 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.finagle.stats.BroadcastStatsReceiver -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.logger.MRLogger -import com.twitter.frigate.common.store -import com.twitter.frigate.common.store.Fail -import com.twitter.frigate.common.store.IbisRequestInfo -import com.twitter.frigate.common.store.IbisResponse -import com.twitter.frigate.common.store.Sent -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.ibis2.service.thriftscala.Flags -import com.twitter.ibis2.service.thriftscala.FlowControl -import com.twitter.ibis2.service.thriftscala.Ibis2Request -import com.twitter.ibis2.service.thriftscala.Ibis2Response -import com.twitter.ibis2.service.thriftscala.Ibis2ResponseStatus -import com.twitter.ibis2.service.thriftscala.Ibis2Service -import com.twitter.ibis2.service.thriftscala.NotificationNotSentCode -import com.twitter.ibis2.service.thriftscala.TargetFanoutResult.NotSentReason -import com.twitter.util.Future - -trait Ibis2Store extends store.Ibis2Store { - def send(ibis2Request: Ibis2Request, candidate: PushCandidate): Future[IbisResponse] -} - -case class PushIbis2Store( - ibisClient: Ibis2Service.MethodPerEndpoint -)( - implicit val statsReceiver: StatsReceiver = NullStatsReceiver) - extends Ibis2Store { - private val log = MRLogger(this.getClass.getSimpleName) - private val stats = statsReceiver.scope("ibis_v2_store") - private val statsByCrt = stats.scope("byCrt") - private val requestsByCrt = statsByCrt.scope("requests") - private val failuresByCrt = statsByCrt.scope("failures") - private val successByCrt = statsByCrt.scope("success") - - private val statsByIbisModel = stats.scope("byIbisModel") - private val requestsByIbisModel = statsByIbisModel.scope("requests") - private val failuresByIbisModel = statsByIbisModel.scope("failures") - private val successByIbisModel = statsByIbisModel.scope("success") - - private[this] def ibisSend( - ibis2Request: Ibis2Request, - commonRecommendationType: CommonRecommendationType - ): Future[IbisResponse] = { - val ibisModel = ibis2Request.modelName - - val bStats = if (ibis2Request.flags.getOrElse(Flags()).darkWrite.contains(true)) { - BroadcastStatsReceiver( - Seq( - stats, - stats.scope("dark_write") - ) - ) - } else BroadcastStatsReceiver(Seq(stats)) - - bStats.counter("requests").incr() - requestsByCrt.counter(commonRecommendationType.name).incr() - requestsByIbisModel.counter(ibisModel).incr() - - retry(ibisClient, ibis2Request, 3, bStats) - .map { response => - bStats.counter(response.status.status.name).incr() - successByCrt.counter(response.status.status.name, commonRecommendationType.name).incr() - successByIbisModel.counter(response.status.status.name, ibisModel).incr() - response.status.status match { - case Ibis2ResponseStatus.SuccessWithDeliveries | - Ibis2ResponseStatus.SuccessNoDeliveries => - IbisResponse(Sent, Some(response)) - case _ => - IbisResponse(Fail, Some(response)) - } - } - .onFailure { ex => - bStats.counter("failures").incr() - val exceptionName = ex.getClass.getCanonicalName - bStats.scope("failures").counter(exceptionName).incr() - failuresByCrt.counter(exceptionName, commonRecommendationType.name).incr() - failuresByIbisModel.counter(exceptionName, ibisModel).incr() - } - } - - private def getNotifNotSentReason( - ibis2Response: Ibis2Response - ): Option[NotificationNotSentCode] = { - ibis2Response.status.fanoutResults match { - case Some(fanoutResult) => - fanoutResult.pushResult.flatMap { pushResult => - pushResult.results.headOption match { - case Some(NotSentReason(notSentInfo)) => Some(notSentInfo.notSentCode) - case _ => None - } - } - case _ => None - } - } - - def send(ibis2Request: Ibis2Request, candidate: PushCandidate): Future[IbisResponse] = { - val requestWithIID = if (ibis2Request.flowControl.exists(_.externalIid.isDefined)) { - ibis2Request - } else { - ibis2Request.copy( - flowControl = Some( - ibis2Request.flowControl - .getOrElse(FlowControl()) - .copy(externalIid = Some(candidate.impressionId)) - ) - ) - } - - val commonRecommendationType = candidate.frigateNotification.commonRecommendationType - - ibisSend(requestWithIID, commonRecommendationType) - .onSuccess { response => - response.ibis2Response.foreach { ibis2Response => - getNotifNotSentReason(ibis2Response).foreach { notifNotSentCode => - stats.scope(ibis2Response.status.status.name).counter(s"$notifNotSentCode").incr() - } - if (ibis2Response.status.status != Ibis2ResponseStatus.SuccessWithDeliveries) { - log.warning( - s"Request dropped on ibis for ${ibis2Request.recipientSelector.recipientId}: $ibis2Response") - } - } - } - .onFailure { ex => - log.warning( - s"Ibis Request failure: ${ex.getClass.getCanonicalName} \n For IbisRequest: $ibis2Request") - log.error(ex, ex.getMessage) - } - } - - // retry request when Ibis2ResponseStatus is PreFanoutError - def retry( - ibisClient: Ibis2Service.MethodPerEndpoint, - request: Ibis2Request, - retryCount: Int, - bStats: StatsReceiver - ): Future[Ibis2Response] = { - ibisClient.sendNotification(request).flatMap { response => - response.status.status match { - case Ibis2ResponseStatus.PreFanoutError if retryCount > 0 => - bStats.scope("requests").counter("retry").incr() - bStats.counter(response.status.status.name).incr() - retry(ibisClient, request, retryCount - 1, bStats) - case _ => - Future.value(response) - } - } - } - - override def send( - ibis2Request: Ibis2Request, - requestInfo: IbisRequestInfo - ): Future[IbisResponse] = { - ibisSend(ibis2Request, requestInfo.commonRecommendationType) - } -} - -case class StagingIbis2Store(remoteIbis2Store: PushIbis2Store) extends Ibis2Store { - - final def addDarkWriteFlagIbis2Request( - isTeamMember: Boolean, - ibis2Request: Ibis2Request - ): Ibis2Request = { - val flags = - ibis2Request.flags.getOrElse(Flags()) - val darkWrite: Boolean = !isTeamMember || flags.darkWrite.getOrElse(false) - ibis2Request.copy(flags = Some(flags.copy(darkWrite = Some(darkWrite)))) - } - - override def send(ibis2Request: Ibis2Request, candidate: PushCandidate): Future[IbisResponse] = { - candidate.target.isTeamMember.flatMap { isTeamMember => - val ibis2Req = addDarkWriteFlagIbis2Request(isTeamMember, ibis2Request) - remoteIbis2Store.send(ibis2Req, candidate) - } - } - - override def send( - ibis2Request: Ibis2Request, - requestInfo: IbisRequestInfo - ): Future[IbisResponse] = { - requestInfo.isTeamMember.flatMap { isTeamMember => - val ibis2Req = addDarkWriteFlagIbis2Request(isTeamMember, ibis2Request) - remoteIbis2Store.send(ibis2Req, requestInfo) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/InterestDiscoveryStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/InterestDiscoveryStore.scala deleted file mode 100644 index 80fc0ea7e..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/InterestDiscoveryStore.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.interests_discovery.thriftscala.InterestsDiscoveryService -import com.twitter.interests_discovery.thriftscala.RecommendedListsRequest -import com.twitter.interests_discovery.thriftscala.RecommendedListsResponse -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -case class InterestDiscoveryStore( - client: InterestsDiscoveryService.MethodPerEndpoint) - extends ReadableStore[RecommendedListsRequest, RecommendedListsResponse] { - - override def get(request: RecommendedListsRequest): Future[Option[RecommendedListsResponse]] = { - client.getListRecos(request).map(Some(_)) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/LabeledPushRecsDecideredStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/LabeledPushRecsDecideredStore.scala deleted file mode 100644 index 73fc28837..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/LabeledPushRecsDecideredStore.scala +++ /dev/null @@ -1,156 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.candidate.TargetDecider -import com.twitter.frigate.common.history.History -import com.twitter.frigate.common.history.HistoryStoreKeyContext -import com.twitter.frigate.common.history.PushServiceHistoryStore -import com.twitter.frigate.data_pipeline.thriftscala._ -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.hermit.store.labeled_push_recs.LabeledPushRecsJoinedWithNotificationHistoryStore -import com.twitter.logging.Logger -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future -import com.twitter.util.Time - -case class LabeledPushRecsVerifyingStoreKey( - historyStoreKey: HistoryStoreKeyContext, - useHydratedDataset: Boolean, - verifyHydratedDatasetResults: Boolean) { - def userId: Long = historyStoreKey.targetUserId -} - -case class LabeledPushRecsVerifyingStoreResponse( - userHistory: UserHistoryValue, - unequalNotificationsUnhydratedToHydrated: Option[ - Map[(Time, FrigateNotification), FrigateNotification] - ], - missingFromHydrated: Option[Map[Time, FrigateNotification]]) - -case class LabeledPushRecsVerifyingStore( - labeledPushRecsStore: ReadableStore[UserHistoryKey, UserHistoryValue], - historyStore: PushServiceHistoryStore -)( - implicit stats: StatsReceiver) - extends ReadableStore[LabeledPushRecsVerifyingStoreKey, LabeledPushRecsVerifyingStoreResponse] { - - private def getByJoiningWithRealHistory( - key: HistoryStoreKeyContext - ): Future[Option[UserHistoryValue]] = { - val historyFut = historyStore.get(key, Some(365.days)) - val toJoinWithRealHistoryFut = labeledPushRecsStore.get(UserHistoryKey.UserId(key.targetUserId)) - Future.join(historyFut, toJoinWithRealHistoryFut).map { - case (_, None) => None - case (History(realtimeHistoryMap), Some(uhValue)) => - Some( - LabeledPushRecsJoinedWithNotificationHistoryStore - .joinLabeledPushRecsSentWithNotificationHistory(uhValue, realtimeHistoryMap, stats) - ) - } - } - - private def processUserHistoryValue(uhValue: UserHistoryValue): Map[Time, FrigateNotification] = { - uhValue.events - .getOrElse(Nil) - .collect { - case Event( - EventType.LabeledPushRecSend, - Some(tsMillis), - Some(EventUnion.LabeledPushRecSendEvent(lprs: LabeledPushRecSendEvent)) - ) if lprs.pushRecSendEvent.frigateNotification.isDefined => - Time.fromMilliseconds(tsMillis) -> lprs.pushRecSendEvent.frigateNotification.get - } - .toMap - } - - override def get( - key: LabeledPushRecsVerifyingStoreKey - ): Future[Option[LabeledPushRecsVerifyingStoreResponse]] = { - val uhKey = UserHistoryKey.UserId(key.userId) - if (!key.useHydratedDataset) { - getByJoiningWithRealHistory(key.historyStoreKey).map { uhValueOpt => - uhValueOpt.map { uhValue => LabeledPushRecsVerifyingStoreResponse(uhValue, None, None) } - } - } else { - labeledPushRecsStore.get(uhKey).flatMap { hydratedValueOpt: Option[UserHistoryValue] => - if (!key.verifyHydratedDatasetResults) { - Future.value(hydratedValueOpt.map { uhValue => - LabeledPushRecsVerifyingStoreResponse(uhValue, None, None) - }) - } else { - getByJoiningWithRealHistory(key.historyStoreKey).map { - joinedWithRealHistoryOpt: Option[UserHistoryValue] => - val joinedWithRealHistoryMap = - joinedWithRealHistoryOpt.map(processUserHistoryValue).getOrElse(Map.empty) - val hydratedMap = hydratedValueOpt.map(processUserHistoryValue).getOrElse(Map.empty) - val unequal = joinedWithRealHistoryMap.flatMap { - case (time, frigateNotif) => - hydratedMap.get(time).collect { - case n if n != frigateNotif => ((time, frigateNotif), n) - } - } - val missing = joinedWithRealHistoryMap.filter { - case (time, frigateNotif) => !hydratedMap.contains(time) - } - hydratedValueOpt.map { hydratedValue => - LabeledPushRecsVerifyingStoreResponse(hydratedValue, Some(unequal), Some(missing)) - } - } - } - } - } - } -} - -case class LabeledPushRecsStoreKey(target: TargetDecider, historyStoreKey: HistoryStoreKeyContext) { - def userId: Long = historyStoreKey.targetUserId -} - -case class LabeledPushRecsDecideredStore( - verifyingStore: ReadableStore[ - LabeledPushRecsVerifyingStoreKey, - LabeledPushRecsVerifyingStoreResponse - ], - useHydratedLabeledSendsDatasetDeciderKey: String, - verifyHydratedLabeledSendsForHistoryDeciderKey: String -)( - implicit globalStats: StatsReceiver) - extends ReadableStore[LabeledPushRecsStoreKey, UserHistoryValue] { - private val log = Logger() - private val stats = globalStats.scope("LabeledPushRecsDecideredStore") - private val numComparisons = stats.counter("num_comparisons") - private val numMissingStat = stats.stat("num_missing") - private val numUnequalStat = stats.stat("num_unequal") - - override def get(key: LabeledPushRecsStoreKey): Future[Option[UserHistoryValue]] = { - val useHydrated = key.target.isDeciderEnabled( - useHydratedLabeledSendsDatasetDeciderKey, - stats, - useRandomRecipient = true - ) - - val verifyHydrated = if (useHydrated) { - key.target.isDeciderEnabled( - verifyHydratedLabeledSendsForHistoryDeciderKey, - stats, - useRandomRecipient = true - ) - } else false - - val newKey = LabeledPushRecsVerifyingStoreKey(key.historyStoreKey, useHydrated, verifyHydrated) - verifyingStore.get(newKey).map { - case None => None - case Some(LabeledPushRecsVerifyingStoreResponse(uhValue, unequalOpt, missingOpt)) => - (unequalOpt, missingOpt) match { - case (Some(unequal), Some(missing)) => - numComparisons.incr() - numMissingStat.add(missing.size) - numUnequalStat.add(unequal.size) - case _ => //no-op - } - Some(uhValue) - } - } - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/LexServiceStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/LexServiceStore.scala deleted file mode 100644 index b11cdc0dd..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/LexServiceStore.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.livevideo.common.ids.EventId -import com.twitter.livevideo.timeline.client.v2.LiveVideoTimelineClient -import com.twitter.livevideo.timeline.domain.v2.Event -import com.twitter.livevideo.timeline.domain.v2.LookupContext -import com.twitter.stitch.storehaus.ReadableStoreOfStitch -import com.twitter.stitch.NotFound -import com.twitter.stitch.Stitch -import com.twitter.storehaus.ReadableStore - -case class EventRequest(eventId: Long, lookupContext: LookupContext = LookupContext.default) - -object LexServiceStore { - def apply( - liveVideoTimelineClient: LiveVideoTimelineClient - ): ReadableStore[EventRequest, Event] = { - ReadableStoreOfStitch { eventRequest => - liveVideoTimelineClient.getEvent( - EventId(eventRequest.eventId), - eventRequest.lookupContext) rescue { - case NotFound => Stitch.NotFound - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/NTabHistoryStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/NTabHistoryStore.scala deleted file mode 100644 index 9e9bc37b7..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/NTabHistoryStore.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.hermit.store.common.ReadableWritableStore -import com.twitter.notificationservice.thriftscala.GenericNotificationOverrideKey -import com.twitter.stitch.Stitch -import com.twitter.storage.client.manhattan.bijections.Bijections.BinaryCompactScalaInjection -import com.twitter.storage.client.manhattan.bijections.Bijections.LongInjection -import com.twitter.storage.client.manhattan.bijections.Bijections.StringInjection -import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpoint -import com.twitter.storage.client.manhattan.kv.impl.Component -import com.twitter.storage.client.manhattan.kv.impl.DescriptorP1L1 -import com.twitter.storage.client.manhattan.kv.impl.KeyDescriptor -import com.twitter.storage.client.manhattan.kv.impl.ValueDescriptor -import com.twitter.util.Future - -case class NTabHistoryStore(mhEndpoint: ManhattanKVEndpoint, dataset: String) - extends ReadableWritableStore[(Long, String), GenericNotificationOverrideKey] { - - private val keyDesc: DescriptorP1L1.EmptyKey[Long, String] = - KeyDescriptor(Component(LongInjection), Component(StringInjection)) - - private val genericNotifKeyValDesc: ValueDescriptor.EmptyValue[GenericNotificationOverrideKey] = - ValueDescriptor[GenericNotificationOverrideKey]( - BinaryCompactScalaInjection(GenericNotificationOverrideKey) - ) - - override def get(key: (Long, String)): Future[Option[GenericNotificationOverrideKey]] = { - val (userId, impressionId) = key - val mhKey = keyDesc.withDataset(dataset).withPkey(userId).withLkey(impressionId) - - Stitch - .run(mhEndpoint.get(mhKey, genericNotifKeyValDesc)) - .map { optionMhValue => - optionMhValue.map(_.contents) - } - } - - override def put(keyValue: ((Long, String), GenericNotificationOverrideKey)): Future[Unit] = { - val ((userId, impressionId), genericNotifOverrideKey) = keyValue - val mhKey = keyDesc.withDataset(dataset).withPkey(userId).withLkey(impressionId) - val mhVal = genericNotifKeyValDesc.withValue(genericNotifOverrideKey) - Stitch.run(mhEndpoint.insert(mhKey, mhVal)) - } - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OCFPromptHistoryStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OCFPromptHistoryStore.scala deleted file mode 100644 index 33b119c79..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OCFPromptHistoryStore.scala +++ /dev/null @@ -1,73 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.onboarding.task.service.thriftscala.FatigueFlowEnrollment -import com.twitter.stitch.Stitch -import com.twitter.storage.client.manhattan.bijections.Bijections.BinaryScalaInjection -import com.twitter.storage.client.manhattan.bijections.Bijections.LongInjection -import com.twitter.storage.client.manhattan.bijections.Bijections.StringInjection -import com.twitter.storage.client.manhattan.kv.impl.Component -import com.twitter.storage.client.manhattan.kv.impl.KeyDescriptor -import com.twitter.storage.client.manhattan.kv.impl.ValueDescriptor -import com.twitter.storage.client.manhattan.kv.ManhattanKVClient -import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams -import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpointBuilder -import com.twitter.storage.client.manhattan.kv.NoMtlsParams -import com.twitter.storehaus.ReadableStore -import com.twitter.storehaus_internal.manhattan.Omega -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.util.Time - -case class OCFHistoryStoreKey(userId: Long, fatigueDuration: Duration, fatigueGroup: String) - -class OCFPromptHistoryStore( - manhattanAppId: String, - dataset: String, - mtlsParams: ManhattanKVClientMtlsParams = NoMtlsParams -)( - implicit stats: StatsReceiver) - extends ReadableStore[OCFHistoryStoreKey, FatigueFlowEnrollment] { - - import ManhattanInjections._ - - private val client = ManhattanKVClient( - appId = manhattanAppId, - dest = Omega.wilyName, - mtlsParams = mtlsParams, - label = "ocf_history_store" - ) - private val endpoint = ManhattanKVEndpointBuilder(client, defaultMaxTimeout = 5.seconds) - .statsReceiver(stats.scope("ocf_history_store")) - .build() - - private val limitResultsTo = 1 - - private val datasetKey = keyDesc.withDataset(dataset) - - override def get(storeKey: OCFHistoryStoreKey): Future[Option[FatigueFlowEnrollment]] = { - val userId = storeKey.userId - val fatigueGroup = storeKey.fatigueGroup - val fatigueLength = storeKey.fatigueDuration.inMilliseconds - val currentTime = Time.now.inMilliseconds - val fullKey = datasetKey - .withPkey(userId) - .from(fatigueGroup) - .to(fatigueGroup, fatigueLength - currentTime) - - Stitch - .run(endpoint.slice(fullKey, valDesc, limit = Some(limitResultsTo))) - .map { results => - if (results.nonEmpty) { - val (_, mhValue) = results.head - Some(mhValue.contents) - } else None - } - } -} - -object ManhattanInjections { - val keyDesc = KeyDescriptor(Component(LongInjection), Component(StringInjection, LongInjection)) - val valDesc = ValueDescriptor(BinaryScalaInjection(FatigueFlowEnrollment)) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OnlineUserHistoryStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OnlineUserHistoryStore.scala deleted file mode 100644 index d7ecfa7e4..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OnlineUserHistoryStore.scala +++ /dev/null @@ -1,81 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.conversions.DurationOps._ -import com.twitter.frigate.common.history.History -import com.twitter.frigate.common.store.RealTimeClientEventStore -import com.twitter.frigate.data_pipeline.common.HistoryJoin -import com.twitter.frigate.data_pipeline.thriftscala.Event -import com.twitter.frigate.data_pipeline.thriftscala.EventUnion -import com.twitter.frigate.data_pipeline.thriftscala.PushRecSendEvent -import com.twitter.frigate.data_pipeline.thriftscala.UserHistoryValue -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.util.Time - -case class OnlineUserHistoryKey( - userId: Long, - offlineUserHistory: Option[UserHistoryValue], - history: Option[History]) - -case class OnlineUserHistoryStore( - realTimeClientEventStore: RealTimeClientEventStore, - duration: Duration = 3.days) - extends ReadableStore[OnlineUserHistoryKey, UserHistoryValue] { - - override def get(key: OnlineUserHistoryKey): Future[Option[UserHistoryValue]] = { - val now = Time.now - - val pushRecSends = key.history - .getOrElse(History(Nil.toMap)) - .sortedPushDmHistory - .filter(_._1 > now - (duration + 1.day)) - .map { - case (time, frigateNotification) => - val pushRecSendEvent = PushRecSendEvent( - frigateNotification = Some(frigateNotification), - impressionId = frigateNotification.impressionId - ) - pushRecSendEvent -> time - } - - realTimeClientEventStore - .get(key.userId, now - duration, now) - .map { attributedEventHistory => - val attributedClientEvents = attributedEventHistory.sortedHistory.flatMap { - case (time, event) => - event.eventUnion match { - case Some(eventUnion: EventUnion.AttributedPushRecClientEvent) => - Some((eventUnion.attributedPushRecClientEvent, event.eventType, time)) - case _ => None - } - } - - val realtimeLabeledSends: Seq[Event] = HistoryJoin.getLabeledPushRecSends( - pushRecSends, - attributedClientEvents, - Seq(), - Seq(), - Seq(), - now - ) - - key.offlineUserHistory.map { offlineUserHistory => - val combinedEvents = offlineUserHistory.events.map { offlineEvents => - (offlineEvents ++ realtimeLabeledSends) - .map { event => - event.timestampMillis -> event - } - .toMap - .values - .toSeq - .sortBy { event => - -1 * event.timestampMillis.getOrElse(0L) - } - } - - offlineUserHistory.copy(events = combinedEvents) - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OpenAppUserStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OpenAppUserStore.scala deleted file mode 100644 index 85d3f5afa..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OpenAppUserStore.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.frigate.common.store.strato.StratoFetchableStore -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.client.Client -import com.twitter.strato.generated.client.rux.open_app.UsersInOpenAppDdgOnUserClientColumn - -object OpenAppUserStore { - def apply(stratoClient: Client): ReadableStore[Long, Boolean] = { - val fetcher = new UsersInOpenAppDdgOnUserClientColumn(stratoClient).fetcher - StratoFetchableStore.withUnitView(fetcher).mapValues(_ => true) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/SocialGraphServiceProcessStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/SocialGraphServiceProcessStore.scala deleted file mode 100644 index 4af473656..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/SocialGraphServiceProcessStore.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.frigate.pushservice.params.PushQPSLimitConstants.SocialGraphServiceBatchSize -import com.twitter.hermit.predicate.socialgraph.RelationEdge -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -case class SocialGraphServiceProcessStore(edgeStore: ReadableStore[RelationEdge, Boolean]) - extends ReadableStore[RelationEdge, Boolean] { - override def multiGet[T <: RelationEdge]( - relationEdges: Set[T] - ): Map[T, Future[Option[Boolean]]] = { - val splitSet = relationEdges.grouped(SocialGraphServiceBatchSize).toSet - splitSet - .map { relationship => - edgeStore.multiGet(relationship) - }.foldLeft(Map.empty[T, Future[Option[Boolean]]]) { (map1, map2) => - map1 ++ map2 - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/SoftUserFollowingStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/SoftUserFollowingStore.scala deleted file mode 100644 index b2de4cf26..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/SoftUserFollowingStore.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.gizmoduck.thriftscala.UserType -import com.twitter.stitch.Stitch -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.client.Client -import com.twitter.strato.client.UserId -import com.twitter.strato.config.FlockCursors.BySource.Begin -import com.twitter.strato.config.FlockCursors.Continue -import com.twitter.strato.config.FlockCursors.End -import com.twitter.strato.config.FlockPage -import com.twitter.strato.generated.client.socialgraph.service.soft_users.softUserFollows.EdgeBySourceClientColumn -import com.twitter.util.Future - -object SoftUserFollowingStore { - type ViewerFollowingCursor = EdgeBySourceClientColumn.Cursor - val MaxPagesToFetch = 2 - val PageLimit = 50 -} - -class SoftUserFollowingStore(stratoClient: Client) extends ReadableStore[User, Seq[Long]] { - import SoftUserFollowingStore._ - private val softUserFollowingEdgesPaginator = new EdgeBySourceClientColumn(stratoClient).paginator - - private def accumulateIds(cursor: ViewerFollowingCursor, pagesToFetch: Int): Stitch[Seq[Long]] = - softUserFollowingEdgesPaginator.paginate(cursor).flatMap { - case FlockPage(data, next, _) => - next match { - case cont: Continue if pagesToFetch > 1 => - Stitch - .join( - Stitch.value(data.map(_.to).map(_.value)), - accumulateIds(cont, pagesToFetch - 1)) - .map { - case (a, b) => a ++ b - } - - case _: End | _: Continue => - // end pagination if last page has been fetched or [[MaxPagesToFetch]] have been fetched - Stitch.value(data.map(_.to).map(_.value)) - } - } - - private def softFollowingFromStrato( - sourceId: Long, - pageLimit: Int, - pagesToFetch: Int - ): Stitch[Seq[Long]] = { - val begin = Begin[UserId, UserId](UserId(sourceId), pageLimit) - accumulateIds(begin, pagesToFetch) - } - - override def get(user: User): Future[Option[Seq[Long]]] = { - user.userType match { - case UserType.Soft => - Stitch.run(softFollowingFromStrato(user.id, PageLimit, MaxPagesToFetch)).map(Option(_)) - case _ => Future.None - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/TweetImpressionsStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/TweetImpressionsStore.scala deleted file mode 100644 index 6acf1d136..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/TweetImpressionsStore.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.frigate.common.store.strato.StratoFetchableStore -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.client.{Client => StratoClient} -import com.twitter.util.Future - -/** - * Store to get inbound Tweet impressions count for a specific Tweet id. - */ -class TweetImpressionsStore(stratoClient: StratoClient) extends ReadableStore[Long, String] { - - private val column = "rux/impression.Tweet" - private val store = StratoFetchableStore.withUnitView[Long, String](stratoClient, column) - - def getCounts(tweetId: Long): Future[Option[Long]] = { - store.get(tweetId).map(_.map(_.toLong)) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/TweetTranslationStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/TweetTranslationStore.scala deleted file mode 100644 index 618d8da32..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/TweetTranslationStore.scala +++ /dev/null @@ -1,211 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.context.TwitterContext -import com.twitter.context.thriftscala.Viewer -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.TwitterContextPermit -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.kujaku.domain.thriftscala.CacheUsageType -import com.twitter.kujaku.domain.thriftscala.MachineTranslation -import com.twitter.kujaku.domain.thriftscala.MachineTranslationResponse -import com.twitter.kujaku.domain.thriftscala.TranslationSource -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.generated.client.translation.service.IsTweetTranslatableClientColumn -import com.twitter.strato.generated.client.translation.service.platform.MachineTranslateTweetClientColumn -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.util.Future -import com.twitter.util.logging.Logging - -object TweetTranslationStore { - case class Key( - target: Target, - tweetId: Long, - tweet: Option[Tweet], - crt: CommonRecommendationType) - - case class Value( - translatedTweetText: String, - localizedSourceLanguage: String) - - val allowedCRTs = Set[CommonRecommendationType]( - CommonRecommendationType.TwistlyTweet - ) -} - -case class TweetTranslationStore( - translateTweetStore: ReadableStore[ - MachineTranslateTweetClientColumn.Key, - MachineTranslationResponse - ], - isTweetTranslatableStore: ReadableStore[IsTweetTranslatableClientColumn.Key, Boolean], - statsReceiver: StatsReceiver) - extends ReadableStore[TweetTranslationStore.Key, TweetTranslationStore.Value] - with Logging { - - private val stats = statsReceiver.scope("tweetTranslationStore") - private val isTranslatableCounter = stats.counter("tweetIsTranslatable") - private val notTranslatableCounter = stats.counter("tweetIsNotTranslatable") - private val protectedUserCounter = stats.counter("protectedUser") - private val notProtectedUserCounter = stats.counter("notProtectedUser") - private val validLanguageCounter = stats.counter("validTweetLanguage") - private val invalidLanguageCounter = stats.counter("invalidTweetLanguage") - private val validCrtCounter = stats.counter("validCrt") - private val invalidCrtCounter = stats.counter("invalidCrt") - private val paramEnabledCounter = stats.counter("paramEnabled") - private val paramDisabledCounter = stats.counter("paramDisabled") - - private val twitterContext = TwitterContext(TwitterContextPermit) - - override def get(k: TweetTranslationStore.Key): Future[Option[TweetTranslationStore.Value]] = { - k.target.inferredUserDeviceLanguage.flatMap { - case Some(deviceLanguage) => - setTwitterContext(k.target, deviceLanguage) { - translateTweet( - target = k.target, - tweetId = k.tweetId, - tweet = k.tweet, - crt = k.crt, - deviceLanguage = deviceLanguage).map { responseOpt => - responseOpt.flatMap { response => - response.translatorLocalizedSourceLanguage - .map { localizedSourceLanguage => - TweetTranslationStore.Value( - translatedTweetText = response.translation, - localizedSourceLanguage = localizedSourceLanguage - ) - }.filter { _ => - response.translationSource == TranslationSource.Google - } - } - } - } - case None => Future.None - } - - } - - // Don't sent protected tweets to external API for translation - private def checkProtectedUser(target: Target): Future[Boolean] = { - target.targetUser.map(_.flatMap(_.safety).forall(_.isProtected)).onSuccess { - case true => protectedUserCounter.incr() - case false => notProtectedUserCounter.incr() - } - } - - private def isTweetTranslatable( - target: Target, - tweetId: Long, - tweet: Option[Tweet], - crt: CommonRecommendationType, - deviceLanguage: String - ): Future[Boolean] = { - val tweetLangOpt = tweet.flatMap(_.language) - val isValidLanguage = tweetLangOpt.exists { tweetLang => - tweetLang.confidence > 0.5 && - tweetLang.language != deviceLanguage - } - - if (isValidLanguage) { - validLanguageCounter.incr() - } else { - invalidLanguageCounter.incr() - } - - val isValidCrt = TweetTranslationStore.allowedCRTs.contains(crt) - if (isValidCrt) { - validCrtCounter.incr() - } else { - invalidCrtCounter.incr() - } - - if (isValidCrt && isValidLanguage && target.params(PushParams.EnableIsTweetTranslatableCheck)) { - checkProtectedUser(target).flatMap { - case false => - val isTweetTranslatableKey = IsTweetTranslatableClientColumn.Key( - tweetId = tweetId, - destinationLanguage = Some(deviceLanguage), - translationSource = Some(TranslationSource.Google.name), - excludePreferredLanguages = Some(true) - ) - isTweetTranslatableStore - .get(isTweetTranslatableKey).map { resultOpt => - resultOpt.getOrElse(false) - }.onSuccess { - case true => isTranslatableCounter.incr() - case false => notTranslatableCounter.incr() - } - case true => - Future.False - } - } else { - Future.False - } - } - - private def translateTweet( - tweetId: Long, - deviceLanguage: String - ): Future[Option[MachineTranslation]] = { - val translateKey = MachineTranslateTweetClientColumn.Key( - tweetId = tweetId, - destinationLanguage = deviceLanguage, - translationSource = TranslationSource.Google, - translatableEntityTypes = Seq(), - onlyCached = false, - cacheUsageType = CacheUsageType.Default - ) - translateTweetStore.get(translateKey).map { - _.collect { - case MachineTranslationResponse.Result(result) => result - } - } - } - - private def translateTweet( - target: Target, - tweetId: Long, - tweet: Option[Tweet], - crt: CommonRecommendationType, - deviceLanguage: String - ): Future[Option[MachineTranslation]] = { - isTweetTranslatable(target, tweetId, tweet, crt, deviceLanguage).flatMap { - case true => - val isEnabledByParam = target.params(PushFeatureSwitchParams.EnableTweetTranslation) - if (isEnabledByParam) { - paramEnabledCounter.incr() - translateTweet(tweetId, deviceLanguage) - } else { - paramDisabledCounter.incr() - Future.None - } - case false => - Future.None - } - } - - private def setTwitterContext[Rep]( - target: Target, - deviceLanguage: String - )( - f: => Future[Rep] - ): Future[Rep] = { - twitterContext() match { - case Some(viewer) if viewer.userId.nonEmpty && viewer.authenticatedUserId.nonEmpty => - // If the context is already setup with a user ID just use it - f - case _ => - // If not, create a new context containing the viewer user id - twitterContext.let( - Viewer( - userId = Some(target.targetId), - requestLanguageCode = Some(deviceLanguage), - authenticatedUserId = Some(target.targetId) - )) { - f - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/UttEntityHydrationStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/UttEntityHydrationStore.scala deleted file mode 100644 index bd96bf690..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/UttEntityHydrationStore.scala +++ /dev/null @@ -1,79 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.escherbird.util.uttclient.CachedUttClientV2 -import com.twitter.escherbird.util.uttclient.InvalidUttEntityException -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.logging.Logger -import com.twitter.stitch.Stitch -import com.twitter.topiclisting.TopicListingViewerContext -import com.twitter.topiclisting.utt.LocalizedEntity -import com.twitter.topiclisting.utt.LocalizedEntityFactory -import com.twitter.util.Future - -/** - * - * @param viewerContext: [[TopicListingViewerContext]] for filtering topic - * @param semanticCoreEntityIds: list of semantic core entities to hydrate - */ -case class UttEntityHydrationQuery( - viewerContext: TopicListingViewerContext, - semanticCoreEntityIds: Seq[Long]) - -/** - * - * @param cachedUttClientV2 - * @param statsReceiver - */ -class UttEntityHydrationStore( - cachedUttClientV2: CachedUttClientV2, - statsReceiver: StatsReceiver, - log: Logger) { - - private val stats = statsReceiver.scope(this.getClass.getSimpleName) - private val uttEntityNotFound = stats.counter("invalid_utt_entity") - private val deviceLanguageMismatch = stats.counter("language_mismatch") - - /** - * SemanticCore recommends setting language and country code to None to fetch all localized topic - * names and apply filtering for locales on our end - * - * We use [[LocalizedEntityFactory]] from [[Topiclisting]] library to filter out topic name based - * on user locale - * - * Some(LocalizedEntity) - LocalizedUttEntity found - * None - LocalizedUttEntity not found - */ - def getLocalizedTopicEntities( - query: UttEntityHydrationQuery - ): Future[Seq[Option[LocalizedEntity]]] = Stitch.run { - Stitch.collect { - query.semanticCoreEntityIds.map { semanticCoreEntityId => - val uttEntity = cachedUttClientV2.cachedGetUttEntity( - language = None, - country = None, - version = None, - entityId = semanticCoreEntityId) - - uttEntity - .map { uttEntityMetadata => - val localizedEntity = LocalizedEntityFactory.getLocalizedEntity( - uttEntityMetadata, - query.viewerContext, - enableInternationalTopics = true, - enableTopicDescription = true) - // update counter - localizedEntity.foreach { entity => - if (!entity.nameMatchesDeviceLanguage) deviceLanguageMismatch.incr() - } - - localizedEntity - }.handle { - case e: InvalidUttEntityException => - log.error(e.getMessage) - uttEntityNotFound.incr() - None - } - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/CandidateNotifier.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/CandidateNotifier.scala deleted file mode 100644 index 1eb8cbc04..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/CandidateNotifier.scala +++ /dev/null @@ -1,160 +0,0 @@ -package com.twitter.frigate.pushservice.take - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.Stats.track -import com.twitter.frigate.common.logger.MRLogger -import com.twitter.frigate.common.store.Fail -import com.twitter.frigate.common.store.IbisResponse -import com.twitter.frigate.common.store.InvalidConfiguration -import com.twitter.frigate.common.store.NoRequest -import com.twitter.frigate.common.store.Sent -import com.twitter.frigate.common.util.CasLock -import com.twitter.frigate.common.util.PushServiceUtil.InvalidConfigResponse -import com.twitter.frigate.common.util.PushServiceUtil.NtabWriteOnlyResponse -import com.twitter.frigate.common.util.PushServiceUtil.SendFailedResponse -import com.twitter.frigate.common.util.PushServiceUtil.SentResponse -import com.twitter.frigate.pushservice.predicate.CasLockPredicate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.take.history._ -import com.twitter.frigate.pushservice.util.CopyUtil -import com.twitter.frigate.pushservice.thriftscala.PushResponse -import com.twitter.frigate.pushservice.thriftscala.PushStatus -import com.twitter.frigate.pushservice.util.OverrideNotificationUtil -import com.twitter.frigate.thriftscala.ChannelName -import com.twitter.util.Future - -class CandidateNotifier( - notificationSender: NotificationSender, - casLock: CasLock, - historyWriter: HistoryWriter, - eventBusWriter: EventBusWriter, - ntabOnlyChannelSelector: NtabOnlyChannelSelector -)( - implicit statsReceiver: StatsReceiver) { - - private lazy val casLockPredicate = - CasLockPredicate(casLock, expiryDuration = 10.minutes)(statsReceiver) - private val candidateNotifierStats = statsReceiver.scope(this.getClass.getSimpleName) - private val historyWriteCounter = - candidateNotifierStats.counter("simply_notifier_history_write_num") - private val loggedOutHistoryWriteCounter = - candidateNotifierStats.counter("logged_out_simply_notifier_history_write_num") - private val notificationSenderLatency = - candidateNotifierStats.scope("notification_sender_send") - private val log = MRLogger("CandidateNotifier") - - private def mapIbisResponse(ibisResponse: IbisResponse): PushResponse = { - ibisResponse match { - case IbisResponse(Sent, _) => SentResponse - case IbisResponse(Fail, _) => SendFailedResponse - case IbisResponse(InvalidConfiguration, _) => InvalidConfigResponse - case IbisResponse(NoRequest, _) => NtabWriteOnlyResponse - } - } - - /** - * - write to history store - * - send the notification - * - scribe the notification - * - * final modifier is to signal that this function cannot be overriden. There's some critical logic - * in this function, and it's helpful to know that no sub-class overrides it. - */ - final def notify( - candidate: PushCandidate, - ): Future[PushResponse] = { - if (candidate.target.isDarkWrite) { - notificationSender.sendIbisDarkWrite(candidate).map(mapIbisResponse) - } else { - casLockPredicate(Seq(candidate)).flatMap { casLockResults => - if (casLockResults.head || candidate.target.pushContext - .exists(_.skipFilters.contains(true))) { - Future - .join( - candidate.target.isSilentPush, - OverrideNotificationUtil - .getOverrideInfo(candidate, candidateNotifierStats), - CopyUtil.getCopyFeatures(candidate, candidateNotifierStats) - ).flatMap { - case (isSilentPush, overrideInfoOpt, copyFeaturesMap) => - val channels = ntabOnlyChannelSelector.selectChannel(candidate) - channels.flatMap { channels => - candidate - .frigateNotificationForPersistence( - channels, - isSilentPush, - overrideInfoOpt, - copyFeaturesMap.keySet).flatMap { frigateNotificationForPersistence => - val result = if (candidate.target.isDarkWrite) { - candidateNotifierStats.counter("dark_write").incr() - Future.Unit - } else { - historyWriteCounter.incr() - historyWriter - .writeSendToHistory(candidate, frigateNotificationForPersistence) - } - result.flatMap { _ => - track(notificationSenderLatency)( - notificationSender - .notify(channels, candidate) - .map { ibisResponse => - eventBusWriter - .writeToEventBus(candidate, frigateNotificationForPersistence) - mapIbisResponse(ibisResponse) - }) - } - } - } - } - } else { - candidateNotifierStats.counter("filtered_by_cas_lock").incr() - Future.value(PushResponse(PushStatus.Filtered, Some(casLockPredicate.name))) - } - } - } - } - - final def loggedOutNotify( - candidate: PushCandidate, - ): Future[PushResponse] = { - if (candidate.target.isDarkWrite) { - notificationSender.sendIbisDarkWrite(candidate).map(mapIbisResponse) - } else { - casLockPredicate(Seq(candidate)).flatMap { casLockResults => - if (casLockResults.head || candidate.target.pushContext - .exists(_.skipFilters.contains(true))) { - val response = candidate.target.isSilentPush.flatMap { isSilentPush => - candidate - .frigateNotificationForPersistence( - Seq(ChannelName.PushNtab), - isSilentPush, - None, - Set.empty).flatMap { frigateNotificationForPersistence => - val result = if (candidate.target.isDarkWrite) { - candidateNotifierStats.counter("logged_out_dark_write").incr() - Future.Unit - } else { - loggedOutHistoryWriteCounter.incr() - historyWriter.writeSendToHistory(candidate, frigateNotificationForPersistence) - } - - result.flatMap { _ => - track(notificationSenderLatency)( - notificationSender - .loggedOutNotify(candidate) - .map { ibisResponse => - mapIbisResponse(ibisResponse) - }) - } - } - } - response - } else { - candidateNotifierStats.counter("filtered_by_cas_lock").incr() - Future.value(PushResponse(PushStatus.Filtered, Some(casLockPredicate.name))) - } - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/LoggedOutRefreshForPushNotifier.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/LoggedOutRefreshForPushNotifier.scala deleted file mode 100644 index 07574f46f..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/LoggedOutRefreshForPushNotifier.scala +++ /dev/null @@ -1,118 +0,0 @@ -package com.twitter.frigate.pushservice.take - -import com.twitter.finagle.stats.BroadcastStatsReceiver -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateResult -import com.twitter.frigate.common.base.Invalid -import com.twitter.frigate.common.base.OK -import com.twitter.frigate.common.base.Response -import com.twitter.frigate.common.base.Result -import com.twitter.frigate.common.base.Stats.track -import com.twitter.frigate.common.config.CommonConstants -import com.twitter.frigate.common.logger.MRLogger -import com.twitter.frigate.common.util.PushServiceUtil.FilteredLoggedOutResponseFut -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.refresh_handler.RFPHStatsRecorder -import com.twitter.frigate.pushservice.thriftscala.LoggedOutResponse -import com.twitter.frigate.pushservice.thriftscala.PushStatus -import com.twitter.util.Future -import com.twitter.util.JavaTimer -import com.twitter.util.Timer - -class LoggedOutRefreshForPushNotifier( - rfphStatsRecorder: RFPHStatsRecorder, - loCandidateNotifier: CandidateNotifier -)( - globalStats: StatsReceiver) { - private implicit val statsReceiver: StatsReceiver = - globalStats.scope("LoggedOutRefreshForPushHandler") - private val loPushStats: StatsReceiver = statsReceiver.scope("logged_out_push") - private val loSendLatency: StatsReceiver = statsReceiver.scope("logged_out_send") - private val processedCandidatesCounter: Counter = - statsReceiver.counter("processed_candidates_count") - private val validCandidatesCounter: Counter = statsReceiver.counter("valid_candidates_count") - private val okayCandidateCounter: Counter = statsReceiver.counter("ok_candidate_count") - private val nonOkayCandidateCounter: Counter = statsReceiver.counter("non_ok_candidate_count") - private val successNotifyCounter: Counter = statsReceiver.counter("success_notify_count") - private val notifyCandidate: Counter = statsReceiver.counter("notify_candidate") - private val noneCandidateResultCounter: Counter = statsReceiver.counter("none_candidate_count") - private val nonOkayPredsResult: Counter = statsReceiver.counter("non_okay_preds_result") - private val invalidResultCounter: Counter = statsReceiver.counter("invalid_result_count") - private val filteredLoggedOutResponse: Counter = statsReceiver.counter("filtered_response_count") - - implicit private val timer: Timer = new JavaTimer(true) - val log = MRLogger("LoggedOutRefreshForNotifier") - - private def notify( - candidatesResult: CandidateResult[PushCandidate, Result] - ): Future[LoggedOutResponse] = { - val candidate = candidatesResult.candidate - if (candidate != null) - notifyCandidate.incr() - val predsResult = candidatesResult.result - if (predsResult != OK) { - nonOkayPredsResult.incr() - val invalidResult = predsResult - invalidResult match { - case Invalid(Some(reason)) => - invalidResultCounter.incr() - Future.value(LoggedOutResponse(PushStatus.Filtered, Some(reason))) - case _ => - filteredLoggedOutResponse.incr() - Future.value(LoggedOutResponse(PushStatus.Filtered, None)) - } - } else { - track(loSendLatency)(loCandidateNotifier.loggedOutNotify(candidate).map { res => - LoggedOutResponse(res.status) - }) - } - } - - def checkResponseAndNotify( - response: Response[PushCandidate, Result] - ): Future[LoggedOutResponse] = { - val receivers = Seq(statsReceiver) - val loggedOutResponse = response match { - case Response(OK, processedCandidates) => - processedCandidatesCounter.incr(processedCandidates.size) - val validCandidates = processedCandidates.filter(_.result == OK) - validCandidatesCounter.incr(validCandidates.size) - - validCandidates.headOption match { - case Some(candidatesResult) => - candidatesResult.result match { - case OK => - okayCandidateCounter.incr() - notify(candidatesResult) - .onSuccess { nr => - successNotifyCounter.incr() - loPushStats.scope("lo_result").counter(nr.status.name).incr() - } - case _ => - nonOkayCandidateCounter.incr() - FilteredLoggedOutResponseFut - } - case _ => - noneCandidateResultCounter.incr() - FilteredLoggedOutResponseFut - } - - case Response(Invalid(reason), _) => - FilteredLoggedOutResponseFut.map(_.copy(filteredBy = reason)) - - case _ => - FilteredLoggedOutResponseFut - } - val bstats = BroadcastStatsReceiver(receivers) - Stat - .timeFuture(bstats.stat("logged_out_latency"))( - loggedOutResponse.raiseWithin(CommonConstants.maxPushRequestDuration) - ) - .onFailure { exception => - rfphStatsRecorder.loggedOutRequestExceptionStats(exception, bstats) - } - loggedOutResponse - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/NotificationSender.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/NotificationSender.scala deleted file mode 100644 index 70a695fb3..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/NotificationSender.scala +++ /dev/null @@ -1,95 +0,0 @@ -package com.twitter.frigate.pushservice.take - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.Stats.track -import com.twitter.frigate.common.store.IbisResponse -import com.twitter.frigate.common.store.Sent -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.take.sender.Ibis2Sender -import com.twitter.frigate.pushservice.take.sender.NtabSender -import com.twitter.frigate.scribe.thriftscala.NotificationScribe -import com.twitter.util.Future -import com.twitter.frigate.thriftscala.ChannelName - -/** - * NotificationSender wraps up all the notification infra send logic, and serves as an abstract layer - * between CandidateNotifier and the respective senders including ntab, ibis, which is being - * gated with both a decider/feature switch - */ -class NotificationSender( - ibis2Sender: Ibis2Sender, - ntabSender: NtabSender, - statsReceiver: StatsReceiver, - notificationScribe: NotificationScribe => Unit) { - - private val notificationNotifierStats = statsReceiver.scope(this.getClass.getSimpleName) - private val ibis2SendLatency = notificationNotifierStats.scope("ibis2_send") - private val loggedOutIbis2SendLatency = notificationNotifierStats.scope("logged_out_ibis2_send") - private val ntabSendLatency = notificationNotifierStats.scope("ntab_send") - - private val ntabWriteThenSkipPushCounter = - notificationNotifierStats.counter("ntab_write_then_skip_push") - private val ntabWriteThenIbisSendCounter = - notificationNotifierStats.counter("ntab_write_then_ibis_send") - notificationNotifierStats.counter("ins_dark_traffic_send") - - private val ntabOnlyChannelSenderV3Counter = - notificationNotifierStats.counter("ntab_only_channel_send_v3") - - def sendIbisDarkWrite(candidate: PushCandidate): Future[IbisResponse] = { - ibis2Sender.sendAsDarkWrite(candidate) - } - - private def isNtabOnlySend( - channels: Seq[ChannelName] - ): Future[Boolean] = { - val isNtabOnlyChannel = channels.contains(ChannelName.NtabOnly) - if (isNtabOnlyChannel) ntabOnlyChannelSenderV3Counter.incr() - - Future.value(isNtabOnlyChannel) - } - - private def isPushOnly(channels: Seq[ChannelName], candidate: PushCandidate): Future[Boolean] = { - Future.value(channels.contains(ChannelName.PushOnly)) - } - - def notify( - channels: Seq[ChannelName], - candidate: PushCandidate - ): Future[IbisResponse] = { - Future - .join(isPushOnly(channels, candidate), isNtabOnlySend(channels)).map { - case (isPushOnly, isNtabOnly) => - if (isPushOnly) { - track(ibis2SendLatency)(ibis2Sender.send(channels, candidate, notificationScribe, None)) - } else { - track(ntabSendLatency)( - ntabSender - .send(candidate, isNtabOnly)) - .flatMap { ntabResponse => - if (isNtabOnly) { - ntabWriteThenSkipPushCounter.incr() - candidate - .scribeData(channels = channels).foreach(notificationScribe).map(_ => - IbisResponse(Sent)) - } else { - ntabWriteThenIbisSendCounter.incr() - track(ibis2SendLatency)( - ibis2Sender.send(channels, candidate, notificationScribe, ntabResponse)) - } - } - - } - }.flatten - } - - def loggedOutNotify( - candidate: PushCandidate - ): Future[IbisResponse] = { - val ibisResponse = { - track(loggedOutIbis2SendLatency)( - ibis2Sender.send(Seq(ChannelName.PushNtab), candidate, notificationScribe, None)) - } - ibisResponse - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/NotificationServiceSender.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/NotificationServiceSender.scala deleted file mode 100644 index c2f729115..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/NotificationServiceSender.scala +++ /dev/null @@ -1,273 +0,0 @@ -package com.twitter.frigate.pushservice.take - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.logger.MRLogger -import com.twitter.frigate.common.ntab.InvalidNTABWriteRequestException -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.notificationservice.thriftscala._ -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi.Param -import com.twitter.util.Future -import scala.util.control.NoStackTrace - -class NtabCopyIdNotFoundException(private val message: String) - extends Exception(message) - with NoStackTrace - -class InvalidNtabCopyIdException(private val message: String) - extends Exception(message) - with NoStackTrace - -object NotificationServiceSender { - - def generateSocialContextTextEntities( - ntabDisplayNamesAndIdsFut: Future[Seq[(String, Long)]], - otherCountFut: Future[Int] - ): Future[Seq[DisplayTextEntity]] = { - Future.join(ntabDisplayNamesAndIdsFut, otherCountFut).map { - case (namesWithIdInOrder, otherCount) => - val displays = namesWithIdInOrder.zipWithIndex.map { - case ((name, id), index) => - DisplayTextEntity( - name = "user" + s"${index + 1}", - value = TextValue.Text(name), - emphasis = true, - userId = Some(id) - ) - } ++ Seq( - DisplayTextEntity(name = "nameCount", value = TextValue.Number(namesWithIdInOrder.size)) - ) - - val otherDisplay = if (otherCount > 0) { - Some( - DisplayTextEntity( - name = "otherCount", - value = TextValue.Number(otherCount) - ) - ) - } else None - displays ++ otherDisplay - } - } - - def getDisplayTextEntityFromUser( - userOpt: Option[User], - fieldName: String, - isBold: Boolean - ): Option[DisplayTextEntity] = { - for { - user <- userOpt - profile <- user.profile - } yield { - DisplayTextEntity( - name = fieldName, - value = TextValue.Text(profile.name), - emphasis = isBold, - userId = Some(user.id) - ) - } - } - - def getDisplayTextEntityFromUser( - user: Future[Option[User]], - fieldName: String, - isBold: Boolean - ): Future[Option[DisplayTextEntity]] = { - user.map { getDisplayTextEntityFromUser(_, fieldName, isBold) } - } -} - -case class NotificationServiceRequest( - candidate: PushCandidate, - impressionId: String, - isBadgeUpdate: Boolean, - overrideId: Option[String] = None) - -class NotificationServiceSender( - send: (Target, CreateGenericNotificationRequest) => Future[CreateGenericNotificationResponse], - enableWritesParam: Param[Boolean], - enableForEmployeesParam: Param[Boolean], - enableForEveryoneParam: Param[Boolean] -)( - implicit globalStats: StatsReceiver) - extends ReadableStore[NotificationServiceRequest, CreateGenericNotificationResponse] { - - val log = MRLogger(this.getClass.getName) - - val stats = globalStats.scope("NotificationServiceSender") - val requestEmpty = stats.scope("request_empty") - val requestNonEmpty = stats.counter("request_non_empty") - - val requestBadgeCount = stats.counter("request_badge_count") - - val successfulWrite = stats.counter("successful_write") - val successfulWriteScope = stats.scope("successful_write") - val failedWriteScope = stats.scope("failed_write") - val gotNonSuccessResponse = stats.counter("got_non_success_response") - val gotEmptyResponse = stats.counter("got_empty_response") - val deciderTurnedOffResponse = stats.scope("decider_turned_off_response") - - val disabledByDeciderForCandidate = stats.scope("model/candidate").counter("disabled_by_decider") - val sentToAlphaUserForCandidate = - stats.scope("model/candidate").counter("send_to_employee_or_team") - val sentToNonBucketedUserForCandidate = - stats.scope("model/candidate").counter("send_to_non_bucketed_decidered_user") - val noSendForCandidate = stats.scope("model/candidate").counter("no_send") - - val ineligibleUsersForCandidate = stats.scope("model/candidate").counter("ineligible_users") - - val darkWriteRequestsForCandidate = stats.scope("model/candidate").counter("dark_write_traffic") - - val heavyUserForCandidateCounter = stats.scope("model/candidate").counter("target_heavy") - val nonHeavyUserForCandidateCounter = stats.scope("model/candidate").counter("target_non_heavy") - - val skipWritingToNTAB = stats.counter("skip_writing_to_ntab") - - val ntabWriteDisabledForCandidate = stats.scope("model/candidate").counter("ntab_write_disabled") - - val ntabOverrideEnabledForCandidate = stats.scope("model/candidate").counter("override_enabled") - val ntabTTLForCandidate = stats.scope("model/candidate").counter("ttl_enabled") - - override def get( - notifRequest: NotificationServiceRequest - ): Future[Option[CreateGenericNotificationResponse]] = { - notifRequest.candidate.target.deviceInfo.flatMap { deviceInfoOpt => - val disableWritingToNtab = - notifRequest.candidate.target.params(PushParams.DisableWritingToNTAB) - - if (disableWritingToNtab) { - skipWritingToNTAB.incr() - Future.None - } else { - if (notifRequest.overrideId.nonEmpty) { ntabOverrideEnabledForCandidate.incr() } - Future - .join( - notifRequest.candidate.ntabRequest, - ntabWritesEnabledForCandidate(notifRequest.candidate)).flatMap { - case (Some(ntabRequest), ntabWritesEnabled) if ntabWritesEnabled => - if (ntabRequest.expiryTimeMillis.nonEmpty) { ntabTTLForCandidate.incr() } - sendNTabRequest( - ntabRequest, - notifRequest.candidate.target, - notifRequest.isBadgeUpdate, - notifRequest.candidate.commonRecType, - isFromCandidate = true, - overrideId = notifRequest.overrideId - ) - case (Some(_), ntabWritesEnabled) if !ntabWritesEnabled => - ntabWriteDisabledForCandidate.incr() - Future.None - case (None, ntabWritesEnabled) => - if (!ntabWritesEnabled) ntabWriteDisabledForCandidate.incr() - requestEmpty.counter(s"candidate_${notifRequest.candidate.commonRecType}").incr() - Future.None - } - } - } - } - - private def sendNTabRequest( - genericNotificationRequest: CreateGenericNotificationRequest, - target: Target, - isBadgeUpdate: Boolean, - crt: CommonRecommendationType, - isFromCandidate: Boolean, - overrideId: Option[String] - ): Future[Option[CreateGenericNotificationResponse]] = { - requestNonEmpty.incr() - val notifSvcReq = - genericNotificationRequest.copy( - sendBadgeCountUpdate = isBadgeUpdate, - overrideId = overrideId - ) - requestBadgeCount.incr() - send(target, notifSvcReq) - .map { response => - if (response.responseType.equals(CreateGenericNotificationResponseType.DecideredOff)) { - deciderTurnedOffResponse.counter(s"$crt").incr() - deciderTurnedOffResponse.counter(s"${genericNotificationRequest.genericType}").incr() - throw InvalidNTABWriteRequestException("Decider is turned off") - } else { - Some(response) - } - } - .onFailure { ex => - stats.counter(s"error_${ex.getClass.getCanonicalName}").incr() - failedWriteScope.counter(s"${crt}").incr() - log - .error( - ex, - s"NTAB failure $notifSvcReq" - ) - } - .onSuccess { - case Some(response) => - successfulWrite.incr() - val successfulWriteScopeString = if (isFromCandidate) "model/candidate" else "envelope" - successfulWriteScope.scope(successfulWriteScopeString).counter(s"$crt").incr() - if (response.responseType != CreateGenericNotificationResponseType.Success) { - gotNonSuccessResponse.incr() - log.warning(s"NTAB dropped $notifSvcReq with response $response") - } - - case _ => - gotEmptyResponse.incr() - } - } - - private def ntabWritesEnabledForCandidate(cand: PushCandidate): Future[Boolean] = { - if (!cand.target.params(enableWritesParam)) { - disabledByDeciderForCandidate.incr() - Future.False - } else { - Future - .join( - cand.target.isAnEmployee, - cand.target.isInNotificationsServiceWhitelist, - cand.target.isTeamMember - ) - .flatMap { - case (isEmployee, isInNotificationsServiceWhitelist, isTeamMember) => - cand.target.deviceInfo.flatMap { deviceInfoOpt => - deviceInfoOpt - .map { deviceInfo => - cand.target.isHeavyUserState.map { isHeavyUser => - val isAlphaTester = (isEmployee && cand.target - .params(enableForEmployeesParam)) || isInNotificationsServiceWhitelist || isTeamMember - if (cand.target.isDarkWrite) { - stats - .scope("model/candidate").counter( - s"dark_write_${cand.commonRecType}").incr() - darkWriteRequestsForCandidate.incr() - false - } else if (isAlphaTester || deviceInfo.isMRinNTabEligible - || cand.target.insertMagicrecsIntoNTabForNonPushableUsers) { - if (isHeavyUser) heavyUserForCandidateCounter.incr() - else nonHeavyUserForCandidateCounter.incr() - - val enabledForDesiredUsers = cand.target.params(enableForEveryoneParam) - if (isAlphaTester) { - sentToAlphaUserForCandidate.incr() - true - } else if (enabledForDesiredUsers) { - sentToNonBucketedUserForCandidate.incr() - true - } else { - noSendForCandidate.incr() - false - } - } else { - ineligibleUsersForCandidate.incr() - false - } - } - }.getOrElse(Future.False) - } - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/SendHandlerNotifier.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/SendHandlerNotifier.scala deleted file mode 100644 index feb65dffe..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/SendHandlerNotifier.scala +++ /dev/null @@ -1,86 +0,0 @@ -package com.twitter.frigate.pushservice.take - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.Invalid -import com.twitter.frigate.common.base.OK -import com.twitter.frigate.common.base.Response -import com.twitter.frigate.common.base.Result -import com.twitter.frigate.common.util.NotificationScribeUtil -import com.twitter.frigate.common.util.PushServiceUtil -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.thriftscala.PushResponse -import com.twitter.frigate.pushservice.thriftscala.PushStatus -import com.twitter.util.Future - -class SendHandlerNotifier( - candidateNotifier: CandidateNotifier, - private val statsReceiver: StatsReceiver) { - - val missingResponseCounter = statsReceiver.counter("missing_response") - val filteredResponseCounter = statsReceiver.counter("filtered") - - /** - * - * @param isScribeInfoRequired: [[Boolean]] to indicate if scribe info is required - * @param candidate: [[PushCandidate]] to build the scribe data from - * @return: scribe response string - */ - private def scribeInfoForResponse( - isScribeInfoRequired: Boolean, - candidate: PushCandidate - ): Future[Option[String]] = { - if (isScribeInfoRequired) { - candidate.scribeData().map { scribedInfo => - Some(NotificationScribeUtil.convertToJsonString(scribedInfo)) - } - } else Future.None - } - - /** - * - * @param response: Candidate validation response - * @param responseWithScribedInfo: boolean indicating if scribe data is expected in push response - * @return: [[PushResponse]] containing final result of send request for [[com.twitter.frigate.pushservice.thriftscala.PushRequest]] - */ - final def checkResponseAndNotify( - response: Response[PushCandidate, Result], - responseWithScribedInfo: Boolean - ): Future[PushResponse] = { - - response match { - case Response(OK, processedCandidates) => - val (validCandidates, invalidCandidates) = processedCandidates.partition(_.result == OK) - validCandidates.headOption match { - case Some(candidateResult) => - val scribeInfo = - scribeInfoForResponse(responseWithScribedInfo, candidateResult.candidate) - scribeInfo.flatMap { scribedData => - val response: Future[PushResponse] = - candidateNotifier.notify(candidateResult.candidate) - response.map(_.copy(notifScribe = scribedData)) - } - - case None => - invalidCandidates.headOption match { - case Some(candidateResult) => - filteredResponseCounter.incr() - val response = candidateResult.result match { - case Invalid(reason) => PushResponse(PushStatus.Filtered, filteredBy = reason) - case _ => PushResponse(PushStatus.Filtered, filteredBy = Some("unknown")) - } - - val scribeInfo = - scribeInfoForResponse(responseWithScribedInfo, candidateResult.candidate) - scribeInfo.map(scribeData => response.copy(notifScribe = scribeData)) - - case None => - missingResponseCounter.incr() - PushServiceUtil.FilteredPushResponseFut - } - } - - case Response(Invalid(reason), _) => - throw new IllegalStateException(s"Unexpected target filtering in SendHandler: $reason") - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/CandidateValidator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/CandidateValidator.scala deleted file mode 100644 index ee85ba590..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/CandidateValidator.scala +++ /dev/null @@ -1,83 +0,0 @@ -package com.twitter.frigate.pushservice.take.candidate_validator - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.logger.MRLogger -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.take.predicates.TakeCommonPredicates -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.hermit.predicate.ConcurrentPredicate -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.hermit.predicate.SequentialPredicate -import com.twitter.util.Future - -trait CandidateValidator extends TakeCommonPredicates { - - override implicit val statsReceiver: StatsReceiver = config.statsReceiver - - protected val log = MRLogger("CandidateValidator") - - private lazy val skipFiltersCounter = statsReceiver.counter("enable_skip_filters") - private lazy val emailUserSkipFiltersCounter = - statsReceiver.counter("email_user_enable_skip_filters") - private lazy val enablePredicatesCounter = statsReceiver.counter("enable_predicates") - - protected def enabledPredicates[C <: PushCandidate]( - candidate: C, - predicates: List[NamedPredicate[C]] - ): List[NamedPredicate[C]] = { - val target = candidate.target - val skipFilters: Boolean = - target.pushContext.flatMap(_.skipFilters).getOrElse(false) || target.params( - PushFeatureSwitchParams.SkipPostRankingFilters) - - if (skipFilters) { - skipFiltersCounter.incr() - if (target.isEmailUser) emailUserSkipFiltersCounter.incr() - - val predicatesToEnable = target.pushContext.flatMap(_.predicatesToEnable).getOrElse(Nil) - if (predicatesToEnable.nonEmpty) enablePredicatesCounter.incr() - - // if we skip predicates on pushContext, only enable the explicitly specified predicates - predicates.filter(predicatesToEnable.contains) - } else predicates - } - - protected def executeSequentialPredicates[C <: PushCandidate]( - candidate: C, - predicates: List[NamedPredicate[C]] - ): Future[Option[Predicate[C]]] = { - val predicatesEnabled = enabledPredicates(candidate, predicates) - val sequentialPredicate = new SequentialPredicate(predicatesEnabled) - - sequentialPredicate.track(Seq(candidate)).map(_.head) - } - - protected def executeConcurrentPredicates[C <: PushCandidate]( - candidate: C, - predicates: List[NamedPredicate[C]] - ): Future[List[Predicate[C]]] = { - val predicatesEnabled = enabledPredicates(candidate, predicates) - val concurrentPredicate: ConcurrentPredicate[C] = new ConcurrentPredicate[C](predicatesEnabled) - concurrentPredicate.track(Seq(candidate)).map(_.head) - } - - protected val candidatePredicatesMap: Map[CommonRecommendationType, List[ - NamedPredicate[_ <: PushCandidate] - ]] - - protected def getCRTPredicates[C <: PushCandidate]( - CRT: CommonRecommendationType - ): List[NamedPredicate[C]] = { - candidatePredicatesMap.get(CRT) match { - case Some(predicates) => - predicates.asInstanceOf[List[NamedPredicate[C]]] - case _ => - throw new IllegalStateException( - s"Unknown CommonRecommendationType for Predicates: ${CRT.name}") - } - } - - def validateCandidate[C <: PushCandidate](candidate: C): Future[Option[Predicate[C]]] -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/RFPHCandidateValidator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/RFPHCandidateValidator.scala deleted file mode 100644 index ecc99cc9e..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/RFPHCandidateValidator.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.frigate.pushservice.take.candidate_validator - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.take.predicates.candidate_map.CandidatePredicatesMap -import com.twitter.hermit.predicate.Predicate -import com.twitter.util.Future - -class RFPHCandidateValidator(override val config: Config) extends CandidateValidator { - private val rFPHCandidateValidatorStats = statsReceiver.scope(this.getClass.getSimpleName) - private val concurrentPredicateCount = rFPHCandidateValidatorStats.counter("concurrent") - private val sequentialPredicateCount = rFPHCandidateValidatorStats.counter("sequential") - - override protected val candidatePredicatesMap = CandidatePredicatesMap(config) - - override def validateCandidate[C <: PushCandidate](candidate: C): Future[Option[Predicate[C]]] = { - val candidatePredicates = getCRTPredicates(candidate.commonRecType) - val predicates = rfphPrePredicates ++ candidatePredicates ++ postPredicates - if (candidate.target.isEmailUser) { - concurrentPredicateCount.incr() - executeConcurrentPredicates(candidate, predicates).map(_.headOption) - } else { - sequentialPredicateCount.incr() - executeSequentialPredicates(candidate, predicates) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/SendHandlerPostCandidateValidator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/SendHandlerPostCandidateValidator.scala deleted file mode 100644 index 096e9f102..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/SendHandlerPostCandidateValidator.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.frigate.pushservice.take.candidate_validator - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.take.predicates.candidate_map.SendHandlerCandidatePredicatesMap -import com.twitter.hermit.predicate.Predicate -import com.twitter.util.Future - -class SendHandlerPostCandidateValidator(override val config: Config) extends CandidateValidator { - - override protected val candidatePredicatesMap = - SendHandlerCandidatePredicatesMap.postCandidatePredicates(config) - - private val sendHandlerPostCandidateValidatorStats = - statsReceiver.counter("sendHandlerPostCandidateValidator_stats") - - override def validateCandidate[C <: PushCandidate](candidate: C): Future[Option[Predicate[C]]] = { - val candidatePredicates = getCRTPredicates(candidate.commonRecType) - val predicates = candidatePredicates ++ postPredicates - - sendHandlerPostCandidateValidatorStats.incr() - - executeConcurrentPredicates(candidate, predicates) - .map(_.headOption) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/SendHandlerPreCandidateValidator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/SendHandlerPreCandidateValidator.scala deleted file mode 100644 index eb0293017..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/SendHandlerPreCandidateValidator.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.frigate.pushservice.take.candidate_validator - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.take.predicates.candidate_map.SendHandlerCandidatePredicatesMap -import com.twitter.hermit.predicate.Predicate -import com.twitter.util.Future - -class SendHandlerPreCandidateValidator(override val config: Config) extends CandidateValidator { - - override protected val candidatePredicatesMap = - SendHandlerCandidatePredicatesMap.preCandidatePredicates(config) - - private val sendHandlerPreCandidateValidatorStats = - statsReceiver.counter("sendHandlerPreCandidateValidator_stats") - - override def validateCandidate[C <: PushCandidate](candidate: C): Future[Option[Predicate[C]]] = { - val candidatePredicates = getCRTPredicates(candidate.commonRecType) - val predicates = sendHandlerPrePredicates ++ candidatePredicates - - sendHandlerPreCandidateValidatorStats.incr() - executeSequentialPredicates(candidate, predicates) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/ChannelCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/ChannelCandidate.scala deleted file mode 100644 index a278ef237..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/ChannelCandidate.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.frigate.pushservice.take - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.thriftscala.ChannelName -import com.twitter.util.Future -import java.util.concurrent.ConcurrentHashMap -import scala.collection.concurrent -import scala.collection.convert.decorateAsScala._ - -/** - * A class to save all the channel related information - */ -trait ChannelForCandidate { - self: PushCandidate => - - // Cache of channel selection result - private[this] val selectedChannels: concurrent.Map[String, Future[Seq[ChannelName]]] = - new ConcurrentHashMap[String, Future[Seq[ChannelName]]]().asScala - - // Returns the channel information from all ChannelSelectors. - def getChannels(): Future[Seq[ChannelName]] = { - Future.collect(selectedChannels.values.toSeq).map { c => c.flatten } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/ChannelSelector.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/ChannelSelector.scala deleted file mode 100644 index 62378a4bc..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/ChannelSelector.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.frigate.pushservice.take - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.thriftscala.ChannelName -import com.twitter.util.Future - -abstract class ChannelSelector { - - // Returns a map of channel name, and the candidates that can be sent on that channel. - def selectChannel( - candidate: PushCandidate - ): Future[Seq[ChannelName]] - - def getSelectorName(): String -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/NtabOnlyChannelSelector.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/NtabOnlyChannelSelector.scala deleted file mode 100644 index e999da9be..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/NtabOnlyChannelSelector.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.frigate.pushservice.take - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.thriftscala.ChannelName -import com.twitter.util.Future - -class NtabOnlyChannelSelector extends ChannelSelector { - val SELECTOR_NAME = "NtabOnlyChannelSelector" - - def getSelectorName(): String = SELECTOR_NAME - - // Returns a map of channel name, and the candidates that can be sent on that channel - def selectChannel( - candidate: PushCandidate - ): Future[Seq[ChannelName]] = { - // Check candidate channel eligible (based on setting, push cap etc - // Decide which candidate can be sent on what channel - val channelName: Future[ChannelName] = Future.value(ChannelName.PushNtab) - channelName.map(channel => Seq(channel)) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/history/EventBusWriter.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/history/EventBusWriter.scala deleted file mode 100644 index 2bdf412ac..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/history/EventBusWriter.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.frigate.pushservice.take.history - -import com.twitter.eventbus.client.EventBusPublisher -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.NotificationScribeUtil -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.scribe.thriftscala.NotificationScribe -import com.twitter.frigate.thriftscala.FrigateNotification - -class EventBusWriter( - eventBusPublisher: EventBusPublisher[NotificationScribe], - stats: StatsReceiver) { - private def writeSendEventToEventBus( - target: PushTypes.Target, - notificationScribe: NotificationScribe - ): Unit = { - if (target.params(PushParams.EnablePushSendEventBus)) { - val result = eventBusPublisher.publish(notificationScribe) - result.onFailure { _ => stats.counter("push_send_eventbus_failure").incr() } - } - } - - def writeToEventBus( - candidate: PushCandidate, - frigateNotificationForPersistence: FrigateNotification - ): Unit = { - val notificationScribe = NotificationScribeUtil.getNotificationScribe( - targetId = candidate.target.targetId, - impressionId = candidate.impressionId, - frigateNotification = frigateNotificationForPersistence, - createdAt = candidate.createdAt - ) - writeSendEventToEventBus(candidate.target, notificationScribe) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/history/HistoryWriter.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/history/HistoryWriter.scala deleted file mode 100644 index ca9fe31bc..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/history/HistoryWriter.scala +++ /dev/null @@ -1,49 +0,0 @@ -package com.twitter.frigate.pushservice.take.history - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.history.HistoryStoreKeyContext -import com.twitter.frigate.common.history.PushServiceHistoryStore -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.conversions.DurationOps._ - -class HistoryWriter(historyStore: PushServiceHistoryStore, stats: StatsReceiver) { - private lazy val historyWriterStats = stats.scope(this.getClass.getSimpleName) - private lazy val historyWriteCounter = historyWriterStats.counter("history_write_num") - private lazy val loggedOutHistoryWriteCounter = - historyWriterStats.counter("logged_out_history_write_num") - - private def writeTtlForHistory(candidate: PushCandidate): Duration = { - if (candidate.target.isLoggedOutUser) { - 60.days - } else if (RecTypes.isTweetType(candidate.commonRecType)) { - candidate.target.params(PushFeatureSwitchParams.FrigateHistoryTweetNotificationWriteTtl) - } else candidate.target.params(PushFeatureSwitchParams.FrigateHistoryOtherNotificationWriteTtl) - } - - def writeSendToHistory( - candidate: PushCandidate, - frigateNotificationForPersistence: FrigateNotification - ): Future[Unit] = { - val historyStoreKeyContext = HistoryStoreKeyContext( - candidate.target.targetId, - candidate.target.pushContext.flatMap(_.useMemcacheForHistory).getOrElse(false) - ) - if (candidate.target.isLoggedOutUser) { - loggedOutHistoryWriteCounter.incr() - } else { - historyWriteCounter.incr() - } - historyStore - .put( - historyStoreKeyContext, - candidate.createdAt, - frigateNotificationForPersistence, - Some(writeTtlForHistory(candidate)) - ) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicRFPHPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicRFPHPredicates.scala deleted file mode 100644 index 99719af99..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicRFPHPredicates.scala +++ /dev/null @@ -1,7 +0,0 @@ -package com.twitter.frigate.pushservice.take.predicates -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.hermit.predicate.NamedPredicate - -trait BasicRFPHPredicates[C <: PushCandidate] { - val predicates: List[NamedPredicate[C]] -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicSendHandlerPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicSendHandlerPredicates.scala deleted file mode 100644 index 591a4df75..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicSendHandlerPredicates.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.frigate.pushservice.take.predicates - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.hermit.predicate.NamedPredicate - -trait BasicSendHandlerPredicates[C <: PushCandidate] { - - // specific predicates per candidate type before basic SendHandler predicates - val preCandidateSpecificPredicates: List[NamedPredicate[C]] = List.empty - - // specific predicates per candidate type after basic SendHandler predicates, could be empty - val postCandidateSpecificPredicates: List[NamedPredicate[C]] = List.empty -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicTweetPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicTweetPredicates.scala deleted file mode 100644 index 0750abd6e..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicTweetPredicates.scala +++ /dev/null @@ -1,104 +0,0 @@ -package com.twitter.frigate.pushservice.take.predicates - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.predicate.BqmlHealthModelPredicates -import com.twitter.frigate.pushservice.predicate.BqmlQualityModelPredicates -import com.twitter.frigate.pushservice.predicate.HealthPredicates -import com.twitter.frigate.pushservice.predicate.OONSpreadControlPredicate -import com.twitter.frigate.pushservice.predicate.OONTweetNegativeFeedbackBasedPredicate -import com.twitter.frigate.pushservice.predicate.OutOfNetworkCandidatesQualityPredicates -import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate -import com.twitter.frigate.pushservice.predicate.PNegMultimodalPredicates -import com.twitter.frigate.pushservice.predicate.TargetEngagementPredicate -import com.twitter.frigate.pushservice.predicate.TweetEngagementRatioPredicate -import com.twitter.frigate.pushservice.predicate.TweetLanguagePredicate -import com.twitter.frigate.pushservice.predicate.TweetWithheldContentPredicate - -trait BasicTweetPredicates { - - def config: Config - - implicit def statsReceiver: StatsReceiver - - final lazy val basicTweetPredicates = - List( - HealthPredicates.sensitiveMediaCategoryPredicate(), - HealthPredicates.profanityPredicate(), - PredicatesForCandidate.disableOutNetworkTweetPredicate(config.edgeStore), - TweetEngagementRatioPredicate.QTtoNtabClickBasedPredicate(), - TweetLanguagePredicate.oonTweeetLanguageMatch(), - HealthPredicates.userHealthSignalsPredicate(config.userHealthSignalStore), - HealthPredicates.authorSensitiveMediaPredicate(config.producerMediaRepresentationStore), - HealthPredicates.authorProfileBasedPredicate(), - PNegMultimodalPredicates.healthSignalScorePNegMultimodalPredicate( - config.tweetHealthScoreStore), - BqmlHealthModelPredicates.healthModelOonPredicate( - config.filteringModelScorer, - config.producerMediaRepresentationStore, - config.userHealthSignalStore, - config.tweetHealthScoreStore), - BqmlQualityModelPredicates.BqmlQualityModelOonPredicate(config.filteringModelScorer), - HealthPredicates.tweetHealthSignalScorePredicate(config.tweetHealthScoreStore), - HealthPredicates - .tweetHealthSignalScorePredicate(config.tweetHealthScoreStore, applyToQuoteTweet = true), - PredicatesForCandidate.nullCastF1ProtectedExperientPredicate( - config.cachedTweetyPieStoreV2 - ), - OONTweetNegativeFeedbackBasedPredicate.ntabDislikeBasedPredicate(), - OONSpreadControlPredicate.oonTweetSpreadControlPredicate(), - OONSpreadControlPredicate.oonAuthorSpreadControlPredicate(), - HealthPredicates.healthSignalScoreMultilingualPnsfwTweetTextPredicate( - config.tweetHealthScoreStore), - PredicatesForCandidate - .recommendedTweetAuthorAcceptableToTargetUser(config.edgeStore), - HealthPredicates.healthSignalScorePnsfwTweetTextPredicate(config.tweetHealthScoreStore), - HealthPredicates.healthSignalScoreSpammyTweetPredicate(config.tweetHealthScoreStore), - OutOfNetworkCandidatesQualityPredicates.NegativeKeywordsPredicate( - config.postRankingFeatureStoreClient), - PredicatesForCandidate.authorNotBeingDeviceFollowed(config.edgeStore), - TweetWithheldContentPredicate(), - PredicatesForCandidate.noOptoutFreeFormInterestPredicate, - PredicatesForCandidate.disableInNetworkTweetPredicate(config.edgeStore), - TweetEngagementRatioPredicate.TweetReplyLikeRatioPredicate(), - TargetEngagementPredicate( - config.userTweetPerspectiveStore, - defaultForMissing = true - ), - ) -} - -/** - * This trait is a new version of BasicTweetPredicates - * Difference from old version is that basicTweetPredicates are different - * basicTweetPredicates here don't include Social Graph Service related predicates - */ -trait BasicTweetPredicatesWithoutSGSPredicates { - - def config: Config - - implicit def statsReceiver: StatsReceiver - - final lazy val basicTweetPredicates = { - List( - HealthPredicates.healthSignalScoreSpammyTweetPredicate(config.tweetHealthScoreStore), - PredicatesForCandidate.nullCastF1ProtectedExperientPredicate( - config.cachedTweetyPieStoreV2 - ), - TweetWithheldContentPredicate(), - TargetEngagementPredicate( - config.userTweetPerspectiveStore, - defaultForMissing = true - ), - PredicatesForCandidate.noOptoutFreeFormInterestPredicate, - HealthPredicates.userHealthSignalsPredicate(config.userHealthSignalStore), - HealthPredicates.tweetHealthSignalScorePredicate(config.tweetHealthScoreStore), - BqmlQualityModelPredicates.BqmlQualityModelOonPredicate(config.filteringModelScorer), - BqmlHealthModelPredicates.healthModelOonPredicate( - config.filteringModelScorer, - config.producerMediaRepresentationStore, - config.userHealthSignalStore, - config.tweetHealthScoreStore), - ) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicTweetPredicatesForRFPH.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicTweetPredicatesForRFPH.scala deleted file mode 100644 index 7d660632a..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicTweetPredicatesForRFPH.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.frigate.pushservice.take.predicates - -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.common.base.TweetDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.hermit.predicate.NamedPredicate - -trait BasicTweetPredicatesForRFPH[C <: PushCandidate with TweetCandidate with TweetDetails] - extends BasicTweetPredicates - with BasicRFPHPredicates[C] { - - // specific predicates per candidate type before basic tweet predicates - def preCandidateSpecificPredicates: List[NamedPredicate[C]] = List.empty - - // specific predicates per candidate type after basic tweet predicates - def postCandidateSpecificPredicates: List[NamedPredicate[C]] = List.empty - - override lazy val predicates: List[NamedPredicate[C]] = - preCandidateSpecificPredicates ++ basicTweetPredicates ++ postCandidateSpecificPredicates -} - -/** - * This trait is a new version of BasicTweetPredicatesForRFPH - * Difference from old version is that basicTweetPredicates are different - * basicTweetPredicates here don't include Social Graph Service related predicates - */ -trait BasicTweetPredicatesForRFPHWithoutSGSPredicates[ - C <: PushCandidate with TweetCandidate with TweetDetails] - extends BasicTweetPredicatesWithoutSGSPredicates - with BasicRFPHPredicates[C] { - - // specific predicates per candidate type before basic tweet predicates - def preCandidateSpecificPredicates: List[NamedPredicate[C]] = List.empty - - // specific predicates per candidate type after basic tweet predicates - def postCandidateSpecificPredicates: List[NamedPredicate[C]] = List.empty - - override lazy val predicates: List[NamedPredicate[C]] = - preCandidateSpecificPredicates ++ basicTweetPredicates ++ postCandidateSpecificPredicates - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/OutOfNetworkTweetPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/OutOfNetworkTweetPredicates.scala deleted file mode 100644 index e85dc95f0..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/OutOfNetworkTweetPredicates.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.frigate.pushservice.take.predicates - -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.common.base.TweetDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate -import com.twitter.hermit.predicate.NamedPredicate - -trait OutOfNetworkTweetPredicates[C <: PushCandidate with TweetCandidate with TweetDetails] - extends BasicTweetPredicatesForRFPH[C] { - - override lazy val preCandidateSpecificPredicates: List[NamedPredicate[C]] = - List( - PredicatesForCandidate.authorNotBeingFollowed(config.edgeStore) - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/TakeCommonPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/TakeCommonPredicates.scala deleted file mode 100644 index e921f3fcf..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/TakeCommonPredicates.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.frigate.pushservice.take.predicates - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.predicate.CrtDeciderPredicate -import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate -import com.twitter.frigate.pushservice.predicate.ScarecrowPredicate -import com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue.NtabCaretClickFatiguePredicate -import com.twitter.hermit.predicate.NamedPredicate - -trait TakeCommonPredicates { - def config: Config - - implicit def statsReceiver: StatsReceiver - - lazy val rfphPrePredicates: List[NamedPredicate[PushCandidate]] = List( - CrtDeciderPredicate(config.decider), - PredicatesForCandidate.isChannelValidPredicate, - ) - - lazy val sendHandlerPrePredicates: List[NamedPredicate[PushCandidate]] = List( - CrtDeciderPredicate(config.decider), - PredicatesForCandidate.enableSendHandlerCandidates, - PredicatesForCandidate.mrWebHoldbackPredicate, - PredicatesForCandidate.targetUserExists, - PredicatesForCandidate.authorInSocialContext, - PredicatesForCandidate.recommendedTweetIsAuthoredBySelf, - PredicatesForCandidate.selfInSocialContext, - NtabCaretClickFatiguePredicate() - ) - - lazy val postPredicates: List[NamedPredicate[PushCandidate]] = List( - ScarecrowPredicate(config.scarecrowCheckEventStore) - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/candidate_map/CandidatePredicatesMap.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/candidate_map/CandidatePredicatesMap.scala deleted file mode 100644 index aa4642cea..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/candidate_map/CandidatePredicatesMap.scala +++ /dev/null @@ -1,75 +0,0 @@ -package com.twitter.frigate.pushservice.take.predicates.candidate_map - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model._ -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.CommonRecommendationType._ -import com.twitter.hermit.predicate.NamedPredicate - -object CandidatePredicatesMap { - - def apply( - implicit config: Config - ): Map[CommonRecommendationType, List[NamedPredicate[_ <: PushCandidate]]] = { - - val trendTweetCandidatePredicates = TrendTweetPredicates(config).predicates - val tripTweetCandidatePredicates = TripTweetCandidatePredicates(config).predicates - val f1TweetCandidatePredicates = F1TweetCandidatePredicates(config).predicates - val oonTweetCandidatePredicates = OutOfNetworkTweetCandidatePredicates(config).predicates - val tweetActionCandidatePredicates = TweetActionCandidatePredicates(config).predicates - val topicProofTweetCandidatePredicates = TopicProofTweetCandidatePredicates(config).predicates - val addressBookPushPredicates = AddressBookPushCandidatePredicates(config).predicates - val completeOnboardingPushPredicates = CompleteOnboardingPushCandidatePredicates( - config).predicates - val popGeoTweetCandidatePredicate = PopGeoTweetCandidatePredicates(config).predicates - val topTweetImpressionsCandidatePredicates = TopTweetImpressionsPushCandidatePredicates( - config).predicates - val listCandidatePredicates = ListRecommendationPredicates(config).predicates - val subscribedSearchTweetCandidatePredicates = SubscribedSearchTweetCandidatePredicates( - config).predicates - - Map( - F1FirstdegreeTweet -> f1TweetCandidatePredicates, - F1FirstdegreePhoto -> f1TweetCandidatePredicates, - F1FirstdegreeVideo -> f1TweetCandidatePredicates, - ElasticTimelineTweet -> oonTweetCandidatePredicates, - ElasticTimelinePhoto -> oonTweetCandidatePredicates, - ElasticTimelineVideo -> oonTweetCandidatePredicates, - TwistlyTweet -> oonTweetCandidatePredicates, - TwistlyPhoto -> oonTweetCandidatePredicates, - TwistlyVideo -> oonTweetCandidatePredicates, - ExploreVideoTweet -> oonTweetCandidatePredicates, - UserInterestinTweet -> oonTweetCandidatePredicates, - UserInterestinPhoto -> oonTweetCandidatePredicates, - UserInterestinVideo -> oonTweetCandidatePredicates, - PastEmailEngagementTweet -> oonTweetCandidatePredicates, - PastEmailEngagementPhoto -> oonTweetCandidatePredicates, - PastEmailEngagementVideo -> oonTweetCandidatePredicates, - TagSpaceTweet -> oonTweetCandidatePredicates, - TwhinTweet -> oonTweetCandidatePredicates, - FrsTweet -> oonTweetCandidatePredicates, - MrModelingBasedTweet -> oonTweetCandidatePredicates, - TrendTweet -> trendTweetCandidatePredicates, - ReverseAddressbookTweet -> oonTweetCandidatePredicates, - ForwardAddressbookTweet -> oonTweetCandidatePredicates, - TripGeoTweet -> oonTweetCandidatePredicates, - TripHqTweet -> tripTweetCandidatePredicates, - DetopicTweet -> oonTweetCandidatePredicates, - CrowdSearchTweet -> oonTweetCandidatePredicates, - TweetFavorite -> tweetActionCandidatePredicates, - TweetFavoritePhoto -> tweetActionCandidatePredicates, - TweetFavoriteVideo -> tweetActionCandidatePredicates, - TweetRetweet -> tweetActionCandidatePredicates, - TweetRetweetPhoto -> tweetActionCandidatePredicates, - TweetRetweetVideo -> tweetActionCandidatePredicates, - TopicProofTweet -> topicProofTweetCandidatePredicates, - SubscribedSearch -> subscribedSearchTweetCandidatePredicates, - AddressBookUploadPush -> addressBookPushPredicates, - CompleteOnboardingPush -> completeOnboardingPushPredicates, - List -> listCandidatePredicates, - GeoPopTweet -> popGeoTweetCandidatePredicate, - TweetImpressions -> topTweetImpressionsCandidatePredicates - ) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/candidate_map/SendHandlerCandidatePredicatesMap.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/candidate_map/SendHandlerCandidatePredicatesMap.scala deleted file mode 100644 index e37a91044..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/candidate_map/SendHandlerCandidatePredicatesMap.scala +++ /dev/null @@ -1,78 +0,0 @@ -package com.twitter.frigate.pushservice.take.predicates.candidate_map - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model._ -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.CommonRecommendationType._ -import com.twitter.hermit.predicate.NamedPredicate - -object SendHandlerCandidatePredicatesMap { - - def preCandidatePredicates( - implicit config: Config - ): Map[CommonRecommendationType, List[NamedPredicate[_ <: PushCandidate]]] = { - val magicFanoutNewsEventCandidatePredicates = - MagicFanoutNewsEventCandidatePredicates(config).preCandidateSpecificPredicates - - val scheduledSpaceSubscriberPredicates = ScheduledSpaceSubscriberCandidatePredicates( - config).preCandidateSpecificPredicates - - val scheduledSpaceSpeakerPredicates = ScheduledSpaceSpeakerCandidatePredicates( - config).preCandidateSpecificPredicates - - val magicFanoutSportsEventCandidatePredicates = - MagicFanoutSportsEventCandidatePredicates(config).preCandidateSpecificPredicates - - val magicFanoutProductLaunchPredicates = MagicFanoutProductLaunchPushCandidatePredicates( - config).preCandidateSpecificPredicates - - val creatorSubscriptionFanoutPredicates = MagicFanouCreatorSubscriptionEventPushPredicates( - config).preCandidateSpecificPredicates - - val newCreatorFanoutPredicates = MagicFanoutNewCreatorEventPushPredicates( - config).preCandidateSpecificPredicates - - Map( - MagicFanoutNewsEvent -> magicFanoutNewsEventCandidatePredicates, - ScheduledSpaceSubscriber -> scheduledSpaceSubscriberPredicates, - ScheduledSpaceSpeaker -> scheduledSpaceSpeakerPredicates, - MagicFanoutSportsEvent -> magicFanoutSportsEventCandidatePredicates, - MagicFanoutProductLaunch -> magicFanoutProductLaunchPredicates, - NewCreator -> newCreatorFanoutPredicates, - CreatorSubscriber -> creatorSubscriptionFanoutPredicates - ) - } - - def postCandidatePredicates( - implicit config: Config - ): Map[CommonRecommendationType, List[NamedPredicate[_ <: PushCandidate]]] = { - val magicFanoutNewsEventCandidatePredicates = - MagicFanoutNewsEventCandidatePredicates(config).postCandidateSpecificPredicates - - val scheduledSpaceSubscriberPredicates = ScheduledSpaceSubscriberCandidatePredicates( - config).postCandidateSpecificPredicates - - val scheduledSpaceSpeakerPredicates = ScheduledSpaceSpeakerCandidatePredicates( - config).postCandidateSpecificPredicates - - val magicFanoutSportsEventCandidatePredicates = - MagicFanoutSportsEventCandidatePredicates(config).postCandidateSpecificPredicates - val magicFanoutProductLaunchPredicates = MagicFanoutProductLaunchPushCandidatePredicates( - config).postCandidateSpecificPredicates - val creatorSubscriptionFanoutPredicates = MagicFanouCreatorSubscriptionEventPushPredicates( - config).postCandidateSpecificPredicates - val newCreatorFanoutPredicates = MagicFanoutNewCreatorEventPushPredicates( - config).postCandidateSpecificPredicates - - Map( - MagicFanoutNewsEvent -> magicFanoutNewsEventCandidatePredicates, - ScheduledSpaceSubscriber -> scheduledSpaceSubscriberPredicates, - ScheduledSpaceSpeaker -> scheduledSpaceSpeakerPredicates, - MagicFanoutSportsEvent -> magicFanoutSportsEventCandidatePredicates, - MagicFanoutProductLaunch -> magicFanoutProductLaunchPredicates, - NewCreator -> newCreatorFanoutPredicates, - CreatorSubscriber -> creatorSubscriptionFanoutPredicates - ) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/sender/Ibis2Sender.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/sender/Ibis2Sender.scala deleted file mode 100644 index a17d7fe1e..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/sender/Ibis2Sender.scala +++ /dev/null @@ -1,185 +0,0 @@ -package com.twitter.frigate.pushservice.take.sender - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.common.base.TweetDetails -import com.twitter.frigate.common.store.IbisResponse -import com.twitter.frigate.common.store.InvalidConfiguration -import com.twitter.frigate.common.store.NoRequest -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} -import com.twitter.frigate.pushservice.store.Ibis2Store -import com.twitter.frigate.pushservice.store.TweetTranslationStore -import com.twitter.frigate.pushservice.util.CopyUtil -import com.twitter.frigate.pushservice.util.FunctionalUtil -import com.twitter.frigate.pushservice.util.InlineActionUtil -import com.twitter.frigate.pushservice.util.OverrideNotificationUtil -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.frigate.scribe.thriftscala.NotificationScribe -import com.twitter.frigate.thriftscala.ChannelName -import com.twitter.frigate.thriftscala.NotificationDisplayLocation -import com.twitter.ibis2.service.thriftscala.Ibis2Request -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationResponse -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -class Ibis2Sender( - pushIbisV2Store: Ibis2Store, - tweetTranslationStore: ReadableStore[TweetTranslationStore.Key, TweetTranslationStore.Value], - statsReceiver: StatsReceiver) { - - private val stats = statsReceiver.scope(getClass.getSimpleName) - private val silentPushCounter = stats.counter("silent_push") - private val ibisSendFailureCounter = stats.scope("ibis_send_failure").counter("failures") - private val buggyAndroidReleaseCounter = stats.counter("is_buggy_android_release") - private val androidPrimaryCounter = stats.counter("android_primary_device") - private val addTranslationModelValuesCounter = stats.counter("with_translation_model_values") - private val patchNtabResponseEnabled = stats.scope("with_ntab_response") - private val noIbisPushStats = stats.counter("no_ibis_push") - - private def ibisSend( - candidate: PushCandidate, - translationModelValues: Option[Map[String, String]] = None, - ntabResponse: Option[CreateGenericNotificationResponse] = None - ): Future[IbisResponse] = { - if (candidate.frigateNotification.notificationDisplayLocation != NotificationDisplayLocation.PushToMobileDevice) { - Future.value(IbisResponse(InvalidConfiguration)) - } else { - candidate.ibis2Request.flatMap { - case Some(request) => - val requestWithTranslationMV = - addTranslationModelValues(request, translationModelValues) - val patchedIbisRequest = { - if (candidate.target.isLoggedOutUser) { - requestWithTranslationMV - } else { - patchNtabResponseToIbisRequest(requestWithTranslationMV, candidate, ntabResponse) - } - } - pushIbisV2Store.send(patchedIbisRequest, candidate) - case _ => - noIbisPushStats.incr() - Future.value(IbisResponse(sendStatus = NoRequest, ibis2Response = None)) - } - } - } - - def sendAsDarkWrite( - candidate: PushCandidate - ): Future[IbisResponse] = { - ibisSend(candidate) - } - - def send( - channels: Seq[ChannelName], - pushCandidate: PushCandidate, - notificationScribe: NotificationScribe => Unit, - ntabResponse: Option[CreateGenericNotificationResponse], - ): Future[IbisResponse] = pushCandidate.target.isSilentPush.flatMap { isSilentPush: Boolean => - if (isSilentPush) silentPushCounter.incr() - pushCandidate.target.deviceInfo.flatMap { deviceInfo => - if (deviceInfo.exists(_.isSim40AndroidVersion)) buggyAndroidReleaseCounter.incr() - if (PushDeviceUtil.isPrimaryDeviceAndroid(deviceInfo)) androidPrimaryCounter.incr() - Future - .join( - OverrideNotificationUtil - .getOverrideInfo(pushCandidate, stats), - CopyUtil.getCopyFeatures(pushCandidate, stats), - getTranslationModelValues(pushCandidate) - ).flatMap { - case (overrideInfoOpt, copyFeaturesMap, translationModelValues) => - ibisSend(pushCandidate, translationModelValues, ntabResponse) - .onSuccess { ibisResponse => - pushCandidate - .scribeData( - ibis2Response = ibisResponse.ibis2Response, - isSilentPush = isSilentPush, - overrideInfoOpt = overrideInfoOpt, - copyFeaturesList = copyFeaturesMap.keySet, - channels = channels - ).foreach(notificationScribe) - }.onFailure { _ => - pushCandidate - .scribeData(channels = channels).foreach { data => - ibisSendFailureCounter.incr() - notificationScribe(data) - } - } - } - } - } - - private def getTranslationModelValues( - candidate: PushCandidate - ): Future[Option[Map[String, String]]] = { - candidate match { - case tweetCandidate: TweetCandidate with TweetDetails => - val key = TweetTranslationStore.Key( - target = candidate.target, - tweetId = tweetCandidate.tweetId, - tweet = tweetCandidate.tweet, - crt = candidate.commonRecType - ) - - tweetTranslationStore - .get(key) - .map { - case Some(value) => - Some( - Map( - "translated_tweet_text" -> value.translatedTweetText, - "localized_source_language" -> value.localizedSourceLanguage - )) - case None => None - } - case _ => Future.None - } - } - - private def addTranslationModelValues( - ibisRequest: Ibis2Request, - translationModelValues: Option[Map[String, String]] - ): Ibis2Request = { - (translationModelValues, ibisRequest.modelValues) match { - case (Some(translationModelVal), Some(existingModelValues)) => - addTranslationModelValuesCounter.incr() - ibisRequest.copy(modelValues = Some(translationModelVal ++ existingModelValues)) - case (Some(translationModelVal), None) => - addTranslationModelValuesCounter.incr() - ibisRequest.copy(modelValues = Some(translationModelVal)) - case (None, _) => ibisRequest - } - } - - private def patchNtabResponseToIbisRequest( - ibis2Req: Ibis2Request, - candidate: PushCandidate, - ntabResponse: Option[CreateGenericNotificationResponse] - ): Ibis2Request = { - if (candidate.target.params(FS.EnableInlineFeedbackOnPush)) { - patchNtabResponseEnabled.counter().incr() - val dislikePosition = candidate.target.params(FS.InlineFeedbackSubstitutePosition) - val dislikeActionOption = ntabResponse - .map(FunctionalUtil.incr(patchNtabResponseEnabled.counter("ntab_response_exist"))) - .flatMap(response => InlineActionUtil.getDislikeInlineAction(candidate, response)) - .map(FunctionalUtil.incr(patchNtabResponseEnabled.counter("dislike_action_generated"))) - - // Only generate patch serialized inline action when original request has existing serialized_inline_actions_v2 - val patchedSerializedActionOption = ibis2Req.modelValues - .flatMap(model => model.get("serialized_inline_actions_v2")) - .map(FunctionalUtil.incr(patchNtabResponseEnabled.counter("inline_action_v2_exists"))) - .map(serialized => - InlineActionUtil - .patchInlineActionAtPosition(serialized, dislikeActionOption, dislikePosition)) - .map(FunctionalUtil.incr(patchNtabResponseEnabled.counter("patch_inline_action_generated"))) - - (ibis2Req.modelValues, patchedSerializedActionOption) match { - case (Some(existingModelValue), Some(patchedActionV2)) => - patchNtabResponseEnabled.scope("patch_applied").counter().incr() - ibis2Req.copy(modelValues = - Some(existingModelValue ++ Map("serialized_inline_actions_v2" -> patchedActionV2))) - case _ => ibis2Req - } - } else ibis2Req - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/sender/NtabSender.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/sender/NtabSender.scala deleted file mode 100644 index 5019aa040..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/sender/NtabSender.scala +++ /dev/null @@ -1,237 +0,0 @@ -package com.twitter.frigate.pushservice.take.sender - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.history.History -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.ibis.PushOverrideInfo -import com.twitter.frigate.pushservice.params.PushConstants -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FSParams} -import com.twitter.frigate.pushservice.take.NotificationServiceRequest -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.hermit.store.common.ReadableWritableStore -import com.twitter.notificationservice.api.thriftscala.DeleteCurrentTimelineForUserRequest -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationResponse -import com.twitter.notificationservice.thriftscala.DeleteGenericNotificationRequest -import com.twitter.notificationservice.thriftscala.GenericNotificationKey -import com.twitter.notificationservice.thriftscala.GenericNotificationOverrideKey -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -object OverrideCandidate extends Enumeration { - val One: String = "overrideEntry1" -} - -class NtabSender( - notificationServiceSender: ReadableStore[ - NotificationServiceRequest, - CreateGenericNotificationResponse - ], - nTabHistoryStore: ReadableWritableStore[(Long, String), GenericNotificationOverrideKey], - nTabDelete: DeleteGenericNotificationRequest => Future[Unit], - nTabDeleteTimeline: DeleteCurrentTimelineForUserRequest => Future[Unit] -)( - implicit statsReceiver: StatsReceiver) { - - private[this] val nTabDeleteRequests = statsReceiver.counter("ntab_delete_request") - private[this] val nTabDeleteTimelineRequests = - statsReceiver.counter("ntab_delete_timeline_request") - private[this] val ntabOverrideImpressionNotFound = - statsReceiver.counter("ntab_impression_not_found") - private[this] val nTabOverrideOverriddenStat = - statsReceiver.counter("ntab_override_overridden") - private[this] val storeGenericNotifOverrideKey = - statsReceiver.counter("ntab_store_generic_notif_key") - private[this] val prevGenericNotifKeyNotFound = - statsReceiver.counter("ntab_prev_generic_notif_key_not_found") - - private[this] val ntabOverride = - statsReceiver.scope("ntab_override") - private[this] val ntabRequestWithOverrideId = - ntabOverride.counter("request") - private[this] val storeGenericNotifOverrideKeyWithOverrideId = - ntabOverride.counter("store_override_key") - - def send( - candidate: PushCandidate, - isNtabOnlyNotification: Boolean - ): Future[Option[CreateGenericNotificationResponse]] = { - if (candidate.target.params(FSParams.EnableOverrideIdNTabRequest)) { - ntabRequestWithOverrideId.incr() - overridePreviousEntry(candidate).flatMap { _ => - if (shouldDisableNtabOverride(candidate)) { - sendNewEntry(candidate, isNtabOnlyNotification, None) - } else { - sendNewEntry(candidate, isNtabOnlyNotification, Some(OverrideCandidate.One)) - } - } - } else { - for { - notificationOverwritten <- overrideNSlot(candidate) - _ <- deleteCachedApiTimeline(candidate, notificationOverwritten) - gnResponse <- sendNewEntry(candidate, isNtabOnlyNotification) - } yield gnResponse - } - } - - private def sendNewEntry( - candidate: PushCandidate, - isNtabOnlyNotif: Boolean, - overrideId: Option[String] = None - ): Future[Option[CreateGenericNotificationResponse]] = { - notificationServiceSender - .get( - NotificationServiceRequest( - candidate, - impressionId = candidate.impressionId, - isBadgeUpdate = isNtabOnlyNotif, - overrideId = overrideId - )).flatMap { - case Some(response) => - storeGenericNotifKey(candidate, response, overrideId).map { _ => Some(response) } - case _ => Future.None - } - } - - private def storeGenericNotifKey( - candidate: PushCandidate, - createGenericNotificationResponse: CreateGenericNotificationResponse, - overrideId: Option[String] - ): Future[Unit] = { - if (candidate.target.params(FSParams.EnableStoringNtabGenericNotifKey)) { - createGenericNotificationResponse.successKey match { - case Some(genericNotificationKey) => - val userId = genericNotificationKey.userId - if (overrideId.nonEmpty) { - storeGenericNotifOverrideKeyWithOverrideId.incr() - } - val gnOverrideKey = GenericNotificationOverrideKey( - userId = userId, - hashKey = genericNotificationKey.hashKey, - timestampMillis = genericNotificationKey.timestampMillis, - overrideId = overrideId - ) - val mhKeyVal = - ((userId, candidate.impressionId), gnOverrideKey) - storeGenericNotifOverrideKey.incr() - nTabHistoryStore.put(mhKeyVal) - case _ => Future.Unit - } - } else Future.Unit - } - - private def candidateEligibleForOverride( - targetHistory: History, - targetEntries: Seq[FrigateNotification], - ): FrigateNotification = { - val timestampToEntriesMap = - targetEntries.map { entry => - PushOverrideInfo - .getTimestampInMillisForFrigateNotification(entry, targetHistory, statsReceiver) - .getOrElse(PushConstants.DefaultLookBackForHistory.ago.inMilliseconds) -> entry - }.toMap - - PushOverrideInfo.getOldestFrigateNotification(timestampToEntriesMap) - } - - private def overrideNSlot(candidate: PushCandidate): Future[Boolean] = { - if (candidate.target.params(FSParams.EnableNslotsForOverrideOnNtab)) { - val targetHistoryFut = candidate.target.history - targetHistoryFut.flatMap { targetHistory => - val nonEligibleOverrideTypes = - Seq(RecTypes.RecommendedSpaceFanoutTypes ++ RecTypes.ScheduledSpaceReminderTypes) - - val overrideNotifs = PushOverrideInfo - .getOverrideEligiblePushNotifications( - targetHistory, - candidate.target.params(FSParams.OverrideNotificationsLookbackDurationForNTab), - statsReceiver - ).filterNot { - case notification => - nonEligibleOverrideTypes.contains(notification.commonRecommendationType) - } - - val maxNumUnreadEntries = - candidate.target.params(FSParams.OverrideNotificationsMaxCountForNTab) - if (overrideNotifs.nonEmpty && overrideNotifs.size >= maxNumUnreadEntries) { - val eligibleOverrideCandidateOpt = candidateEligibleForOverride( - targetHistory, - overrideNotifs - ) - eligibleOverrideCandidateOpt match { - case overrideCandidate if overrideCandidate.impressionId.nonEmpty => - deleteNTabEntryFromGenericNotificationStore( - candidate.target.targetId, - eligibleOverrideCandidateOpt.impressionId.head) - case _ => - ntabOverrideImpressionNotFound.incr() - Future.False - } - } else Future.False - } - } else { - Future.False - } - } - - private def shouldDisableNtabOverride(candidate: PushCandidate): Boolean = - RecTypes.isSendHandlerType(candidate.commonRecType) - - private def overridePreviousEntry(candidate: PushCandidate): Future[Boolean] = { - - if (shouldDisableNtabOverride(candidate)) { - nTabOverrideOverriddenStat.incr() - Future.False - } else { - val targetHistoryFut = candidate.target.history - targetHistoryFut.flatMap { targetHistory => - val impressionIds = PushOverrideInfo.getImpressionIdsOfPrevEligiblePushNotif( - targetHistory, - candidate.target.params(FSParams.OverrideNotificationsLookbackDurationForImpressionId), - statsReceiver) - - if (impressionIds.nonEmpty) { - deleteNTabEntryFromGenericNotificationStore(candidate.target.targetId, impressionIds.head) - } else { - ntabOverrideImpressionNotFound.incr() - Future.False // no deletes issued - } - } - } - } - - private def deleteCachedApiTimeline( - candidate: PushCandidate, - isNotificationOverridden: Boolean - ): Future[Unit] = { - if (isNotificationOverridden && candidate.target.params(FSParams.EnableDeletingNtabTimeline)) { - val deleteTimelineRequest = DeleteCurrentTimelineForUserRequest(candidate.target.targetId) - nTabDeleteTimelineRequests.incr() - nTabDeleteTimeline(deleteTimelineRequest) - } else { - Future.Unit - } - } - - private def deleteNTabEntryFromGenericNotificationStore( - targetUserId: Long, - targetImpressionId: String - ): Future[Boolean] = { - val mhKey = (targetUserId, targetImpressionId) - val genericNotificationKeyFut = nTabHistoryStore.get(mhKey) - genericNotificationKeyFut.flatMap { - case Some(genericNotifOverrideKey) => - val gnKey = GenericNotificationKey( - userId = genericNotifOverrideKey.userId, - hashKey = genericNotifOverrideKey.hashKey, - timestampMillis = genericNotifOverrideKey.timestampMillis - ) - val deleteEntryRequest = DeleteGenericNotificationRequest(gnKey) - nTabDeleteRequests.incr() - nTabDelete(deleteEntryRequest).map(_ => true) - case _ => - prevGenericNotifKeyNotFound.incr() - Future.False - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/CustomFSFields.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/CustomFSFields.scala deleted file mode 100644 index 9690e9bad..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/CustomFSFields.scala +++ /dev/null @@ -1,98 +0,0 @@ -package com.twitter.frigate.pushservice.target - -import com.twitter.featureswitches.FSCustomMapInput -import com.twitter.featureswitches.parsing.DynMap -import com.twitter.frigate.common.store.deviceinfo.DeviceInfo -import com.twitter.frigate.pushservice.util.NsfwInfo -import com.twitter.gizmoduck.thriftscala.User - -object CustomFSFields { - private val IsReturningUser = "is_returning_user" - private val DaysSinceSignup = "days_since_signup" - private val DaysSinceLogin = "days_since_login" - private val DaysSinceReactivation = "days_since_reactivation" - private val ReactivationDate = "reactivation_date" - private val FollowGraphSize = "follow_graph_size" - private val GizmoduckUserType = "gizmoduck_user_type" - private val UserAge = "mr_user_age" - private val SensitiveOptIn = "sensitive_opt_in" - private val NsfwFollowRatio = "nsfw_follow_ratio" - private val TotalFollows = "follow_count" - private val NsfwRealGraphScore = "nsfw_real_graph_score" - private val NsfwProfileVisit = "nsfw_profile_visit" - private val TotalSearches = "total_searches" - private val NsfwSearchScore = "nsfw_search_score" - private val HasReportedNsfw = "nsfw_reported" - private val HasDislikedNsfw = "nsfw_disliked" - private val UserState = "user_state" - private val MrUserState = "mr_user_state" - private val NumDaysReceivedPushInLast30Days = - "num_days_received_push_in_last_30_days" - private val RecommendationsSetting = "recommendations_setting" - private val TopicsSetting = "topics_setting" - private val SpacesSetting = "spaces_setting" - private val NewsSetting = "news_setting" - private val LiveVideoSetting = "live_video_setting" - private val HasRecentPushableRebDevice = "has_recent_pushable_rweb_device" - private val RequestSource = "request_source" -} - -case class CustomFSFields( - isReactivatedUser: Boolean, - daysSinceSignup: Int, - numDaysReceivedPushInLast30Days: Int, - daysSinceLogin: Option[Int], - daysSinceReactivation: Option[Int], - user: Option[User], - userState: Option[String], - mrUserState: Option[String], - reactivationDate: Option[String], - requestSource: Option[String], - userAge: Option[Int], - nsfwInfo: Option[NsfwInfo], - deviceInfo: Option[DeviceInfo]) { - - import CustomFSFields._ - - private val keyValMap: Map[String, Any] = Map( - IsReturningUser -> isReactivatedUser, - DaysSinceSignup -> daysSinceSignup, - DaysSinceLogin -> daysSinceLogin, - NumDaysReceivedPushInLast30Days -> numDaysReceivedPushInLast30Days - ) ++ - daysSinceReactivation.map(DaysSinceReactivation -> _) ++ - reactivationDate.map(ReactivationDate -> _) ++ - user.flatMap(_.counts.map(counts => FollowGraphSize -> counts.following)) ++ - user.map(u => GizmoduckUserType -> u.userType.name) ++ - userState.map(UserState -> _) ++ - mrUserState.map(MrUserState -> _) ++ - requestSource.map(RequestSource -> _) ++ - userAge.map(UserAge -> _) ++ - nsfwInfo.flatMap(_.senstiveOptIn).map(SensitiveOptIn -> _) ++ - nsfwInfo - .map { nsInfo => - Map[String, Any]( - NsfwFollowRatio -> nsInfo.nsfwFollowRatio, - TotalFollows -> nsInfo.totalFollowCount, - NsfwRealGraphScore -> nsInfo.realGraphScore, - NsfwProfileVisit -> nsInfo.nsfwProfileVisits, - TotalSearches -> nsInfo.totalSearches, - NsfwSearchScore -> nsInfo.searchNsfwScore, - HasReportedNsfw -> nsInfo.hasReported, - HasDislikedNsfw -> nsInfo.hasDisliked - ) - }.getOrElse(Map.empty[String, Any]) ++ - deviceInfo - .map { deviceInfo => - Map[String, Boolean]( - RecommendationsSetting -> deviceInfo.isRecommendationsEligible, - TopicsSetting -> deviceInfo.isTopicsEligible, - SpacesSetting -> deviceInfo.isSpacesEligible, - LiveVideoSetting -> deviceInfo.isBroadcastsEligible, - NewsSetting -> deviceInfo.isNewsEligible, - HasRecentPushableRebDevice -> deviceInfo.hasRecentPushableRWebDevice - ) - }.getOrElse(Map.empty[String, Boolean]) - - val fsMap = FSCustomMapInput(DynMap(keyValMap)) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/LoggedOutPushTargetUserBuilder.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/LoggedOutPushTargetUserBuilder.scala deleted file mode 100644 index facbaede0..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/LoggedOutPushTargetUserBuilder.scala +++ /dev/null @@ -1,182 +0,0 @@ -package com.twitter.frigate.pushservice.target - -import com.twitter.abdecider.LoggingABDecider -import com.twitter.conversions.DurationOps._ -import com.twitter.decider.Decider -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.FeatureMap -import com.twitter.frigate.common.history.History -import com.twitter.frigate.common.history.HistoryStoreKeyContext -import com.twitter.frigate.common.history.MagicFanoutReasonHistory -import com.twitter.frigate.common.history.PushServiceHistoryStore -import com.twitter.frigate.common.history.RecItems -import com.twitter.frigate.common.store.deviceinfo.DeviceInfo -import com.twitter.frigate.common.util.ABDeciderWithOverride -import com.twitter.frigate.common.util.LanguageLocaleUtil -import com.twitter.frigate.data_pipeline.features_common.MrRequestContextForFeatureStore -import com.twitter.frigate.data_pipeline.thriftscala.UserHistoryValue -import com.twitter.frigate.dau_model.thriftscala.DauProbability -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.thriftscala.PushContext -import com.twitter.frigate.thriftscala.UserForPushTargeting -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.hermit.stp.thriftscala.STPResult -import com.twitter.interests.thriftscala.InterestId -import com.twitter.notificationservice.genericfeedbackstore.FeedbackPromptValue -import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails -import com.twitter.nrel.hydration.push.HydrationContext -import com.twitter.permissions_storage.thriftscala.AppPermission -import com.twitter.service.metastore.gen.thriftscala.Location -import com.twitter.service.metastore.gen.thriftscala.UserLanguages -import com.twitter.stitch.Stitch -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.columns.frigate.logged_out_web_notifications.thriftscala.LOWebNotificationMetadata -import com.twitter.timelines.configapi -import com.twitter.timelines.configapi.Params -import com.twitter.timelines.real_graph.v1.thriftscala.RealGraphFeatures -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.wtf.scalding.common.thriftscala.UserFeatures - -case class LoggedOutPushTargetUserBuilder( - historyStore: PushServiceHistoryStore, - inputDecider: Decider, - inputAbDecider: LoggingABDecider, - loggedOutPushInfoStore: ReadableStore[Long, LOWebNotificationMetadata] -)( - globalStatsReceiver: StatsReceiver) { - private val stats = globalStatsReceiver.scope("LORefreshForPushHandler") - private val noHistoryCounter = stats.counter("no_logged_out_history") - private val historyFoundCounter = stats.counter("logged_out_history_counter") - private val noLoggedOutUserCounter = stats.counter("no_logged_out_user") - private val countryCodeCounter = stats.counter("country_counter") - private val noCountryCodeCounter = stats.counter("no_country_counter") - private val noLanguageCodeCounter = stats.counter("no_language_counter") - - def buildTarget( - guestId: Long, - inputPushContext: Option[PushContext] - ): Future[Target] = { - - val historyStoreKeyContext = HistoryStoreKeyContext( - guestId, - inputPushContext.flatMap(_.useMemcacheForHistory).getOrElse(false) - ) - if (historyStore.get(historyStoreKeyContext, Some(30.days)) == Future.None) { - noHistoryCounter.incr() - } else { - historyFoundCounter.incr() - - } - if (loggedOutPushInfoStore.get(guestId) == Future.None) { - noLoggedOutUserCounter.incr() - } - Future - .join( - historyStore.get(historyStoreKeyContext, Some(30.days)), - loggedOutPushInfoStore.get(guestId) - ).map { - case (loNotifHistory, loggedOutUserPushInfo) => - new Target { - override lazy val stats: StatsReceiver = globalStatsReceiver - override val targetId: Long = guestId - override val targetGuestId = Some(guestId) - override lazy val decider: Decider = inputDecider - override lazy val loggedOutMetadata = Future.value(loggedOutUserPushInfo) - val rawLanguageFut = loggedOutMetadata.map { metadata => metadata.map(_.language) } - override val targetLanguage: Future[Option[String]] = rawLanguageFut.map { rawLang => - if (rawLang.isDefined) { - val lang = LanguageLocaleUtil.getStandardLanguageCode(rawLang.get) - if (lang.isEmpty) { - noLanguageCodeCounter.incr() - None - } else { - Option(lang) - } - } else None - } - val country = loggedOutMetadata.map(_.map(_.countryCode)) - if (country.isDefined) { - countryCodeCounter.incr() - } else { - noCountryCodeCounter.incr() - } - if (loNotifHistory == null) { - noHistoryCounter.incr() - } else { - historyFoundCounter.incr() - } - override lazy val location: Future[Option[Location]] = country.map { - case Some(code) => - Some( - Location( - city = "", - region = "", - countryCode = code, - confidence = 0.0, - lat = None, - lon = None, - metro = None, - placeIds = None, - weightedLocations = None, - createdAtMsec = None, - ip = None, - isSignupIp = None, - placeMap = None - )) - case _ => None - } - - override lazy val pushContext: Option[PushContext] = inputPushContext - override lazy val history: Future[History] = Future.value(loNotifHistory) - override lazy val magicFanoutReasonHistory30Days: Future[MagicFanoutReasonHistory] = - Future.value(null) - override lazy val globalStats: StatsReceiver = globalStatsReceiver - override lazy val pushTargeting: Future[Option[UserForPushTargeting]] = Future.None - override lazy val appPermissions: Future[Option[AppPermission]] = Future.None - override lazy val lastHTLVisitTimestamp: Future[Option[Long]] = Future.None - override lazy val pushRecItems: Future[RecItems] = Future.value(null) - - override lazy val isNewSignup: Boolean = false - override lazy val metastoreLanguages: Future[Option[UserLanguages]] = Future.None - override lazy val optOutUserInterests: Future[Option[Seq[InterestId]]] = Future.None - override lazy val mrRequestContextForFeatureStore: MrRequestContextForFeatureStore = - null - override lazy val targetUser: Future[Option[User]] = Future.None - override lazy val notificationFeedbacks: Future[Option[Seq[FeedbackPromptValue]]] = - Future.None - override lazy val promptFeedbacks: Stitch[Seq[FeedbackPromptValue]] = null - override lazy val seedsWithWeight: Future[Option[Map[Long, Double]]] = Future.None - override lazy val tweetImpressionResults: Future[Seq[Long]] = Future.Nil - override lazy val params: configapi.Params = Params.Empty - override lazy val deviceInfo: Future[Option[DeviceInfo]] = Future.None - override lazy val userFeatures: Future[Option[UserFeatures]] = Future.None - override lazy val isOpenAppExperimentUser: Future[Boolean] = Future.False - override lazy val featureMap: Future[FeatureMap] = Future.value(null) - override lazy val dauProbability: Future[Option[DauProbability]] = Future.None - override lazy val labeledPushRecsHydrated: Future[Option[UserHistoryValue]] = - Future.None - override lazy val onlineLabeledPushRecs: Future[Option[UserHistoryValue]] = Future.None - override lazy val realGraphFeatures: Future[Option[RealGraphFeatures]] = Future.None - override lazy val stpResult: Future[Option[STPResult]] = Future.None - override lazy val globalOptoutProbabilities: Seq[Future[Option[Double]]] = Seq.empty - override lazy val bucketOptoutProbability: Future[Option[Double]] = Future.None - override lazy val utcOffset: Future[Option[Duration]] = Future.None - override lazy val abDecider: ABDeciderWithOverride = - ABDeciderWithOverride(inputAbDecider, ddgOverrideOption)(globalStatsReceiver) - override lazy val resurrectionDate: Future[Option[String]] = Future.None - override lazy val isResurrectedUser: Boolean = false - override lazy val timeSinceResurrection: Option[Duration] = None - override lazy val inlineActionHistory: Future[Seq[(Long, String)]] = Future.Nil - override lazy val caretFeedbacks: Future[Option[Seq[CaretFeedbackDetails]]] = - Future.None - - override def targetHydrationContext: Future[HydrationContext] = Future.value(null) - override def isBlueVerified: Future[Option[Boolean]] = Future.None - override def isVerified: Future[Option[Boolean]] = Future.None - override def isSuperFollowCreator: Future[Option[Boolean]] = Future.None - } - } - } - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/PushTargetUserBuilder.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/PushTargetUserBuilder.scala deleted file mode 100644 index 8378500af..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/PushTargetUserBuilder.scala +++ /dev/null @@ -1,694 +0,0 @@ -package com.twitter.frigate.pushservice.target - -import com.twitter.abdecider.LoggingABDecider -import com.twitter.conversions.DurationOps._ -import com.twitter.decider.Decider -import com.twitter.discovery.common.configapi.ConfigParamsBuilder -import com.twitter.discovery.common.configapi.ExperimentOverride -import com.twitter.featureswitches.Recipient -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.history._ -import com.twitter.frigate.common.logger.MRLogger -import com.twitter.frigate.common.store.FeedbackRequest -import com.twitter.frigate.common.store.PushRecItemsKey -import com.twitter.frigate.common.store.deviceinfo.DeviceInfo -import com.twitter.frigate.common.store.interests.UserId -import com.twitter.frigate.common.util._ -import com.twitter.frigate.data_pipeline.features_common.MrRequestContextForFeatureStore -import com.twitter.frigate.data_pipeline.thriftscala.UserHistoryValue -import com.twitter.frigate.dau_model.thriftscala.DauProbability -import com.twitter.frigate.pushcap.thriftscala.PushcapInfo -import com.twitter.frigate.pushcap.thriftscala.PushcapUserHistory -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.ml.HydrationContextBuilder -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.store.LabeledPushRecsStoreKey -import com.twitter.frigate.pushservice.store.OnlineUserHistoryKey -import com.twitter.frigate.pushservice.util.NsfwInfo -import com.twitter.frigate.pushservice.util.NsfwPersonalizationUtil -import com.twitter.frigate.pushservice.util.PushAppPermissionUtil -import com.twitter.frigate.pushservice.util.PushCapUtil.getMinimumRestrictedPushcapInfo -import com.twitter.frigate.pushservice.thriftscala.PushContext -import com.twitter.frigate.pushservice.thriftscala.RequestSource -import com.twitter.frigate.thriftscala.SecondaryAccountsByUserState -import com.twitter.frigate.thriftscala.UserForPushTargeting -import com.twitter.frigate.user_states.thriftscala.MRUserHmmState -import com.twitter.frigate.user_states.thriftscala.{UserState => MrUserState} -import com.twitter.frontpage.stream.util.SnowflakeUtil -import com.twitter.geoduck.common.thriftscala.Place -import com.twitter.geoduck.service.thriftscala.LocationResponse -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.hermit.model.user_state.UserState -import com.twitter.hermit.model.user_state.UserState.UserState -import com.twitter.hermit.stp.thriftscala.STPResult -import com.twitter.ibis.thriftscala.ContentRecData -import com.twitter.interests.thriftscala.InterestId -import com.twitter.notificationservice.feedback.thriftscala.FeedbackInteraction -import com.twitter.notificationservice.genericfeedbackstore.FeedbackPromptValue -import com.twitter.notificationservice.genericfeedbackstore.GenericFeedbackStore -import com.twitter.notificationservice.genericfeedbackstore.GenericFeedbackStoreException -import com.twitter.notificationservice.model.service.DismissMenuFeedbackAction -import com.twitter.notificationservice.scribe.manhattan.GenericNotificationsFeedbackRequest -import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails -import com.twitter.nrel.heavyranker.FeatureHydrator -import com.twitter.nrel.hydration.push.HydrationContext -import com.twitter.permissions_storage.thriftscala.AppPermission -import com.twitter.rux.common.strato.thriftscala.UserTargetingProperty -import com.twitter.scio.nsfw_user_segmentation.thriftscala.NSFWUserSegmentation -import com.twitter.service.metastore.gen.thriftscala.Location -import com.twitter.service.metastore.gen.thriftscala.UserLanguages -import com.twitter.stitch.Stitch -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi -import com.twitter.timelines.real_graph.thriftscala.{RealGraphFeatures => RealGraphFeaturesUnion} -import com.twitter.timelines.real_graph.v1.thriftscala.RealGraphFeatures -import com.twitter.ubs.thriftscala.SellerApplicationState -import com.twitter.ubs.thriftscala.SellerTrack -import com.twitter.user_session_store.thriftscala.UserSession -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.util.Time -import com.twitter.wtf.scalding.common.thriftscala.UserFeatures - -case class PushTargetUserBuilder( - historyStore: PushServiceHistoryStore, - emailHistoryStore: PushServiceHistoryStore, - labeledPushRecsStore: ReadableStore[LabeledPushRecsStoreKey, UserHistoryValue], - onlineUserHistoryStore: ReadableStore[OnlineUserHistoryKey, UserHistoryValue], - pushRecItemsStore: ReadableStore[PushRecItemsKey, RecItems], - userStore: ReadableStore[Long, User], - pushInfoStore: ReadableStore[Long, UserForPushTargeting], - userCountryStore: ReadableStore[Long, Location], - userUtcOffsetStore: ReadableStore[Long, Duration], - dauProbabilityStore: ReadableStore[Long, DauProbability], - nsfwConsumerStore: ReadableStore[Long, NSFWUserSegmentation], - userFeatureStore: ReadableStore[Long, UserFeatures], - userTargetingPropertyStore: ReadableStore[Long, UserTargetingProperty], - mrUserStateStore: ReadableStore[Long, MRUserHmmState], - tweetImpressionStore: ReadableStore[Long, Seq[Long]], - ntabCaretFeedbackStore: ReadableStore[GenericNotificationsFeedbackRequest, Seq[ - CaretFeedbackDetails - ]], - genericFeedbackStore: ReadableStore[FeedbackRequest, Seq[FeedbackPromptValue]], - genericNotificationFeedbackStore: GenericFeedbackStore, - timelinesUserSessionStore: ReadableStore[Long, UserSession], - cachedTweetyPieStore: ReadableStore[Long, TweetyPieResult], - strongTiesStore: ReadableStore[Long, STPResult], - userHTLLastVisitStore: ReadableStore[Long, Seq[Long]], - userLanguagesStore: ReadableStore[Long, UserLanguages], - inputDecider: Decider, - inputAbDecider: LoggingABDecider, - realGraphScoresTop500InStore: ReadableStore[Long, Map[Long, Double]], - recentFollowsStore: ReadableStore[Long, Seq[Long]], - resurrectedUserStore: ReadableStore[Long, String], - configParamsBuilder: ConfigParamsBuilder, - optOutUserInterestsStore: ReadableStore[UserId, Seq[InterestId]], - deviceInfoStore: ReadableStore[Long, DeviceInfo], - pushcapDynamicPredictionStore: ReadableStore[Long, PushcapUserHistory], - appPermissionStore: ReadableStore[(Long, (String, String)), AppPermission], - optoutModelScorer: PushMLModelScorer, - inlineActionHistoryStore: ReadableStore[Long, Seq[(Long, String)]], - featureHydrator: FeatureHydrator, - openAppUserStore: ReadableStore[Long, Boolean], - openedPushByHourAggregatedStore: ReadableStore[Long, Map[Int, Int]], - geoduckStoreV2: ReadableStore[Long, LocationResponse], - superFollowEligibilityUserStore: ReadableStore[Long, Boolean], - superFollowApplicationStatusStore: ReadableStore[(Long, SellerTrack), SellerApplicationState] -)( - globalStatsReceiver: StatsReceiver) { - - implicit val statsReceiver: StatsReceiver = globalStatsReceiver - - private val log = MRLogger("PushTargetUserBuilder") - private val recentFollowscounter = statsReceiver.counter("query_recent_follows") - private val isModelTrainingDataCounter = - statsReceiver.scope("TargetUserBuilder").counter("is_model_training") - private val feedbackStoreGenerationErr = statsReceiver.counter("feedback_store_generation_error") - private val newSignUpUserStats = statsReceiver.counter("new_signup_user") - private val pushcapSelectionStat = statsReceiver.scope("pushcap_modeling") - private val dormantUserCount = statsReceiver.counter("dormant_user_counter") - private val optoutModelStat = statsReceiver.scope("optout_modeling") - private val placeFoundStat = statsReceiver.scope("geoduck_v2").stat("places_found") - private val placesNotFound = statsReceiver.scope("geoduck_v2").counter("places_not_found") - // Email history store stats - private val emailHistoryStats = statsReceiver.scope("email_tweet_history") - private val emptyEmailHistoryCounter = emailHistoryStats.counter("empty") - private val nonEmptyEmailHistoryCounter = emailHistoryStats.counter("non_empty") - - private val MagicRecsCategory = "MagicRecs" - private val MomentsViaMagicRecsCategory = "MomentsViaMagicRecs" - private val MomentsCategory = "Moments" - - def buildTarget( - userId: Long, - inputPushContext: Option[PushContext], - forcedFeatureValues: Option[Map[String, configapi.FeatureValue]] = None - ): Future[Target] = { - val historyStoreKeyContext = HistoryStoreKeyContext( - userId, - inputPushContext.flatMap(_.useMemcacheForHistory).getOrElse(false) - ) - Future - .join( - userStore.get(userId), - deviceInfoStore.get(userId), - pushInfoStore.get(userId), - historyStore.get(historyStoreKeyContext, Some(30.days)), - emailHistoryStore.get( - HistoryStoreKeyContext(userId, useStoreB = false), - Some(7.days) // we only keep 7 days of email tweet history - ) - ).flatMap { - case (userOpt, deviceInfoOpt, userForPushTargetingInfoOpt, notifHistory, emailHistory) => - getCustomFSFields( - userId, - userOpt, - deviceInfoOpt, - userForPushTargetingInfoOpt, - notifHistory, - inputPushContext.flatMap(_.requestSource)).map { customFSField => - new Target { - - override lazy val stats: StatsReceiver = statsReceiver - - override val targetId: Long = userId - - override val targetUser: Future[Option[User]] = Future.value(userOpt) - - override val isEmailUser: Boolean = - inputPushContext.flatMap(_.requestSource) match { - case Some(source) if source == RequestSource.Email => true - case _ => false - } - - override val pushContext = inputPushContext - - override def globalStats: StatsReceiver = globalStatsReceiver - - override lazy val abDecider: ABDeciderWithOverride = - ABDeciderWithOverride(inputAbDecider, ddgOverrideOption) - - override lazy val pushRecItems: Future[RecItems] = - pushRecItemsStore - .get(PushRecItemsKey(historyStoreKeyContext, history)) - .map(_.getOrElse(RecItems.empty)) - - // List of past tweet candidates sent in the past through email with timestamp - override lazy val emailRecItems: Future[Seq[(Time, Long)]] = { - Future.value { - emailHistory.sortedEmailHistory.flatMap { - case (timeStamp, notification) => - notification.contentRecsNotification - .map { notification => - notification.recommendations.contentRecCollections.flatMap { - contentRecs => - contentRecs.contentRecModules.flatMap { contentRecModule => - contentRecModule.recData match { - case ContentRecData.TweetRec(tweetRec) => - nonEmptyEmailHistoryCounter.incr() - Seq(tweetRec.tweetId) - case _ => - emptyEmailHistoryCounter.incr() - Nil - } - } - } - }.getOrElse { - emptyEmailHistoryCounter.incr() - Nil - }.map(timeStamp -> _) - } - } - } - - override lazy val history: Future[History] = Future.value(notifHistory) - - override lazy val pushTargeting: Future[Option[UserForPushTargeting]] = - Future.value(userForPushTargetingInfoOpt) - - override lazy val decider: Decider = inputDecider - - override lazy val location: Future[Option[Location]] = - userCountryStore.get(userId) - - override lazy val deviceInfo: Future[Option[DeviceInfo]] = - Future.value(deviceInfoOpt) - - override lazy val targetLanguage: Future[Option[String]] = targetUser map { userOpt => - userOpt.flatMap(_.account.map(_.language)) - } - - override lazy val targetAgeInYears: Future[Option[Int]] = - Future.value(customFSField.userAge) - - override lazy val metastoreLanguages: Future[Option[UserLanguages]] = - userLanguagesStore.get(targetId) - - override lazy val utcOffset: Future[Option[Duration]] = - userUtcOffsetStore.get(targetId) - - override lazy val userFeatures: Future[Option[UserFeatures]] = - userFeatureStore.get(targetId) - - override lazy val targetUserState: Future[Option[UserState]] = - Future.value( - customFSField.userState - .flatMap(userState => UserState.valueOf(userState))) - - override lazy val targetMrUserState: Future[Option[MrUserState]] = - Future.value( - customFSField.mrUserState - .flatMap(mrUserState => MrUserState.valueOf(mrUserState))) - - override lazy val accountStateWithDeviceInfo: Future[ - Option[SecondaryAccountsByUserState] - ] = Future.None - - override lazy val dauProbability: Future[Option[DauProbability]] = { - dauProbabilityStore.get(targetId) - } - - override lazy val labeledPushRecsHydrated: Future[Option[UserHistoryValue]] = - labeledPushRecsStore.get(LabeledPushRecsStoreKey(this, historyStoreKeyContext)) - - override lazy val onlineLabeledPushRecs: Future[Option[UserHistoryValue]] = - labeledPushRecsHydrated.flatMap { labeledPushRecs => - history.flatMap { history => - onlineUserHistoryStore.get( - OnlineUserHistoryKey(targetId, labeledPushRecs, Some(history)) - ) - } - } - - override lazy val tweetImpressionResults: Future[Seq[Long]] = - tweetImpressionStore.get(targetId).map { - case Some(impressionList) => - impressionList - case _ => Nil - } - - override lazy val realGraphFeatures: Future[Option[RealGraphFeatures]] = - timelinesUserSessionStore.get(targetId).map { userSessionOpt => - userSessionOpt.flatMap { userSession => - userSession.realGraphFeatures.collect { - case RealGraphFeaturesUnion.V1(rGFeatures) => - rGFeatures - } - } - } - - override lazy val stpResult: Future[Option[STPResult]] = - strongTiesStore.get(targetId) - - override lazy val lastHTLVisitTimestamp: Future[Option[Long]] = - userHTLLastVisitStore.get(targetId).map { - case Some(lastVisitTimestamps) if lastVisitTimestamps.nonEmpty => - Some(lastVisitTimestamps.max) - case _ => None - } - - override lazy val caretFeedbacks: Future[Option[Seq[CaretFeedbackDetails]]] = { - val scribeHistoryLookbackPeriod = 365.days - val now = Time.now - val request = GenericNotificationsFeedbackRequest( - userId = targetId, - eventStartTimestamp = now - scribeHistoryLookbackPeriod, - eventEndTimestamp = now, - filterCategory = - Some(Set(MagicRecsCategory, MomentsViaMagicRecsCategory, MomentsCategory)), - filterFeedbackActionText = - Some(Set(DismissMenuFeedbackAction.FeedbackActionTextSeeLessOften)) - ) - ntabCaretFeedbackStore.get(request) - } - - override lazy val notificationFeedbacks: Future[ - Option[Seq[FeedbackPromptValue]] - ] = { - val scribeHistoryLookbackPeriod = 30.days - val now = Time.now - val request = FeedbackRequest( - userId = targetId, - oldestTimestamp = scribeHistoryLookbackPeriod.ago, - newestTimestamp = Time.now, - feedbackInteraction = FeedbackInteraction.Feedback - ) - genericFeedbackStore.get(request) - } - - // DEPRECATED: Use notificationFeedbacks instead. - // This method will increase latency dramatically. - override lazy val promptFeedbacks: Stitch[Seq[FeedbackPromptValue]] = { - val scribeHistoryLookbackPeriod = 7.days - - genericNotificationFeedbackStore - .getAll( - userId = targetId, - oldestTimestamp = scribeHistoryLookbackPeriod.ago, - newestTimestamp = Time.now, - feedbackInteraction = FeedbackInteraction.Feedback - ).handle { - case _: GenericFeedbackStoreException => { - feedbackStoreGenerationErr.incr() - Seq.empty[FeedbackPromptValue] - } - } - } - - override lazy val optOutUserInterests: Future[Option[Seq[InterestId]]] = { - optOutUserInterestsStore.get(targetId) - } - - private val experimentOverride = ddgOverrideOption.map { - case DDGOverride(Some(exp), Some(bucket)) => - Set(ExperimentOverride(exp, bucket)) - case _ => Set.empty[ExperimentOverride] - } - - override val signupCountryCode = - Future.value(userOpt.flatMap(_.safety.flatMap(_.signupCountryCode))) - - override lazy val params: configapi.Params = { - val fsRecipient = Recipient( - userId = Some(targetId), - userRoles = userOpt.flatMap(_.roles.map(_.roles.toSet)), - clientApplicationId = deviceInfoOpt.flatMap(_.guessedPrimaryClientAppId), - userAgent = deviceInfoOpt.flatMap(_.guessedPrimaryDeviceUserAgent), - countryCode = - userOpt.flatMap(_.account.flatMap(_.countryCode.map(_.toUpperCase))), - customFields = Some(customFSField.fsMap), - signupCountryCode = - userOpt.flatMap(_.safety.flatMap(_.signupCountryCode.map(_.toUpperCase))), - languageCode = deviceInfoOpt.flatMap { - _.deviceLanguages.flatMap(IbisAppPushDeviceSettingsUtil.inferredDeviceLanguage) - } - ) - - configParamsBuilder.build( - userId = Some(targetId), - experimentOverrides = experimentOverride, - featureRecipient = Some(fsRecipient), - forcedFeatureValues = forcedFeatureValues.getOrElse(Map.empty), - ) - } - - override lazy val mrRequestContextForFeatureStore = - MrRequestContextForFeatureStore(targetId, params, isModelTrainingData) - - override lazy val dynamicPushcap: Future[Option[PushcapInfo]] = { - // Get the pushcap from the pushcap model prediction store - if (params(PushParams.EnableModelBasedPushcapAssignments)) { - val originalPushcapInfoFut = - PushCapUtil.getPushcapFromUserHistory( - userId, - pushcapDynamicPredictionStore, - params(FeatureSwitchParams.PushcapModelType), - params(FeatureSwitchParams.PushcapModelPredictionVersion), - pushcapSelectionStat - ) - // Modify the push cap info if there is a restricted min value for predicted push caps. - val restrictedPushcap = params(PushFeatureSwitchParams.RestrictedMinModelPushcap) - originalPushcapInfoFut.map { - case Some(originalPushcapInfo) => - Some( - getMinimumRestrictedPushcapInfo( - restrictedPushcap, - originalPushcapInfo, - pushcapSelectionStat)) - case _ => None - } - } else Future.value(None) - } - - override lazy val targetHydrationContext: Future[HydrationContext] = - HydrationContextBuilder.build(this) - - override lazy val featureMap: Future[FeatureMap] = - targetHydrationContext.flatMap { hydrationContext => - featureHydrator.hydrateTarget( - hydrationContext, - this.params, - this.mrRequestContextForFeatureStore) - } - - override lazy val globalOptoutProbabilities: Seq[Future[Option[Double]]] = { - params(PushFeatureSwitchParams.GlobalOptoutModelParam).map { model_id => - optoutModelScorer - .singlePredictionForTargetLevel(model_id, targetId, featureMap) - } - } - - override lazy val bucketOptoutProbability: Future[Option[Double]] = { - Future - .collect(globalOptoutProbabilities).map { - _.zip(params(PushFeatureSwitchParams.GlobalOptoutThresholdParam)) - .exists { - case (Some(score), threshold) => score >= threshold - case _ => false - } - }.flatMap { - case true => - optoutModelScorer.singlePredictionForTargetLevel( - params(PushFeatureSwitchParams.BucketOptoutModelParam), - targetId, - featureMap) - case _ => Future.None - } - } - - override lazy val optoutAdjustedPushcap: Future[Option[Short]] = { - if (params(PushFeatureSwitchParams.EnableOptoutAdjustedPushcap)) { - bucketOptoutProbability.map { - case Some(score) => - val idx = params(PushFeatureSwitchParams.BucketOptoutSlotThresholdParam) - .indexWhere(score <= _) - if (idx >= 0) { - val pushcap = - params(PushFeatureSwitchParams.BucketOptoutSlotPushcapParam)(idx).toShort - optoutModelStat.scope("adjusted_pushcap").counter(f"$pushcap").incr() - if (pushcap >= 0) Some(pushcap) - else None - } else None - case _ => None - } - } else Future.None - } - - override lazy val seedsWithWeight: Future[Option[Map[Long, Double]]] = { - Future - .join( - realGraphScoresTop500InStore.get(userId), - targetUserState, - targetUser - ) - .flatMap { - case (seedSetOpt, userState, gizmoduckUser) => - val seedSet = seedSetOpt.getOrElse(Map.empty[Long, Double]) - - //If new sign_up or New user, combine recent_follows with real graph seedset - val isNewUserEnabled = { - val isNewerThan7days = customFSField.daysSinceSignup <= 7 - val isNewUserState = userState.contains(UserState.New) - isNewUserState || isNewSignup || isNewerThan7days - } - - val nonSeedSetFollowsFut = gizmoduckUser match { - case Some(user) if isNewUserEnabled => - recentFollowscounter.incr() - recentFollowsStore.get(user.id) - - case Some(user) if this.isModelTrainingData => - recentFollowscounter.incr() - isModelTrainingDataCounter.incr() - recentFollowsStore.get(user.id) - - case _ => Future.None - } - nonSeedSetFollowsFut.map { nonSeedSetFollows => - Some( - SeedsetUtil.combineRecentFollowsWithWeightedSeedset( - seedSet, - nonSeedSetFollows.getOrElse(Nil) - ) - ) - } - } - } - - override def magicFanoutReasonHistory30Days: Future[MagicFanoutReasonHistory] = - history.map(history => MagicFanoutReasonHistory(history)) - - override val isNewSignup: Boolean = - pushContext.flatMap(_.isFromNewUserLoopProcessor).getOrElse(false) - - override lazy val resurrectionDate: Future[Option[String]] = - Future.value(customFSField.reactivationDate) - - override lazy val isResurrectedUser: Boolean = - customFSField.daysSinceReactivation.isDefined - - override lazy val timeSinceResurrection: Option[Duration] = - customFSField.daysSinceReactivation.map(Duration.fromDays) - - override lazy val appPermissions: Future[Option[AppPermission]] = - PushAppPermissionUtil.getAppPermission( - userId, - PushAppPermissionUtil.AddressBookPermissionKey, - deviceInfo, - appPermissionStore) - - override lazy val inlineActionHistory: Future[Seq[(Long, String)]] = { - inlineActionHistoryStore - .get(userId).map { - case Some(sortedInlineActionHistory) => sortedInlineActionHistory - case _ => Seq.empty - } - } - - lazy val isOpenAppExperimentUser: Future[Boolean] = - openAppUserStore.get(userId).map(_.contains(true)) - - override lazy val openedPushByHourAggregated: Future[Option[Map[Int, Int]]] = - openedPushByHourAggregatedStore.get(userId) - - override lazy val places: Future[Seq[Place]] = { - geoduckStoreV2 - .get(targetId) - .map(_.flatMap(_.places)) - .map { - case Some(placeSeq) if placeSeq.nonEmpty => - placeFoundStat.add(placeSeq.size) - placeSeq - case _ => - placesNotFound.incr() - Seq.empty - } - } - - override val isBlueVerified: Future[Option[Boolean]] = - Future.value(userOpt.flatMap(_.safety.flatMap(_.isBlueVerified))) - - override val isVerified: Future[Option[Boolean]] = - Future.value(userOpt.flatMap(_.safety.map(_.verified))) - - override lazy val isSuperFollowCreator: Future[Option[Boolean]] = - superFollowEligibilityUserStore.get(targetId) - } - } - } - } - - /** - * Provide general way to add needed FS for target user, and package them in CustomFSFields. - * Custom Fields is a powerful feature that allows Feature Switch library users to define and - * match against any arbitrary fields. - **/ - private def getCustomFSFields( - userId: Long, - userOpt: Option[User], - deviceInfo: Option[DeviceInfo], - userForPushTargetingInfo: Option[UserForPushTargeting], - notifHistory: History, - requestSource: Option[RequestSource] - ): Future[CustomFSFields] = { - val reactivationDateFutOpt: Future[Option[String]] = resurrectedUserStore.get(userId) - val reactivationTimeFutOpt: Future[Option[Time]] = - reactivationDateFutOpt.map(_.map(dateStr => DateUtil.dateStrToTime(dateStr))) - - val isReactivatedUserFut: Future[Boolean] = reactivationTimeFutOpt.map { timeOpt => - timeOpt - .exists { time => Time.now - time < 30.days } - } - - val daysSinceReactivationFut: Future[Option[Int]] = - reactivationTimeFutOpt.map(_.map(time => Time.now.since(time).inDays)) - - val daysSinceSignup: Int = (Time.now - SnowflakeUtil.timeFromId(userId)).inDays - if (daysSinceSignup < 14) newSignUpUserStats.incr() - - val targetAgeInYears = userOpt.flatMap(_.extendedProfile.flatMap(_.ageInYears)) - - val lastLoginFut: Future[Option[Long]] = - userHTLLastVisitStore.get(userId).map { - case Some(lastHTLVisitTimes) => - val latestHTLVisitTime = lastHTLVisitTimes.max - userForPushTargetingInfo.flatMap( - _.lastActiveOnAppTimestamp - .map(_.max(latestHTLVisitTime)).orElse(Some(latestHTLVisitTime))) - case None => - userForPushTargetingInfo.flatMap(_.lastActiveOnAppTimestamp) - } - - val daysSinceLoginFut = lastLoginFut.map { - _.map { lastLoginTimestamp => - val timeSinceLogin = Time.now - Time.fromMilliseconds(lastLoginTimestamp) - if (timeSinceLogin.inDays > 21) { - dormantUserCount.incr() - } - timeSinceLogin.inDays - } - } - - /* Could add more custom FS here */ - val userNSFWInfoFut: Future[Option[NsfwInfo]] = - nsfwConsumerStore - .get(userId).map(_.map(nsfwUserSegmentation => NsfwInfo(nsfwUserSegmentation))) - - val userStateFut: Future[Option[String]] = userFeatureStore.get(userId).map { userFeaturesOpt => - userFeaturesOpt.flatMap { uFeats => - uFeats.userState.map(uState => uState.name) - } - } - - val mrUserStateFut: Future[Option[String]] = - mrUserStateStore.get(userId).map { mrUserStateOpt => - mrUserStateOpt.flatMap { mrUserState => - mrUserState.userState.map(_.name) - } - } - - Future - .join( - reactivationDateFutOpt, - isReactivatedUserFut, - userStateFut, - mrUserStateFut, - daysSinceLoginFut, - daysSinceReactivationFut, - userNSFWInfoFut - ).map { - case ( - reactivationDate, - isReactivatedUser, - userState, - mrUserState, - daysSinceLogin, - daysSinceReactivation, - userNSFWInfo) => - val numDaysReceivedPushInLast30Days: Int = - notifHistory.history.keys.map(_.inDays).toSet.size - - NsfwPersonalizationUtil.computeNsfwUserStats(userNSFWInfo) - - CustomFSFields( - isReactivatedUser, - daysSinceSignup, - numDaysReceivedPushInLast30Days, - daysSinceLogin, - daysSinceReactivation, - userOpt, - userState, - mrUserState, - reactivationDate, - requestSource.map(_.name), - targetAgeInYears, - userNSFWInfo, - deviceInfo - ) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/RFPHTargetPredicateGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/RFPHTargetPredicateGenerator.scala deleted file mode 100644 index 745e061fb..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/RFPHTargetPredicateGenerator.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.frigate.pushservice.target - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.predicate.TargetPromptFeedbackFatiguePredicate -import com.twitter.frigate.common.predicate.TargetUserPredicates -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushConstants -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.predicate.TargetNtabCaretClickFatiguePredicate -import com.twitter.frigate.pushservice.predicate.TargetPredicates -import com.twitter.hermit.predicate.NamedPredicate - -class RFPHTargetPredicateGenerator(implicit statsReceiver: StatsReceiver) { - val predicates: List[NamedPredicate[Target]] = List( - TargetPredicates.magicRecsMinDurationSinceSent(), - TargetPredicates.targetHTLVisitPredicate(), - TargetPredicates.inlineActionFatiguePredicate(), - TargetPredicates.targetFatiguePredicate(), - TargetUserPredicates.secondaryDormantAccountPredicate(), - TargetPredicates.targetValidMobileSDKPredicate, - TargetPredicates.targetPushBitEnabledPredicate, - TargetUserPredicates.targetUserExists(), - TargetPredicates.paramPredicate(PushFeatureSwitchParams.EnablePushRecommendationsParam), - TargetPromptFeedbackFatiguePredicate.responseNoPredicate( - PushParams.EnablePromptFeedbackFatigueResponseNoPredicate, - PushConstants.AcceptableTimeSinceLastNegativeResponse), - TargetPredicates.teamExceptedPredicate(TargetNtabCaretClickFatiguePredicate.apply()), - TargetPredicates.optoutProbPredicate(), - TargetPredicates.webNotifsHoldback() - ) -} - -object RFPHTargetPredicates { - def apply(implicit statsReceiver: StatsReceiver): List[NamedPredicate[Target]] = - new RFPHTargetPredicateGenerator().predicates -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/TargetAppPermissions.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/TargetAppPermissions.scala deleted file mode 100644 index c84af1286..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/TargetAppPermissions.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.frigate.pushservice.target - -import com.twitter.permissions_storage.thriftscala.AppPermission -import com.twitter.util.Future - -trait TargetAppPermissions { - - def appPermissions: Future[Option[AppPermission]] - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/TargetScoringDetails.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/TargetScoringDetails.scala deleted file mode 100644 index 2a9c26e8b..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/TargetScoringDetails.scala +++ /dev/null @@ -1,121 +0,0 @@ -package com.twitter.frigate.pushservice.target - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.FeatureMap -import com.twitter.frigate.common.base.TargetUser -import com.twitter.frigate.common.candidate.TargetABDecider -import com.twitter.frigate.common.candidate.TargetDecider -import com.twitter.frigate.common.candidate.UserDetails -import com.twitter.frigate.data_pipeline.thriftscala.UserHistoryValue -import com.twitter.frigate.dau_model.thriftscala.DauProbability -import com.twitter.frigate.scribe.thriftscala.SkipModelInfo -import com.twitter.hermit.stp.thriftscala.STPResult -import com.twitter.timelines.real_graph.v1.thriftscala.RealGraphFeatures -import com.twitter.util.Future -import com.twitter.util.Time -import com.twitter.frigate.pushservice.params.DeciderKey -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.params.WeightedOpenOrNtabClickModel -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.nrel.hydration.push.HydrationContext -import com.twitter.timelines.configapi.FSParam - -trait TargetScoringDetails { - tuc: TargetUser with TargetDecider with TargetABDecider with UserDetails => - - def stats: StatsReceiver - - /* - * We have 3 types of model training data: - * 1, skip ranker and model predicates - * controlled by decider frigate_notifier_quality_model_training_data - * the data distribution is same to the distribution in ranking - * 2, skip model predicates only - * controlled by decider skip_ml_model_predicate - * the data distribution is same to the distribution in filtering - * 3, no skip, only scribe features - * controlled by decider scribe_model_features - * the data distribution is same to production traffic - * The "miscellaneous" is used to store all misc information for selecting the data offline (e.g., ddg-bucket information) - * */ - lazy val skipModelInfo: Option[SkipModelInfo] = { - val trainingDataDeciderKey = DeciderKey.trainingDataDeciderKey.toString - val skipMlModelPredicateDeciderKey = DeciderKey.skipMlModelPredicateDeciderKey.toString - val scribeModelFeaturesDeciderKey = DeciderKey.scribeModelFeaturesDeciderKey.toString - val miscellaneous = None - - if (isDeciderEnabled(trainingDataDeciderKey, stats, useRandomRecipient = true)) { - Some( - SkipModelInfo( - skipPushOpenPredicate = Some(true), - skipPushRanker = Some(true), - miscellaneous = miscellaneous)) - } else if (isDeciderEnabled(skipMlModelPredicateDeciderKey, stats, useRandomRecipient = true)) { - Some( - SkipModelInfo( - skipPushOpenPredicate = Some(true), - skipPushRanker = Some(false), - miscellaneous = miscellaneous)) - } else if (isDeciderEnabled(scribeModelFeaturesDeciderKey, stats, useRandomRecipient = true)) { - Some(SkipModelInfo(noSkipButScribeFeatures = Some(true), miscellaneous = miscellaneous)) - } else { - Some(SkipModelInfo(miscellaneous = miscellaneous)) - } - } - - lazy val scribeFeatureForRequestScribe = - isDeciderEnabled( - DeciderKey.scribeModelFeaturesForRequestScribe.toString, - stats, - useRandomRecipient = true) - - lazy val rankingModelParam: Future[FSParam[WeightedOpenOrNtabClickModel.ModelNameType]] = - tuc.deviceInfo.map { deviceInfoOpt => - if (PushDeviceUtil.isPrimaryDeviceAndroid(deviceInfoOpt) && - tuc.params(PushParams.AndroidOnlyRankingExperimentParam)) { - PushFeatureSwitchParams.WeightedOpenOrNtabClickRankingModelForAndroidParam - } else { - PushFeatureSwitchParams.WeightedOpenOrNtabClickRankingModelParam - } - } - - lazy val filteringModelParam: FSParam[WeightedOpenOrNtabClickModel.ModelNameType] = - PushFeatureSwitchParams.WeightedOpenOrNtabClickFilteringModelParam - - def skipMlRanker: Boolean = skipModelInfo.exists(_.skipPushRanker.contains(true)) - - def skipModelPredicate: Boolean = skipModelInfo.exists(_.skipPushOpenPredicate.contains(true)) - - def noSkipButScribeFeatures: Boolean = - skipModelInfo.exists(_.noSkipButScribeFeatures.contains(true)) - - def isModelTrainingData: Boolean = skipMlRanker || skipModelPredicate || noSkipButScribeFeatures - - def scribeFeatureWithoutHydratingNewFeatures: Boolean = - isDeciderEnabled( - DeciderKey.scribeModelFeaturesWithoutHydratingNewFeaturesDeciderKey.toString, - stats, - useRandomRecipient = true - ) - - def targetHydrationContext: Future[HydrationContext] - - def featureMap: Future[FeatureMap] - - def dauProbability: Future[Option[DauProbability]] - - def labeledPushRecsHydrated: Future[Option[UserHistoryValue]] - - def onlineLabeledPushRecs: Future[Option[UserHistoryValue]] - - def realGraphFeatures: Future[Option[RealGraphFeatures]] - - def stpResult: Future[Option[STPResult]] - - def globalOptoutProbabilities: Seq[Future[Option[Double]]] - - def bucketOptoutProbability: Future[Option[Double]] - - val sendTime: Long = Time.now.inMillis -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/AdaptorUtils.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/AdaptorUtils.scala deleted file mode 100644 index dad918023..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/AdaptorUtils.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.FutureOps -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -object AdaptorUtils { - def getTweetyPieResults( - tweetIds: Set[Long], - tweetyPieStore: ReadableStore[Long, TweetyPieResult], - ): Future[Map[Long, Option[TweetyPieResult]]] = - FutureOps - .mapCollect(tweetyPieStore.multiGet(tweetIds)) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/AdhocStatsUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/AdhocStatsUtil.scala deleted file mode 100644 index 1d3ff461e..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/AdhocStatsUtil.scala +++ /dev/null @@ -1,104 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.common.base.CandidateResult -import com.twitter.frigate.common.base.Invalid -import com.twitter.frigate.common.base.OK -import com.twitter.frigate.common.base.Result -import com.twitter.frigate.common.base.TweetAuthor -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams.ListOfAdhocIdsForStatsTracking - -class AdhocStatsUtil(stats: StatsReceiver) { - - private def getAdhocIds(candidate: PushCandidate): Set[Long] = - candidate.target.params(ListOfAdhocIdsForStatsTracking) - - private def isAdhocTweetCandidate(candidate: PushCandidate): Boolean = { - candidate match { - case tweetCandidate: RawCandidate with TweetCandidate with TweetAuthor => - tweetCandidate.authorId.exists(id => getAdhocIds(candidate).contains(id)) - case _ => false - } - } - - def getCandidateSourceStats(hydratedCandidates: Seq[CandidateDetails[PushCandidate]]): Unit = { - hydratedCandidates.foreach { hydratedCandidate => - if (isAdhocTweetCandidate(hydratedCandidate.candidate)) { - stats.scope("candidate_source").counter(hydratedCandidate.source).incr() - } - } - } - - def getPreRankingFilterStats( - preRankingFilteredCandidates: Seq[CandidateResult[PushCandidate, Result]] - ): Unit = { - preRankingFilteredCandidates.foreach { filteredCandidate => - if (isAdhocTweetCandidate(filteredCandidate.candidate)) { - filteredCandidate.result match { - case Invalid(reason) => - stats.scope("preranking_filter").counter(reason.getOrElse("unknown_reason")).incr() - case _ => - } - } - } - } - - def getLightRankingStats(lightRankedCandidates: Seq[CandidateDetails[PushCandidate]]): Unit = { - lightRankedCandidates.foreach { lightRankedCandidate => - if (isAdhocTweetCandidate(lightRankedCandidate.candidate)) { - stats.scope("light_ranker").counter("passed_light_ranking").incr() - } - } - } - - def getRankingStats(rankedCandidates: Seq[CandidateDetails[PushCandidate]]): Unit = { - rankedCandidates.zipWithIndex.foreach { - case (rankedCandidate, index) => - val rankerStats = stats.scope("heavy_ranker") - if (isAdhocTweetCandidate(rankedCandidate.candidate)) { - rankerStats.counter("ranked_candidates").incr() - rankerStats.stat("rank").add(index.toFloat) - rankedCandidate.candidate.modelScores.map { modelScores => - modelScores.foreach { - case (modelName, score) => - // mutiply score by 1000 to not lose precision while converting to Float - val precisionScore = (score * 100000).toFloat - rankerStats.stat(modelName).add(precisionScore) - } - } - } - } - } - def getReRankingStats(rankedCandidates: Seq[CandidateDetails[PushCandidate]]): Unit = { - rankedCandidates.zipWithIndex.foreach { - case (rankedCandidate, index) => - val rankerStats = stats.scope("re_ranking") - if (isAdhocTweetCandidate(rankedCandidate.candidate)) { - rankerStats.counter("re_ranked_candidates").incr() - rankerStats.stat("re_rank").add(index.toFloat) - } - } - } - - def getTakeCandidateResultStats( - allTakeCandidateResults: Seq[CandidateResult[PushCandidate, Result]] - ): Unit = { - val takeStats = stats.scope("take_step") - allTakeCandidateResults.foreach { candidateResult => - if (isAdhocTweetCandidate(candidateResult.candidate)) { - candidateResult.result match { - case OK => - takeStats.counter("sent").incr() - case Invalid(reason) => - takeStats.counter(reason.getOrElse("unknown_reason")).incr() - case _ => - takeStats.counter("unknown_filter").incr() - } - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/Candidate2FrigateNotification.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/Candidate2FrigateNotification.scala deleted file mode 100644 index 6fa0bc288..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/Candidate2FrigateNotification.scala +++ /dev/null @@ -1,119 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.frigate.thriftscala.NotificationDisplayLocation - -object Candidate2FrigateNotification { - - def getFrigateNotification( - candidate: PushCandidate - )( - implicit statsReceiver: StatsReceiver - ): FrigateNotification = { - candidate match { - - case topicTweetCandidate: PushCandidate with BaseTopicTweetCandidate => - PushAdaptorUtil.getFrigateNotificationForTweet( - crt = topicTweetCandidate.commonRecType, - tweetId = topicTweetCandidate.tweetId, - scActions = Nil, - authorIdOpt = topicTweetCandidate.authorId, - pushCopyId = topicTweetCandidate.pushCopyId, - ntabCopyId = topicTweetCandidate.ntabCopyId, - simclusterId = None, - semanticCoreEntityIds = topicTweetCandidate.semanticCoreEntityId.map(List(_)), - candidateContent = topicTweetCandidate.content, - trendId = None - ) - - case trendTweetCandidate: PushCandidate with TrendTweetCandidate => - PushAdaptorUtil.getFrigateNotificationForTweet( - trendTweetCandidate.commonRecType, - trendTweetCandidate.tweetId, - Nil, - trendTweetCandidate.authorId, - trendTweetCandidate.pushCopyId, - trendTweetCandidate.ntabCopyId, - None, - None, - trendTweetCandidate.content, - Some(trendTweetCandidate.trendId) - ) - - case tripTweetCandidate: PushCandidate with OutOfNetworkTweetCandidate with TripCandidate => - PushAdaptorUtil.getFrigateNotificationForTweet( - crt = tripTweetCandidate.commonRecType, - tweetId = tripTweetCandidate.tweetId, - scActions = Nil, - authorIdOpt = tripTweetCandidate.authorId, - pushCopyId = tripTweetCandidate.pushCopyId, - ntabCopyId = tripTweetCandidate.ntabCopyId, - simclusterId = None, - semanticCoreEntityIds = None, - candidateContent = tripTweetCandidate.content, - trendId = None, - tweetTripDomain = tripTweetCandidate.tripDomain - ) - - case outOfNetworkTweetCandidate: PushCandidate with OutOfNetworkTweetCandidate => - PushAdaptorUtil.getFrigateNotificationForTweet( - crt = outOfNetworkTweetCandidate.commonRecType, - tweetId = outOfNetworkTweetCandidate.tweetId, - scActions = Nil, - authorIdOpt = outOfNetworkTweetCandidate.authorId, - pushCopyId = outOfNetworkTweetCandidate.pushCopyId, - ntabCopyId = outOfNetworkTweetCandidate.ntabCopyId, - simclusterId = None, - semanticCoreEntityIds = None, - candidateContent = outOfNetworkTweetCandidate.content, - trendId = None - ) - - case userCandidate: PushCandidate with UserCandidate with SocialContextActions => - PushAdaptorUtil.getFrigateNotificationForUser( - userCandidate.commonRecType, - userCandidate.userId, - userCandidate.socialContextActions, - userCandidate.pushCopyId, - userCandidate.ntabCopyId - ) - - case userCandidate: PushCandidate with UserCandidate => - PushAdaptorUtil.getFrigateNotificationForUser( - userCandidate.commonRecType, - userCandidate.userId, - Nil, - userCandidate.pushCopyId, - userCandidate.ntabCopyId - ) - - case tweetCandidate: PushCandidate with TweetCandidate with TweetDetails with SocialContextActions => - PushAdaptorUtil.getFrigateNotificationForTweetWithSocialContextActions( - tweetCandidate.commonRecType, - tweetCandidate.tweetId, - tweetCandidate.socialContextActions, - tweetCandidate.authorId, - tweetCandidate.pushCopyId, - tweetCandidate.ntabCopyId, - candidateContent = tweetCandidate.content, - semanticCoreEntityIds = None, - trendId = None - ) - case pushCandidate: PushCandidate => - FrigateNotification( - commonRecommendationType = pushCandidate.commonRecType, - notificationDisplayLocation = NotificationDisplayLocation.PushToMobileDevice, - pushCopyId = pushCandidate.pushCopyId, - ntabCopyId = pushCandidate.ntabCopyId - ) - - case _ => - statsReceiver - .scope(s"${candidate.commonRecType}").counter("frigate_notification_error").incr() - throw new IllegalStateException("Incorrect candidate type when create FrigateNotification") - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CandidateHydrationUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CandidateHydrationUtil.scala deleted file mode 100644 index 8b737fe67..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CandidateHydrationUtil.scala +++ /dev/null @@ -1,439 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.channels.common.thriftscala.ApiList -import com.twitter.escherbird.common.thriftscala.Domains -import com.twitter.escherbird.metadata.thriftscala.EntityMegadata -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext -import com.twitter.frigate.magic_events.thriftscala.FanoutEvent -import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason -import com.twitter.frigate.magic_events.thriftscala.TargetID -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model._ -import com.twitter.frigate.pushservice.model.FanoutReasonEntities -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.store.EventRequest -import com.twitter.frigate.pushservice.store.UttEntityHydrationStore -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.hermit.predicate.socialgraph.RelationEdge -import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery -import com.twitter.interests.thriftscala.UserInterests -import com.twitter.livevideo.timeline.domain.v2.{Event => LiveEvent} -import com.twitter.simclusters_v2.thriftscala.SimClustersInferredEntities -import com.twitter.storehaus.FutureOps -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.client.UserId -import com.twitter.ubs.thriftscala.AudioSpace -import com.twitter.util.Future - -object CandidateHydrationUtil { - - def getAuthorIdFromTweetCandidate(tweetCandidate: TweetCandidate): Option[Long] = { - tweetCandidate match { - case candidate: TweetCandidate with TweetAuthor => - candidate.authorId - case _ => None - } - } - - private def getCandidateAuthorFromUserMap( - tweetCandidate: TweetCandidate, - userMap: Map[Long, User] - ): Option[User] = { - getAuthorIdFromTweetCandidate(tweetCandidate) match { - case Some(id) => - userMap.get(id) - case _ => - None - } - } - - private def getRelationshipMapForInNetworkCandidate( - candidate: RawCandidate with TweetAuthor, - relationshipMap: Map[RelationEdge, Boolean] - ): Map[RelationEdge, Boolean] = { - val relationEdges = - RelationshipUtil.getPreCandidateRelationshipsForInNetworkTweets(candidate).toSet - relationEdges.map { relationEdge => - (relationEdge, relationshipMap(relationEdge)) - }.toMap - } - - private def getTweetCandidateSocialContextUsers( - candidate: RawCandidate with SocialContextActions, - userMap: Map[Long, User] - ): Map[Long, Option[User]] = { - candidate.socialContextUserIds.map { userId => userId -> userMap.get(userId) }.toMap - } - - type TweetWithSocialContextTraits = TweetCandidate with TweetDetails with SocialContextActions - - def getHydratedCandidateForTweetRetweet( - candidate: RawCandidate with TweetWithSocialContextTraits, - userMap: Map[Long, User], - copyIds: CopyIds - )( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer - ): TweetRetweetPushCandidate = { - new TweetRetweetPushCandidate( - candidate = candidate, - socialContextUserMap = Future.value(getTweetCandidateSocialContextUsers(candidate, userMap)), - author = Future.value(getCandidateAuthorFromUserMap(candidate, userMap)), - copyIds: CopyIds - ) - } - - def getHydratedCandidateForTweetFavorite( - candidate: RawCandidate with TweetWithSocialContextTraits, - userMap: Map[Long, User], - copyIds: CopyIds - )( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer - ): TweetFavoritePushCandidate = { - new TweetFavoritePushCandidate( - candidate = candidate, - socialContextUserMap = Future.value(getTweetCandidateSocialContextUsers(candidate, userMap)), - author = Future.value(getCandidateAuthorFromUserMap(candidate, userMap)), - copyIds = copyIds - ) - } - - def getHydratedCandidateForF1FirstDegreeTweet( - candidate: RawCandidate with F1FirstDegree, - userMap: Map[Long, User], - relationshipMap: Map[RelationEdge, Boolean], - copyIds: CopyIds - )( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer - ): F1TweetPushCandidate = { - new F1TweetPushCandidate( - candidate = candidate, - author = Future.value(getCandidateAuthorFromUserMap(candidate, userMap)), - socialGraphServiceResultMap = - getRelationshipMapForInNetworkCandidate(candidate, relationshipMap), - copyIds = copyIds - ) - } - def getHydratedTopicProofTweetCandidate( - candidate: RawCandidate with TopicProofTweetCandidate, - userMap: Map[Long, User], - copyIds: CopyIds - )( - implicit stats: StatsReceiver, - pushMLModelScorer: PushMLModelScorer - ): TopicProofTweetPushCandidate = - new TopicProofTweetPushCandidate( - candidate, - getCandidateAuthorFromUserMap(candidate, userMap), - copyIds - ) - - def getHydratedSubscribedSearchTweetCandidate( - candidate: RawCandidate with SubscribedSearchTweetCandidate, - userMap: Map[Long, User], - copyIds: CopyIds - )( - implicit stats: StatsReceiver, - pushMLModelScorer: PushMLModelScorer - ): SubscribedSearchTweetPushCandidate = - new SubscribedSearchTweetPushCandidate( - candidate, - getCandidateAuthorFromUserMap(candidate, userMap), - copyIds) - - def getHydratedListCandidate( - apiListStore: ReadableStore[Long, ApiList], - candidate: RawCandidate with ListPushCandidate, - copyIds: CopyIds - )( - implicit stats: StatsReceiver, - pushMLModelScorer: PushMLModelScorer - ): ListRecommendationPushCandidate = { - new ListRecommendationPushCandidate(apiListStore, candidate, copyIds) - } - - def getHydratedCandidateForOutOfNetworkTweetCandidate( - candidate: RawCandidate with OutOfNetworkTweetCandidate with TopicCandidate, - userMap: Map[Long, User], - copyIds: CopyIds - )( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer - ): OutOfNetworkTweetPushCandidate = { - new OutOfNetworkTweetPushCandidate( - candidate: RawCandidate with OutOfNetworkTweetCandidate with TopicCandidate, - author = Future.value(getCandidateAuthorFromUserMap(candidate, userMap)), - copyIds: CopyIds - ) - } - - def getHydratedCandidateForTripTweetCandidate( - candidate: RawCandidate with OutOfNetworkTweetCandidate with TripCandidate, - userMap: Map[Long, User], - copyIds: CopyIds - )( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer - ): TripTweetPushCandidate = { - new TripTweetPushCandidate( - candidate: RawCandidate with OutOfNetworkTweetCandidate with TripCandidate, - author = Future.value(getCandidateAuthorFromUserMap(candidate, userMap)), - copyIds: CopyIds - ) - } - - def getHydratedCandidateForDiscoverTwitterCandidate( - candidate: RawCandidate with DiscoverTwitterCandidate, - copyIds: CopyIds - )( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer - ): DiscoverTwitterPushCandidate = { - new DiscoverTwitterPushCandidate( - candidate = candidate, - copyIds = copyIds - ) - } - - /** - * /* - * This method can be reusable for hydrating event candidates - **/ - * @param candidate - * @param fanoutMetadataStore - * @param semanticCoreMegadataStore - * @return (hydratedEvent, hydratedFanoutEvent, hydratedSemanticEntityResults, hydratedSemanticCoreMegadata) - */ - private def hydrateMagicFanoutEventCandidate( - candidate: RawCandidate with MagicFanoutEventCandidate, - fanoutMetadataStore: ReadableStore[(Long, Long), FanoutEvent], - semanticCoreMegadataStore: ReadableStore[SemanticEntityForQuery, EntityMegadata] - ): Future[MagicFanoutEventHydratedInfo] = { - - val fanoutEventFut = fanoutMetadataStore.get((candidate.eventId, candidate.pushId)) - - val semanticEntityForQueries: Seq[SemanticEntityForQuery] = { - val semanticCoreEntityIdQueries = candidate.candidateMagicEventsReasons match { - case magicEventsReasons: Seq[MagicEventsReason] => - magicEventsReasons.map(_.reason).collect { - case TargetID.SemanticCoreID(scInterest) => - SemanticEntityForQuery(domainId = scInterest.domainId, entityId = scInterest.entityId) - } - case _ => Seq.empty - } - val eventEntityQuery = SemanticEntityForQuery( - domainId = Domains.EventsEntityService.value, - entityId = candidate.eventId) - semanticCoreEntityIdQueries :+ eventEntityQuery - } - - val semanticEntityResultsFut = FutureOps.mapCollect( - semanticCoreMegadataStore.multiGet(semanticEntityForQueries.toSet) - ) - - Future - .join(fanoutEventFut, semanticEntityResultsFut).map { - case (fanoutEvent, semanticEntityResults) => - MagicFanoutEventHydratedInfo( - fanoutEvent, - semanticEntityResults - ) - case _ => - throw new IllegalArgumentException( - "event candidate hydration errors" + candidate.frigateNotification.toString) - } - } - - def getHydratedCandidateForMagicFanoutNewsEvent( - candidate: RawCandidate with MagicFanoutNewsEventCandidate, - copyIds: CopyIds, - lexServiceStore: ReadableStore[EventRequest, LiveEvent], - fanoutMetadataStore: ReadableStore[(Long, Long), FanoutEvent], - semanticCoreMegadataStore: ReadableStore[SemanticEntityForQuery, EntityMegadata], - simClusterToEntityStore: ReadableStore[Int, SimClustersInferredEntities], - interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests], - uttEntityHydrationStore: UttEntityHydrationStore - )( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer - ): Future[MagicFanoutNewsEventPushCandidate] = { - val magicFanoutEventHydratedInfoFut = hydrateMagicFanoutEventCandidate( - candidate, - fanoutMetadataStore, - semanticCoreMegadataStore - ) - - lazy val simClusterToEntityMappingFut: Future[Map[Int, Option[SimClustersInferredEntities]]] = - Future.collect { - simClusterToEntityStore.multiGet( - FanoutReasonEntities - .from(candidate.candidateMagicEventsReasons.map(_.reason)).simclusterIds.map( - _.clusterId) - ) - } - - Future - .join( - magicFanoutEventHydratedInfoFut, - simClusterToEntityMappingFut - ).map { - case (magicFanoutEventHydratedInfo, simClusterToEntityMapping) => - new MagicFanoutNewsEventPushCandidate( - candidate = candidate, - copyIds = copyIds, - fanoutEvent = magicFanoutEventHydratedInfo.fanoutEvent, - semanticEntityResults = magicFanoutEventHydratedInfo.semanticEntityResults, - simClusterToEntities = simClusterToEntityMapping, - lexServiceStore = lexServiceStore, - interestsLookupStore = interestsLookupStore, - uttEntityHydrationStore = uttEntityHydrationStore - ) - } - } - - def getHydratedCandidateForMagicFanoutSportsEvent( - candidate: RawCandidate - with MagicFanoutSportsEventCandidate - with MagicFanoutSportsScoreInformation, - copyIds: CopyIds, - lexServiceStore: ReadableStore[EventRequest, LiveEvent], - fanoutMetadataStore: ReadableStore[(Long, Long), FanoutEvent], - semanticCoreMegadataStore: ReadableStore[SemanticEntityForQuery, EntityMegadata], - interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests], - uttEntityHydrationStore: UttEntityHydrationStore - )( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer - ): Future[MagicFanoutSportsPushCandidate] = { - val magicFanoutEventHydratedInfoFut = hydrateMagicFanoutEventCandidate( - candidate, - fanoutMetadataStore, - semanticCoreMegadataStore - ) - - magicFanoutEventHydratedInfoFut.map { magicFanoutEventHydratedInfo => - new MagicFanoutSportsPushCandidate( - candidate = candidate, - copyIds = copyIds, - fanoutEvent = magicFanoutEventHydratedInfo.fanoutEvent, - semanticEntityResults = magicFanoutEventHydratedInfo.semanticEntityResults, - simClusterToEntities = Map.empty, - lexServiceStore = lexServiceStore, - interestsLookupStore = interestsLookupStore, - uttEntityHydrationStore = uttEntityHydrationStore - ) - } - } - - def getHydratedCandidateForMagicFanoutProductLaunch( - candidate: RawCandidate with MagicFanoutProductLaunchCandidate, - copyIds: CopyIds - )( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer - ): Future[MagicFanoutProductLaunchPushCandidate] = - Future.value(new MagicFanoutProductLaunchPushCandidate(candidate, copyIds)) - - def getHydratedCandidateForMagicFanoutCreatorEvent( - candidate: RawCandidate with MagicFanoutCreatorEventCandidate, - safeUserStore: ReadableStore[Long, User], - copyIds: CopyIds, - creatorTweetCountStore: ReadableStore[UserId, Int] - )( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer - ): Future[MagicFanoutCreatorEventPushCandidate] = { - safeUserStore.get(candidate.creatorId).map { hydratedCreatorUser => - new MagicFanoutCreatorEventPushCandidate( - candidate, - hydratedCreatorUser, - copyIds, - creatorTweetCountStore) - } - } - - def getHydratedCandidateForScheduledSpaceSubscriber( - candidate: RawCandidate with ScheduledSpaceSubscriberCandidate, - safeUserStore: ReadableStore[Long, User], - copyIds: CopyIds, - audioSpaceStore: ReadableStore[String, AudioSpace] - )( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer - ): Future[ScheduledSpaceSubscriberPushCandidate] = { - - candidate.hostId match { - case Some(spaceHostId) => - safeUserStore.get(spaceHostId).map { hydratedHost => - new ScheduledSpaceSubscriberPushCandidate( - candidate = candidate, - hostUser = hydratedHost, - copyIds = copyIds, - audioSpaceStore = audioSpaceStore - ) - } - case _ => - Future.exception( - new IllegalStateException( - "Missing Space Host Id for hydrating ScheduledSpaceSubscriberCandidate")) - } - } - - def getHydratedCandidateForScheduledSpaceSpeaker( - candidate: RawCandidate with ScheduledSpaceSpeakerCandidate, - safeUserStore: ReadableStore[Long, User], - copyIds: CopyIds, - audioSpaceStore: ReadableStore[String, AudioSpace] - )( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer - ): Future[ScheduledSpaceSpeakerPushCandidate] = { - - candidate.hostId match { - case Some(spaceHostId) => - safeUserStore.get(spaceHostId).map { hydratedHost => - new ScheduledSpaceSpeakerPushCandidate( - candidate = candidate, - hostUser = hydratedHost, - copyIds = copyIds, - audioSpaceStore = audioSpaceStore - ) - } - case _ => - Future.exception( - new RuntimeException( - "Missing Space Host Id for hydrating ScheduledSpaceSpeakerCandidate")) - } - } - - def getHydratedCandidateForTopTweetImpressionsCandidate( - candidate: RawCandidate with TopTweetImpressionsCandidate, - copyIds: CopyIds - )( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer - ): TopTweetImpressionsPushCandidate = { - new TopTweetImpressionsPushCandidate( - candidate = candidate, - copyIds = copyIds - ) - } - - def isNsfwAccount(user: User, nsfwTokens: Seq[String]): Boolean = { - def hasNsfwToken(str: String): Boolean = nsfwTokens.exists(str.toLowerCase().contains(_)) - - val name = user.profile.map(_.name).getOrElse("") - val screenName = user.profile.map(_.screenName).getOrElse("") - val location = user.profile.map(_.location).getOrElse("") - val description = user.profile.map(_.description).getOrElse("") - val hasNsfwFlag = - user.safety.map(safety => safety.nsfwUser || safety.nsfwAdmin).getOrElse(false) - hasNsfwToken(name) || hasNsfwToken(screenName) || hasNsfwToken(location) || hasNsfwToken( - description) || hasNsfwFlag - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CandidateUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CandidateUtil.scala deleted file mode 100644 index a4802da45..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CandidateUtil.scala +++ /dev/null @@ -1,138 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.contentrecommender.thriftscala.MetricTag -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.OutOfNetworkTweetCandidate -import com.twitter.frigate.common.base.SocialContextAction -import com.twitter.frigate.common.base.SocialContextActions -import com.twitter.frigate.common.base.TargetInfo -import com.twitter.frigate.common.base.TargetUser -import com.twitter.frigate.common.base.TopicProofTweetCandidate -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.candidate.TargetABDecider -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.params.CrtGroupEnum -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.CommonRecommendationType.TripGeoTweet -import com.twitter.frigate.thriftscala.CommonRecommendationType.TripHqTweet -import com.twitter.frigate.thriftscala.{SocialContextAction => TSocialContextAction} -import com.twitter.util.Future - -object CandidateUtil { - private val mrTwistlyMetricTags = - Seq(MetricTag.PushOpenOrNtabClick, MetricTag.RequestHealthFilterPushOpenBasedTweetEmbedding) - - def getSocialContextActionsFromCandidate(candidate: RawCandidate): Seq[TSocialContextAction] = { - candidate match { - case candidateWithSocialContex: RawCandidate with SocialContextActions => - candidateWithSocialContex.socialContextActions.map { scAction => - TSocialContextAction( - scAction.userId, - scAction.timestampInMillis, - scAction.tweetId - ) - } - case _ => Seq.empty - } - } - - /** - * Ranking Social Context based on the Real Graph weight - * @param socialContextActions Sequence of Social Context Actions - * @param seedsWithWeight Real Graph map consisting of User ID as key and RG weight as the value - * @param defaultToRecency Boolean to represent if we should use the timestamp of the SC to rank - * @return Returns the ranked sequence of SC Actions - */ - def getRankedSocialContext( - socialContextActions: Seq[SocialContextAction], - seedsWithWeight: Future[Option[Map[Long, Double]]], - defaultToRecency: Boolean - ): Future[Seq[SocialContextAction]] = { - seedsWithWeight.map { - case Some(followingsMap) => - socialContextActions.sortBy { action => -followingsMap.getOrElse(action.userId, 0.0) } - case _ => - if (defaultToRecency) socialContextActions.sortBy(-_.timestampInMillis) - else socialContextActions - } - } - - def shouldApplyHealthQualityFiltersForPrerankingPredicates( - candidate: TweetAuthorDetails with TargetInfo[TargetUser with TargetABDecider] - )( - implicit stats: StatsReceiver - ): Future[Boolean] = { - candidate.tweetAuthor.map { - case Some(user) => - val numFollowers: Double = user.counts.map(_.followers.toDouble).getOrElse(0.0) - numFollowers < candidate.target - .params(PushFeatureSwitchParams.NumFollowerThresholdForHealthAndQualityFiltersPreranking) - case _ => true - } - } - - def shouldApplyHealthQualityFilters( - candidate: PushCandidate - )( - implicit stats: StatsReceiver - ): Boolean = { - val numFollowers = - candidate.numericFeatures.getOrElse("RecTweetAuthor.User.ActiveFollowers", 0.0) - numFollowers < candidate.target - .params(PushFeatureSwitchParams.NumFollowerThresholdForHealthAndQualityFilters) - } - - def useAggressiveHealthThresholds(cand: PushCandidate): Boolean = - isMrTwistlyCandidate(cand) || - (cand.commonRecType == CommonRecommendationType.GeoPopTweet && cand.target.params( - PushFeatureSwitchParams.PopGeoTweetEnableAggressiveThresholds)) - - def isMrTwistlyCandidate(cand: PushCandidate): Boolean = - cand match { - case oonCandidate: PushCandidate with OutOfNetworkTweetCandidate => - oonCandidate.tagsCR - .getOrElse(Seq.empty).intersect(mrTwistlyMetricTags).nonEmpty && oonCandidate.tagsCR - .map(_.toSet.size).getOrElse(0) == 1 - case oonCandidate: PushCandidate with TopicProofTweetCandidate - if cand.target.params(PushFeatureSwitchParams.EnableHealthFiltersForTopicProofTweet) => - oonCandidate.tagsCR - .getOrElse(Seq.empty).intersect(mrTwistlyMetricTags).nonEmpty && oonCandidate.tagsCR - .map(_.toSet.size).getOrElse(0) == 1 - case _ => false - } - - def getTagsCRCount(cand: PushCandidate): Double = - cand match { - case oonCandidate: PushCandidate with OutOfNetworkTweetCandidate => - oonCandidate.tagsCR.map(_.toSet.size).getOrElse(0).toDouble - case oonCandidate: PushCandidate with TopicProofTweetCandidate - if cand.target.params(PushFeatureSwitchParams.EnableHealthFiltersForTopicProofTweet) => - oonCandidate.tagsCR.map(_.toSet.size).getOrElse(0).toDouble - case _ => 0.0 - } - - def isRelatedToMrTwistlyCandidate(cand: PushCandidate): Boolean = - cand match { - case oonCandidate: PushCandidate with OutOfNetworkTweetCandidate => - oonCandidate.tagsCR.getOrElse(Seq.empty).intersect(mrTwistlyMetricTags).nonEmpty - case oonCandidate: PushCandidate with TopicProofTweetCandidate - if cand.target.params(PushFeatureSwitchParams.EnableHealthFiltersForTopicProofTweet) => - oonCandidate.tagsCR.getOrElse(Seq.empty).intersect(mrTwistlyMetricTags).nonEmpty - case _ => false - } - - def getCrtGroup(commonRecType: CommonRecommendationType): CrtGroupEnum.Value = { - commonRecType match { - case crt if RecTypes.twistlyTweets(crt) => CrtGroupEnum.Twistly - case crt if RecTypes.frsTypes(crt) => CrtGroupEnum.Frs - case crt if RecTypes.f1RecTypes(crt) => CrtGroupEnum.F1 - case crt if crt == TripGeoTweet || crt == TripHqTweet => CrtGroupEnum.Trip - case crt if RecTypes.TopicTweetTypes(crt) => CrtGroupEnum.Topic - case crt if RecTypes.isGeoPopTweetType(crt) => CrtGroupEnum.GeoPop - case _ => CrtGroupEnum.Other - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CopyUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CopyUtil.scala deleted file mode 100644 index 95208c35e..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CopyUtil.scala +++ /dev/null @@ -1,448 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushConstants -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} -import com.twitter.ibis2.lib.util.JsonMarshal -import com.twitter.util.Future -import com.twitter.util.Time - -object CopyUtil { - - /** - * Get a list of history feature copy alone with metadata in the look back period, the metadata - * can be used to calculate number of copy pushed after the current feature copy - * @param candidate the candidate to be pushed to the user - * @return Future[Seq((..,))], which is a seq of the history FEATURE copy along with - * metadata within the look back period. In the tuple, the 4 elements represents: - * 1. Timestamp of the past feature copy - * 2. Option[Seq()] of copy feature names of the past copy - * 3. Index of the particular feature copy in look back history if normal copy presents - */ - private def getPastCopyFeaturesList( - candidate: PushCandidate - ): Future[Seq[(Time, Option[Seq[String]], Int)]] = { - val target = candidate.target - - target.history.map { targetHistory => - val historyLookbackDuration = target.params(FS.CopyFeaturesHistoryLookbackDuration) - val notificationHistoryInLookbackDuration = targetHistory.sortedHistory - .takeWhile { - case (notifTimestamp, _) => historyLookbackDuration.ago < notifTimestamp - } - notificationHistoryInLookbackDuration.zipWithIndex - .filter { - case ((_, notification), _) => - notification.copyFeatures match { - case Some(copyFeatures) => copyFeatures.nonEmpty - case _ => false - } - } - .collect { - case ((timestamp, notification), notificationIndex) => - (timestamp, notification.copyFeatures, notificationIndex) - } - } - } - - private def getPastCopyFeaturesListForF1( - candidate: PushCandidate - ): Future[Seq[(Time, Option[Seq[String]], Int)]] = { - val target = candidate.target - target.history.map { targetHistory => - val historyLookbackDuration = target.params(FS.CopyFeaturesHistoryLookbackDuration) - val notificationHistoryInLookbackDuration = targetHistory.sortedHistory - .takeWhile { - case (notifTimestamp, _) => historyLookbackDuration.ago < notifTimestamp - } - notificationHistoryInLookbackDuration.zipWithIndex - .filter { - case ((_, notification), _) => - notification.copyFeatures match { - case Some(copyFeatures) => - RecTypes.isF1Type(notification.commonRecommendationType) && copyFeatures.nonEmpty - case _ => false - } - } - .collect { - case ((timestamp, notification), notificationIndex) => - (timestamp, notification.copyFeatures, notificationIndex) - } - } - } - - private def getPastCopyFeaturesListForOON( - candidate: PushCandidate - ): Future[Seq[(Time, Option[Seq[String]], Int)]] = { - val target = candidate.target - target.history.map { targetHistory => - val historyLookbackDuration = target.params(FS.CopyFeaturesHistoryLookbackDuration) - val notificationHistoryInLookbackDuration = targetHistory.sortedHistory - .takeWhile { - case (notifTimestamp, _) => historyLookbackDuration.ago < notifTimestamp - } - notificationHistoryInLookbackDuration.zipWithIndex - .filter { - case ((_, notification), _) => - notification.copyFeatures match { - case Some(copyFeatures) => - !RecTypes.isF1Type(notification.commonRecommendationType) && copyFeatures.nonEmpty - - case _ => false - } - } - .collect { - case ((timestamp, notification), notificationIndex) => - (timestamp, notification.copyFeatures, notificationIndex) - } - } - } - private def getEmojiFeaturesMap( - candidate: PushCandidate, - copyFeatureHistory: Seq[(Time, Option[Seq[String]], Int)], - lastHTLVisitTimestamp: Option[Long], - stats: StatsReceiver - ): Map[String, String] = { - val (emojiFatigueDuration, emojiFatigueNumOfPushes) = { - if (RecTypes.isF1Type(candidate.commonRecType)) { - ( - candidate.target.params(FS.F1EmojiCopyFatigueDuration), - candidate.target.params(FS.F1EmojiCopyNumOfPushesFatigue)) - } else { - ( - candidate.target.params(FS.OonEmojiCopyFatigueDuration), - candidate.target.params(FS.OonEmojiCopyNumOfPushesFatigue)) - } - } - - val scopedStats = stats - .scope("getEmojiFeaturesMap").scope(candidate.commonRecType.toString).scope( - emojiFatigueDuration.toString) - val addedEmojiCopyFeature = scopedStats.counter("added_emoji") - val fatiguedEmojiCopyFeature = scopedStats.counter("no_emoji") - - val copyFeatureType = PushConstants.EmojiFeatureNameForIbis2ModelValues - - val durationFatigueCarryFunc = () => - isUnderDurationFatigue(copyFeatureHistory, copyFeatureType, emojiFatigueDuration) - - val enableHTLBasedFatigueBasicRule = candidate.target.params(FS.EnableHTLBasedFatigueBasicRule) - val minDuration = candidate.target.params(FS.MinFatigueDurationSinceLastHTLVisit) - val lastHTLVisitBasedNonFatigueWindow = - candidate.target.params(FS.LastHTLVisitBasedNonFatigueWindow) - val htlBasedCopyFatigueCarryFunc = () => - isUnderHTLBasedFatigue(lastHTLVisitTimestamp, minDuration, lastHTLVisitBasedNonFatigueWindow) - - val isUnderFatigue = getIsUnderFatigue( - Seq( - (durationFatigueCarryFunc, true), - (htlBasedCopyFatigueCarryFunc, enableHTLBasedFatigueBasicRule), - ), - scopedStats - ) - - if (!isUnderFatigue) { - addedEmojiCopyFeature.incr() - Map(PushConstants.EmojiFeatureNameForIbis2ModelValues -> "true") - } else { - fatiguedEmojiCopyFeature.incr() - Map.empty[String, String] - } - } - - private def getTargetFeaturesMap( - candidate: PushCandidate, - copyFeatureHistory: Seq[(Time, Option[Seq[String]], Int)], - lastHTLVisitTimestamp: Option[Long], - stats: StatsReceiver - ): Map[String, String] = { - val targetFatigueDuration = { - if (RecTypes.isF1Type(candidate.commonRecType)) { - candidate.target.params(FS.F1TargetCopyFatigueDuration) - } else { - candidate.target.params(FS.OonTargetCopyFatigueDuration) - } - } - - val scopedStats = stats - .scope("getTargetFeaturesMap").scope(candidate.commonRecType.toString).scope( - targetFatigueDuration.toString) - val addedTargetCopyFeature = scopedStats.counter("added_target") - val fatiguedTargetCopyFeature = scopedStats.counter("no_target") - - val featureCopyType = PushConstants.TargetFeatureNameForIbis2ModelValues - val durationFatigueCarryFunc = () => - isUnderDurationFatigue(copyFeatureHistory, featureCopyType, targetFatigueDuration) - - val enableHTLBasedFatigueBasicRule = candidate.target.params(FS.EnableHTLBasedFatigueBasicRule) - val minDuration = candidate.target.params(FS.MinFatigueDurationSinceLastHTLVisit) - val lastHTLVisitBasedNonFatigueWindow = - candidate.target.params(FS.LastHTLVisitBasedNonFatigueWindow) - val htlBasedCopyFatigueCarryFunc = () => - isUnderHTLBasedFatigue(lastHTLVisitTimestamp, minDuration, lastHTLVisitBasedNonFatigueWindow) - - val isUnderFatigue = getIsUnderFatigue( - Seq( - (durationFatigueCarryFunc, true), - (htlBasedCopyFatigueCarryFunc, enableHTLBasedFatigueBasicRule), - ), - scopedStats - ) - - if (!isUnderFatigue) { - addedTargetCopyFeature.incr() - Map(PushConstants.TargetFeatureNameForIbis2ModelValues -> "true") - } else { - - fatiguedTargetCopyFeature.incr() - Map.empty[String, String] - } - } - - type FatigueRuleFlag = Boolean - type FatigueRuleFunc = () => Boolean - - def getIsUnderFatigue( - fatigueRulesWithFlags: Seq[(FatigueRuleFunc, FatigueRuleFlag)], - statsReceiver: StatsReceiver, - ): Boolean = { - val defaultFatigue = true - val finalFatigueRes = - fatigueRulesWithFlags.zipWithIndex.foldLeft(defaultFatigue)( - (fatigueSoFar, fatigueRuleFuncWithFlagAndIndex) => { - val ((fatigueRuleFunc, flag), index) = fatigueRuleFuncWithFlagAndIndex - val funcScopedStats = statsReceiver.scope(s"fatigueFunction${index}") - if (flag) { - val shouldFatigueForTheRule = fatigueRuleFunc() - funcScopedStats.scope(s"eval_${shouldFatigueForTheRule}").counter().incr() - val f = fatigueSoFar && shouldFatigueForTheRule - f - } else { - fatigueSoFar - } - }) - statsReceiver.scope(s"final_fatigue_${finalFatigueRes}").counter().incr() - finalFatigueRes - } - - private def isUnderDurationFatigue( - copyFeatureHistory: Seq[(Time, Option[Seq[String]], Int)], - copyFeatureType: String, - fatigueDuration: com.twitter.util.Duration, - ): Boolean = { - copyFeatureHistory.exists { - case (notifTimestamp, Some(copyFeatures), _) if copyFeatures.contains(copyFeatureType) => - notifTimestamp > fatigueDuration.ago - case _ => false - } - } - - private def isUnderHTLBasedFatigue( - lastHTLVisitTimestamp: Option[Long], - minDurationSinceLastHTLVisit: com.twitter.util.Duration, - lastHTLVisitBasedNonFatigueWindow: com.twitter.util.Duration, - ): Boolean = { - val lastHTLVisit = lastHTLVisitTimestamp.map(t => Time.fromMilliseconds(t)).getOrElse(Time.now) - val first = Time.now < (lastHTLVisit + minDurationSinceLastHTLVisit) - val second = - Time.now > (lastHTLVisit + minDurationSinceLastHTLVisit + lastHTLVisitBasedNonFatigueWindow) - first || second - } - - def getOONCBasedFeature( - candidate: PushCandidate, - stats: StatsReceiver - ): Future[Map[String, String]] = { - val target = candidate.target - val metric = stats.scope("getOONCBasedFeature") - if (target.params(FS.EnableOONCBasedCopy)) { - candidate.mrWeightedOpenOrNtabClickRankingProbability.map { - case Some(score) if score >= target.params(FS.HighOONCThresholdForCopy) => - metric.counter("high_OONC").incr() - metric.counter(FS.HighOONCTweetFormat.toString).incr() - Map( - "whole_template" -> JsonMarshal.toJson( - Map( - target.params(FS.HighOONCTweetFormat).toString -> true - ))) - case Some(score) if score <= target.params(FS.LowOONCThresholdForCopy) => - metric.counter("low_OONC").incr() - metric.counter(FS.LowOONCThresholdForCopy.toString).incr() - Map( - "whole_template" -> JsonMarshal.toJson( - Map( - target.params(FS.LowOONCTweetFormat).toString -> true - ))) - case _ => - metric.counter("not_in_OONC_range").incr() - Map.empty[String, String] - } - } else { - Future.value(Map.empty[String, String]) - } - } - - def getCopyFeatures( - candidate: PushCandidate, - stats: StatsReceiver, - ): Future[Map[String, String]] = { - if (candidate.target.isLoggedOutUser) { - Future.value(Map.empty[String, String]) - } else { - val featureMaps = getCopyBodyFeatures(candidate, stats) - for { - titleFeat <- getCopyTitleFeatures(candidate, stats) - nsfwFeat <- getNsfwCopyFeatures(candidate, stats) - ooncBasedFeature <- getOONCBasedFeature(candidate, stats) - } yield { - titleFeat ++ featureMaps ++ nsfwFeat ++ ooncBasedFeature - } - } - } - - private def getCopyTitleFeatures( - candidate: PushCandidate, - stats: StatsReceiver - ): Future[Map[String, String]] = { - val scopedStats = stats.scope("CopyUtil").scope("getCopyTitleFeatures") - - val target = candidate.target - - if ((RecTypes.isSimClusterBasedType(candidate.commonRecType) && target.params( - FS.EnableCopyFeaturesForOon)) || (RecTypes.isF1Type(candidate.commonRecType) && target - .params(FS.EnableCopyFeaturesForF1))) { - - val enableTargetAndEmojiSplitFatigue = target.params(FS.EnableTargetAndEmojiSplitFatigue) - val isTargetF1Type = RecTypes.isF1Type(candidate.commonRecType) - - val copyFeatureHistoryFuture = if (enableTargetAndEmojiSplitFatigue && isTargetF1Type) { - getPastCopyFeaturesListForF1(candidate) - } else if (enableTargetAndEmojiSplitFatigue && !isTargetF1Type) { - getPastCopyFeaturesListForOON(candidate) - } else { - getPastCopyFeaturesList(candidate) - } - - Future - .join( - copyFeatureHistoryFuture, - target.lastHTLVisitTimestamp, - ).map { - case (copyFeatureHistory, lastHTLVisitTimestamp) => - val emojiFeatures = { - if ((RecTypes.isF1Type(candidate.commonRecType) && target.params( - FS.EnableEmojiInF1Copy)) - || RecTypes.isSimClusterBasedType(candidate.commonRecType) && target.params( - FS.EnableEmojiInOonCopy)) { - getEmojiFeaturesMap( - candidate, - copyFeatureHistory, - lastHTLVisitTimestamp, - scopedStats) - } else Map.empty[String, String] - } - - val targetFeatures = { - if ((RecTypes.isF1Type(candidate.commonRecType) && target.params( - FS.EnableTargetInF1Copy)) || (RecTypes.isSimClusterBasedType( - candidate.commonRecType) && target.params(FS.EnableTargetInOonCopy))) { - getTargetFeaturesMap( - candidate, - copyFeatureHistory, - lastHTLVisitTimestamp, - scopedStats) - } else Map.empty[String, String] - } - - val baseCopyFeaturesMap = - if (emojiFeatures.nonEmpty || targetFeatures.nonEmpty) - Map(PushConstants.EnableCopyFeaturesForIbis2ModelValues -> "true") - else Map.empty[String, String] - baseCopyFeaturesMap ++ emojiFeatures ++ targetFeatures - case _ => - Map.empty[String, String] - } - } else Future.value(Map.empty[String, String]) - } - - private def getCopyBodyTruncateFeatures( - candidate: PushCandidate, - ): Map[String, String] = { - if (candidate.target.params(FS.EnableIosCopyBodyTruncate)) { - Map("enable_body_truncate_ios" -> "true") - } else { - Map.empty[String, String] - } - } - - private def getNsfwCopyFeatures( - candidate: PushCandidate, - stats: StatsReceiver - ): Future[Map[String, String]] = { - val scopedStats = stats.scope("CopyUtil").scope("getNsfwCopyBodyFeatures") - val hasNsfwScoreF1Counter = scopedStats.counter("f1_has_nsfw_score") - val hasNsfwScoreOonCounter = scopedStats.counter("oon_has_nsfw_score") - val noNsfwScoreCounter = scopedStats.counter("no_nsfw_score") - val nsfwScoreF1 = scopedStats.stat("f1_nsfw_score") - val nsfwScoreOon = scopedStats.stat("oon_nsfw_score") - val isNsfwF1Counter = scopedStats.counter("is_f1_nsfw") - val isNsfwOonCounter = scopedStats.counter("is_oon_nsfw") - - val target = candidate.target - val nsfwScoreFut = if (target.params(FS.EnableNsfwCopy)) { - candidate.mrNsfwScore - } else Future.None - - nsfwScoreFut.map { - case Some(nsfwScore) => - if (RecTypes.isF1Type(candidate.commonRecType)) { - hasNsfwScoreF1Counter.incr() - nsfwScoreF1.add(nsfwScore.toFloat * 10000) - if (nsfwScore > target.params(FS.NsfwScoreThresholdForF1Copy)) { - isNsfwF1Counter.incr() - Map("is_f1_nsfw" -> "true") - } else { - Map.empty[String, String] - } - } else if (RecTypes.isOutOfNetworkTweetRecType(candidate.commonRecType)) { - nsfwScoreOon.add(nsfwScore.toFloat * 10000) - hasNsfwScoreOonCounter.incr() - if (nsfwScore > target.params(FS.NsfwScoreThresholdForOONCopy)) { - isNsfwOonCounter.incr() - Map("is_oon_nsfw" -> "true") - } else { - Map.empty[String, String] - } - } else { - Map.empty[String, String] - } - case _ => - noNsfwScoreCounter.incr() - Map.empty[String, String] - } - } - - private def getCopyBodyFeatures( - candidate: PushCandidate, - stats: StatsReceiver - ): Map[String, String] = { - val target = candidate.target - val scopedStats = stats.scope("CopyUtil").scope("getCopyBodyFeatures") - - val copyBodyFeatures = { - if (RecTypes.isF1Type(candidate.commonRecType) && target.params(FS.EnableF1CopyBody)) { - scopedStats.counter("f1BodyExpEnabled").incr() - Map(PushConstants.CopyBodyExpIbisModelValues -> "true") - } else if (RecTypes.isOutOfNetworkTweetRecType(candidate.commonRecType) && target.params( - FS.EnableOONCopyBody)) { - scopedStats.counter("oonBodyExpEnabled").incr() - Map(PushConstants.CopyBodyExpIbisModelValues -> "true") - } else - Map.empty[String, String] - } - val copyBodyTruncateFeatures = getCopyBodyTruncateFeatures(candidate) - copyBodyFeatures ++ copyBodyTruncateFeatures - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/EmailLandingPageExperimentUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/EmailLandingPageExperimentUtil.scala deleted file mode 100644 index 34d5b9bae..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/EmailLandingPageExperimentUtil.scala +++ /dev/null @@ -1,92 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.frigate.common.store.deviceinfo.DeviceInfo -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams.EnableRuxLandingPage -import com.twitter.frigate.pushservice.params.PushParams.EnableRuxLandingPageAndroidParam -import com.twitter.frigate.pushservice.params.PushParams.EnableRuxLandingPageIOSParam -import com.twitter.frigate.pushservice.params.PushParams.RuxLandingPageExperimentKeyAndroidParam -import com.twitter.frigate.pushservice.params.PushParams.RuxLandingPageExperimentKeyIOSParam -import com.twitter.frigate.pushservice.params.PushParams.ShowRuxLandingPageAsModalOnIOS -import com.twitter.rux.common.context.thriftscala.MagicRecsNTabTweet -import com.twitter.rux.common.context.thriftscala.MagicRecsPushTweet -import com.twitter.rux.common.context.thriftscala.RuxContext -import com.twitter.rux.common.context.thriftscala.Source -import com.twitter.rux.common.encode.RuxContextEncoder - -/** - * This class provides utility functions for email landing page for push - */ -object EmailLandingPageExperimentUtil { - val ruxCxtEncoder = new RuxContextEncoder() - - def getIbis2ModelValue( - deviceInfoOpt: Option[DeviceInfo], - target: Target, - tweetId: Long - ): Map[String, String] = { - val enable = enablePushEmailLanding(deviceInfoOpt, target) - if (enable) { - val ruxCxt = if (deviceInfoOpt.exists(_.isRuxLandingPageEligible)) { - val encodedCxt = getRuxContext(tweetId, target, deviceInfoOpt) - Map("rux_cxt" -> encodedCxt) - } else Map.empty[String, String] - val enableModal = if (showModalForIOS(deviceInfoOpt, target)) { - Map("enable_modal" -> "true") - } else Map.empty[String, String] - - Map("land_on_email_landing_page" -> "true") ++ ruxCxt ++ enableModal - } else Map.empty[String, String] - } - - def createNTabRuxLandingURI(screenName: String, tweetId: Long): String = { - val encodedCxt = - ruxCxtEncoder.encode(RuxContext(Some(Source.MagicRecsNTabTweet(MagicRecsNTabTweet(tweetId))))) - s"$screenName/status/${tweetId.toString}?cxt=$encodedCxt" - } - - private def getRuxContext( - tweetId: Long, - target: Target, - deviceInfoOpt: Option[DeviceInfo] - ): String = { - val isDeviceIOS = PushDeviceUtil.isPrimaryDeviceIOS(deviceInfoOpt) - val isDeviceAndroid = PushDeviceUtil.isPrimaryDeviceAndroid(deviceInfoOpt) - val keyOpt = if (isDeviceIOS) { - target.params(RuxLandingPageExperimentKeyIOSParam) - } else if (isDeviceAndroid) { - target.params(RuxLandingPageExperimentKeyAndroidParam) - } else None - val context = RuxContext(Some(Source.MagicRecsTweet(MagicRecsPushTweet(tweetId))), None, keyOpt) - ruxCxtEncoder.encode(context) - } - - private def enablePushEmailLanding( - deviceInfoOpt: Option[DeviceInfo], - target: Target - ): Boolean = - deviceInfoOpt.exists(deviceInfo => - if (deviceInfo.isEmailLandingPageEligible) { - val isRuxLandingPageEnabled = target.params(EnableRuxLandingPage) - isRuxLandingPageEnabled && isRuxLandingEnabledBasedOnDeviceInfo(deviceInfoOpt, target) - } else false) - - private def showModalForIOS(deviceInfoOpt: Option[DeviceInfo], target: Target): Boolean = { - deviceInfoOpt.exists { deviceInfo => - deviceInfo.isRuxLandingPageAsModalEligible && target.params(ShowRuxLandingPageAsModalOnIOS) - } - } - - private def isRuxLandingEnabledBasedOnDeviceInfo( - deviceInfoOpt: Option[DeviceInfo], - target: Target - ): Boolean = { - val isDeviceIOS = PushDeviceUtil.isPrimaryDeviceIOS(deviceInfoOpt) - val isDeviceAndroid = PushDeviceUtil.isPrimaryDeviceAndroid(deviceInfoOpt) - if (isDeviceIOS) { - target.params(EnableRuxLandingPageIOSParam) - } else if (isDeviceAndroid) { - target.params(EnableRuxLandingPageAndroidParam) - } else true - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/FunctionalUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/FunctionalUtil.scala deleted file mode 100644 index a2721873e..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/FunctionalUtil.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.finagle.stats.Counter - -object FunctionalUtil { - def incr[T](counter: Counter): T => T = { x => - { - counter.incr() - x - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/IbisScribeTargets.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/IbisScribeTargets.scala deleted file mode 100644 index de1108f56..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/IbisScribeTargets.scala +++ /dev/null @@ -1,55 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.CommonRecommendationType._ - -object IbisScribeTargets { - val User2 = "magic_rec_user_2" - val User4 = "magic_rec_user_4" - val Tweet2 = "magic_rec_tweet_2" - val Tweet4 = "magic_rec_tweet_4" - val Tweet5 = "magic_rec_tweet_5" - val Tweet9 = "magic_rec_tweet_9" - val Tweet10 = "magic_rec_tweet_10" - val Tweet11 = "magic_rec_tweet_11" - val Tweet12 = "magic_rec_tweet_12" - val Tweet16 = "magic_rec_tweet_16" - val Hashtag = "magic_rec_hashtag" - val UnreadBadgeCount17 = "magic_rec_unread_badge_count_17" - val Highlights = "highlights" - val TweetAnalytics = "magic_rec_tweet_analytics" - val Untracked = "untracked" - - def crtToScribeTarget(crt: CommonRecommendationType): String = crt match { - case UserFollow => - User2 - case HermitUser => - User4 - case TweetRetweet | TweetFavorite => - Tweet2 - case TweetRetweetPhoto | TweetFavoritePhoto => - Tweet4 - case TweetRetweetVideo | TweetFavoriteVideo => - Tweet5 - case UrlTweetLanding => - Tweet9 - case F1FirstdegreeTweet | F1FirstdegreePhoto | F1FirstdegreeVideo => - Tweet10 - case AuthorTargetingTweet => - Tweet11 - case PeriscopeShare => - Tweet12 - case CommonRecommendationType.Highlights => - Highlights - case HashtagTweet | HashtagTweetRetweet => - Hashtag - case PinnedTweet => - Tweet16 - case UnreadBadgeCount => - UnreadBadgeCount17 - case TweetImpressions => - TweetAnalytics - case _ => - Untracked - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/InlineActionUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/InlineActionUtil.scala deleted file mode 100644 index 2900b7418..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/InlineActionUtil.scala +++ /dev/null @@ -1,219 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.google.common.io.BaseEncoding -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.InlineActionsEnum -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.ibis2.lib.util.JsonMarshal -import com.twitter.notifications.platform.thriftscala._ -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationResponse -import com.twitter.scrooge.BinaryThriftStructSerializer -import com.twitter.util.Future - -/** - * This class provides utility functions for inline action for push - */ -object InlineActionUtil { - - def scopedStats(statsReceiver: StatsReceiver): StatsReceiver = - statsReceiver.scope(getClass.getSimpleName) - - /** - * Util function to build web inline actions for Ibis - * @param actions list of inline actions to be hydrated depending on the CRT - * @param enableForDesktopWeb if web inline actions should be shown on desktop RWeb, for experimentation purpose - * @param enableForMobileWeb if web inline actions should be shwon on mobile RWeb, for experimentation purpose - * @return Params for web inline actions to be consumed by `smart.inline.actions.web.mustache` in Ibis - */ - def getGeneratedTweetInlineActionsForWeb( - actions: Seq[InlineActionsEnum.Value], - enableForDesktopWeb: Boolean, - enableForMobileWeb: Boolean - ): Map[String, String] = { - if (!enableForDesktopWeb && !enableForMobileWeb) { - Map.empty - } else { - val inlineActions = buildEnrichedInlineActionsMap(actions) ++ Map( - "enable_for_desktop_web" -> enableForDesktopWeb.toString, - "enable_for_mobile_web" -> enableForMobileWeb.toString - ) - Map( - "inline_action_details_web" -> JsonMarshal.toJson(inlineActions), - ) - } - } - - def getGeneratedTweetInlineActionsV1( - actions: Seq[InlineActionsEnum.Value] - ): Map[String, String] = { - val inlineActions = buildEnrichedInlineActionsMap(actions) - Map( - "inline_action_details" -> JsonMarshal.toJson(inlineActions) - ) - } - - private def buildEnrichedInlineActionsMap( - actions: Seq[InlineActionsEnum.Value] - ): Map[String, Seq[Map[String, Any]]] = { - Map( - "actions" -> actions - .map(_.toString.toLowerCase) - .zipWithIndex - .map { - case (a: String, i: Int) => - Map("action" -> a) ++ Map( - s"use_${a}_stringcenter_key" -> true, - "last" -> (i == (actions.length - 1)) - ) - }.seq - ) - } - - def getGeneratedTweetInlineActionsV2( - actions: Seq[InlineActionsEnum.Value] - ): Map[String, String] = { - val v2CustomActions = actions - .map { - case InlineActionsEnum.Favorite => - NotificationCustomAction( - Some("mr_inline_favorite_title"), - CustomActionData.LegacyAction(LegacyAction(ActionIdentifier.Favorite)) - ) - case InlineActionsEnum.Follow => - NotificationCustomAction( - Some("mr_inline_follow_title"), - CustomActionData.LegacyAction(LegacyAction(ActionIdentifier.Follow))) - case InlineActionsEnum.Reply => - NotificationCustomAction( - Some("mr_inline_reply_title"), - CustomActionData.LegacyAction(LegacyAction(ActionIdentifier.Reply))) - case InlineActionsEnum.Retweet => - NotificationCustomAction( - Some("mr_inline_retweet_title"), - CustomActionData.LegacyAction(LegacyAction(ActionIdentifier.Retweet))) - case _ => - NotificationCustomAction( - Some("mr_inline_favorite_title"), - CustomActionData.LegacyAction(LegacyAction(ActionIdentifier.Favorite)) - ) - } - val notifications = NotificationCustomActions(v2CustomActions) - Map("serialized_inline_actions_v2" -> serializeActionsToBase64(notifications)) - } - - def getDislikeInlineAction( - candidate: PushCandidate, - ntabResponse: CreateGenericNotificationResponse - ): Option[NotificationCustomAction] = { - ntabResponse.successKey.map(successKey => { - val urlParams = Map[String, String]( - "answer" -> "dislike", - "notification_hash" -> successKey.hashKey.toString, - "upstream_uid" -> candidate.impressionId, - "notification_timestamp" -> successKey.timestampMillis.toString - ) - val urlParamsString = urlParams.map(kvp => f"${kvp._1}=${kvp._2}").mkString("&") - - val httpPostRequest = HttpRequest.PostRequest( - PostRequest(url = f"/2/notifications/feedback.json?$urlParamsString", bodyParams = None)) - val httpRequestAction = HttpRequestAction( - httpRequest = httpPostRequest, - scribeAction = Option("dislike_scribe_action"), - isAuthorizationRequired = Option(true), - isDestructive = Option(false), - undoable = None - ) - val dislikeAction = CustomActionData.HttpRequestAction(httpRequestAction) - NotificationCustomAction(title = Option("mr_inline_dislike_title"), action = dislikeAction) - }) - } - - /** - * Given a serialized inline action v2, update the action at index to the given new action. - * If given index is bigger than current action length, append the given inline action at the end. - * @param serialized_inline_actions_v2 the original action in serialized version - * @param actionOption an Option of the new action to replace the old one - * @param index the position where the old action will be replaced - * @return a new serialized inline action v2 - */ - def patchInlineActionAtPosition( - serialized_inline_actions_v2: String, - actionOption: Option[NotificationCustomAction], - index: Int - ): String = { - val originalActions: Seq[NotificationCustomAction] = deserializeActionsFromString( - serialized_inline_actions_v2).actions - val newActions = actionOption match { - case Some(action) if index >= originalActions.size => originalActions ++ Seq(action) - case Some(action) => originalActions.updated(index, action) - case _ => originalActions - } - serializeActionsToBase64(NotificationCustomActions(newActions)) - } - - /** - * Return list of available inline actions for ibis2 model - */ - def getGeneratedTweetInlineActions( - target: Target, - statsReceiver: StatsReceiver, - actions: Seq[InlineActionsEnum.Value], - ): Map[String, String] = { - val scopedStatsReceiver = scopedStats(statsReceiver) - val useV1 = target.params(PushFeatureSwitchParams.UseInlineActionsV1) - val useV2 = target.params(PushFeatureSwitchParams.UseInlineActionsV2) - if (useV1 && useV2) { - scopedStatsReceiver.counter("use_v1_and_use_v2").incr() - getGeneratedTweetInlineActionsV1(actions) ++ getGeneratedTweetInlineActionsV2(actions) - } else if (useV1 && !useV2) { - scopedStatsReceiver.counter("only_use_v1").incr() - getGeneratedTweetInlineActionsV1(actions) - } else if (!useV1 && useV2) { - scopedStatsReceiver.counter("only_use_v2").incr() - getGeneratedTweetInlineActionsV2(actions) - } else { - scopedStatsReceiver.counter("use_neither_v1_nor_v2").incr() - Map.empty[String, String] - } - } - - /** - * Return Tweet inline action ibis2 model values after applying experiment logic - */ - def getTweetInlineActionValue(target: Target): Future[Map[String, String]] = { - if (target.isLoggedOutUser) { - Future( - Map( - "show_inline_action" -> "false" - ) - ) - } else { - val showInlineAction: Boolean = target.params(PushParams.MRAndroidInlineActionOnPushCopyParam) - Future( - Map( - "show_inline_action" -> s"$showInlineAction" - ) - ) - } - } - - private val binaryThriftStructSerializer: BinaryThriftStructSerializer[ - NotificationCustomActions - ] = BinaryThriftStructSerializer.apply(NotificationCustomActions) - private val base64Encoding = BaseEncoding.base64() - - def serializeActionsToBase64(notificationCustomActions: NotificationCustomActions): String = { - val actionsAsByteArray: Array[Byte] = - binaryThriftStructSerializer.toBytes(notificationCustomActions) - base64Encoding.encode(actionsAsByteArray) - } - - def deserializeActionsFromString(serializedInlineActionV2: String): NotificationCustomActions = { - val actionAsByteArray = base64Encoding.decode(serializedInlineActionV2) - binaryThriftStructSerializer.fromBytes(actionAsByteArray) - } - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MediaAnnotationsUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MediaAnnotationsUtil.scala deleted file mode 100644 index a3a3ecf50..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MediaAnnotationsUtil.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate - -object MediaAnnotationsUtil { - - val mediaIdToCategoryMapping = Map("0" -> "0") - - val nudityCategoryId = "0" - val beautyCategoryId = "0" - val singlePersonCategoryId = "0" - val sensitiveMediaCategoryFeatureName = - "tweet.mediaunderstanding.tweet_annotations.sensitive_category_probabilities" - - def updateMediaCategoryStats( - candidates: Seq[CandidateDetails[PushCandidate]] - )( - implicit statsReceiver: StatsReceiver - ) = { - - val statScope = statsReceiver.scope("mediaStats") - val filteredCandidates = candidates.filter { candidate => - !candidate.candidate.sparseContinuousFeatures - .getOrElse(sensitiveMediaCategoryFeatureName, Map.empty[String, Double]).contains( - nudityCategoryId) - } - - if (filteredCandidates.isEmpty) - statScope.counter("emptyCandidateListAfterNudityFilter").incr() - else - statScope.counter("nonEmptyCandidateListAfterNudityFilter").incr() - candidates.foreach { candidate => - statScope.counter("totalCandidates").incr() - val mediaFeature = candidate.candidate.sparseContinuousFeatures - .getOrElse(sensitiveMediaCategoryFeatureName, Map.empty[String, Double]) - if (mediaFeature.nonEmpty) { - val mediaCategoryByMaxScore = mediaFeature.maxBy(_._2)._1 - statScope - .scope("mediaCategoryByMaxScore").counter(mediaIdToCategoryMapping - .getOrElse(mediaCategoryByMaxScore, "undefined")).incr() - - mediaFeature.keys.map { feature => - statScope - .scope("mediaCategory").counter(mediaIdToCategoryMapping - .getOrElse(feature, "undefined")).incr() - } - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MinDurationModifierCalculator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MinDurationModifierCalculator.scala deleted file mode 100644 index e96ecb817..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MinDurationModifierCalculator.scala +++ /dev/null @@ -1,187 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.TimeUtil -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushConstants -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FSParams} -import com.twitter.util.Future -import com.twitter.util.Time -import java.util.Calendar -import java.util.TimeZone - -case class MinDurationModifierCalculator() { - - private def mapCountryCodeToTimeZone( - countryCode: String, - stats: StatsReceiver - ): Option[Calendar] = { - PushConstants.countryCodeToTimeZoneMap - .get(countryCode.toUpperCase).map(timezone => - Calendar.getInstance(TimeZone.getTimeZone(timezone))) - } - - private def transformToHour( - dayOfHour: Int - ): Int = { - if (dayOfHour < 0) dayOfHour + 24 - else dayOfHour - } - - private def getMinDurationByHourOfDay( - hourOfDay: Int, - startTimeList: Seq[Int], - endTimeList: Seq[Int], - minDurationTimeModifierConst: Seq[Int], - stats: StatsReceiver - ): Option[Int] = { - val scopedStats = stats.scope("getMinDurationByHourOfDay") - scopedStats.counter("request").incr() - val durationOpt = (startTimeList, endTimeList, minDurationTimeModifierConst).zipped.toList - .filter { - case (startTime, endTime, _) => - if (startTime <= endTime) hourOfDay >= startTime && hourOfDay < endTime - else (hourOfDay >= startTime) || hourOfDay < endTime - case _ => false - }.map { - case (_, _, modifier) => modifier - }.headOption - durationOpt match { - case Some(duration) => scopedStats.counter(s"$duration.minutes").incr() - case _ => scopedStats.counter("none").incr() - } - durationOpt - } - - def getMinDurationModifier( - target: Target, - calendar: Calendar, - stats: StatsReceiver - ): Option[Int] = { - val startTimeList = target.params(FSParams.MinDurationModifierStartHourList) - val endTimeList = target.params(FSParams.MinDurationModifierEndHourList) - val minDurationTimeModifierConst = target.params(FSParams.MinDurationTimeModifierConst) - if (startTimeList.length != endTimeList.length || minDurationTimeModifierConst.length != startTimeList.length) { - None - } else { - val hourOfDay = calendar.get(Calendar.HOUR_OF_DAY) - getMinDurationByHourOfDay( - hourOfDay, - startTimeList, - endTimeList, - minDurationTimeModifierConst, - stats) - } - } - - def getMinDurationModifier( - target: Target, - countryCodeOpt: Option[String], - stats: StatsReceiver - ): Option[Int] = { - val scopedStats = stats - .scope("getMinDurationModifier") - scopedStats.counter("total_requests").incr() - - countryCodeOpt match { - case Some(countryCode) => - scopedStats - .counter("country_code_exists").incr() - val calendarOpt = mapCountryCodeToTimeZone(countryCode, scopedStats) - calendarOpt.flatMap(calendar => getMinDurationModifier(target, calendar, scopedStats)) - case _ => None - } - } - - def getMinDurationModifier(target: Target, stats: StatsReceiver): Future[Option[Int]] = { - val scopedStats = stats - .scope("getMinDurationModifier") - scopedStats.counter("total_requests").incr() - - val startTimeList = target.params(FSParams.MinDurationModifierStartHourList) - val endTimeList = target.params(FSParams.MinDurationModifierEndHourList) - val minDurationTimeModifierConst = target.params(FSParams.MinDurationTimeModifierConst) - if (startTimeList.length != endTimeList.length || minDurationTimeModifierConst.length != startTimeList.length) { - Future.value(None) - } else { - target.localTimeInHHMM.map { - case (hourOfDay, _) => - getMinDurationByHourOfDay( - hourOfDay, - startTimeList, - endTimeList, - minDurationTimeModifierConst, - scopedStats) - case _ => None - } - } - } - - def getMinDurationModifierByUserOpenedHistory( - target: Target, - openedPushByHourAggregatedOpt: Option[Map[Int, Int]], - stats: StatsReceiver - ): Option[Int] = { - val scopedStats = stats - .scope("getMinDurationModifierByUserOpenedHistory") - scopedStats.counter("total_requests").incr() - openedPushByHourAggregatedOpt match { - case Some(openedPushByHourAggregated) => - if (openedPushByHourAggregated.isEmpty) { - scopedStats.counter("openedPushByHourAggregated_empty").incr() - None - } else { - val currentUTCHour = TimeUtil.hourOfDay(Time.now) - val utcHourWithMaxOpened = if (target.params(FSParams.EnableRandomHourForQuickSend)) { - (target.targetId % 24).toInt - } else { - openedPushByHourAggregated.maxBy(_._2)._1 - } - val numOfMaxOpened = openedPushByHourAggregated.maxBy(_._2)._2 - if (numOfMaxOpened >= target.params(FSParams.SendTimeByUserHistoryMaxOpenedThreshold)) { - scopedStats.counter("pass_experiment_bucket_threshold").incr() - if (numOfMaxOpened >= target - .params(FSParams.SendTimeByUserHistoryMaxOpenedThreshold)) { // only update if number of opened pushes meet threshold - scopedStats.counter("pass_max_threshold").incr() - val quickSendBeforeHours = - target.params(FSParams.SendTimeByUserHistoryQuickSendBeforeHours) - val quickSendAfterHours = - target.params(FSParams.SendTimeByUserHistoryQuickSendAfterHours) - - val hoursToLessSend = target.params(FSParams.SendTimeByUserHistoryNoSendsHours) - - val quickSendTimeMinDurationInMinute = - target.params(FSParams.SendTimeByUserHistoryQuickSendMinDurationInMinute) - val noSendTimeMinDuration = - target.params(FSParams.SendTimeByUserHistoryNoSendMinDuration) - - val startTimeForNoSend = transformToHour( - utcHourWithMaxOpened - quickSendBeforeHours - hoursToLessSend) - val startTimeForQuickSend = transformToHour( - utcHourWithMaxOpened - quickSendBeforeHours) - val endTimeForNoSend = - transformToHour(utcHourWithMaxOpened - quickSendBeforeHours) - val endTimeForQuickSend = - transformToHour(utcHourWithMaxOpened + quickSendAfterHours) + 1 - - val startTimeList = Seq(startTimeForNoSend, startTimeForQuickSend) - val endTimeList = Seq(endTimeForNoSend, endTimeForQuickSend) - val minDurationTimeModifierConst = - Seq(noSendTimeMinDuration, quickSendTimeMinDurationInMinute) - - getMinDurationByHourOfDay( - currentUTCHour, - startTimeList, - endTimeList, - minDurationTimeModifierConst, - scopedStats) - - } else None - } else None - } - case _ => - None - } - } - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MrUserStateUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MrUserStateUtil.scala deleted file mode 100644 index 33333c4c4..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MrUserStateUtil.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.TargetUser - -object MrUserStateUtil { - def updateMrUserStateStats(target: TargetUser)(implicit statsReceiver: StatsReceiver) = { - statsReceiver.counter("AllUserStates").incr() - target.targetMrUserState.map { - case Some(state) => - statsReceiver.counter(state.name).incr() - case _ => - statsReceiver.counter("UnknownUserState").incr() - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/NsfwPersonalizationUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/NsfwPersonalizationUtil.scala deleted file mode 100644 index 01c8d0a72..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/NsfwPersonalizationUtil.scala +++ /dev/null @@ -1,126 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.scio.nsfw_user_segmentation.thriftscala.NSFWUserSegmentation - -object NsfwPersonalizationUtil { - def computeNsfwUserStats( - targetNsfwInfo: Option[NsfwInfo] - )( - implicit statsReceiver: StatsReceiver - ): Unit = { - - def computeNsfwProfileVisitStats(sReceiver: StatsReceiver, nsfwProfileVisits: Long): Unit = { - if (nsfwProfileVisits >= 1) - sReceiver.counter("nsfwProfileVisits_gt_1").incr() - if (nsfwProfileVisits >= 2) - sReceiver.counter("nsfwProfileVisits_gt_2").incr() - if (nsfwProfileVisits >= 3) - sReceiver.counter("nsfwProfileVisits_gt_3").incr() - if (nsfwProfileVisits >= 5) - sReceiver.counter("nsfwProfileVisits_gt_5").incr() - if (nsfwProfileVisits >= 8) - sReceiver.counter("nsfwProfileVisits_gt_8").incr() - } - - def computeRatioStats( - sReceiver: StatsReceiver, - ratio: Int, - statName: String, - intervals: List[Double] = List(0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9) - ): Unit = { - intervals.foreach { i => - if (ratio > i * 10000) - sReceiver.counter(f"${statName}_greater_than_${i}").incr() - } - } - val sReceiver = statsReceiver.scope("nsfw_personalization") - sReceiver.counter("AllUsers").incr() - - (targetNsfwInfo) match { - case (Some(nsfwInfo)) => - val sensitive = nsfwInfo.senstiveOptIn.getOrElse(false) - val nsfwFollowRatio = - nsfwInfo.nsfwFollowRatio - val totalFollows = nsfwInfo.totalFollowCount - val numNsfwProfileVisits = nsfwInfo.nsfwProfileVisits - val nsfwRealGraphScore = nsfwInfo.realGraphScore - val nsfwSearchScore = nsfwInfo.searchNsfwScore - val totalSearches = nsfwInfo.totalSearches - val realGraphScore = nsfwInfo.realGraphScore - val searchScore = nsfwInfo.searchNsfwScore - - if (sensitive) - sReceiver.counter("sensitiveOptInEnabled").incr() - else - sReceiver.counter("sensitiveOptInDisabled").incr() - - computeRatioStats(sReceiver, nsfwFollowRatio, "nsfwRatio") - computeNsfwProfileVisitStats(sReceiver, numNsfwProfileVisits) - computeRatioStats(sReceiver, nsfwRealGraphScore.toInt, "nsfwRealGraphScore") - - if (totalSearches >= 10) - computeRatioStats(sReceiver, nsfwSearchScore.toInt, "nsfwSearchScore") - if (searchScore == 0) - sReceiver.counter("lowSearchScore").incr() - if (realGraphScore < 500) - sReceiver.counter("lowRealScore").incr() - if (numNsfwProfileVisits == 0) - sReceiver.counter("lowProfileVisit").incr() - if (nsfwFollowRatio == 0) - sReceiver.counter("lowFollowScore").incr() - - if (totalSearches > 10 && searchScore > 5000) - sReceiver.counter("highSearchScore").incr() - if (realGraphScore > 7000) - sReceiver.counter("highRealScore").incr() - if (numNsfwProfileVisits > 5) - sReceiver.counter("highProfileVisit").incr() - if (totalFollows > 10 && nsfwFollowRatio > 7000) - sReceiver.counter("highFollowScore").incr() - - if (searchScore == 0 && realGraphScore <= 500 && numNsfwProfileVisits == 0 && nsfwFollowRatio == 0) - sReceiver.counter("lowIntent").incr() - if ((totalSearches > 10 && searchScore > 5000) || realGraphScore > 7000 || numNsfwProfileVisits > 5 || (totalFollows > 10 && nsfwFollowRatio > 7000)) - sReceiver.counter("highIntent").incr() - case _ => - } - } -} - -case class NsfwInfo(nsfwUserSegmentation: NSFWUserSegmentation) { - - val scalingFactor = 10000 // to convert float to int as custom fields cannot be float - val senstiveOptIn: Option[Boolean] = nsfwUserSegmentation.nsfwView - val totalFollowCount: Long = nsfwUserSegmentation.totalFollowCnt.getOrElse(0L) - val nsfwFollowCnt: Long = - nsfwUserSegmentation.nsfwAdminOrHighprecOrAgathaGtP98FollowsCnt.getOrElse(0L) - val nsfwFollowRatio: Int = { - if (totalFollowCount != 0) { - (nsfwFollowCnt * scalingFactor / totalFollowCount).toInt - } else 0 - } - val nsfwProfileVisits: Long = - nsfwUserSegmentation.nsfwAdminOrHighPrecOrAgathaGtP98Visits - .map(_.numProfilesInLast14Days).getOrElse(0L) - val realGraphScore: Int = - nsfwUserSegmentation.realGraphMetrics - .map { rm => - if (rm.totalOutboundRGScore != 0) - rm.totalNsfwAdmHPAgthGtP98OutboundRGScore * scalingFactor / rm.totalOutboundRGScore - else 0d - }.getOrElse(0d).toInt - val totalSearches: Long = - nsfwUserSegmentation.searchMetrics.map(_.numNonTrndSrchInLast14Days).getOrElse(0L) - val searchNsfwScore: Int = nsfwUserSegmentation.searchMetrics - .map { sm => - if (sm.numNonTrndNonHshtgSrchInLast14Days != 0) - sm.numNonTrndNonHshtgGlobalNsfwSrchInLast14Days.toDouble * scalingFactor / sm.numNonTrndNonHshtgSrchInLast14Days - else 0 - }.getOrElse(0d).toInt - val hasReported: Boolean = - nsfwUserSegmentation.notifFeedbackMetrics.exists(_.notifReportMetrics.exists(_.countTotal != 0)) - val hasDisliked: Boolean = - nsfwUserSegmentation.notifFeedbackMetrics - .exists(_.notifDislikeMetrics.exists(_.countTotal != 0)) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/OverrideNotificationUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/OverrideNotificationUtil.scala deleted file mode 100644 index ac4aba8a7..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/OverrideNotificationUtil.scala +++ /dev/null @@ -1,230 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.MagicFanoutEventCandidate -import com.twitter.frigate.common.history.History -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.common.store.deviceinfo.DeviceInfo -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes -import com.twitter.frigate.pushservice.model.ibis.PushOverrideInfo -import com.twitter.frigate.pushservice.params.PushConstants -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FSParams} -import com.twitter.frigate.thriftscala.CollapseInfo -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.CommonRecommendationType.MagicFanoutSportsEvent -import com.twitter.frigate.thriftscala.OverrideInfo -import com.twitter.util.Future -import java.util.UUID - -object OverrideNotificationUtil { - - /** - * Gets Override Info for the current notification. - * @param candidate [[PushCandidate]] object representing the recommendation candidate - * @param stats StatsReceiver to track stats for this function as well as the subsequent funcs. called - * @return Returns OverrideInfo if CollapseInfo exists, else None - */ - - def getOverrideInfo( - candidate: PushCandidate, - stats: StatsReceiver - ): Future[Option[OverrideInfo]] = { - if (candidate.target.isLoggedOutUser) { - Future.None - } else if (isOverrideEnabledForCandidate(candidate)) - getCollapseInfo(candidate, stats).map(_.map(OverrideInfo(_))) - else Future.None - } - - private def getCollapseInfo( - candidate: PushCandidate, - stats: StatsReceiver - ): Future[Option[CollapseInfo]] = { - val target = candidate.target - for { - targetHistory <- target.history - deviceInfo <- target.deviceInfo - } yield getCollapseInfo(target, targetHistory, deviceInfo, stats) - } - - /** - * Get Collapse Info for the current notification. - * @param target Push Target - recipient of the notification - * @param targetHistory Target's History - * @param deviceInfoOpt `Option` of the Target's Device Info - * @param stats StatsReceiver to track stats for this function as well as the subsequent funcs. called - * @return Returns CollapseInfo if the Target is eligible for Override Notifs, else None - */ - def getCollapseInfo( - target: PushTypes.Target, - targetHistory: History, - deviceInfoOpt: Option[DeviceInfo], - stats: StatsReceiver - ): Option[CollapseInfo] = { - val overrideInfoOfLastNotif = - PushOverrideInfo.getOverrideInfoOfLastEligiblePushNotif( - targetHistory, - target.params(FSParams.OverrideNotificationsLookbackDurationForOverrideInfo), - stats) - overrideInfoOfLastNotif match { - case Some(prevOverrideInfo) if isOverrideEnabled(target, deviceInfoOpt, stats) => - val notifsInLastOverrideChain = - PushOverrideInfo.getMrPushNotificationsInOverrideChain( - targetHistory, - prevOverrideInfo.collapseInfo.overrideChainId, - stats) - val numNotifsInLastOverrideChain = notifsInLastOverrideChain.size - val timestampOfFirstNotifInOverrideChain = - PushOverrideInfo - .getTimestampInMillisForFrigateNotification( - notifsInLastOverrideChain.last, - targetHistory, - stats).getOrElse(PushConstants.DefaultLookBackForHistory.ago.inMilliseconds) - if (numNotifsInLastOverrideChain < target.params(FSParams.MaxMrPushSends24HoursParam) && - timestampOfFirstNotifInOverrideChain > PushConstants.DefaultLookBackForHistory.ago.inMilliseconds) { - Some(prevOverrideInfo.collapseInfo) - } else { - val prevCollapseId = prevOverrideInfo.collapseInfo.collapseId - val newOverrideChainId = UUID.randomUUID.toString.replaceAll("-", "") - Some(CollapseInfo(prevCollapseId, newOverrideChainId)) - } - case None if isOverrideEnabled(target, deviceInfoOpt, stats) => - val newOverrideChainId = UUID.randomUUID.toString.replaceAll("-", "") - Some(CollapseInfo("", newOverrideChainId)) - case _ => None // Override is disabled for everything else - } - } - - /** - * Gets the collapse and impression identifier for the current override notification - * @param target Push Target - recipient of the notification - * @param stats StatsReceiver to track stats for this function as well as the subsequent funcs. called - * @return A Future of Collapse ID as well as the Impression ID. - */ - def getCollapseAndImpressionIdForOverride( - candidate: PushCandidate - ): Future[Option[(String, Seq[String])]] = { - if (isOverrideEnabledForCandidate(candidate)) { - val target = candidate.target - val stats = candidate.statsReceiver - Future.join(target.history, target.deviceInfo).map { - case (targetHistory, deviceInfoOpt) => - val collapseInfoOpt = getCollapseInfo(target, targetHistory, deviceInfoOpt, stats) - - val impressionIds = candidate.commonRecType match { - case MagicFanoutSportsEvent - if target.params(FSParams.EnableEventIdBasedOverrideForSportsCandidates) => - PushOverrideInfo.getImpressionIdsForPrevEligibleMagicFanoutEventCandidates( - targetHistory, - target.params(FSParams.OverrideNotificationsLookbackDurationForImpressionId), - stats, - MagicFanoutSportsEvent, - candidate - .asInstanceOf[RawCandidate with MagicFanoutEventCandidate].eventId - ) - case _ => - PushOverrideInfo.getImpressionIdsOfPrevEligiblePushNotif( - targetHistory, - target.params(FSParams.OverrideNotificationsLookbackDurationForImpressionId), - stats) - } - - collapseInfoOpt match { - case Some(collapseInfo) if impressionIds.nonEmpty => - val notifsInLastOverrideChain = - PushOverrideInfo.getMrPushNotificationsInOverrideChain( - targetHistory, - collapseInfo.overrideChainId, - stats) - stats - .scope("OverrideNotificationUtil").stat("number_of_notifications_sent").add( - notifsInLastOverrideChain.size + 1) - Some((collapseInfo.collapseId, impressionIds)) - case _ => None - } - case _ => None - } - } else Future.None - } - - /** - * Checks to see if override notifications are enabled based on the Target's Device Info and Params - * @param target Push Target - recipient of the notification - * @param deviceInfoOpt `Option` of the Target's Device Info - * @param stats StatsReceiver to track stats for this function - * @return Returns True if Override Notifications are enabled for the provided - * Target, else False. - */ - private def isOverrideEnabled( - target: PushTypes.Target, - deviceInfoOpt: Option[DeviceInfo], - stats: StatsReceiver - ): Boolean = { - val scopedStats = stats.scope("OverrideNotificationUtil").scope("isOverrideEnabled") - val enabledForAndroidCounter = scopedStats.counter("android_enabled") - val disabledForAndroidCounter = scopedStats.counter("android_disabled") - val enabledForIosCounter = scopedStats.counter("ios_enabled") - val disabledForIosCounter = scopedStats.counter("ios_disabled") - val disabledForOtherDevicesCounter = scopedStats.counter("other_disabled") - - val isPrimaryDeviceAndroid = PushDeviceUtil.isPrimaryDeviceAndroid(deviceInfoOpt) - val isPrimaryDeviceIos = PushDeviceUtil.isPrimaryDeviceIOS(deviceInfoOpt) - - lazy val validAndroidDevice = - isPrimaryDeviceAndroid && target.params(FSParams.EnableOverrideNotificationsForAndroid) - lazy val validIosDevice = - isPrimaryDeviceIos && target.params(FSParams.EnableOverrideNotificationsForIos) - - if (isPrimaryDeviceAndroid) { - if (validAndroidDevice) enabledForAndroidCounter.incr() else disabledForAndroidCounter.incr() - } else if (isPrimaryDeviceIos) { - if (validIosDevice) enabledForIosCounter.incr() else disabledForIosCounter.incr() - } else { - disabledForOtherDevicesCounter.incr() - } - - validAndroidDevice || validIosDevice - } - - /** - * Checks if override is enabled for the currently supported types for SendHandler or not. - * This method is package private for unit testing. - * @param candidate [[PushCandidate]] - * @param stats StatsReceiver to track statistics for this function - * @return Returns True if override notifications are enabled for the current type, otherwise False. - */ - private def isOverrideEnabledForSendHandlerCandidate( - candidate: PushCandidate - ): Boolean = { - val scopedStats = candidate.statsReceiver - .scope("OverrideNotificationUtil").scope("isOverrideEnabledForSendHandlerType") - - val overrideSupportedTypesForSpaces: Set[CommonRecommendationType] = Set( - CommonRecommendationType.SpaceSpeaker, - CommonRecommendationType.SpaceHost - ) - - val isOverrideSupportedForSpaces = { - overrideSupportedTypesForSpaces.contains(candidate.commonRecType) && - candidate.target.params(FSParams.EnableOverrideForSpaces) - } - - val isOverrideSupportedForSports = { - candidate.commonRecType == CommonRecommendationType.MagicFanoutSportsEvent && - candidate.target - .params(PushFeatureSwitchParams.EnableOverrideForSportsCandidates) - } - - val isOverrideSupported = isOverrideSupportedForSpaces || isOverrideSupportedForSports - - scopedStats.counter(s"$isOverrideSupported").incr() - isOverrideSupported - } - - private[util] def isOverrideEnabledForCandidate(candidate: PushCandidate) = - !RecTypes.isSendHandlerType( - candidate.commonRecType) || isOverrideEnabledForSendHandlerCandidate(candidate) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushAdaptorUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushAdaptorUtil.scala deleted file mode 100644 index 7bc29cf01..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushAdaptorUtil.scala +++ /dev/null @@ -1,151 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.contentrecommender.thriftscala.MetricTag -import com.twitter.frigate.common.base.AlgorithmScore -import com.twitter.frigate.common.base.OutOfNetworkTweetCandidate -import com.twitter.frigate.common.base.SocialContextAction -import com.twitter.frigate.common.base.TopicCandidate -import com.twitter.frigate.common.base.TripCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.thriftscala.{SocialContextAction => TSocialContextAction} -import com.twitter.frigate.thriftscala.{CommonRecommendationType => CRT} -import com.twitter.frigate.thriftscala._ -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.topiclisting.utt.LocalizedEntity -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain -import scala.collection.Seq - -case class MediaCRT( - crt: CRT, - photoCRT: CRT, - videoCRT: CRT) - -object PushAdaptorUtil { - - def getFrigateNotificationForUser( - crt: CRT, - userId: Long, - scActions: Seq[SocialContextAction], - pushCopyId: Option[Int], - ntabCopyId: Option[Int] - ): FrigateNotification = { - - val thriftSCActions = scActions.map { scAction => - TSocialContextAction( - scAction.userId, - scAction.timestampInMillis, - scAction.tweetId - ) - } - FrigateNotification( - crt, - NotificationDisplayLocation.PushToMobileDevice, - userNotification = Some(UserNotification(userId, thriftSCActions)), - pushCopyId = pushCopyId, - ntabCopyId = ntabCopyId - ) - } - - def getFrigateNotificationForTweet( - crt: CRT, - tweetId: Long, - scActions: Seq[TSocialContextAction], - authorIdOpt: Option[Long], - pushCopyId: Option[Int], - ntabCopyId: Option[Int], - simclusterId: Option[Int], - semanticCoreEntityIds: Option[List[Long]], - candidateContent: Option[CandidateContent], - trendId: Option[String], - tweetTripDomain: Option[scala.collection.Set[TripDomain]] = None - ): FrigateNotification = { - FrigateNotification( - crt, - NotificationDisplayLocation.PushToMobileDevice, - tweetNotification = Some( - TweetNotification( - tweetId, - scActions, - authorIdOpt, - simclusterId, - semanticCoreEntityIds, - trendId, - tripDomain = tweetTripDomain) - ), - pushCopyId = pushCopyId, - ntabCopyId = ntabCopyId, - candidateContent = candidateContent - ) - } - - def getFrigateNotificationForTweetWithSocialContextActions( - crt: CRT, - tweetId: Long, - scActions: Seq[SocialContextAction], - authorIdOpt: Option[Long], - pushCopyId: Option[Int], - ntabCopyId: Option[Int], - candidateContent: Option[CandidateContent], - semanticCoreEntityIds: Option[List[Long]], - trendId: Option[String] - ): FrigateNotification = { - - val thriftSCActions = scActions.map { scAction => - TSocialContextAction( - scAction.userId, - scAction.timestampInMillis, - scAction.tweetId - ) - } - - getFrigateNotificationForTweet( - crt = crt, - tweetId = tweetId, - scActions = thriftSCActions, - authorIdOpt = authorIdOpt, - pushCopyId = pushCopyId, - ntabCopyId = ntabCopyId, - simclusterId = None, - candidateContent = candidateContent, - semanticCoreEntityIds = semanticCoreEntityIds, - trendId = trendId - ) - } - - def generateOutOfNetworkTweetCandidates( - inputTarget: Target, - id: Long, - mediaCRT: MediaCRT, - result: Option[TweetyPieResult], - localizedEntity: Option[LocalizedEntity] = None, - isMrBackfillFromCR: Option[Boolean] = None, - tagsFromCR: Option[Seq[MetricTag]] = None, - score: Option[Double] = None, - algorithmTypeCR: Option[String] = None, - tripTweetDomain: Option[scala.collection.Set[TripDomain]] = None - ): RawCandidate - with OutOfNetworkTweetCandidate - with TopicCandidate - with TripCandidate - with AlgorithmScore = { - new RawCandidate - with OutOfNetworkTweetCandidate - with TopicCandidate - with TripCandidate - with AlgorithmScore { - override val tweetId: Long = id - override val target: Target = inputTarget - override val tweetyPieResult: Option[TweetyPieResult] = result - override val localizedUttEntity: Option[LocalizedEntity] = localizedEntity - override val semanticCoreEntityId: Option[Long] = localizedEntity.map(_.entityId) - override def commonRecType: CRT = - getMediaBasedCRT(mediaCRT.crt, mediaCRT.photoCRT, mediaCRT.videoCRT) - override def isMrBackfillCR: Option[Boolean] = isMrBackfillFromCR - override def tagsCR: Option[Seq[MetricTag]] = tagsFromCR - override def algorithmScore: Option[Double] = score - override def algorithmCR: Option[String] = algorithmTypeCR - override def tripDomain: Option[collection.Set[TripDomain]] = tripTweetDomain - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushAppPermissionUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushAppPermissionUtil.scala deleted file mode 100644 index 0afa90fed..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushAppPermissionUtil.scala +++ /dev/null @@ -1,49 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.frigate.common.store.deviceinfo.DeviceInfo -import com.twitter.onboarding.task.service.models.external.PermissionState -import com.twitter.permissions_storage.thriftscala.AppPermission -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -object PushAppPermissionUtil { - - final val AddressBookPermissionKey = "addressBook" - final val SyncStateKey = "syncState" - final val SyncStateOnValue = "on" - - /** - * Obtains the specified target's App Permissions, based on their primary device. - * @param targetId Target's Identifier - * @param permissionName The permission type we are querying for (address book, geolocation, etc.) - * @param deviceInfoFut Device info of the Target, presented as a Future - * @param appPermissionStore Readable Store which allows us to query the App Permission Strato Column - * @return Returns the AppPermission of the Target, presented as a Future - */ - def getAppPermission( - targetId: Long, - permissionName: String, - deviceInfoFut: Future[Option[DeviceInfo]], - appPermissionStore: ReadableStore[(Long, (String, String)), AppPermission] - ): Future[Option[AppPermission]] = { - deviceInfoFut.flatMap { deviceInfoOpt => - val primaryDeviceIdOpt = deviceInfoOpt.flatMap(_.primaryDeviceId) - primaryDeviceIdOpt match { - case Some(primaryDeviceId) => - val queryKey = (targetId, (primaryDeviceId, permissionName)) - appPermissionStore.get(queryKey) - case _ => Future.None - } - } - } - - def hasTargetUploadedAddressBook( - appPermissionOpt: Option[AppPermission] - ): Boolean = { - appPermissionOpt.exists { appPermission => - val syncState = appPermission.metadata.get(SyncStateKey) - appPermission.systemPermissionState == PermissionState.On && syncState - .exists(_.equalsIgnoreCase(SyncStateOnValue)) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushCapUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushCapUtil.scala deleted file mode 100644 index d5d79c4fd..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushCapUtil.scala +++ /dev/null @@ -1,184 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.TargetUser -import com.twitter.frigate.common.candidate.FrigateHistory -import com.twitter.frigate.common.candidate.ResurrectedUserDetails -import com.twitter.frigate.common.candidate.TargetABDecider -import com.twitter.frigate.common.candidate.UserDetails -import com.twitter.frigate.pushcap.thriftscala.ModelType -import com.twitter.frigate.pushcap.thriftscala.PushcapInfo -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.scribe.thriftscala.PushCapInfo -import com.twitter.util.Duration -import com.twitter.util.Future - -case class PushCapFatigueInfo( - pushcap: Int, - fatigueInterval: Duration) {} - -object PushCapUtil { - - def getDefaultPushCap(target: Target): Future[Int] = { - Future.value(target.params(PushFeatureSwitchParams.MaxMrPushSends24HoursParam)) - } - - def getMinimumRestrictedPushcapInfo( - restrictedPushcap: Int, - originalPushcapInfo: PushcapInfo, - statsReceiver: StatsReceiver - ): PushcapInfo = { - if (originalPushcapInfo.pushcap < restrictedPushcap) { - statsReceiver - .scope("minModelPushcapRestrictions").counter( - f"num_users_adjusted_from_${originalPushcapInfo.pushcap}_to_${restrictedPushcap}").incr() - PushcapInfo( - pushcap = restrictedPushcap.toShort, - modelType = ModelType.NoModel, - timestamp = 0L, - fatigueMinutes = Some((24L / restrictedPushcap) * 60L) - ) - } else originalPushcapInfo - } - - def getPushCapFatigue( - target: Target, - statsReceiver: StatsReceiver - ): Future[PushCapFatigueInfo] = { - val pushCapStats = statsReceiver.scope("pushcap_stats") - target.dynamicPushcap - .map { dynamicPushcapOpt => - val pushCap: Int = dynamicPushcapOpt match { - case Some(pushcapInfo) => pushcapInfo.pushcap - case _ => target.params(PushFeatureSwitchParams.MaxMrPushSends24HoursParam) - } - - pushCapStats.stat("pushCapValueStats").add(pushCap) - pushCapStats - .scope("pushCapValueCount").counter(f"num_users_with_pushcap_$pushCap").incr() - - target.finalPushcapAndFatigue += "pushPushCap" -> PushCapInfo("pushPushCap", pushCap.toByte) - - PushCapFatigueInfo(pushCap, 24.hours) - } - } - - def getMinDurationsSincePushWithoutUsingPushCap( - target: TargetUser - with TargetABDecider - with FrigateHistory - with UserDetails - with ResurrectedUserDetails - )( - implicit statsReceiver: StatsReceiver - ): Duration = { - val minDurationSincePush = - if (target.params(PushFeatureSwitchParams.EnableGraduallyRampUpNotification)) { - val daysInterval = - target.params(PushFeatureSwitchParams.GraduallyRampUpPhaseDurationDays).inDays.toDouble - val daysSinceActivation = - if (target.isResurrectedUser && target.timeSinceResurrection.isDefined) { - target.timeSinceResurrection.map(_.inDays.toDouble).get - } else { - target.timeElapsedAfterSignup.inDays.toDouble - } - val phaseInterval = - Math.max( - 1, - Math.ceil(daysSinceActivation / daysInterval).toInt - ) - val minDuration = 24 / phaseInterval - val finalMinDuration = - Math.max(4, minDuration).hours - statsReceiver - .scope("GraduallyRampUpFinalMinDuration").counter(s"$finalMinDuration.hours").incr() - finalMinDuration - } else { - target.params(PushFeatureSwitchParams.MinDurationSincePushParam) - } - statsReceiver - .scope("minDurationsSincePushWithoutUsingPushCap").counter( - s"$minDurationSincePush.hours").incr() - minDurationSincePush - } - - def getMinDurationSincePush( - target: Target, - statsReceiver: StatsReceiver - ): Future[Duration] = { - val minDurationStats: StatsReceiver = statsReceiver.scope("pushcapMinDuration_stats") - val minDurationModifierCalculator = - MinDurationModifierCalculator() - val openedPushByHourAggregatedFut = - if (target.params(PushFeatureSwitchParams.EnableQueryUserOpenedHistory)) - target.openedPushByHourAggregated - else Future.None - Future - .join( - target.dynamicPushcap, - target.accountCountryCode, - openedPushByHourAggregatedFut - ) - .map { - case (dynamicPushcapOpt, countryCodeOpt, openedPushByHourAggregated) => - val minDurationSincePush: Duration = { - val isGraduallyRampingUpResurrected = target.isResurrectedUser && target.params( - PushFeatureSwitchParams.EnableGraduallyRampUpNotification) - if (isGraduallyRampingUpResurrected || target.params( - PushFeatureSwitchParams.EnableExplicitPushCap)) { - getMinDurationsSincePushWithoutUsingPushCap(target)(minDurationStats) - } else { - dynamicPushcapOpt match { - case Some(pushcapInfo) => - pushcapInfo.fatigueMinutes match { - case Some(fatigueMinutes) => (fatigueMinutes / 60).hours - case _ if pushcapInfo.pushcap > 0 => (24 / pushcapInfo.pushcap).hours - case _ => getMinDurationsSincePushWithoutUsingPushCap(target)(minDurationStats) - } - case _ => - getMinDurationsSincePushWithoutUsingPushCap(target)(minDurationStats) - } - } - } - - val modifiedMinDurationSincePush = - if (target.params(PushFeatureSwitchParams.EnableMinDurationModifier)) { - val modifierHourOpt = - minDurationModifierCalculator.getMinDurationModifier( - target, - countryCodeOpt, - statsReceiver.scope("MinDuration")) - modifierHourOpt match { - case Some(modifierHour) => modifierHour.hours - case _ => minDurationSincePush - } - } else if (target.params( - PushFeatureSwitchParams.EnableMinDurationModifierByUserHistory)) { - val modifierMinuteOpt = - minDurationModifierCalculator.getMinDurationModifierByUserOpenedHistory( - target, - openedPushByHourAggregated, - statsReceiver.scope("MinDuration")) - - modifierMinuteOpt match { - case Some(modifierMinute) => modifierMinute.minutes - case _ => minDurationSincePush - } - } else minDurationSincePush - - target.finalPushcapAndFatigue += "pushFatigue" -> PushCapInfo( - "pushFatigue", - modifiedMinDurationSincePush.inHours.toByte) - - minDurationStats - .stat("minDurationSincePushValueStats").add(modifiedMinDurationSincePush.inHours) - minDurationStats - .scope("minDurationSincePushValueCount").counter( - s"$modifiedMinDurationSincePush").incr() - - modifiedMinDurationSincePush - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushDeviceUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushDeviceUtil.scala deleted file mode 100644 index d191d742a..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushDeviceUtil.scala +++ /dev/null @@ -1,57 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.frigate.common.store.deviceinfo.DeviceInfo -import com.twitter.frigate.common.store.deviceinfo.MobileClientType -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.util.Future -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver - -object PushDeviceUtil { - - def isPrimaryDeviceAndroid(deviceInfoOpt: Option[DeviceInfo]): Boolean = { - deviceInfoOpt.exists { - _.guessedPrimaryClient.exists { clientType => - (clientType == MobileClientType.Android) || (clientType == MobileClientType.AndroidLite) - } - } - } - - def isPrimaryDeviceIOS(deviceInfoOpt: Option[DeviceInfo]): Boolean = { - deviceInfoOpt.exists { - _.guessedPrimaryClient.exists { clientType => - (clientType == MobileClientType.Iphone) || (clientType == MobileClientType.Ipad) - } - } - } - - def isPushRecommendationsEligible(target: Target): Future[Boolean] = - target.deviceInfo.map(_.exists(_.isRecommendationsEligible)) - - def isTopicsEligible( - target: Target, - statsReceiver: StatsReceiver = NullStatsReceiver - ): Future[Boolean] = { - val isTopicsSkipFatigue = Future.True - - Future.join(isTopicsSkipFatigue, target.deviceInfo.map(_.exists(_.isTopicsEligible))).map { - case (isTopicsNotFatigue, isTopicsEligibleSetting) => - isTopicsNotFatigue && isTopicsEligibleSetting - } - } - - def isSpacesEligible(target: Target): Future[Boolean] = - target.deviceInfo.map(_.exists(_.isSpacesEligible)) - - def isNtabOnlyEligible: Future[Boolean] = { - Future.False - } - - def isRecommendationsEligible(target: Target): Future[Boolean] = { - Future.join(isPushRecommendationsEligible(target), isNtabOnlyEligible).map { - case (isPushRecommendation, isNtabOnly) => isPushRecommendation || isNtabOnly - case _ => false - } - } - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushIbisUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushIbisUtil.scala deleted file mode 100644 index 7567726bf..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushIbisUtil.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.util.Future - -object PushIbisUtil { - - def getSocialContextModelValues(socialContextUserIds: Seq[Long]): Map[String, String] = { - - val socialContextSize = socialContextUserIds.size - - val (displaySocialContexts, otherCount) = { - if (socialContextSize < 3) (socialContextUserIds, 0) - else (socialContextUserIds.take(1), socialContextSize - 1) - } - - val usersValue = displaySocialContexts.map(_.toString).mkString(",") - - if (otherCount > 0) Map("social_users" -> s"$usersValue+$otherCount") - else Map("social_users" -> usersValue) - } - - def mergeFutModelValues( - mvFut1: Future[Map[String, String]], - mvFut2: Future[Map[String, String]] - ): Future[Map[String, String]] = { - Future.join(mvFut1, mvFut2).map { - case (mv1, mv2) => mv1 ++ mv2 - } - } - - def mergeModelValues( - mvFut1: Future[Map[String, String]], - mv2: Map[String, String] - ): Future[Map[String, String]] = - mvFut1.map { mv1 => mv1 ++ mv2 } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushToHomeUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushToHomeUtil.scala deleted file mode 100644 index 8f9fd63c3..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushToHomeUtil.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.store.deviceinfo.DeviceInfo -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -object PushToHomeUtil { - def getIbis2ModelValue( - deviceInfoOpt: Option[DeviceInfo], - target: Target, - stats: StatsReceiver - ): Option[Map[String, String]] = { - deviceInfoOpt.flatMap { deviceInfo => - val isAndroidEnabled = deviceInfo.isLandOnHomeAndroid && target.params( - PushFeatureSwitchParams.EnableTweetPushToHomeAndroid) - val isIOSEnabled = deviceInfo.isLandOnHomeiOS && target.params( - PushFeatureSwitchParams.EnableTweetPushToHomeiOS) - if (isAndroidEnabled || isIOSEnabled) { - stats.counter("enable_push_to_home").incr() - Some(Map("is_land_on_home" -> "true")) - } else None - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/RFPHTakeStepUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/RFPHTakeStepUtil.scala deleted file mode 100644 index 015e065ec..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/RFPHTakeStepUtil.scala +++ /dev/null @@ -1,114 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.Invalid -import com.twitter.frigate.common.base.OK -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.refresh_handler.ResultWithDebugInfo -import com.twitter.frigate.pushservice.predicate.BigFilteringEpsilonGreedyExplorationPredicate -import com.twitter.frigate.pushservice.predicate.MlModelsHoldbackExperimentPredicate -import com.twitter.frigate.pushservice.take.candidate_validator.RFPHCandidateValidator -import com.twitter.frigate.pushservice.thriftscala.PushStatus -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.util.Future - -class RFPHTakeStepUtil()(globalStats: StatsReceiver) { - - implicit val statsReceiver: StatsReceiver = - globalStats.scope("RefreshForPushHandler") - private val takeStats: StatsReceiver = statsReceiver.scope("take") - private val notifierStats = takeStats.scope("notifier") - private val validatorStats = takeStats.scope("validator") - private val validatorLatency: Stat = validatorStats.stat("latency") - - private val executedPredicatesInTandem: Counter = - takeStats.counter("predicates_executed_in_tandem") - - private val bigFilteringEpsGreedyPredicate: NamedPredicate[PushCandidate] = - BigFilteringEpsilonGreedyExplorationPredicate()(takeStats) - private val bigFilteringEpsGreedyStats: StatsReceiver = - takeStats.scope("big_filtering_eps_greedy_predicate") - - private val modelPredicate: NamedPredicate[PushCandidate] = - MlModelsHoldbackExperimentPredicate()(takeStats) - private val mlPredicateStats: StatsReceiver = takeStats.scope("ml_predicate") - - private def updateFilteredStatusExptStats(candidate: PushCandidate, predName: String): Unit = { - - val recTypeStat = globalStats.scope( - candidate.commonRecType.toString - ) - - recTypeStat.counter(PushStatus.Filtered.toString).incr() - recTypeStat - .scope(PushStatus.Filtered.toString) - .counter(predName) - .incr() - } - - def isCandidateValid( - candidate: PushCandidate, - candidateValidator: RFPHCandidateValidator - ): Future[ResultWithDebugInfo] = { - val predResultFuture = Stat.timeFuture(validatorLatency) { - Future - .join( - bigFilteringEpsGreedyPredicate.apply(Seq(candidate)), - modelPredicate.apply(Seq(candidate)) - ).flatMap { - case (Seq(true), Seq(true)) => - executedPredicatesInTandem.incr() - - bigFilteringEpsGreedyStats - .scope(candidate.commonRecType.toString) - .counter("passed") - .incr() - - mlPredicateStats - .scope(candidate.commonRecType.toString) - .counter("passed") - .incr() - candidateValidator.validateCandidate(candidate).map((_, Nil)) - case (Seq(false), _) => - bigFilteringEpsGreedyStats - .scope(candidate.commonRecType.toString) - .counter("filtered") - .incr() - Future.value((Some(bigFilteringEpsGreedyPredicate), Nil)) - case (_, _) => - mlPredicateStats - .scope(candidate.commonRecType.toString) - .counter("filtered") - .incr() - Future.value((Some(modelPredicate), Nil)) - } - } - - predResultFuture.map { - case (Some(pred: NamedPredicate[_]), candPredicateResults) => - takeStats.counter("filtered_by_named_general_predicate").incr() - updateFilteredStatusExptStats(candidate, pred.name) - ResultWithDebugInfo( - Invalid(Some(pred.name)), - candPredicateResults - ) - - case (Some(_), candPredicateResults) => - takeStats.counter("filtered_by_unnamed_general_predicate").incr() - updateFilteredStatusExptStats(candidate, predName = "unk") - ResultWithDebugInfo( - Invalid(Some("unnamed_candidate_predicate")), - candPredicateResults - ) - - case (None, candPredicateResults) => - takeStats.counter("accepted_push_ok").incr() - ResultWithDebugInfo( - OK, - candPredicateResults - ) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/RelationshipUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/RelationshipUtil.scala deleted file mode 100644 index 8f24756ae..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/RelationshipUtil.scala +++ /dev/null @@ -1,66 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.frigate.common.base.TweetAuthor -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.hermit.predicate.socialgraph.Edge -import com.twitter.hermit.predicate.socialgraph.RelationEdge -import com.twitter.socialgraph.thriftscala.RelationshipType - -/** - * This class provides utility functions for relationshipEdge for each Candidate type. - */ -object RelationshipUtil { - - /** - * Form relationEdges - * @param candidate PushCandidate - * @param relationship relationshipTypes for different candidate types - * @return relationEdges for different candidate types - */ - private def formRelationEdgeWithTargetIdAndAuthorId( - candidate: RawCandidate, - relationship: List[RelationshipType with Product] - ): List[RelationEdge] = { - candidate match { - case candidate: RawCandidate with TweetAuthor => - candidate.authorId match { - case Some(authorId) => - val edge = Edge(candidate.target.targetId, authorId) - for { - r <- relationship - } yield RelationEdge(edge, r) - case _ => List.empty[RelationEdge] - } - case _ => List.empty[RelationEdge] - } - } - - /** - * Form all relationshipEdges for basicTweetRelationShips - * @param candidate PushCandidate - * @return List of relationEdges for basicTweetRelationShips - */ - def getBasicTweetRelationships(candidate: RawCandidate): List[RelationEdge] = { - val relationship = List( - RelationshipType.DeviceFollowing, - RelationshipType.Blocking, - RelationshipType.BlockedBy, - RelationshipType.HideRecommendations, - RelationshipType.Muting) - formRelationEdgeWithTargetIdAndAuthorId(candidate, relationship) - } - - /** - * Form all relationshipEdges for F1tweetsRelationships - * @param candidate PushCandidate - * @return List of relationEdges for F1tweetsRelationships - */ - def getPreCandidateRelationshipsForInNetworkTweets( - candidate: RawCandidate - ): List[RelationEdge] = { - val relationship = List(RelationshipType.Following) - getBasicTweetRelationships(candidate) ++ formRelationEdgeWithTargetIdAndAuthorId( - candidate, - relationship) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/ResponseStatsTrackUtils.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/ResponseStatsTrackUtils.scala deleted file mode 100644 index 1b16ec8c0..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/ResponseStatsTrackUtils.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.finagle.stats.BroadcastStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.thriftscala.PushResponse -import com.twitter.frigate.pushservice.thriftscala.PushStatus -import com.twitter.frigate.thriftscala.CommonRecommendationType - -object ResponseStatsTrackUtils { - def trackStatsForResponseToRequest( - crt: CommonRecommendationType, - target: Target, - response: PushResponse, - receivers: Seq[StatsReceiver] - )( - originalStats: StatsReceiver - ): Unit = { - val newReceivers = Seq( - originalStats - .scope("is_model_training_data") - .scope(target.isModelTrainingData.toString), - originalStats.scope("scribe_target").scope(IbisScribeTargets.crtToScribeTarget(crt)) - ) - - val broadcastStats = BroadcastStatsReceiver(receivers) - val broadcastStatsWithExpts = BroadcastStatsReceiver(newReceivers ++ receivers) - - if (response.status == PushStatus.Sent) { - if (target.isModelTrainingData) { - broadcastStats.counter("num_training_data_recs_sent").incr() - } - } - broadcastStatsWithExpts.counter(response.status.toString).incr() - if (response.status == PushStatus.Filtered) { - broadcastStats - .scope(response.status.toString) - .counter(response.filteredBy.getOrElse("None")) - .incr() - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/SendHandlerPredicateUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/SendHandlerPredicateUtil.scala deleted file mode 100644 index 4174fa21c..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/SendHandlerPredicateUtil.scala +++ /dev/null @@ -1,129 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.common.base.CandidateResult -import com.twitter.frigate.common.base.Invalid -import com.twitter.frigate.common.base.OK -import com.twitter.frigate.common.base.Result -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.refresh_handler.ResultWithDebugInfo -import com.twitter.frigate.pushservice.take.candidate_validator.SendHandlerPostCandidateValidator -import com.twitter.frigate.pushservice.take.candidate_validator.SendHandlerPreCandidateValidator -import com.twitter.frigate.pushservice.thriftscala.PushStatus -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.util.Future - -class SendHandlerPredicateUtil()(globalStats: StatsReceiver) { - implicit val statsReceiver: StatsReceiver = - globalStats.scope("SendHandler") - private val validateStats: StatsReceiver = statsReceiver.scope("validate") - - private def updateFilteredStatusExptStats(candidate: PushCandidate, predName: String): Unit = { - - val recTypeStat = globalStats.scope( - candidate.commonRecType.toString - ) - - recTypeStat.counter(PushStatus.Filtered.toString).incr() - recTypeStat - .scope(PushStatus.Filtered.toString) - .counter(predName) - .incr() - } - - /** - * Parsing the candidateValidtor result into desired format for preValidation before ml filtering - * @param hydratedCandidates - * @param candidateValidator - * @return - */ - def preValidationForCandidate( - hydratedCandidates: Seq[CandidateDetails[PushCandidate]], - candidateValidator: SendHandlerPreCandidateValidator - ): Future[ - (Seq[CandidateDetails[PushCandidate]], Seq[CandidateResult[PushCandidate, Result]]) - ] = { - val predResultFuture = - Future.collect( - hydratedCandidates.map(hydratedCandidate => - candidateValidator.validateCandidate(hydratedCandidate.candidate)) - ) - - predResultFuture.map { results => - results - .zip(hydratedCandidates) - .foldLeft( - ( - Seq.empty[CandidateDetails[PushCandidate]], - Seq.empty[CandidateResult[PushCandidate, Result]] - ) - ) { - case ((goodCandidates, filteredCandidates), (result, candidateDetails)) => - result match { - case None => - (goodCandidates :+ candidateDetails, filteredCandidates) - case Some(pred: NamedPredicate[_]) => - val r = Invalid(Some(pred.name)) - ( - goodCandidates, - filteredCandidates :+ CandidateResult[PushCandidate, Result]( - candidateDetails.candidate, - candidateDetails.source, - r - ) - ) - case Some(_) => - val r = Invalid(Some("Filtered by un-named predicate")) - ( - goodCandidates, - filteredCandidates :+ CandidateResult[PushCandidate, Result]( - candidateDetails.candidate, - candidateDetails.source, - r - ) - ) - } - } - } - } - - /** - * Parsing the candidateValidtor result into desired format for postValidation including and after ml filtering - * @param candidate - * @param candidateValidator - * @return - */ - def postValidationForCandidate( - candidate: PushCandidate, - candidateValidator: SendHandlerPostCandidateValidator - ): Future[ResultWithDebugInfo] = { - val predResultFuture = - candidateValidator.validateCandidate(candidate) - - predResultFuture.map { - case (Some(pred: NamedPredicate[_])) => - validateStats.counter("filtered_by_named_general_predicate").incr() - updateFilteredStatusExptStats(candidate, pred.name) - ResultWithDebugInfo( - Invalid(Some(pred.name)), - Nil - ) - - case Some(_) => - validateStats.counter("filtered_by_unnamed_general_predicate").incr() - updateFilteredStatusExptStats(candidate, predName = "unk") - ResultWithDebugInfo( - Invalid(Some("unnamed_candidate_predicate")), - Nil - ) - - case _ => - validateStats.counter("accepted_push_ok").incr() - ResultWithDebugInfo( - OK, - Nil - ) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/TopicsUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/TopicsUtil.scala deleted file mode 100644 index a5b7cf2c5..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/TopicsUtil.scala +++ /dev/null @@ -1,340 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.contentrecommender.thriftscala.DisplayLocation -import com.twitter.finagle.stats.Stat -import com.twitter.frigate.common.base.TargetUser -import com.twitter.frigate.common.predicate.CommonOutNetworkTweetCandidatesSourcePredicates.authorNotBeingFollowedPredicate -import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.model.PushTypes -import com.twitter.frigate.pushservice.store.UttEntityHydrationQuery -import com.twitter.frigate.pushservice.store.UttEntityHydrationStore -import com.twitter.hermit.predicate.Predicate -import com.twitter.hermit.predicate.socialgraph.RelationEdge -import com.twitter.interests.thriftscala.InterestRelationType -import com.twitter.interests.thriftscala.InterestRelationship -import com.twitter.interests.thriftscala.InterestedInInterestLookupContext -import com.twitter.interests.thriftscala.InterestedInInterestModel -import com.twitter.interests.thriftscala.ProductId -import com.twitter.interests.thriftscala.UserInterest -import com.twitter.interests.thriftscala.UserInterestData -import com.twitter.interests.thriftscala.UserInterests -import com.twitter.interests.thriftscala.{TopicListingViewerContext => TopicListingViewerContextCR} -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi.Param -import com.twitter.topiclisting.TopicListingViewerContext -import com.twitter.topiclisting.utt.LocalizedEntity -import com.twitter.tsp.thriftscala.TopicListingSetting -import com.twitter.tsp.thriftscala.TopicSocialProofRequest -import com.twitter.tsp.thriftscala.TopicSocialProofResponse -import com.twitter.tsp.thriftscala.TopicWithScore -import com.twitter.util.Future -import scala.collection.Map - -case class TweetWithTopicProof( - tweetId: Long, - topicId: Long, - authorId: Option[Long], - score: Double, - tweetyPieResult: TweetyPieResult, - topicListingSetting: String, - algorithmCR: Option[String], - isOON: Boolean) - -object TopicsUtil { - - /** - * Obtains the Localized Entities for the provided SC Entity IDs - * @param target The target user for which we're obtaining candidates - * @param semanticCoreEntityIds The seq. of entity ids for which we would like to obtain the Localized Entities - * @param uttEntityHydrationStore Store to query the actual LocalizedEntities - * @return A Future Map consisting of the entity id as the key and LocalizedEntity as the value - */ - def getLocalizedEntityMap( - target: Target, - semanticCoreEntityIds: Set[Long], - uttEntityHydrationStore: UttEntityHydrationStore - ): Future[Map[Long, LocalizedEntity]] = { - buildTopicListingViewerContext(target) - .flatMap { topicListingViewerContext => - val query = UttEntityHydrationQuery(topicListingViewerContext, semanticCoreEntityIds.toSeq) - val localizedTopicEntitiesFut = - uttEntityHydrationStore.getLocalizedTopicEntities(query).map(_.flatten) - localizedTopicEntitiesFut.map { localizedTopicEntities => - localizedTopicEntities.map { localizedTopicEntity => - localizedTopicEntity.entityId -> localizedTopicEntity - }.toMap - } - } - } - - /** - * Fetch explict followed interests i.e Topics for targetUser - * - * @param targetUser: [[Target]] object representing a user eligible for MagicRecs notification - * @return: list of all Topics(Interests) Followed by targetUser - */ - def getTopicsFollowedByUser( - targetUser: Target, - interestsWithLookupContextStore: ReadableStore[ - InterestsLookupRequestWithContext, - UserInterests - ], - followedTopicsStats: Stat - ): Future[Option[Seq[UserInterest]]] = { - buildTopicListingViewerContext(targetUser).flatMap { topicListingViewerContext => - // explicit interests relation query - val explicitInterestsLookupRequest = InterestsLookupRequestWithContext( - targetUser.targetId, - Some( - InterestedInInterestLookupContext( - explicitContext = None, - inferredContext = None, - productId = Some(ProductId.Followable), - topicListingViewerContext = Some(topicListingViewerContext.toThrift), - disableExplicit = None, - disableImplicit = Some(true) - ) - ) - ) - - // filter explicit follow relationships from response - interestsWithLookupContextStore.get(explicitInterestsLookupRequest).map { - _.flatMap { userInterests => - val followedTopics = userInterests.interests.map { - _.filter { - case UserInterest(_, Some(interestData)) => - interestData match { - case UserInterestData.InterestedIn(interestedIn) => - interestedIn.exists { - case InterestedInInterestModel.ExplicitModel(explicitModel) => - explicitModel match { - case InterestRelationship.V1(v1) => - v1.relation == InterestRelationType.Followed - - case _ => false - } - - case _ => false - } - - case _ => false - } - - case _ => false // interestData unavailable - } - } - followedTopicsStats.add(followedTopics.getOrElse(Seq.empty[UserInterest]).size) - followedTopics - } - } - } - } - - /** - * - * @param target : [[Target]] object respresenting MagicRecs user - * - * @return: [[TopicListingViewerContext]] for querying topics - */ - def buildTopicListingViewerContext(target: Target): Future[TopicListingViewerContext] = { - Future.join(target.inferredUserDeviceLanguage, target.countryCode, target.targetUser).map { - case (inferredLanguage, countryCode, userInfo) => - TopicListingViewerContext( - userId = Some(target.targetId), - guestId = None, - deviceId = None, - clientApplicationId = None, - userAgent = None, - languageCode = inferredLanguage, - countryCode = countryCode, - userRoles = userInfo.flatMap(_.roles.map(_.roles.toSet)) - ) - } - } - - /** - * - * @param target : [[Target]] object respresenting MagicRecs user - * - * @return: [[TopicListingViewerContext]] for querying topics - */ - def buildTopicListingViewerContextForCR(target: Target): Future[TopicListingViewerContextCR] = { - TopicsUtil.buildTopicListingViewerContext(target).map(_.toThrift) - } - - /** - * - * @param target : [[Target]] object respresenting MagicRecs user - * @param tweets : [[Seq[TweetyPieResult]]] object representing Tweets to get TSP for - * @param topicSocialProofServiceStore: [[ReadableStore[TopicSocialProofRequest, TopicSocialProofResponse]]] - * @param edgeStore: [[ReadableStore[RelationEdge, Boolean]]]] - * - * @return: [[Future[Seq[TweetWithTopicProof]]]] Tweets with topic proof - */ - def getTopicSocialProofs( - inputTarget: Target, - tweets: Seq[TweetyPieResult], - topicSocialProofServiceStore: ReadableStore[TopicSocialProofRequest, TopicSocialProofResponse], - edgeStore: ReadableStore[RelationEdge, Boolean], - scoreThresholdParam: Param[Double] - ): Future[Seq[TweetWithTopicProof]] = { - buildTopicListingViewerContextForCR(inputTarget).flatMap { topicListingContext => - val tweetIds: Set[Long] = tweets.map(_.tweet.id).toSet - val tweetIdsToTweetyPie = tweets.map(tp => tp.tweet.id -> tp).toMap - val topicSocialProofRequest = - TopicSocialProofRequest( - inputTarget.targetId, - tweetIds, - DisplayLocation.MagicRecsRecommendTopicTweets, - TopicListingSetting.Followable, - topicListingContext) - - topicSocialProofServiceStore - .get(topicSocialProofRequest).flatMap { - case Some(topicSocialProofResponse) => - val topicProofCandidates = topicSocialProofResponse.socialProofs.collect { - case (tweetId, topicsWithScore) - if topicsWithScore.nonEmpty && topicsWithScore - .maxBy(_.score).score >= inputTarget - .params(scoreThresholdParam) => - // Get the topic with max score if there are any topics returned - val topicWithScore = topicsWithScore.maxBy(_.score) - TweetWithTopicProof( - tweetId, - topicWithScore.topicId, - tweetIdsToTweetyPie(tweetId).tweet.coreData.map(_.userId), - topicWithScore.score, - tweetIdsToTweetyPie(tweetId), - topicWithScore.topicFollowType.map(_.name).getOrElse(""), - topicWithScore.algorithmType.map(_.name), - isOON = true - ) - }.toSeq - - hydrateTopicProofCandidatesWithEdgeStore(inputTarget, topicProofCandidates, edgeStore) - case _ => Future.value(Seq.empty[TweetWithTopicProof]) - } - } - } - - /** - * Obtain TopicWithScores for provided tweet candidates and target - * @param target target user - * @param Tweets tweet candidates represented in a (tweetId, TweetyPieResult) map - * @param topicSocialProofServiceStore store to query topic social proof - * @param enableTopicAnnotation whether to enable topic annotation - * @param topicScoreThreshold threshold for topic score - * @return a (tweetId, TopicWithScore) map where the topic with highest topic score (if exists) is chosen - */ - def getTopicsWithScoreMap( - target: PushTypes.Target, - Tweets: Map[Long, Option[TweetyPieResult]], - topicSocialProofServiceStore: ReadableStore[TopicSocialProofRequest, TopicSocialProofResponse], - enableTopicAnnotation: Boolean, - topicScoreThreshold: Double - ): Future[Option[Map[Long, TopicWithScore]]] = { - - if (enableTopicAnnotation) { - TopicsUtil - .buildTopicListingViewerContextForCR(target).flatMap { topicListingContext => - val tweetIds = Tweets.keySet - val topicSocialProofRequest = - TopicSocialProofRequest( - target.targetId, - tweetIds, - DisplayLocation.MagicRecsRecommendTopicTweets, - TopicListingSetting.Followable, - topicListingContext) - - topicSocialProofServiceStore - .get(topicSocialProofRequest).map { - _.map { topicSocialProofResponse => - topicSocialProofResponse.socialProofs - .collect { - case (tweetId, topicsWithScore) - if topicsWithScore.nonEmpty && Tweets(tweetId).nonEmpty - && topicsWithScore.maxBy(_.score).score >= topicScoreThreshold => - tweetId -> topicsWithScore.maxBy(_.score) - } - - } - } - } - } else { - Future.None - } - - } - - /** - * Obtain LocalizedEntities for provided tweet candidates and target - * @param target target user - * @param Tweets tweet candidates represented in a (tweetId, TweetyPieResult) map - * @param uttEntityHydrationStore store to query the actual LocalizedEntities - * @param topicSocialProofServiceStore store to query topic social proof - * @param enableTopicAnnotation whether to enable topic annotation - * @param topicScoreThreshold threshold for topic score - * @return a (tweetId, LocalizedEntity Option) Future map that stores Localized Entity (can be empty) for given tweetId - */ - def getTweetIdLocalizedEntityMap( - target: PushTypes.Target, - Tweets: Map[Long, Option[TweetyPieResult]], - uttEntityHydrationStore: UttEntityHydrationStore, - topicSocialProofServiceStore: ReadableStore[TopicSocialProofRequest, TopicSocialProofResponse], - enableTopicAnnotation: Boolean, - topicScoreThreshold: Double - ): Future[Map[Long, Option[LocalizedEntity]]] = { - - val topicWithScoreMap = getTopicsWithScoreMap( - target, - Tweets, - topicSocialProofServiceStore, - enableTopicAnnotation, - topicScoreThreshold) - - topicWithScoreMap.flatMap { topicWithScores => - topicWithScores match { - case Some(topics) => - val topicIds = topics.collect { case (_, topic) => topic.topicId }.toSet - val LocalizedEntityMapFut = - getLocalizedEntityMap(target, topicIds, uttEntityHydrationStore) - - LocalizedEntityMapFut.map { LocalizedEntityMap => - topics.map { - case (tweetId, topic) => - tweetId -> LocalizedEntityMap.get(topic.topicId) - } - } - case _ => Future.value(Map[Long, Option[LocalizedEntity]]()) - } - } - - } - - /** - * Hydrate TweetWithTopicProof candidates with isOON field info, - * based on the following relationship between target user and candidate author in edgeStore - * @return TweetWithTopicProof candidates with isOON field populated - */ - def hydrateTopicProofCandidatesWithEdgeStore( - inputTarget: TargetUser, - topicProofCandidates: Seq[TweetWithTopicProof], - edgeStore: ReadableStore[RelationEdge, Boolean], - ): Future[Seq[TweetWithTopicProof]] = { - // IDs of all authors of TopicProof candidates that are OON with respect to inputTarget - val validOONAuthorIdsFut = - Predicate.filter( - topicProofCandidates.flatMap(_.authorId).distinct, - authorNotBeingFollowedPredicate(inputTarget, edgeStore)) - - validOONAuthorIdsFut.map { validOONAuthorIds => - topicProofCandidates.map(candidate => { - candidate.copy(isOON = - candidate.authorId.isDefined && validOONAuthorIds.contains(candidate.authorId.get)) - }) - } - } - -}