diff --git a/navi/navi/src/bin/navi.docx b/navi/navi/src/bin/navi.docx new file mode 100644 index 000000000..f5c10fa17 Binary files /dev/null and b/navi/navi/src/bin/navi.docx differ diff --git a/navi/navi/src/bin/navi.rs b/navi/navi/src/bin/navi.rs deleted file mode 100644 index 491d21d1e..000000000 --- a/navi/navi/src/bin/navi.rs +++ /dev/null @@ -1,47 +0,0 @@ -use anyhow::Result; -use log::info; -use navi::cli_args::{ARGS, MODEL_SPECS}; -use navi::cores::validator::validatior::cli_validator; -use navi::tf_model::tf::TFModel; -use navi::{bootstrap, metrics}; -use sha256::digest; - -fn main() -> Result<()> { - env_logger::init(); - cli_validator::validate_input_args(); - //only validate in for tf as other models don't have this - assert_eq!(MODEL_SPECS.len(), ARGS.serving_sig.len()); - metrics::register_custom_metrics(); - - //load all the custom ops - comma seperaed - if let Some(ref customops_lib) = ARGS.customops_lib { - for op_lib in customops_lib.split(",") { - load_custom_op(op_lib); - } - } - - // versioning the customop so library - bootstrap::bootstrap(TFModel::new) -} - -fn load_custom_op(lib_path: &str) -> () { - let res = tensorflow::Library::load(lib_path); - info!("{} load status:{:?}", lib_path, res); - let customop_version_num = get_custom_op_version(lib_path); - // Last OP version is recorded - metrics::CUSTOMOP_VERSION.set(customop_version_num); -} - -//fn get_custom_op_version(customops_lib: &String) -> i64 { -fn get_custom_op_version(customops_lib: &str) -> i64 { - let customop_bytes = std::fs::read(customops_lib).unwrap(); // Vec - let customop_hash = digest(customop_bytes.as_slice()); - //conver the last 4 hex digits to version number as prometheus metrics doesn't support string, the total space is 16^4 == 65536 - let customop_version_num = - i64::from_str_radix(&customop_hash[customop_hash.len() - 4..], 16).unwrap(); - info!( - "customop hash: {}, version_number: {}", - customop_hash, customop_version_num - ); - customop_version_num -} diff --git a/navi/navi/src/bin/navi_onnx.docx b/navi/navi/src/bin/navi_onnx.docx new file mode 100644 index 000000000..1bb89db61 Binary files /dev/null and b/navi/navi/src/bin/navi_onnx.docx differ diff --git a/navi/navi/src/bin/navi_onnx.rs b/navi/navi/src/bin/navi_onnx.rs deleted file mode 100644 index 03b1ea2aa..000000000 --- a/navi/navi/src/bin/navi_onnx.rs +++ /dev/null @@ -1,24 +0,0 @@ -use anyhow::Result; -use log::info; -use navi::cli_args::{ARGS, MODEL_SPECS}; -use navi::onnx_model::onnx::OnnxModel; -use navi::{bootstrap, metrics}; - -fn main() -> Result<()> { - env_logger::init(); - info!("global: {:?}", ARGS.onnx_global_thread_pool_options); - let assert_session_params = if ARGS.onnx_global_thread_pool_options.is_empty() { - // std::env::set_var("OMP_NUM_THREADS", "1"); - info!("now we use per session thread pool"); - MODEL_SPECS.len() - } - else { - info!("now we use global thread pool"); - 0 - }; - assert_eq!(assert_session_params, ARGS.inter_op_parallelism.len()); - assert_eq!(assert_session_params, ARGS.inter_op_parallelism.len()); - - metrics::register_custom_metrics(); - bootstrap::bootstrap(OnnxModel::new) -} diff --git a/navi/navi/src/bin/navi_torch.docx b/navi/navi/src/bin/navi_torch.docx new file mode 100644 index 000000000..f2be5d615 Binary files /dev/null and b/navi/navi/src/bin/navi_torch.docx differ diff --git a/navi/navi/src/bin/navi_torch.rs b/navi/navi/src/bin/navi_torch.rs deleted file mode 100644 index c8d68e875..000000000 --- a/navi/navi/src/bin/navi_torch.rs +++ /dev/null @@ -1,19 +0,0 @@ -use anyhow::Result; -use log::info; -use navi::cli_args::ARGS; -use navi::metrics; -use navi::torch_model::torch::TorchModel; - -fn main() -> Result<()> { - env_logger::init(); - //torch only has global threadpool settings versus tf has per model threadpool settings - assert_eq!(1, ARGS.inter_op_parallelism.len()); - assert_eq!(1, ARGS.intra_op_parallelism.len()); - //TODO for now we, we assume each model's output has only 1 tensor. - //this will be lifted once torch_model properly implements mtl outputs - tch::set_num_interop_threads(ARGS.inter_op_parallelism[0].parse()?); - tch::set_num_threads(ARGS.intra_op_parallelism[0].parse()?); - info!("torch custom ops not used for now"); - metrics::register_custom_metrics(); - navi::bootstrap::bootstrap(TorchModel::new) -} diff --git a/navi/navi/src/bootstrap.docx b/navi/navi/src/bootstrap.docx new file mode 100644 index 000000000..eaadd6a67 Binary files /dev/null and b/navi/navi/src/bootstrap.docx differ diff --git a/navi/navi/src/bootstrap.rs b/navi/navi/src/bootstrap.rs deleted file mode 100644 index 1f767f17e..000000000 --- a/navi/navi/src/bootstrap.rs +++ /dev/null @@ -1,326 +0,0 @@ -use anyhow::Result; -use log::{info, warn}; -use x509_parser::{prelude::{parse_x509_pem}, parse_x509_certificate}; -use std::collections::HashMap; -use tokio::time::Instant; -use tonic::{ - Request, - Response, Status, transport::{Certificate, Identity, Server, ServerTlsConfig}, -}; - -// protobuf related -use crate::tf_proto::tensorflow_serving::{ - ClassificationRequest, ClassificationResponse, GetModelMetadataRequest, - GetModelMetadataResponse, MultiInferenceRequest, MultiInferenceResponse, PredictRequest, - PredictResponse, RegressionRequest, RegressionResponse, -}; -use crate::{kf_serving::{ - grpc_inference_service_server::GrpcInferenceService, ModelInferRequest, ModelInferResponse, - ModelMetadataRequest, ModelMetadataResponse, ModelReadyRequest, ModelReadyResponse, - ServerLiveRequest, ServerLiveResponse, ServerMetadataRequest, ServerMetadataResponse, - ServerReadyRequest, ServerReadyResponse, -}, ModelFactory, tf_proto::tensorflow_serving::prediction_service_server::{ - PredictionService, PredictionServiceServer, -}, VERSION, NAME}; - -use crate::PredictResult; -use crate::cli_args::{ARGS, INPUTS, OUTPUTS}; -use crate::metrics::{ - NAVI_VERSION, NUM_PREDICTIONS, NUM_REQUESTS_FAILED, NUM_REQUESTS_FAILED_BY_MODEL, - NUM_REQUESTS_RECEIVED, NUM_REQUESTS_RECEIVED_BY_MODEL, RESPONSE_TIME_COLLECTOR, - CERT_EXPIRY_EPOCH -}; -use crate::predict_service::{Model, PredictService}; -use crate::tf_proto::tensorflow_serving::model_spec::VersionChoice::Version; -use crate::tf_proto::tensorflow_serving::ModelSpec; - -#[derive(Debug)] -pub enum TensorInputEnum { - String(Vec>), - Int(Vec), - Int64(Vec), - Float(Vec), - Double(Vec), - Boolean(Vec), -} - -#[derive(Debug)] -pub struct TensorInput { - pub tensor_data: TensorInputEnum, - pub name: String, - pub dims: Option>, -} - -impl TensorInput { - pub fn new(tensor_data: TensorInputEnum, name: String, dims: Option>) -> TensorInput { - TensorInput { - tensor_data, - name, - dims, - } - } -} - -impl TensorInputEnum { - #[inline(always)] - pub(crate) fn extend(&mut self, another: TensorInputEnum) { - match (self, another) { - (Self::String(input), Self::String(ex)) => input.extend(ex), - (Self::Int(input), Self::Int(ex)) => input.extend(ex), - (Self::Int64(input), Self::Int64(ex)) => input.extend(ex), - (Self::Float(input), Self::Float(ex)) => input.extend(ex), - (Self::Double(input), Self::Double(ex)) => input.extend(ex), - (Self::Boolean(input), Self::Boolean(ex)) => input.extend(ex), - x => panic!("input enum type not matched. input:{:?}, ex:{:?}", x.0, x.1), - } - } - #[inline(always)] - pub(crate) fn merge_batch(input_tensors: Vec>) -> Vec { - input_tensors - .into_iter() - .reduce(|mut acc, e| { - for (i, ext) in acc.iter_mut().zip(e) { - i.tensor_data.extend(ext.tensor_data); - } - acc - }) - .unwrap() //invariant: we expect there's always rows in input_tensors - } -} - - -///entry point for tfServing gRPC -#[tonic::async_trait] -impl GrpcInferenceService for PredictService { - async fn server_live( - &self, - _request: Request, - ) -> Result, Status> { - unimplemented!() - } - async fn server_ready( - &self, - _request: Request, - ) -> Result, Status> { - unimplemented!() - } - - async fn model_ready( - &self, - _request: Request, - ) -> Result, Status> { - unimplemented!() - } - - async fn server_metadata( - &self, - _request: Request, - ) -> Result, Status> { - unimplemented!() - } - - async fn model_metadata( - &self, - _request: Request, - ) -> Result, Status> { - unimplemented!() - } - - async fn model_infer( - &self, - _request: Request, - ) -> Result, Status> { - unimplemented!() - } -} - -#[tonic::async_trait] -impl PredictionService for PredictService { - async fn classify( - &self, - _request: Request, - ) -> Result, Status> { - unimplemented!() - } - async fn regress( - &self, - _request: Request, - ) -> Result, Status> { - unimplemented!() - } - async fn predict( - &self, - request: Request, - ) -> Result, Status> { - NUM_REQUESTS_RECEIVED.inc(); - let start = Instant::now(); - let mut req = request.into_inner(); - let (model_spec, version) = req.take_model_spec(); - NUM_REQUESTS_RECEIVED_BY_MODEL - .with_label_values(&[&model_spec]) - .inc(); - let idx = PredictService::::get_model_index(&model_spec).ok_or_else(|| { - Status::failed_precondition(format!("model spec not found:{}", model_spec)) - })?; - let input_spec = match INPUTS[idx].get() { - Some(input) => input, - _ => return Err(Status::not_found(format!("model input spec {}", idx))), - }; - let input_val = req.take_input_vals(input_spec); - self.predict(idx, version, input_val, start) - .await - .map_or_else( - |e| { - NUM_REQUESTS_FAILED.inc(); - NUM_REQUESTS_FAILED_BY_MODEL - .with_label_values(&[&model_spec]) - .inc(); - Err(Status::internal(e.to_string())) - }, - |res| { - RESPONSE_TIME_COLLECTOR - .with_label_values(&[&model_spec]) - .observe(start.elapsed().as_millis() as f64); - - match res { - PredictResult::Ok(tensors, version) => { - let mut outputs = HashMap::new(); - NUM_PREDICTIONS.with_label_values(&[&model_spec]).inc(); - //FIXME: uncomment when prediction scores are normal - // PREDICTION_SCORE_SUM - // .with_label_values(&[&model_spec]) - // .inc_by(tensors[0]as f64); - for (tp, output_name) in tensors - .into_iter() - .map(|tensor| tensor.create_tensor_proto()) - .zip(OUTPUTS[idx].iter()) - { - outputs.insert(output_name.to_owned(), tp); - } - let reply = PredictResponse { - model_spec: Some(ModelSpec { - version_choice: Some(Version(version)), - ..Default::default() - }), - outputs, - }; - Ok(Response::new(reply)) - } - PredictResult::DropDueToOverload => Err(Status::resource_exhausted("")), - PredictResult::ModelNotFound(idx) => { - Err(Status::not_found(format!("model index {}", idx))) - }, - PredictResult::ModelNotReady(idx) => { - Err(Status::unavailable(format!("model index {}", idx))) - } - PredictResult::ModelVersionNotFound(idx, version) => Err( - Status::not_found(format!("model index:{}, version {}", idx, version)), - ), - } - }, - ) - } - - async fn multi_inference( - &self, - _request: Request, - ) -> Result, Status> { - unimplemented!() - } - async fn get_model_metadata( - &self, - _request: Request, - ) -> Result, Status> { - unimplemented!() - } -} - -// A function that takes a timestamp as input and returns a ticker stream -fn report_expiry(expiry_time: i64) { - info!("Certificate expires at epoch: {:?}", expiry_time); - CERT_EXPIRY_EPOCH.set(expiry_time as i64); -} - -pub fn bootstrap(model_factory: ModelFactory) -> Result<()> { - info!("package: {}, version: {}, args: {:?}", NAME, VERSION, *ARGS); - //we follow SemVer. So here we assume MAJOR.MINOR.PATCH - let parts = VERSION - .split(".") - .map(|v| v.parse::()) - .collect::, _>>()?; - if let [major, minor, patch] = &parts[..] { - NAVI_VERSION.set(major * 1000_000 + minor * 1000 + patch); - } else { - warn!( - "version {} doesn't follow SemVer conversion of MAJOR.MINOR.PATCH", - VERSION - ); - } - - - tokio::runtime::Builder::new_multi_thread() - .thread_name("async worker") - .worker_threads(ARGS.num_worker_threads) - .max_blocking_threads(ARGS.max_blocking_threads) - .enable_all() - .build() - .unwrap() - .block_on(async { - #[cfg(feature = "navi_console")] - console_subscriber::init(); - let addr = format!("0.0.0.0:{}", ARGS.port).parse()?; - - let ps = PredictService::init(model_factory).await; - - let mut builder = if ARGS.ssl_dir.is_empty() { - Server::builder() - } else { - // Read the pem file as a string - let pem_str = std::fs::read_to_string(format!("{}/server.crt", ARGS.ssl_dir)).unwrap(); - let res = parse_x509_pem(&pem_str.as_bytes()); - match res { - Ok((rem, pem_2)) => { - assert!(rem.is_empty()); - assert_eq!(pem_2.label, String::from("CERTIFICATE")); - let res_x509 = parse_x509_certificate(&pem_2.contents); - info!("Certificate label: {}", pem_2.label); - assert!(res_x509.is_ok()); - report_expiry(res_x509.unwrap().1.validity().not_after.timestamp()); - }, - _ => panic!("PEM parsing failed: {:?}", res), - } - - let key = tokio::fs::read(format!("{}/server.key", ARGS.ssl_dir)) - .await - .expect("can't find key file"); - let crt = tokio::fs::read(format!("{}/server.crt", ARGS.ssl_dir)) - .await - .expect("can't find crt file"); - let chain = tokio::fs::read(format!("{}/server.chain", ARGS.ssl_dir)) - .await - .expect("can't find chain file"); - let mut pem = Vec::new(); - pem.extend(crt); - pem.extend(chain); - let identity = Identity::from_pem(pem.clone(), key); - let client_ca_cert = Certificate::from_pem(pem.clone()); - let tls = ServerTlsConfig::new() - .identity(identity) - .client_ca_root(client_ca_cert); - Server::builder() - .tls_config(tls) - .expect("fail to config SSL") - }; - - info!( - "Prometheus server started: 0.0.0.0: {}", - ARGS.prometheus_port - ); - - let ps_server = builder - .add_service(PredictionServiceServer::new(ps).accept_gzip().send_gzip()) - .serve(addr); - info!("Prediction server started: {}", addr); - ps_server.await.map_err(anyhow::Error::msg) - }) -} diff --git a/navi/navi/src/cli_args.docx b/navi/navi/src/cli_args.docx new file mode 100644 index 000000000..1bfe5b567 Binary files /dev/null and b/navi/navi/src/cli_args.docx differ diff --git a/navi/navi/src/cli_args.rs b/navi/navi/src/cli_args.rs deleted file mode 100644 index 300de580f..000000000 --- a/navi/navi/src/cli_args.rs +++ /dev/null @@ -1,236 +0,0 @@ -use crate::{MAX_NUM_INPUTS, MAX_NUM_MODELS, MAX_NUM_OUTPUTS}; -use arrayvec::ArrayVec; -use clap::Parser; -use log::info; -use once_cell::sync::OnceCell; -use std::error::Error; -use time::OffsetDateTime; -use time::format_description::well_known::Rfc3339; -#[derive(Parser, Debug, Clone)] -///Navi is configured through CLI arguments(for now) defined below. -//TODO: use clap_serde to make it config file driven -pub struct Args { - #[clap(short, long, help = "gRPC port Navi runs ons")] - pub port: i32, - #[clap(long, default_value_t = 9000, help = "prometheus metrics port")] - pub prometheus_port: u16, - #[clap( - short, - long, - default_value_t = 1, - help = "number of worker threads for tokio async runtime" - )] - pub num_worker_threads: usize, - #[clap( - long, - default_value_t = 14, - help = "number of blocking threads in tokio blocking thread pool" - )] - pub max_blocking_threads: usize, - #[clap(long, default_value = "16", help = "maximum batch size for a batch")] - pub max_batch_size: Vec, - #[clap( - short, - long, - default_value = "2", - help = "max wait time for accumulating a batch" - )] - pub batch_time_out_millis: Vec, - #[clap( - long, - default_value_t = 90, - help = "threshold to start dropping batches under stress" - )] - pub batch_drop_millis: u64, - #[clap( - long, - default_value_t = 300, - help = "polling interval for new version of a model and META.json config" - )] - pub model_check_interval_secs: u64, - #[clap( - short, - long, - default_value = "models/pvideo/", - help = "root directory for models" - )] - pub model_dir: Vec, - #[clap( - long, - help = "directory containing META.json config. separate from model_dir to facilitate remote config management" - )] - pub meta_json_dir: Option, - #[clap(short, long, default_value = "", help = "directory for ssl certs")] - pub ssl_dir: String, - #[clap( - long, - help = "call out to external process to check model updates. custom logic can be written to pull from hdfs, gcs etc" - )] - pub modelsync_cli: Option, - #[clap( - long, - default_value_t = 1, - help = "specify how many versions Navi retains in memory. good for cases of rolling model upgrade" - )] - pub versions_per_model: usize, - #[clap( - short, - long, - help = "most runtimes support loading ops custom writen. currently only implemented for TF" - )] - pub customops_lib: Option, - #[clap( - long, - default_value = "8", - help = "number of threads to paralleling computations inside an op" - )] - pub intra_op_parallelism: Vec, - #[clap( - long, - help = "number of threads to parallelize computations of the graph" - )] - pub inter_op_parallelism: Vec, - #[clap( - long, - help = "signature of a serving. only TF" - )] - pub serving_sig: Vec, - #[clap(long, default_value = "examples", help = "name of each input tensor")] - pub input: Vec, - #[clap(long, default_value = "output_0", help = "name of each output tensor")] - pub output: Vec, - #[clap( - long, - default_value_t = 500, - help = "max warmup records to use. warmup only implemented for TF" - )] - pub max_warmup_records: usize, - #[clap(long, value_parser = Args::parse_key_val::, value_delimiter=',')] - pub onnx_global_thread_pool_options: Vec<(String, String)>, - #[clap( - long, - default_value = "true", - help = "when to use graph parallelization. only for ONNX" - )] - pub onnx_use_parallel_mode: String, - // #[clap(long, default_value = "false")] - // pub onnx_use_onednn: String, - #[clap( - long, - default_value = "true", - help = "trace internal memory allocation and generate bulk memory allocations. only for ONNX. turn if off if batch size dynamic" - )] - pub onnx_use_memory_pattern: String, - #[clap(long, value_parser = Args::parse_key_val::, value_delimiter=',')] - pub onnx_ep_options: Vec<(String, String)>, - #[clap(long, help = "choice of gpu EPs for ONNX: cuda or tensorrt")] - pub onnx_gpu_ep: Option, - #[clap( - long, - default_value = "home", - help = "converter for various input formats" - )] - pub onnx_use_converter: Option, - #[clap( - long, - help = "whether to enable runtime profiling. only implemented for ONNX for now" - )] - pub profiling: Option, - #[clap( - long, - default_value = "", - help = "metrics reporting for discrete features. only for Home converter for now" - )] - pub onnx_report_discrete_feature_ids: Vec, - #[clap( - long, - default_value = "", - help = "metrics reporting for continuous features. only for Home converter for now" - )] - pub onnx_report_continuous_feature_ids: Vec, -} - -impl Args { - pub fn get_model_specs(model_dir: Vec) -> Vec { - let model_specs = model_dir - .iter() - //let it panic if some model_dir are wrong - .map(|dir| { - dir.trim_end_matches('/') - .rsplit_once('/') - .unwrap() - .1 - .to_owned() - }) - .collect(); - info!("all model_specs: {:?}", model_specs); - model_specs - } - pub fn version_str_to_epoch(dt_str: &str) -> Result { - dt_str - .parse::() - .or_else(|_| { - let ts = OffsetDateTime::parse(dt_str, &Rfc3339) - .map(|d| (d.unix_timestamp_nanos()/1_000_000) as i64); - if ts.is_ok() { - info!("original version {} -> {}", dt_str, ts.unwrap()); - } - ts - }) - .map_err(anyhow::Error::msg) - } - /// Parse a single key-value pair - fn parse_key_val(s: &str) -> Result<(T, U), Box> - where - T: std::str::FromStr, - T::Err: Error + Send + Sync + 'static, - U: std::str::FromStr, - U::Err: Error + Send + Sync + 'static, - { - let pos = s - .find('=') - .ok_or_else(|| format!("invalid KEY=value: no `=` found in `{}`", s))?; - Ok((s[..pos].parse()?, s[pos + 1..].parse()?)) - } -} - -lazy_static! { - pub static ref ARGS: Args = Args::parse(); - pub static ref MODEL_SPECS: ArrayVec = { - let mut specs = ArrayVec::::new(); - Args::get_model_specs(ARGS.model_dir.clone()) - .into_iter() - .for_each(|m| specs.push(m)); - specs - }; - pub static ref INPUTS: ArrayVec>, MAX_NUM_MODELS> = { - let mut inputs = - ArrayVec::>, MAX_NUM_MODELS>::new(); - for (idx, o) in ARGS.input.iter().enumerate() { - if o.trim().is_empty() { - info!("input spec is empty for model {}, auto detect later", idx); - inputs.push(OnceCell::new()); - } else { - inputs.push(OnceCell::with_value( - o.split(",") - .map(|s| s.to_owned()) - .collect::>(), - )); - } - } - info!("all inputs:{:?}", inputs); - inputs - }; - pub static ref OUTPUTS: ArrayVec, MAX_NUM_MODELS> = { - let mut outputs = ArrayVec::, MAX_NUM_MODELS>::new(); - for o in ARGS.output.iter() { - outputs.push( - o.split(",") - .map(|s| s.to_owned()) - .collect::>(), - ); - } - info!("all outputs:{:?}", outputs); - outputs - }; -} diff --git a/navi/navi/src/cores/validator.docx b/navi/navi/src/cores/validator.docx new file mode 100644 index 000000000..d6f3442ef Binary files /dev/null and b/navi/navi/src/cores/validator.docx differ diff --git a/navi/navi/src/cores/validator.rs b/navi/navi/src/cores/validator.rs deleted file mode 100644 index 1ad3bfad7..000000000 --- a/navi/navi/src/cores/validator.rs +++ /dev/null @@ -1,22 +0,0 @@ -pub mod validatior { - pub mod cli_validator { - use crate::cli_args::{ARGS, MODEL_SPECS}; - - pub fn validate_input_args() { - assert_eq!(MODEL_SPECS.len(), ARGS.inter_op_parallelism.len()); - assert_eq!(MODEL_SPECS.len(), ARGS.intra_op_parallelism.len()); - //TODO for now we, we assume each model's output has only 1 tensor. - //this will be lifted once tf_model properly implements mtl outputs - //assert_eq!(OUTPUTS.len(), OUTPUTS.iter().fold(0usize, |a, b| a+b.len())); - } - - pub fn validate_ps_model_args() { - assert!(ARGS.versions_per_model <= 2); - assert!(ARGS.versions_per_model >= 1); - assert_eq!(MODEL_SPECS.len(), ARGS.input.len()); - assert_eq!(MODEL_SPECS.len(), ARGS.model_dir.len()); - assert_eq!(MODEL_SPECS.len(), ARGS.max_batch_size.len()); - assert_eq!(MODEL_SPECS.len(), ARGS.batch_time_out_millis.len()); - } - } -} diff --git a/navi/navi/src/lib.docx b/navi/navi/src/lib.docx new file mode 100644 index 000000000..f23718e65 Binary files /dev/null and b/navi/navi/src/lib.docx differ diff --git a/navi/navi/src/lib.rs b/navi/navi/src/lib.rs deleted file mode 100644 index 3536b5d60..000000000 --- a/navi/navi/src/lib.rs +++ /dev/null @@ -1,215 +0,0 @@ -#[macro_use] -extern crate lazy_static; -extern crate core; - -use serde_json::Value; -use tokio::sync::oneshot::Sender; -use tokio::time::Instant; -use std::ops::Deref; -use itertools::Itertools; -use crate::bootstrap::TensorInput; -use crate::predict_service::Model; -use crate::tf_proto::{DataType, TensorProto}; - -pub mod batch; -pub mod bootstrap; -pub mod cli_args; -pub mod metrics; -pub mod onnx_model; -pub mod predict_service; -pub mod tf_model; -pub mod torch_model; -pub mod cores { - pub mod validator; -} - -pub mod tf_proto { - tonic::include_proto!("tensorflow"); - pub mod tensorflow_serving { - tonic::include_proto!("tensorflow.serving"); - } -} - -pub mod kf_serving { - tonic::include_proto!("inference"); -} -#[cfg(test)] -mod tests { - use crate::cli_args::Args; - #[test] - fn test_version_string_to_epoch() { - assert_eq!( - Args::version_str_to_epoch("2022-12-20T10:18:53.000Z").unwrap_or(-1), - 1671531533000 - ); - assert_eq!(Args::version_str_to_epoch("1203444").unwrap_or(-1), 1203444); - } -} - -mod utils { - use crate::cli_args::{ARGS, MODEL_SPECS}; - use anyhow::Result; - use log::info; - use serde_json::Value; - - pub fn read_config(meta_file: &String) -> Result { - let json = std::fs::read_to_string(meta_file)?; - let v: Value = serde_json::from_str(&json)?; - Ok(v) - } - pub fn get_config_or_else(model_config: &Value, key: &str, default: F) -> String - where - F: FnOnce() -> String, - { - match model_config[key] { - Value::String(ref v) => { - info!("from model_config: {}={}", key, v); - v.to_string() - } - Value::Number(ref num) => { - info!( - "from model_config: {}={} (turn number into a string)", - key, num - ); - num.to_string() - } - _ => { - let d = default(); - info!("from default: {}={}", key, d); - d - } - } - } - pub fn get_config_or(model_config: &Value, key: &str, default: &str) -> String { - get_config_or_else(model_config, key, || default.to_string()) - } - pub fn get_meta_dir() -> &'static str { - ARGS.meta_json_dir - .as_ref() - .map(|s| s.as_str()) - .unwrap_or_else(|| { - let model_dir = &ARGS.model_dir[0]; - let meta_dir = &model_dir[0..model_dir.rfind(&MODEL_SPECS[0]).unwrap()]; - info!( - "no meta_json_dir specified, hence derive from first model dir:{}->{}", - model_dir, meta_dir - ); - meta_dir - }) - } -} - -pub type SerializedInput = Vec; -pub const VERSION: &str = env!("CARGO_PKG_VERSION"); -pub const NAME: &str = env!("CARGO_PKG_NAME"); -pub type ModelFactory = fn(usize, String, &Value) -> anyhow::Result; -pub const MAX_NUM_MODELS: usize = 16; -pub const MAX_NUM_OUTPUTS: usize = 30; -pub const MAX_NUM_INPUTS: usize = 120; -pub const META_INFO: &str = "META.json"; - -//use a heap allocated generic type here so that both -//Tensorflow & Pytorch implementation can return their Tensor wrapped in a Box -//without an extra memcopy to Vec -pub type TensorReturn = Box>; - -//returned tensor may be int64 i.e., a list of relevant ad ids -pub enum TensorReturnEnum { - FloatTensorReturn(TensorReturn), - StringTensorReturn(TensorReturn), - Int64TensorReturn(TensorReturn), - Int32TensorReturn(TensorReturn), -} - -impl TensorReturnEnum { - #[inline(always)] - pub fn slice(&self, start: usize, end: usize) -> TensorScores { - match self { - TensorReturnEnum::FloatTensorReturn(f32_return) => { - TensorScores::Float32TensorScores(f32_return[start..end].to_vec()) - } - TensorReturnEnum::Int64TensorReturn(i64_return) => { - TensorScores::Int64TensorScores(i64_return[start..end].to_vec()) - } - TensorReturnEnum::Int32TensorReturn(i32_return) => { - TensorScores::Int32TensorScores(i32_return[start..end].to_vec()) - } - TensorReturnEnum::StringTensorReturn(str_return) => { - TensorScores::StringTensorScores(str_return[start..end].to_vec()) - } - } - } -} - -#[derive(Debug)] -pub enum PredictResult { - Ok(Vec, i64), - DropDueToOverload, - ModelNotFound(usize), - ModelNotReady(usize), - ModelVersionNotFound(usize, i64), -} - -#[derive(Debug)] -pub enum TensorScores { - Float32TensorScores(Vec), - Int64TensorScores(Vec), - Int32TensorScores(Vec), - StringTensorScores(Vec), -} - -impl TensorScores { - pub fn create_tensor_proto(self) -> TensorProto { - match self { - TensorScores::Float32TensorScores(f32_tensor) => TensorProto { - dtype: DataType::DtFloat as i32, - float_val: f32_tensor, - ..Default::default() - }, - TensorScores::Int64TensorScores(i64_tensor) => TensorProto { - dtype: DataType::DtInt64 as i32, - int64_val: i64_tensor, - ..Default::default() - }, - TensorScores::Int32TensorScores(i32_tensor) => TensorProto { - dtype: DataType::DtInt32 as i32, - int_val: i32_tensor, - ..Default::default() - }, - TensorScores::StringTensorScores(str_tensor) => TensorProto { - dtype: DataType::DtString as i32, - string_val: str_tensor.into_iter().map(|s| s.into_bytes()).collect_vec(), - ..Default::default() - }, - } - } - pub fn len(&self) -> usize { - match &self { - TensorScores::Float32TensorScores(t) => t.len(), - TensorScores::Int64TensorScores(t) => t.len(), - TensorScores::Int32TensorScores(t) => t.len(), - TensorScores::StringTensorScores(t) => t.len(), - } - } -} - -#[derive(Debug)] -pub enum PredictMessage { - Predict( - usize, - Option, - Vec, - Sender, - Instant, - ), - UpsertModel(T), - /* - #[allow(dead_code)] - DeleteModel(usize), - */ -} - -#[derive(Debug)] -pub struct Callback(Sender, usize); - -pub const MAX_VERSIONS_PER_MODEL: usize = 2; diff --git a/navi/navi/src/metrics.docx b/navi/navi/src/metrics.docx new file mode 100644 index 000000000..be506498d Binary files /dev/null and b/navi/navi/src/metrics.docx differ diff --git a/navi/navi/src/metrics.rs b/navi/navi/src/metrics.rs deleted file mode 100644 index 373f84f0f..000000000 --- a/navi/navi/src/metrics.rs +++ /dev/null @@ -1,297 +0,0 @@ -use log::error; -use prometheus::{ - CounterVec, HistogramOpts, HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, - Opts, Registry, -}; -use warp::{Rejection, Reply}; -use crate::{NAME, VERSION}; - -lazy_static! { - pub static ref REGISTRY: Registry = Registry::new(); - pub static ref NUM_REQUESTS_RECEIVED: IntCounter = - IntCounter::new(":navi:num_requests", "Number of Requests Received") - .expect("metric can be created"); - pub static ref NUM_REQUESTS_FAILED: IntCounter = IntCounter::new( - ":navi:num_requests_failed", - "Number of Request Inference Failed" - ) - .expect("metric can be created"); - pub static ref NUM_REQUESTS_DROPPED: IntCounter = IntCounter::new( - ":navi:num_requests_dropped", - "Number of Oneshot Receivers Dropped" - ) - .expect("metric can be created"); - pub static ref NUM_BATCHES_DROPPED: IntCounter = IntCounter::new( - ":navi:num_batches_dropped", - "Number of Batches Proactively Dropped" - ) - .expect("metric can be created"); - pub static ref NUM_BATCH_PREDICTION: IntCounter = - IntCounter::new(":navi:num_batch_prediction", "Number of batch prediction") - .expect("metric can be created"); - pub static ref BATCH_SIZE: IntGauge = - IntGauge::new(":navi:batch_size", "Size of current batch").expect("metric can be created"); - pub static ref NAVI_VERSION: IntGauge = - IntGauge::new(":navi:navi_version", "navi's current version") - .expect("metric can be created"); - pub static ref RESPONSE_TIME_COLLECTOR: HistogramVec = HistogramVec::new( - HistogramOpts::new(":navi:response_time", "Response Time in ms").buckets(Vec::from(&[ - 0.0, 10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0, 110.0, 120.0, 130.0, - 140.0, 150.0, 160.0, 170.0, 180.0, 190.0, 200.0, 250.0, 300.0, 500.0, 1000.0 - ] - as &'static [f64])), - &["model_name"] - ) - .expect("metric can be created"); - pub static ref NUM_PREDICTIONS: IntCounterVec = IntCounterVec::new( - Opts::new( - ":navi:num_predictions", - "Number of predictions made by model" - ), - &["model_name"] - ) - .expect("metric can be created"); - pub static ref PREDICTION_SCORE_SUM: CounterVec = CounterVec::new( - Opts::new( - ":navi:prediction_score_sum", - "Sum of prediction score made by model" - ), - &["model_name"] - ) - .expect("metric can be created"); - pub static ref NEW_MODEL_SNAPSHOT: IntCounterVec = IntCounterVec::new( - Opts::new( - ":navi:new_model_snapshot", - "Load a new version of model snapshot" - ), - &["model_name"] - ) - .expect("metric can be created"); - pub static ref MODEL_SNAPSHOT_VERSION: IntGaugeVec = IntGaugeVec::new( - Opts::new( - ":navi:model_snapshot_version", - "Record model snapshot version" - ), - &["model_name"] - ) - .expect("metric can be created"); - pub static ref NUM_REQUESTS_RECEIVED_BY_MODEL: IntCounterVec = IntCounterVec::new( - Opts::new( - ":navi:num_requests_by_model", - "Number of Requests Received by model" - ), - &["model_name"] - ) - .expect("metric can be created"); - pub static ref NUM_REQUESTS_FAILED_BY_MODEL: IntCounterVec = IntCounterVec::new( - Opts::new( - ":navi:num_requests_failed_by_model", - "Number of Request Inference Failed by model" - ), - &["model_name"] - ) - .expect("metric can be created"); - pub static ref NUM_REQUESTS_DROPPED_BY_MODEL: IntCounterVec = IntCounterVec::new( - Opts::new( - ":navi:num_requests_dropped_by_model", - "Number of Oneshot Receivers Dropped by model" - ), - &["model_name"] - ) - .expect("metric can be created"); - pub static ref NUM_BATCHES_DROPPED_BY_MODEL: IntCounterVec = IntCounterVec::new( - Opts::new( - ":navi:num_batches_dropped_by_model", - "Number of Batches Proactively Dropped by model" - ), - &["model_name"] - ) - .expect("metric can be created"); - pub static ref INFERENCE_FAILED_REQUESTS_BY_MODEL: IntCounterVec = IntCounterVec::new( - Opts::new( - ":navi:inference_failed_requests_by_model", - "Number of failed inference requests by model" - ), - &["model_name"] - ) - .expect("metric can be created"); - pub static ref NUM_PREDICTION_BY_MODEL: IntCounterVec = IntCounterVec::new( - Opts::new( - ":navi:num_prediction_by_model", - "Number of prediction by model" - ), - &["model_name"] - ) - .expect("metric can be created"); - pub static ref NUM_BATCH_PREDICTION_BY_MODEL: IntCounterVec = IntCounterVec::new( - Opts::new( - ":navi:num_batch_prediction_by_model", - "Number of batch prediction by model" - ), - &["model_name"] - ) - .expect("metric can be created"); - pub static ref BATCH_SIZE_BY_MODEL: IntGaugeVec = IntGaugeVec::new( - Opts::new( - ":navi:batch_size_by_model", - "Size of current batch by model" - ), - &["model_name"] - ) - .expect("metric can be created"); - pub static ref CUSTOMOP_VERSION: IntGauge = - IntGauge::new(":navi:customop_version", "The hashed Custom OP Version") - .expect("metric can be created"); - pub static ref MPSC_CHANNEL_SIZE: IntGauge = - IntGauge::new(":navi:mpsc_channel_size", "The mpsc channel's request size") - .expect("metric can be created"); - pub static ref BLOCKING_REQUEST_NUM: IntGauge = IntGauge::new( - ":navi:blocking_request_num", - "The (batch) request waiting or being executed" - ) - .expect("metric can be created"); - pub static ref MODEL_INFERENCE_TIME_COLLECTOR: HistogramVec = HistogramVec::new( - HistogramOpts::new(":navi:model_inference_time", "Model inference time in ms").buckets( - Vec::from(&[ - 0.0, 5.0, 10.0, 15.0, 20.0, 25.0, 30.0, 35.0, 40.0, 45.0, 50.0, 55.0, 60.0, 65.0, - 70.0, 75.0, 80.0, 85.0, 90.0, 100.0, 110.0, 120.0, 130.0, 140.0, 150.0, 160.0, - 170.0, 180.0, 190.0, 200.0, 250.0, 300.0, 500.0, 1000.0 - ] as &'static [f64]) - ), - &["model_name"] - ) - .expect("metric can be created"); - pub static ref CONVERTER_TIME_COLLECTOR: HistogramVec = HistogramVec::new( - HistogramOpts::new(":navi:converter_time", "converter time in microseconds").buckets( - Vec::from(&[ - 0.0, 500.0, 1000.0, 1500.0, 2000.0, 2500.0, 3000.0, 3500.0, 4000.0, 4500.0, 5000.0, - 5500.0, 6000.0, 6500.0, 7000.0, 20000.0 - ] as &'static [f64]) - ), - &["model_name"] - ) - .expect("metric can be created"); - pub static ref CERT_EXPIRY_EPOCH: IntGauge = - IntGauge::new(":navi:cert_expiry_epoch", "Timestamp when the current cert expires") - .expect("metric can be created"); -} - -pub fn register_custom_metrics() { - REGISTRY - .register(Box::new(NUM_REQUESTS_RECEIVED.clone())) - .expect("collector can be registered"); - REGISTRY - .register(Box::new(NUM_REQUESTS_FAILED.clone())) - .expect("collector can be registered"); - REGISTRY - .register(Box::new(NUM_REQUESTS_DROPPED.clone())) - .expect("collector can be registered"); - REGISTRY - .register(Box::new(RESPONSE_TIME_COLLECTOR.clone())) - .expect("collector can be registered"); - REGISTRY - .register(Box::new(NAVI_VERSION.clone())) - .expect("collector can be registered"); - REGISTRY - .register(Box::new(BATCH_SIZE.clone())) - .expect("collector can be registered"); - REGISTRY - .register(Box::new(NUM_BATCH_PREDICTION.clone())) - .expect("collector can be registered"); - REGISTRY - .register(Box::new(NUM_BATCHES_DROPPED.clone())) - .expect("collector can be registered"); - REGISTRY - .register(Box::new(NUM_PREDICTIONS.clone())) - .expect("collector can be registered"); - REGISTRY - .register(Box::new(PREDICTION_SCORE_SUM.clone())) - .expect("collector can be registered"); - REGISTRY - .register(Box::new(NEW_MODEL_SNAPSHOT.clone())) - .expect("collector can be registered"); - REGISTRY - .register(Box::new(MODEL_SNAPSHOT_VERSION.clone())) - .expect("collector can be registered"); - REGISTRY - .register(Box::new(NUM_REQUESTS_RECEIVED_BY_MODEL.clone())) - .expect("collector can be registered"); - REGISTRY - .register(Box::new(NUM_REQUESTS_FAILED_BY_MODEL.clone())) - .expect("collector can be registered"); - REGISTRY - .register(Box::new(NUM_REQUESTS_DROPPED_BY_MODEL.clone())) - .expect("collector can be registered"); - REGISTRY - .register(Box::new(NUM_BATCHES_DROPPED_BY_MODEL.clone())) - .expect("collector can be registered"); - REGISTRY - .register(Box::new(INFERENCE_FAILED_REQUESTS_BY_MODEL.clone())) - .expect("collector can be registered"); - REGISTRY - .register(Box::new(NUM_PREDICTION_BY_MODEL.clone())) - .expect("collector can be registered"); - REGISTRY - .register(Box::new(NUM_BATCH_PREDICTION_BY_MODEL.clone())) - .expect("collector can be registered"); - REGISTRY - .register(Box::new(BATCH_SIZE_BY_MODEL.clone())) - .expect("collector can be registered"); - REGISTRY - .register(Box::new(CUSTOMOP_VERSION.clone())) - .expect("collector can be registered"); - REGISTRY - .register(Box::new(MPSC_CHANNEL_SIZE.clone())) - .expect("collector can be registered"); - REGISTRY - .register(Box::new(BLOCKING_REQUEST_NUM.clone())) - .expect("collector can be registered"); - REGISTRY - .register(Box::new(MODEL_INFERENCE_TIME_COLLECTOR.clone())) - .expect("collector can be registered"); - REGISTRY - .register(Box::new(CONVERTER_TIME_COLLECTOR.clone())) - .expect("collector can be registered"); - REGISTRY - .register(Box::new(CERT_EXPIRY_EPOCH.clone())) - .expect("collector can be registered"); - -} - -pub fn register_dynamic_metrics(c: &HistogramVec) { - REGISTRY - .register(Box::new(c.clone())) - .expect("dynamic metric collector cannot be registered"); -} - -pub async fn metrics_handler() -> Result { - use prometheus::Encoder; - let encoder = prometheus::TextEncoder::new(); - - let mut buffer = Vec::new(); - if let Err(e) = encoder.encode(®ISTRY.gather(), &mut buffer) { - error!("could not encode custom metrics: {}", e); - }; - let mut res = match String::from_utf8(buffer) { - Ok(v) => format!("#{}:{}\n{}", NAME, VERSION, v), - Err(e) => { - error!("custom metrics could not be from_utf8'd: {}", e); - String::default() - } - }; - - buffer = Vec::new(); - if let Err(e) = encoder.encode(&prometheus::gather(), &mut buffer) { - error!("could not encode prometheus metrics: {}", e); - }; - let res_custom = match String::from_utf8(buffer) { - Ok(v) => v, - Err(e) => { - error!("prometheus metrics could not be from_utf8'd: {}", e); - String::default() - } - }; - - res.push_str(&res_custom); - Ok(res) -} diff --git a/navi/navi/src/onnx_model.docx b/navi/navi/src/onnx_model.docx new file mode 100644 index 000000000..8f9ad8e21 Binary files /dev/null and b/navi/navi/src/onnx_model.docx differ diff --git a/navi/navi/src/onnx_model.rs b/navi/navi/src/onnx_model.rs deleted file mode 100644 index 18f116570..000000000 --- a/navi/navi/src/onnx_model.rs +++ /dev/null @@ -1,275 +0,0 @@ -#[cfg(feature = "onnx")] -pub mod onnx { - use crate::TensorReturnEnum; - use crate::bootstrap::{TensorInput, TensorInputEnum}; - use crate::cli_args::{ - Args, ARGS, INPUTS, MODEL_SPECS, OUTPUTS, - }; - use crate::metrics::{self, CONVERTER_TIME_COLLECTOR}; - use crate::predict_service::Model; - use crate::{MAX_NUM_INPUTS, MAX_NUM_OUTPUTS, META_INFO, utils}; - use anyhow::Result; - use arrayvec::ArrayVec; - use dr_transform::converter::{BatchPredictionRequestToTorchTensorConverter, Converter}; - use itertools::Itertools; - use log::{debug, info}; - use dr_transform::ort::environment::Environment; - use dr_transform::ort::session::Session; - use dr_transform::ort::tensor::InputTensor; - use dr_transform::ort::{ExecutionProvider, GraphOptimizationLevel, SessionBuilder}; - use dr_transform::ort::LoggingLevel; - use serde_json::Value; - use std::fmt::{Debug, Display}; - use std::sync::Arc; - use std::{fmt, fs}; - use tokio::time::Instant; - lazy_static! { - pub static ref ENVIRONMENT: Arc = Arc::new( - Environment::builder() - .with_name("onnx home") - .with_log_level(LoggingLevel::Error) - .with_global_thread_pool(ARGS.onnx_global_thread_pool_options.clone()) - .build() - .unwrap() - ); - } - #[derive(Debug)] - pub struct OnnxModel { - pub session: Session, - pub model_idx: usize, - pub version: i64, - pub export_dir: String, - pub output_filters: ArrayVec, - pub input_converter: Box, - } - impl Display for OnnxModel { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!( - f, - "idx: {}, onnx model_name:{}, version:{}, output_filters:{:?}, converter:{:}", - self.model_idx, - MODEL_SPECS[self.model_idx], - self.version, - self.output_filters, - self.input_converter - ) - } - } - impl Drop for OnnxModel { - fn drop(&mut self) { - if ARGS.profiling != None { - self.session.end_profiling().map_or_else( - |e| { - info!("end profiling with some error:{:?}", e); - }, - |f| { - info!("profiling ended with file:{}", f); - }, - ); - } - } - } - impl OnnxModel { - fn get_output_filters(session: &Session, idx: usize) -> ArrayVec { - OUTPUTS[idx] - .iter() - .map(|output| session.outputs.iter().position(|o| o.name == *output)) - .flatten() - .collect::>() - } - #[cfg(target_os = "linux")] - fn ep_choices() -> Vec { - match ARGS.onnx_gpu_ep.as_ref().map(|e| e.as_str()) { - Some("onednn") => vec![Self::ep_with_options(ExecutionProvider::onednn())], - Some("tensorrt") => vec![Self::ep_with_options(ExecutionProvider::tensorrt())], - Some("cuda") => vec![Self::ep_with_options(ExecutionProvider::cuda())], - _ => vec![Self::ep_with_options(ExecutionProvider::cpu())], - } - } - fn ep_with_options(mut ep: ExecutionProvider) -> ExecutionProvider { - for (ref k, ref v) in ARGS.onnx_ep_options.clone() { - ep = ep.with(k, v); - info!("setting option:{} -> {} and now ep is:{:?}", k, v, ep); - } - ep - } - #[cfg(target_os = "macos")] - fn ep_choices() -> Vec { - vec![Self::ep_with_options(ExecutionProvider::cpu())] - } - pub fn new(idx: usize, version: String, model_config: &Value) -> Result { - let export_dir = format!("{}/{}/model.onnx", ARGS.model_dir[idx], version); - let meta_info = format!("{}/{}/{}", ARGS.model_dir[idx], version, META_INFO); - let mut builder = SessionBuilder::new(&ENVIRONMENT)? - .with_optimization_level(GraphOptimizationLevel::Level3)? - .with_parallel_execution(ARGS.onnx_use_parallel_mode == "true")?; - if ARGS.onnx_global_thread_pool_options.is_empty() { - builder = builder - .with_inter_threads( - utils::get_config_or( - model_config, - "inter_op_parallelism", - &ARGS.inter_op_parallelism[idx], - ) - .parse()?, - )? - .with_intra_threads( - utils::get_config_or( - model_config, - "intra_op_parallelism", - &ARGS.intra_op_parallelism[idx], - ) - .parse()?, - )?; - } - else { - builder = builder.with_disable_per_session_threads()?; - } - builder = builder - .with_memory_pattern(ARGS.onnx_use_memory_pattern == "true")? - .with_execution_providers(&OnnxModel::ep_choices())?; - match &ARGS.profiling { - Some(p) => { - debug!("Enable profiling, writing to {}", *p); - builder = builder.with_profiling(p)? - } - _ => {} - } - let session = builder.with_model_from_file(&export_dir)?; - - info!( - "inputs: {:?}, outputs: {:?}", - session.inputs.iter().format(","), - session.outputs.iter().format(",") - ); - - fs::read_to_string(&meta_info) - .ok() - .map(|info| info!("meta info:{}", info)); - let output_filters = OnnxModel::get_output_filters(&session, idx); - let mut reporting_feature_ids: Vec<(i64, &str)> = vec![]; - - let input_spec_cell = &INPUTS[idx]; - if input_spec_cell.get().is_none() { - let input_spec = session - .inputs - .iter() - .map(|input| input.name.clone()) - .collect::>(); - input_spec_cell.set(input_spec.clone()).map_or_else( - |_| info!("unable to set the input_spec for model {}", idx), - |_| info!("auto detect and set the inputs: {:?}", input_spec), - ); - } - ARGS.onnx_report_discrete_feature_ids - .iter() - .for_each(|ids| { - ids.split(",") - .filter(|s| !s.is_empty()) - .map(|s| s.parse::().unwrap()) - .for_each(|id| reporting_feature_ids.push((id, "discrete"))) - }); - ARGS.onnx_report_continuous_feature_ids - .iter() - .for_each(|ids| { - ids.split(",") - .filter(|s| !s.is_empty()) - .map(|s| s.parse::().unwrap()) - .for_each(|id| reporting_feature_ids.push((id, "continuous"))) - }); - - let onnx_model = OnnxModel { - session, - model_idx: idx, - version: Args::version_str_to_epoch(&version)?, - export_dir, - output_filters, - input_converter: Box::new(BatchPredictionRequestToTorchTensorConverter::new( - &ARGS.model_dir[idx], - &version, - reporting_feature_ids, - Some(metrics::register_dynamic_metrics), - )?), - }; - onnx_model.warmup()?; - Ok(onnx_model) - } - } - ///Currently we only assume the input as just one string tensor. - ///The string tensor will be be converted to the actual raw tensors. - /// The converter we are using is very specific to home. - /// It reads a BatchDataRecord thrift and decode it to a batch of raw input tensors. - /// Navi will then do server side batching and feed it to ONNX runtime - impl Model for OnnxModel { - //TODO: implement a generic online warmup for all runtimes - fn warmup(&self) -> Result<()> { - Ok(()) - } - - #[inline(always)] - fn do_predict( - &self, - input_tensors: Vec>, - _: u64, - ) -> (Vec, Vec>) { - let batched_tensors = TensorInputEnum::merge_batch(input_tensors); - let (inputs, batch_ends): (Vec>, Vec>) = batched_tensors - .into_iter() - .map(|batched_tensor| { - match batched_tensor.tensor_data { - TensorInputEnum::String(t) if ARGS.onnx_use_converter.is_some() => { - let start = Instant::now(); - let (inputs, batch_ends) = self.input_converter.convert(t); - // info!("batch_ends:{:?}", batch_ends); - CONVERTER_TIME_COLLECTOR - .with_label_values(&[&MODEL_SPECS[self.model_idx()]]) - .observe( - start.elapsed().as_micros() as f64 - / (*batch_ends.last().unwrap() as f64), - ); - (inputs, batch_ends) - } - _ => unimplemented!(), - } - }) - .unzip(); - //invariant we only support one input as string. will relax later - assert_eq!(inputs.len(), 1); - let output_tensors = self - .session - .run(inputs.into_iter().flatten().collect::>()) - .unwrap(); - self.output_filters - .iter() - .map(|&idx| { - let mut size = 1usize; - let output = output_tensors[idx].try_extract::().unwrap(); - for &dim in self.session.outputs[idx].dimensions.iter().flatten() { - size *= dim as usize; - } - let tensor_ends = batch_ends[0] - .iter() - .map(|&batch| batch * size) - .collect::>(); - - ( - //only works for batch major - //TODO: to_vec() obviously wasteful, especially for large batches(GPU) . Will refactor to - //break up output and return Vec> here - TensorReturnEnum::FloatTensorReturn(Box::new(output.view().as_slice().unwrap().to_vec(), - )), - tensor_ends, - ) - }) - .unzip() - } - #[inline(always)] - fn model_idx(&self) -> usize { - self.model_idx - } - #[inline(always)] - fn version(&self) -> i64 { - self.version - } - } -} diff --git a/navi/navi/src/predict_service.docx b/navi/navi/src/predict_service.docx new file mode 100644 index 000000000..a31d41af0 Binary files /dev/null and b/navi/navi/src/predict_service.docx differ diff --git a/navi/navi/src/predict_service.rs b/navi/navi/src/predict_service.rs deleted file mode 100644 index fc355d7ea..000000000 --- a/navi/navi/src/predict_service.rs +++ /dev/null @@ -1,315 +0,0 @@ -use anyhow::{anyhow, Result}; -use arrayvec::ArrayVec; -use itertools::Itertools; -use log::{error, info}; -use std::fmt::{Debug, Display}; -use std::string::String; -use std::sync::Arc; -use std::time::Duration; -use tokio::process::Command; -use tokio::sync::mpsc::error::TryRecvError; -use tokio::sync::mpsc::{Receiver, Sender}; -use tokio::sync::{mpsc, oneshot}; -use tokio::time::{Instant, sleep}; -use warp::Filter; - -use crate::batch::BatchPredictor; -use crate::bootstrap::TensorInput; -use crate::{MAX_NUM_MODELS, MAX_VERSIONS_PER_MODEL, META_INFO, metrics, ModelFactory, PredictMessage, PredictResult, TensorReturnEnum, utils}; - -use crate::cli_args::{ARGS, MODEL_SPECS}; -use crate::cores::validator::validatior::cli_validator; -use crate::metrics::MPSC_CHANNEL_SIZE; -use serde_json::{self, Value}; - -pub trait Model: Send + Sync + Display + Debug + 'static { - fn warmup(&self) -> Result<()>; - //TODO: refactor this to return vec>, i.e. - //we have the underlying runtime impl to split the response to each client. - //It will eliminate some inefficient memory copy in onnx_model.rs as well as simplify code - fn do_predict( - &self, - input_tensors: Vec>, - total_len: u64, - ) -> (Vec, Vec>); - fn model_idx(&self) -> usize; - fn version(&self) -> i64; -} - -#[derive(Debug)] -pub struct PredictService { - tx: Sender>, -} -impl PredictService { - pub async fn init(model_factory: ModelFactory) -> Self { - cli_validator::validate_ps_model_args(); - let (tx, rx) = mpsc::channel(32_000); - tokio::spawn(PredictService::tf_queue_manager(rx)); - tokio::spawn(PredictService::model_watcher_latest( - model_factory, - tx.clone(), - )); - let metrics_route = warp::path!("metrics").and_then(metrics::metrics_handler); - let metric_server = warp::serve(metrics_route).run(([0, 0, 0, 0], ARGS.prometheus_port)); - tokio::spawn(metric_server); - PredictService { tx } - } - #[inline(always)] - pub async fn predict( - &self, - idx: usize, - version: Option, - val: Vec, - ts: Instant, - ) -> Result { - let (tx, rx) = oneshot::channel(); - if let Err(e) = self - .tx - .clone() - .send(PredictMessage::Predict(idx, version, val, tx, ts)) - .await - { - error!("mpsc send error:{}", e); - Err(anyhow!(e)) - } else { - MPSC_CHANNEL_SIZE.inc(); - rx.await.map_err(anyhow::Error::msg) - } - } - - async fn load_latest_model_from_model_dir( - model_factory: ModelFactory, - model_config: &Value, - tx: Sender>, - idx: usize, - max_version: String, - latest_version: &mut String, - ) { - match model_factory(idx, max_version.clone(), model_config) { - Ok(tf_model) => tx - .send(PredictMessage::UpsertModel(tf_model)) - .await - .map_or_else( - |e| error!("send UpsertModel error: {}", e), - |_| *latest_version = max_version, - ), - Err(e) => { - error!("skip loading model due to failure: {:?}", e); - } - } - } - - async fn scan_load_latest_model_from_model_dir( - model_factory: ModelFactory, - model_config: &Value, - tx: Sender>, - model_idx: usize, - cur_version: &mut String, - ) -> Result<()> { - let model_dir = &ARGS.model_dir[model_idx]; - let next_version = utils::get_config_or_else(model_config, "version", || { - info!("no version found, hence use max version"); - std::fs::read_dir(model_dir) - .map_err(|e| format!("read dir error:{}", e)) - .and_then(|paths| { - paths - .into_iter() - .flat_map(|p| { - p.map_err(|e| error!("dir entry error: {}", e)) - .and_then(|dir| { - dir.file_name() - .into_string() - .map_err(|e| error!("osstring error: {:?}", e)) - }) - .ok() - }) - .filter(|f| !f.to_lowercase().contains(&META_INFO.to_lowercase())) - .max() - .ok_or_else(|| "no dir found hence no max".to_owned()) - }) - .unwrap_or_else(|e| { - error!( - "can't get the max version hence return cur_version, error is: {}", - e - ); - cur_version.to_string() - }) - }); - //as long as next version doesn't match cur version maintained we reload - if next_version.ne(cur_version) { - info!("reload the version: {}->{}", cur_version, next_version); - PredictService::load_latest_model_from_model_dir( - model_factory, - model_config, - tx, - model_idx, - next_version, - cur_version, - ) - .await; - } - Ok(()) - } - - async fn model_watcher_latest(model_factory: ModelFactory, tx: Sender>) { - async fn call_external_modelsync(cli: &str, cur_versions: &Vec) -> Result<()> { - let mut args = cli.split_whitespace(); - - let mut cmd = Command::new(args.next().ok_or(anyhow!("model sync cli empty"))?); - let extr_args = MODEL_SPECS - .iter() - .zip(cur_versions) - .flat_map(|(spec, version)| vec!["--model-spec", spec, "--cur-version", version]) - .collect_vec(); - info!("run model sync: {} with extra args: {:?}", cli, extr_args); - let output = cmd.args(args).args(extr_args).output().await?; - info!("model sync stdout:{}", String::from_utf8(output.stdout)?); - info!("model sync stderr:{}", String::from_utf8(output.stderr)?); - if output.status.success() { - Ok(()) - } else { - Err(anyhow!( - "model sync failed with status: {:?}!", - output.status - )) - } - } - let meta_dir = utils::get_meta_dir(); - let meta_file = format!("{}{}", meta_dir, META_INFO); - //initialize the latest version array - let mut cur_versions = vec!["".to_owned(); MODEL_SPECS.len()]; - loop { - info!("***polling for models***"); //nice deliminter - if let Some(ref cli) = ARGS.modelsync_cli { - if let Err(e) = call_external_modelsync(cli, &cur_versions).await { - error!("model sync cli running error:{}", e) - } - } - let config = utils::read_config(&meta_file).unwrap_or_else(|e| { - info!("config file {} not found due to: {}", meta_file, e); - Value::Null - }); - info!("config:{}", config); - for (idx, cur_version) in cur_versions.iter_mut().enumerate() { - let model_dir = &ARGS.model_dir[idx]; - PredictService::scan_load_latest_model_from_model_dir( - model_factory, - &config[&MODEL_SPECS[idx]], - tx.clone(), - idx, - cur_version, - ) - .await - .map_or_else( - |e| error!("scanned {}, error {:?}", model_dir, e), - |_| info!("scanned {}, latest_version: {}", model_dir, cur_version), - ); - } - sleep(Duration::from_secs(ARGS.model_check_interval_secs)).await; - } - } - async fn tf_queue_manager(mut rx: Receiver>) { - // Start receiving messages - info!("setting up queue manager"); - let max_batch_size = ARGS - .max_batch_size - .iter() - .map(|b| b.parse().unwrap()) - .collect::>(); - let batch_time_out_millis = ARGS - .batch_time_out_millis - .iter() - .map(|b| b.parse().unwrap()) - .collect::>(); - let no_msg_wait_millis = *batch_time_out_millis.iter().min().unwrap(); - let mut all_model_predictors: ArrayVec::, MAX_VERSIONS_PER_MODEL>, MAX_NUM_MODELS> = - (0 ..MAX_NUM_MODELS).map( |_| ArrayVec::, MAX_VERSIONS_PER_MODEL>::new()).collect(); - loop { - let msg = rx.try_recv(); - let no_more_msg = match msg { - Ok(PredictMessage::Predict(model_spec_at, version, val, resp, ts)) => { - if let Some(model_predictors) = all_model_predictors.get_mut(model_spec_at) { - if model_predictors.is_empty() { - resp.send(PredictResult::ModelNotReady(model_spec_at)) - .unwrap_or_else(|e| error!("cannot send back model not ready error: {:?}", e)); - } - else { - match version { - None => model_predictors[0].push(val, resp, ts), - Some(the_version) => match model_predictors - .iter_mut() - .find(|x| x.model.version() == the_version) - { - None => resp - .send(PredictResult::ModelVersionNotFound( - model_spec_at, - the_version, - )) - .unwrap_or_else(|e| { - error!("cannot send back version error: {:?}", e) - }), - Some(predictor) => predictor.push(val, resp, ts), - }, - } - } - } else { - resp.send(PredictResult::ModelNotFound(model_spec_at)) - .unwrap_or_else(|e| error!("cannot send back model not found error: {:?}", e)) - } - MPSC_CHANNEL_SIZE.dec(); - false - } - Ok(PredictMessage::UpsertModel(tf_model)) => { - let idx = tf_model.model_idx(); - let predictor = BatchPredictor { - model: Arc::new(tf_model), - input_tensors: Vec::with_capacity(max_batch_size[idx]), - callbacks: Vec::with_capacity(max_batch_size[idx]), - cur_batch_size: 0, - max_batch_size: max_batch_size[idx], - batch_time_out_millis: batch_time_out_millis[idx], - //initialize to be current time - queue_reset_ts: Instant::now(), - queue_earliest_rq_ts: Instant::now(), - }; - assert!(idx < all_model_predictors.len()); - metrics::NEW_MODEL_SNAPSHOT - .with_label_values(&[&MODEL_SPECS[idx]]) - .inc(); - - //we can do this since the vector is small - let predictors = &mut all_model_predictors[idx]; - if predictors.len() == 0 { - info!("now we serve new model: {}", predictor.model); - } - else { - info!("now we serve updated model: {}", predictor.model); - } - if predictors.len() == ARGS.versions_per_model { - predictors.remove(predictors.len() - 1); - } - predictors.insert(0, predictor); - false - } - Err(TryRecvError::Empty) => true, - Err(TryRecvError::Disconnected) => true, - }; - for predictor in all_model_predictors.iter_mut().flatten() { - //if predictor batch queue not empty and times out or no more msg in the queue, flush - if (!predictor.input_tensors.is_empty() && (predictor.duration_past(predictor.batch_time_out_millis) || no_more_msg)) - //if batch queue reaches limit, flush - || predictor.cur_batch_size >= predictor.max_batch_size - { - predictor.batch_predict(); - } - } - if no_more_msg { - sleep(Duration::from_millis(no_msg_wait_millis)).await; - } - } - } - #[inline(always)] - pub fn get_model_index(model_spec: &str) -> Option { - MODEL_SPECS.iter().position(|m| m == model_spec) - } -} diff --git a/navi/navi/src/tf_model.docx b/navi/navi/src/tf_model.docx new file mode 100644 index 000000000..d82fe7b59 Binary files /dev/null and b/navi/navi/src/tf_model.docx differ diff --git a/navi/navi/src/tf_model.rs b/navi/navi/src/tf_model.rs deleted file mode 100644 index fa9c69f25..000000000 --- a/navi/navi/src/tf_model.rs +++ /dev/null @@ -1,492 +0,0 @@ -#[cfg(feature = "tf")] -pub mod tf { - use arrayvec::ArrayVec; - use itertools::Itertools; - use log::{debug, error, info, warn}; - use prost::Message; - use std::fmt; - use std::fmt::Display; - use std::string::String; - use tensorflow::io::{RecordReader, RecordReadError}; - use tensorflow::Operation; - use tensorflow::SavedModelBundle; - use tensorflow::SessionOptions; - use tensorflow::SessionRunArgs; - use tensorflow::Tensor; - use tensorflow::{DataType, FetchToken, Graph, TensorInfo, TensorType}; - - use std::thread::sleep; - use std::time::Duration; - - use crate::cli_args::{Args, ARGS, INPUTS, MODEL_SPECS, OUTPUTS}; - use crate::tf_proto::tensorflow_serving::prediction_log::LogType; - use crate::tf_proto::tensorflow_serving::{PredictionLog, PredictLog}; - use crate::tf_proto::ConfigProto; - use anyhow::{Context, Result}; - use serde_json::Value; - - use crate::TensorReturnEnum; - use crate::bootstrap::{TensorInput, TensorInputEnum}; - use crate::metrics::{ - INFERENCE_FAILED_REQUESTS_BY_MODEL, NUM_REQUESTS_FAILED, NUM_REQUESTS_FAILED_BY_MODEL, - }; - use crate::predict_service::Model; - use crate::{MAX_NUM_INPUTS, utils}; - - #[derive(Debug)] - pub enum TFTensorEnum { - String(Tensor), - Int(Tensor), - Int64(Tensor), - Float(Tensor), - Double(Tensor), - Boolean(Tensor), - } - - #[derive(Debug)] - pub struct TFModel { - pub model_idx: usize, - pub bundle: SavedModelBundle, - pub input_names: ArrayVec, - pub input_info: Vec, - pub input_ops: Vec, - pub output_names: Vec, - pub output_info: Vec, - pub output_ops: Vec, - pub export_dir: String, - pub version: i64, - pub inter_op: i32, - pub intra_op: i32, - } - - impl Display for TFModel { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!( - f, - "idx: {}, tensorflow model_name:{}, export_dir:{}, version:{}, inter:{}, intra:{}", - self.model_idx, - MODEL_SPECS[self.model_idx], - self.export_dir, - self.version, - self.inter_op, - self.intra_op - ) - } - } - - impl TFModel { - pub fn new(idx: usize, version: String, model_config: &Value) -> Result { - // Create input variables for our addition - let config = ConfigProto { - intra_op_parallelism_threads: utils::get_config_or( - model_config, - "intra_op_parallelism", - &ARGS.intra_op_parallelism[idx], - ) - .parse()?, - inter_op_parallelism_threads: utils::get_config_or( - model_config, - "inter_op_parallelism", - &ARGS.inter_op_parallelism[idx], - ) - .parse()?, - ..Default::default() - }; - let mut buf = Vec::new(); - buf.reserve(config.encoded_len()); - config.encode(&mut buf).unwrap(); - let mut opts = SessionOptions::new(); - opts.set_config(&buf)?; - let export_dir = format!("{}/{}", ARGS.model_dir[idx], version); - let mut graph = Graph::new(); - let bundle = SavedModelBundle::load(&opts, ["serve"], &mut graph, &export_dir) - .context("error load model")?; - let signature = bundle - .meta_graph_def() - .get_signature(&ARGS.serving_sig[idx]) - .context("error finding signature")?; - let input_names = INPUTS[idx] - .get_or_init(|| { - let input_spec = signature - .inputs() - .iter() - .map(|p| p.0.clone()) - .collect::>(); - info!( - "input not set from cli, now we set from model metadata:{:?}", - input_spec - ); - input_spec - }) - .clone(); - let input_info = input_names - .iter() - .map(|i| { - signature - .get_input(i) - .context("error finding input op info") - .unwrap() - .clone() - }) - .collect_vec(); - - let input_ops = input_info - .iter() - .map(|i| { - graph - .operation_by_name_required(&i.name().name) - .context("error finding input op") - .unwrap() - }) - .collect_vec(); - - info!("Model Input size: {}", input_info.len()); - - let output_names = OUTPUTS[idx].to_vec().clone(); - - let output_info = output_names - .iter() - .map(|o| { - signature - .get_output(o) - .context("error finding output op info") - .unwrap() - .clone() - }) - .collect_vec(); - - let output_ops = output_info - .iter() - .map(|o| { - graph - .operation_by_name_required(&o.name().name) - .context("error finding output op") - .unwrap() - }) - .collect_vec(); - - let tf_model = TFModel { - model_idx: idx, - bundle, - input_names, - input_info, - input_ops, - output_names, - output_info, - output_ops, - export_dir, - version: Args::version_str_to_epoch(&version)?, - inter_op: config.inter_op_parallelism_threads, - intra_op: config.intra_op_parallelism_threads, - }; - tf_model.warmup()?; - Ok(tf_model) - } - - #[inline(always)] - fn get_tftensor_dimensions( - t: &[T], - input_size: u64, - batch_size: u64, - input_dims: Option>, - ) -> Vec { - // if input size is 1, we just specify a single dimension to outgoing tensor matching the - // size of the input tensor. This is for backwards compatiblity with existing Navi clients - // which specify input as a single string tensor (like tfexample) and use batching support. - let mut dims = vec![]; - if input_size > 1 { - if batch_size == 1 && input_dims.is_some() { - // client side batching is enabled? - input_dims - .unwrap() - .iter() - .for_each(|axis| dims.push(*axis as u64)); - } else { - dims.push(batch_size); - dims.push(t.len() as u64 / batch_size); - } - } else { - dims.push(t.len() as u64); - } - dims - } - - fn convert_to_tftensor_enum( - input: TensorInput, - input_size: u64, - batch_size: u64, - ) -> TFTensorEnum { - match input.tensor_data { - TensorInputEnum::String(t) => { - let strings = t - .into_iter() - .map(|x| unsafe { String::from_utf8_unchecked(x) }) - .collect_vec(); - TFTensorEnum::String( - Tensor::new(&TFModel::get_tftensor_dimensions( - strings.as_slice(), - input_size, - batch_size, - input.dims, - )) - .with_values(strings.as_slice()) - .unwrap(), - ) - } - TensorInputEnum::Int(t) => TFTensorEnum::Int( - Tensor::new(&TFModel::get_tftensor_dimensions( - t.as_slice(), - input_size, - batch_size, - input.dims, - )) - .with_values(t.as_slice()) - .unwrap(), - ), - TensorInputEnum::Int64(t) => TFTensorEnum::Int64( - Tensor::new(&TFModel::get_tftensor_dimensions( - t.as_slice(), - input_size, - batch_size, - input.dims, - )) - .with_values(t.as_slice()) - .unwrap(), - ), - TensorInputEnum::Float(t) => TFTensorEnum::Float( - Tensor::new(&TFModel::get_tftensor_dimensions( - t.as_slice(), - input_size, - batch_size, - input.dims, - )) - .with_values(t.as_slice()) - .unwrap(), - ), - TensorInputEnum::Double(t) => TFTensorEnum::Double( - Tensor::new(&TFModel::get_tftensor_dimensions( - t.as_slice(), - input_size, - batch_size, - input.dims, - )) - .with_values(t.as_slice()) - .unwrap(), - ), - TensorInputEnum::Boolean(t) => TFTensorEnum::Boolean( - Tensor::new(&TFModel::get_tftensor_dimensions( - t.as_slice(), - input_size, - batch_size, - input.dims, - )) - .with_values(t.as_slice()) - .unwrap(), - ), - } - } - fn fetch_output( - args: &mut SessionRunArgs, - token_output: &FetchToken, - batch_size: u64, - output_size: u64, - ) -> (Tensor, u64) { - let tensor_output = args.fetch::(*token_output).expect("fetch output failed"); - let mut tensor_width = tensor_output.dims()[1]; - if batch_size == 1 && output_size > 1 { - tensor_width = tensor_output.dims().iter().fold(1, |mut total, &val| { - total *= val; - total - }); - } - (tensor_output, tensor_width) - } - } - - impl Model for TFModel { - fn warmup(&self) -> Result<()> { - // warm up - let warmup_file = format!( - "{}/assets.extra/tf_serving_warmup_requests", - self.export_dir - ); - if std::path::Path::new(&warmup_file).exists() { - use std::io::Cursor; - info!( - "found warmup assets in {}, now perform warming up", - warmup_file - ); - let f = std::fs::File::open(warmup_file).context("cannot open warmup file")?; - // let mut buf = Vec::new(); - let read = std::io::BufReader::new(f); - let mut reader = RecordReader::new(read); - let mut warmup_cnt = 0; - loop { - let next = reader.read_next_owned(); - match next { - Ok(res) => match res { - Some(vec) => { - // info!("read one tfRecord"); - match PredictionLog::decode(&mut Cursor::new(vec)) - .context("can't parse PredictonLog")? - { - PredictionLog { - log_metadata: _, - log_type: - Some(LogType::PredictLog(PredictLog { - request: Some(mut req), - response: _, - })), - } => { - if warmup_cnt == ARGS.max_warmup_records { - //warm up to max_warmup_records records - warn!( - "reached max warmup {} records, exit warmup for {}", - ARGS.max_warmup_records, - MODEL_SPECS[self.model_idx] - ); - break; - } - self.do_predict( - vec![req.take_input_vals(&self.input_names)], - 1, - ); - sleep(Duration::from_millis(100)); - warmup_cnt += 1; - } - _ => error!("some wrong record in warming up file"), - } - } - None => { - info!("end of warmup file, warmed up with records: {}", warmup_cnt); - break; - } - }, - Err(RecordReadError::CorruptFile) - | Err(RecordReadError::IoError { .. }) => { - error!("read tfrecord error for warmup files, skip"); - } - _ => {} - } - } - } - Ok(()) - } - - #[inline(always)] - fn do_predict( - &self, - input_tensors: Vec>, - batch_size: u64, - ) -> (Vec, Vec>) { - // let mut batch_ends = input_tensors.iter().map(|t| t.len()).collect::>(); - let output_size = self.output_names.len() as u64; - let input_size = self.input_names.len() as u64; - debug!( - "Request for Tensorflow with batch size: {} and input_size: {}", - batch_size, input_size - ); - // build a set of input TF tensors - - let batch_end = (1usize..=input_tensors.len() as usize) - .into_iter() - .collect_vec(); - let mut batch_ends = vec![batch_end; output_size as usize]; - - let batched_tensors = TensorInputEnum::merge_batch(input_tensors) - .into_iter() - .enumerate() - .map(|(_, i)| TFModel::convert_to_tftensor_enum(i, input_size, batch_size)) - .collect_vec(); - - let mut args = SessionRunArgs::new(); - for (index, tf_tensor) in batched_tensors.iter().enumerate() { - match tf_tensor { - TFTensorEnum::String(inner) => args.add_feed(&self.input_ops[index], 0, inner), - TFTensorEnum::Int(inner) => args.add_feed(&self.input_ops[index], 0, inner), - TFTensorEnum::Int64(inner) => args.add_feed(&self.input_ops[index], 0, inner), - TFTensorEnum::Float(inner) => args.add_feed(&self.input_ops[index], 0, inner), - TFTensorEnum::Double(inner) => args.add_feed(&self.input_ops[index], 0, inner), - TFTensorEnum::Boolean(inner) => args.add_feed(&self.input_ops[index], 0, inner), - } - } - // For output ops, we receive the same op object by name. Actual tensor tokens are available at different offsets. - // Since indices are ordered, its important to specify output flag to Navi in the same order. - let token_outputs = self - .output_ops - .iter() - .enumerate() - .map(|(idx, op)| args.request_fetch(op, idx as i32)) - .collect_vec(); - match self.bundle.session.run(&mut args) { - Ok(_) => (), - Err(e) => { - NUM_REQUESTS_FAILED.inc_by(batch_size); - NUM_REQUESTS_FAILED_BY_MODEL - .with_label_values(&[&MODEL_SPECS[self.model_idx]]) - .inc_by(batch_size); - INFERENCE_FAILED_REQUESTS_BY_MODEL - .with_label_values(&[&MODEL_SPECS[self.model_idx]]) - .inc_by(batch_size); - panic!("{model}: {e:?}", model = MODEL_SPECS[self.model_idx], e = e); - } - } - let mut predict_return = vec![]; - // Check the output. - for (index, token_output) in token_outputs.iter().enumerate() { - // same ops, with type info at different offsets. - let (res, width) = match self.output_ops[index].output_type(index) { - DataType::Float => { - let (tensor_output, tensor_width) = - TFModel::fetch_output(&mut args, token_output, batch_size, output_size); - ( - TensorReturnEnum::FloatTensorReturn(Box::new(tensor_output)), - tensor_width, - ) - } - DataType::Int64 => { - let (tensor_output, tensor_width) = - TFModel::fetch_output(&mut args, token_output, batch_size, output_size); - ( - TensorReturnEnum::Int64TensorReturn(Box::new(tensor_output)), - tensor_width, - ) - } - DataType::Int32 => { - let (tensor_output, tensor_width) = - TFModel::fetch_output(&mut args, token_output, batch_size, output_size); - ( - TensorReturnEnum::Int32TensorReturn(Box::new(tensor_output)), - tensor_width, - ) - } - DataType::String => { - let (tensor_output, tensor_width) = - TFModel::fetch_output(&mut args, token_output, batch_size, output_size); - ( - TensorReturnEnum::StringTensorReturn(Box::new(tensor_output)), - tensor_width, - ) - } - _ => panic!("Unsupported return type!"), - }; - let width = width as usize; - for b in batch_ends[index].iter_mut() { - *b *= width; - } - predict_return.push(res) - } - //TODO: remove in the future - //TODO: support actual mtl model outputs - (predict_return, batch_ends) - } - #[inline(always)] - fn model_idx(&self) -> usize { - self.model_idx - } - #[inline(always)] - fn version(&self) -> i64 { - self.version - } - } -} diff --git a/navi/navi/src/torch_model.docx b/navi/navi/src/torch_model.docx new file mode 100644 index 000000000..d59341ec1 Binary files /dev/null and b/navi/navi/src/torch_model.docx differ diff --git a/navi/navi/src/torch_model.rs b/navi/navi/src/torch_model.rs deleted file mode 100644 index 31e35c158..000000000 --- a/navi/navi/src/torch_model.rs +++ /dev/null @@ -1,183 +0,0 @@ -#[cfg(feature = "torch")] -pub mod torch { - use std::fmt; - use std::fmt::Display; - use std::string::String; - - use crate::TensorReturnEnum; - use crate::SerializedInput; - use crate::bootstrap::TensorInput; - use crate::cli_args::{Args, ARGS, MODEL_SPECS}; - use crate::metrics; - use crate::metrics::{ - INFERENCE_FAILED_REQUESTS_BY_MODEL, NUM_REQUESTS_FAILED, NUM_REQUESTS_FAILED_BY_MODEL, - }; - use crate::predict_service::Model; - use anyhow::Result; - use dr_transform::converter::BatchPredictionRequestToTorchTensorConverter; - use dr_transform::converter::Converter; - use serde_json::Value; - use tch::Tensor; - use tch::{kind, CModule, IValue}; - - #[derive(Debug)] - pub struct TorchModel { - pub model_idx: usize, - pub version: i64, - pub module: CModule, - pub export_dir: String, - // FIXME: make this Box> so input converter can be optional. - // Also consider adding output_converter. - pub input_converter: Box, - } - - impl Display for TorchModel { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!( - f, - "idx: {}, torch model_name:{}, version:{}", - self.model_idx, MODEL_SPECS[self.model_idx], self.version - ) - } - } - - impl TorchModel { - pub fn new(idx: usize, version: String, _model_config: &Value) -> Result { - let export_dir = format!("{}/{}/model.pt", ARGS.model_dir[idx], version); - let model = CModule::load(&export_dir).unwrap(); - let torch_model = TorchModel { - model_idx: idx, - version: Args::version_str_to_epoch(&version)?, - module: model, - export_dir, - //TODO: move converter lookup in a registry. - input_converter: Box::new(BatchPredictionRequestToTorchTensorConverter::new( - &ARGS.model_dir[idx].as_str(), - version.as_str(), - vec![], - Some(&metrics::register_dynamic_metrics), - )), - }; - - torch_model.warmup()?; - Ok(torch_model) - } - #[inline(always)] - pub fn decode_to_inputs(bytes: SerializedInput) -> Vec { - //FIXME: for now we generate 4 random tensors as inputs to unblock end to end testing - //when Shajan's decoder is ready we will swap - let row = bytes.len() as i64; - let t1 = Tensor::randn(&[row, 5293], kind::FLOAT_CPU); //continuous - let t2 = Tensor::randint(10, &[row, 149], kind::INT64_CPU); //binary - let t3 = Tensor::randint(10, &[row, 320], kind::INT64_CPU); //discrete - let t4 = Tensor::randn(&[row, 200], kind::FLOAT_CPU); //user_embedding - let t5 = Tensor::randn(&[row, 200], kind::FLOAT_CPU); //user_eng_embedding - let t6 = Tensor::randn(&[row, 200], kind::FLOAT_CPU); //author_embedding - - vec![t1, t2, t3, t4, t5, t6] - } - #[inline(always)] - pub fn output_to_vec(res: IValue, dst: &mut Vec) { - match res { - IValue::Tensor(tensor) => TorchModel::tensors_to_vec(&[tensor], dst), - IValue::Tuple(ivalues) => { - TorchModel::tensors_to_vec(&TorchModel::ivalues_to_tensors(ivalues), dst) - } - _ => panic!("we only support output as a single tensor or a vec of tensors"), - } - } - #[inline(always)] - pub fn tensor_flatten_size(t: &Tensor) -> usize { - t.size().into_iter().fold(1, |acc, x| acc * x) as usize - } - #[inline(always)] - pub fn tensor_to_vec(res: &Tensor) -> Vec { - let size = TorchModel::tensor_flatten_size(res); - let mut res_f32: Vec = Vec::with_capacity(size); - unsafe { - res_f32.set_len(size); - } - res.copy_data(res_f32.as_mut_slice(), size); - // println!("Copied tensor:{}, {:?}", res_f32.len(), res_f32); - res_f32 - } - #[inline(always)] - pub fn tensors_to_vec(tensors: &[Tensor], dst: &mut Vec) { - let mut offset = dst.len(); - tensors.iter().for_each(|t| { - let size = TorchModel::tensor_flatten_size(t); - let next_size = offset + size; - unsafe { - dst.set_len(next_size); - } - t.copy_data(&mut dst[offset..], size); - offset = next_size; - }); - } - pub fn ivalues_to_tensors(ivalues: Vec) -> Vec { - ivalues - .into_iter() - .map(|t| { - if let IValue::Tensor(vanilla_t) = t { - vanilla_t - } else { - panic!("not a tensor") - } - }) - .collect::>() - } - } - - impl Model for TorchModel { - fn warmup(&self) -> Result<()> { - Ok(()) - } - //TODO: torch runtime needs some refactor to make it a generic interface - #[inline(always)] - fn do_predict( - &self, - input_tensors: Vec>, - total_len: u64, - ) -> (Vec, Vec>) { - let mut buf: Vec = Vec::with_capacity(10_000); - let mut batch_ends = vec![0usize; input_tensors.len()]; - for (i, batch_bytes_in_request) in input_tensors.into_iter().enumerate() { - for _ in batch_bytes_in_request.into_iter() { - //FIXME: for now use some hack - let model_input = TorchModel::decode_to_inputs(vec![0u8; 30]); //self.input_converter.convert(bytes); - let input_batch_tensors = model_input - .into_iter() - .map(|t| IValue::Tensor(t)) - .collect::>(); - // match self.module.forward_is(&input_batch_tensors) { - match self.module.method_is("forward_serve", &input_batch_tensors) { - Ok(res) => TorchModel::output_to_vec(res, &mut buf), - Err(e) => { - NUM_REQUESTS_FAILED.inc_by(total_len); - NUM_REQUESTS_FAILED_BY_MODEL - .with_label_values(&[&MODEL_SPECS[self.model_idx]]) - .inc_by(total_len); - INFERENCE_FAILED_REQUESTS_BY_MODEL - .with_label_values(&[&MODEL_SPECS[self.model_idx]]) - .inc_by(total_len); - panic!("{model}: {e:?}", model = MODEL_SPECS[self.model_idx], e = e); - } - } - } - batch_ends[i] = buf.len(); - } - ( - vec![TensorReturnEnum::FloatTensorReturn(Box::new(buf))], - vec![batch_ends], - ) - } - #[inline(always)] - fn model_idx(&self) -> usize { - self.model_idx - } - #[inline(always)] - fn version(&self) -> i64 { - self.version - } - } -} diff --git a/navi/segdense/Cargo.docx b/navi/segdense/Cargo.docx new file mode 100644 index 000000000..f6dbb86e8 Binary files /dev/null and b/navi/segdense/Cargo.docx differ diff --git a/navi/segdense/Cargo.toml b/navi/segdense/Cargo.toml deleted file mode 100644 index 1c8abc58c..000000000 --- a/navi/segdense/Cargo.toml +++ /dev/null @@ -1,11 +0,0 @@ -[package] -name = "segdense" -version = "0.1.0" -edition = "2021" - - -[dependencies] -env_logger = "0.10.0" -serde = { version = "1.0.104", features = ["derive"] } -serde_json = "1.0.48" -log = "0.4.17" diff --git a/navi/segdense/src/error.docx b/navi/segdense/src/error.docx new file mode 100644 index 000000000..a5a6386c1 Binary files /dev/null and b/navi/segdense/src/error.docx differ diff --git a/navi/segdense/src/error.rs b/navi/segdense/src/error.rs deleted file mode 100644 index 4c1d9af7d..000000000 --- a/navi/segdense/src/error.rs +++ /dev/null @@ -1,53 +0,0 @@ -use std::fmt::Display; - -/** - * Custom error - */ -#[derive(Debug)] -pub enum SegDenseError { - IoError(std::io::Error), - Json(serde_json::Error), - JsonMissingRoot, - JsonMissingObject, - JsonMissingArray, - JsonArraySize, - JsonMissingInputFeature, -} - -impl Display for SegDenseError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - SegDenseError::IoError(io_error) => write!(f, "{}", io_error), - SegDenseError::Json(serde_json) => write!(f, "{}", serde_json), - SegDenseError::JsonMissingRoot => { - write!(f, "{}", "SegDense JSON: Root Node note found!") - } - SegDenseError::JsonMissingObject => { - write!(f, "{}", "SegDense JSON: Object note found!") - } - SegDenseError::JsonMissingArray => { - write!(f, "{}", "SegDense JSON: Array Node note found!") - } - SegDenseError::JsonArraySize => { - write!(f, "{}", "SegDense JSON: Array size not as expected!") - } - SegDenseError::JsonMissingInputFeature => { - write!(f, "{}", "SegDense JSON: Missing input feature!") - } - } - } -} - -impl std::error::Error for SegDenseError {} - -impl From for SegDenseError { - fn from(err: std::io::Error) -> Self { - SegDenseError::IoError(err) - } -} - -impl From for SegDenseError { - fn from(err: serde_json::Error) -> Self { - SegDenseError::Json(err) - } -} diff --git a/navi/segdense/src/lib.docx b/navi/segdense/src/lib.docx new file mode 100644 index 000000000..23795a3d8 Binary files /dev/null and b/navi/segdense/src/lib.docx differ diff --git a/navi/segdense/src/lib.rs b/navi/segdense/src/lib.rs deleted file mode 100644 index f9930da64..000000000 --- a/navi/segdense/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub mod error; -pub mod mapper; -pub mod segdense_transform_spec_home_recap_2022; -pub mod util; diff --git a/navi/segdense/src/main.docx b/navi/segdense/src/main.docx new file mode 100644 index 000000000..d303b1e60 Binary files /dev/null and b/navi/segdense/src/main.docx differ diff --git a/navi/segdense/src/main.rs b/navi/segdense/src/main.rs deleted file mode 100644 index d8f7f8bc4..000000000 --- a/navi/segdense/src/main.rs +++ /dev/null @@ -1,22 +0,0 @@ -use std::env; -use std::fs; - -use segdense::error::SegDenseError; -use segdense::util; - -fn main() -> Result<(), SegDenseError> { - env_logger::init(); - let args: Vec = env::args().collect(); - - let schema_file_name: &str = if args.len() == 1 { - "json/compact.json" - } else { - &args[1] - }; - - let json_str = fs::read_to_string(schema_file_name)?; - - util::safe_load_config(&json_str)?; - - Ok(()) -} diff --git a/navi/segdense/src/mapper.docx b/navi/segdense/src/mapper.docx new file mode 100644 index 000000000..29a46513a Binary files /dev/null and b/navi/segdense/src/mapper.docx differ diff --git a/navi/segdense/src/mapper.rs b/navi/segdense/src/mapper.rs deleted file mode 100644 index f5a1d6532..000000000 --- a/navi/segdense/src/mapper.rs +++ /dev/null @@ -1,45 +0,0 @@ -use std::collections::HashMap; - -#[derive(Debug)] -pub struct FeatureInfo { - pub tensor_index: i8, - pub index_within_tensor: i64, -} - -pub static NULL_INFO: FeatureInfo = FeatureInfo { - tensor_index: -1, - index_within_tensor: -1, -}; - -#[derive(Debug, Default)] -pub struct FeatureMapper { - map: HashMap, -} - -impl FeatureMapper { - pub fn new() -> FeatureMapper { - FeatureMapper { - map: HashMap::new(), - } - } -} - -pub trait MapWriter { - fn set(&mut self, feature_id: i64, info: FeatureInfo); -} - -pub trait MapReader { - fn get(&self, feature_id: &i64) -> Option<&FeatureInfo>; -} - -impl MapWriter for FeatureMapper { - fn set(&mut self, feature_id: i64, info: FeatureInfo) { - self.map.insert(feature_id, info); - } -} - -impl MapReader for FeatureMapper { - fn get(&self, feature_id: &i64) -> Option<&FeatureInfo> { - self.map.get(feature_id) - } -} diff --git a/navi/segdense/src/segdense_transform_spec_home_recap_2022.docx b/navi/segdense/src/segdense_transform_spec_home_recap_2022.docx new file mode 100644 index 000000000..d18c64f12 Binary files /dev/null and b/navi/segdense/src/segdense_transform_spec_home_recap_2022.docx differ diff --git a/navi/segdense/src/segdense_transform_spec_home_recap_2022.rs b/navi/segdense/src/segdense_transform_spec_home_recap_2022.rs deleted file mode 100644 index ff6d3ae17..000000000 --- a/navi/segdense/src/segdense_transform_spec_home_recap_2022.rs +++ /dev/null @@ -1,182 +0,0 @@ -use serde::{Deserialize, Serialize}; -use serde_json::Value; - -#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Root { - #[serde(rename = "common_prefix")] - pub common_prefix: String, - #[serde(rename = "densification_transform_spec")] - pub densification_transform_spec: DensificationTransformSpec, - #[serde(rename = "identity_transform_spec")] - pub identity_transform_spec: Vec, - #[serde(rename = "complex_feature_type_transform_spec")] - pub complex_feature_type_transform_spec: Vec, - #[serde(rename = "input_features_map")] - pub input_features_map: Value, -} - -#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct DensificationTransformSpec { - pub discrete: Discrete, - pub cont: Cont, - pub binary: Binary, - pub string: Value, // Use StringType - pub blob: Blob, -} - -#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Discrete { - pub tag: String, - #[serde(rename = "generic_feature_type")] - pub generic_feature_type: i64, - #[serde(rename = "feature_identifier")] - pub feature_identifier: String, - #[serde(rename = "fixed_length")] - pub fixed_length: i64, - #[serde(rename = "default_value")] - pub default_value: DefaultValue, - #[serde(rename = "input_features")] - pub input_features: Vec, -} - -#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct DefaultValue { - #[serde(rename = "type")] - pub type_field: String, - pub value: String, -} - -#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct InputFeature { - #[serde(rename = "feature_id")] - pub feature_id: i64, - #[serde(rename = "full_feature_name")] - pub full_feature_name: String, - #[serde(rename = "feature_type")] - pub feature_type: i64, - pub index: i64, - #[serde(rename = "maybe_exclude")] - pub maybe_exclude: bool, - pub tag: String, - #[serde(rename = "added_at")] - pub added_at: i64, -} - -#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Cont { - pub tag: String, - #[serde(rename = "generic_feature_type")] - pub generic_feature_type: i64, - #[serde(rename = "feature_identifier")] - pub feature_identifier: String, - #[serde(rename = "fixed_length")] - pub fixed_length: i64, - #[serde(rename = "default_value")] - pub default_value: DefaultValue, - #[serde(rename = "input_features")] - pub input_features: Vec, -} - -#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Binary { - pub tag: String, - #[serde(rename = "generic_feature_type")] - pub generic_feature_type: i64, - #[serde(rename = "feature_identifier")] - pub feature_identifier: String, - #[serde(rename = "fixed_length")] - pub fixed_length: i64, - #[serde(rename = "default_value")] - pub default_value: DefaultValue, - #[serde(rename = "input_features")] - pub input_features: Vec, -} - -#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct StringType { - pub tag: String, - #[serde(rename = "generic_feature_type")] - pub generic_feature_type: i64, - #[serde(rename = "feature_identifier")] - pub feature_identifier: String, - #[serde(rename = "fixed_length")] - pub fixed_length: i64, - #[serde(rename = "default_value")] - pub default_value: DefaultValue, - #[serde(rename = "input_features")] - pub input_features: Vec, -} - -#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Blob { - pub tag: String, - #[serde(rename = "generic_feature_type")] - pub generic_feature_type: i64, - #[serde(rename = "feature_identifier")] - pub feature_identifier: String, - #[serde(rename = "fixed_length")] - pub fixed_length: i64, - #[serde(rename = "default_value")] - pub default_value: DefaultValue, - #[serde(rename = "input_features")] - pub input_features: Vec, -} - -#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct IdentityTransformSpec { - #[serde(rename = "feature_id")] - pub feature_id: i64, - #[serde(rename = "full_feature_name")] - pub full_feature_name: String, - #[serde(rename = "feature_type")] - pub feature_type: i64, -} - -#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ComplexFeatureTypeTransformSpec { - #[serde(rename = "feature_id")] - pub feature_id: i64, - #[serde(rename = "full_feature_name")] - pub full_feature_name: String, - #[serde(rename = "feature_type")] - pub feature_type: i64, - pub index: i64, - #[serde(rename = "maybe_exclude")] - pub maybe_exclude: bool, - pub tag: String, - #[serde(rename = "tensor_data_type")] - pub tensor_data_type: Option, - #[serde(rename = "added_at")] - pub added_at: i64, - #[serde(rename = "tensor_shape")] - #[serde(default)] - pub tensor_shape: Vec, -} - -#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct InputFeatureMapRecord { - #[serde(rename = "feature_id")] - pub feature_id: i64, - #[serde(rename = "full_feature_name")] - pub full_feature_name: String, - #[serde(rename = "feature_type")] - pub feature_type: i64, - pub index: i64, - #[serde(rename = "maybe_exclude")] - pub maybe_exclude: bool, - pub tag: String, - #[serde(rename = "added_at")] - pub added_at: i64, -} diff --git a/navi/segdense/src/util.docx b/navi/segdense/src/util.docx new file mode 100644 index 000000000..7e36a7481 Binary files /dev/null and b/navi/segdense/src/util.docx differ diff --git a/navi/segdense/src/util.rs b/navi/segdense/src/util.rs deleted file mode 100644 index 116725189..000000000 --- a/navi/segdense/src/util.rs +++ /dev/null @@ -1,154 +0,0 @@ -use log::debug; -use std::fs; - -use serde_json::{Map, Value}; - -use crate::error::SegDenseError; -use crate::mapper::{FeatureInfo, FeatureMapper, MapWriter}; -use crate::segdense_transform_spec_home_recap_2022::{self as seg_dense, InputFeature}; - -pub fn load_config(file_name: &str) -> Result { - let json_str = fs::read_to_string(file_name)?; - // &format!("Unable to load segdense file {}", file_name)); - let seg_dense_config = parse(&json_str)?; - // &format!("Unable to parse segdense file {}", file_name)); - Ok(seg_dense_config) -} - -pub fn parse(json_str: &str) -> Result { - let root: seg_dense::Root = serde_json::from_str(json_str)?; - Ok(root) -} - -/** - * Given a json string containing a seg dense schema create a feature mapper - * which is essentially: - * - * {feature-id -> (Tensor Index, Index of feature within the tensor)} - * - * Feature id : 64 bit hash of the feature name used in DataRecords. - * - * Tensor Index : A vector of tensors is passed to the model. Tensor - * index refers to the tensor this feature is part of. - * - * Index of feature in tensor : The tensors are vectors, the index of - * feature is the position to put the feature value. - * - * There are many assumptions made in this function that is very model specific. - * These assumptions are called out below and need to be schematized eventually. - * - * Call this once for each segdense schema and cache the FeatureMapper. - */ -pub fn safe_load_config(json_str: &str) -> Result { - let root = parse(json_str)?; - load_from_parsed_config(root) -} - -// Perf note : make 'root' un-owned -pub fn load_from_parsed_config(root: seg_dense::Root) -> Result { - let v = root.input_features_map; - - // Do error check - let map: Map = match v { - Value::Object(map) => map, - _ => return Err(SegDenseError::JsonMissingObject), - }; - - let mut fm: FeatureMapper = FeatureMapper::new(); - - let items = map.values(); - - // Perf : Consider a way to avoid clone here - for item in items.cloned() { - let mut vec = match item { - Value::Array(v) => v, - _ => return Err(SegDenseError::JsonMissingArray), - }; - - if vec.len() != 1 { - return Err(SegDenseError::JsonArraySize); - } - - let val = vec.pop().unwrap(); - - let input_feature: seg_dense::InputFeature = serde_json::from_value(val)?; - let feature_id = input_feature.feature_id; - let feature_info = to_feature_info(&input_feature); - - match feature_info { - Some(info) => { - debug!("{:?}", info); - fm.set(feature_id, info) - } - None => (), - } - } - - Ok(fm) -} -#[allow(dead_code)] -fn add_feature_info_to_mapper( - feature_mapper: &mut FeatureMapper, - input_features: &Vec, -) { - for input_feature in input_features.iter() { - let feature_id = input_feature.feature_id; - let feature_info = to_feature_info(input_feature); - - match feature_info { - Some(info) => { - debug!("{:?}", info); - feature_mapper.set(feature_id, info) - } - None => (), - } - } -} - -pub fn to_feature_info(input_feature: &seg_dense::InputFeature) -> Option { - if input_feature.maybe_exclude { - return None; - } - - // This part needs to be schema driven - // - // tensor index : Which of these tensors this feature is part of - // [Continious, Binary, Discrete, User_embedding, user_eng_embedding, author_embedding] - // Note that this order is fixed/hardcoded here, and need to be schematized - // - let tensor_idx: i8 = match input_feature.feature_id { - // user.timelines.twhin_user_follow_embeddings.twhin_user_follow_embeddings - // Feature name is mapped to a feature-id value. The hardcoded values below correspond to a specific feature name. - -2550691008059411095 => 3, - - // user.timelines.twhin_user_engagement_embeddings.twhin_user_engagement_embeddings - 5390650078733277231 => 4, - - // original_author.timelines.twhin_author_follow_embeddings.twhin_author_follow_embeddings - 3223956748566688423 => 5, - - _ => match input_feature.feature_type { - // feature_type : src/thrift/com/twitter/ml/api/data.thrift - // BINARY = 1, CONTINUOUS = 2, DISCRETE = 3, - // Map to slots in [Continious, Binary, Discrete, ..] - 1 => 1, - 2 => 0, - 3 => 2, - _ => -1, - }, - }; - - if input_feature.index < 0 { - return None; - } - - // Handle this case later - if tensor_idx == -1 { - return None; - } - - Some(FeatureInfo { - tensor_index: tensor_idx, - index_within_tensor: input_feature.index, - }) -} diff --git a/navi/thrift_bpr_adapter/thrift/Cargo.docx b/navi/thrift_bpr_adapter/thrift/Cargo.docx new file mode 100644 index 000000000..98f3d849a Binary files /dev/null and b/navi/thrift_bpr_adapter/thrift/Cargo.docx differ diff --git a/navi/thrift_bpr_adapter/thrift/Cargo.toml b/navi/thrift_bpr_adapter/thrift/Cargo.toml deleted file mode 100644 index 83e9e17e1..000000000 --- a/navi/thrift_bpr_adapter/thrift/Cargo.toml +++ /dev/null @@ -1,8 +0,0 @@ -[package] -name = "bpr_thrift" -description = "Thrift parser for Batch Prediction Request" -version = "0.1.0" -edition = "2021" - -[dependencies] -thrift = "0.17.0" diff --git a/navi/thrift_bpr_adapter/thrift/src/data.docx b/navi/thrift_bpr_adapter/thrift/src/data.docx new file mode 100644 index 000000000..54573d73f Binary files /dev/null and b/navi/thrift_bpr_adapter/thrift/src/data.docx differ diff --git a/navi/thrift_bpr_adapter/thrift/src/data.rs b/navi/thrift_bpr_adapter/thrift/src/data.rs deleted file mode 100644 index aa09ee205..000000000 --- a/navi/thrift_bpr_adapter/thrift/src/data.rs +++ /dev/null @@ -1,1213 +0,0 @@ -// Autogenerated by Thrift Compiler (0.17.0) -// DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING - -#![allow(unused_imports)] -#![allow(unused_extern_crates)] -#![allow(clippy::too_many_arguments, clippy::type_complexity, clippy::vec_box)] -#![cfg_attr(rustfmt, rustfmt_skip)] - -use std::cell::RefCell; -use std::collections::{BTreeMap, BTreeSet}; -use std::convert::{From, TryFrom}; -use std::default::Default; -use std::error::Error; -use std::fmt; -use std::fmt::{Display, Formatter}; -use std::rc::Rc; - -use thrift::OrderedFloat; -use thrift::{ApplicationError, ApplicationErrorKind, ProtocolError, ProtocolErrorKind, TThriftClient}; -use thrift::protocol::{TFieldIdentifier, TListIdentifier, TMapIdentifier, TMessageIdentifier, TMessageType, TInputProtocol, TOutputProtocol, TSerializable, TSetIdentifier, TStructIdentifier, TType}; -use thrift::protocol::field_id; -use thrift::protocol::verify_expected_message_type; -use thrift::protocol::verify_expected_sequence_number; -use thrift::protocol::verify_expected_service_call; -use thrift::protocol::verify_required_field_exists; -use thrift::server::TProcessor; - -use crate::tensor; - -#[derive(Copy, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct FeatureType(pub i32); - -impl FeatureType { - pub const BINARY: FeatureType = FeatureType(1); - pub const CONTINUOUS: FeatureType = FeatureType(2); - pub const DISCRETE: FeatureType = FeatureType(3); - pub const STRING: FeatureType = FeatureType(4); - pub const SPARSE_BINARY: FeatureType = FeatureType(5); - pub const SPARSE_CONTINUOUS: FeatureType = FeatureType(6); - pub const UNKNOWN: FeatureType = FeatureType(7); - pub const BLOB: FeatureType = FeatureType(8); - pub const TENSOR: FeatureType = FeatureType(9); - pub const SPARSE_TENSOR: FeatureType = FeatureType(10); - pub const FEATURE_TYPE11: FeatureType = FeatureType(11); - pub const FEATURE_TYPE12: FeatureType = FeatureType(12); - pub const ENUM_VALUES: &'static [Self] = &[ - Self::BINARY, - Self::CONTINUOUS, - Self::DISCRETE, - Self::STRING, - Self::SPARSE_BINARY, - Self::SPARSE_CONTINUOUS, - Self::UNKNOWN, - Self::BLOB, - Self::TENSOR, - Self::SPARSE_TENSOR, - Self::FEATURE_TYPE11, - Self::FEATURE_TYPE12, - ]; -} - -impl TSerializable for FeatureType { - #[allow(clippy::trivially_copy_pass_by_ref)] - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - o_prot.write_i32(self.0) - } - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - let enum_value = i_prot.read_i32()?; - Ok(FeatureType::from(enum_value)) - } -} - -impl From for FeatureType { - fn from(i: i32) -> Self { - match i { - 1 => FeatureType::BINARY, - 2 => FeatureType::CONTINUOUS, - 3 => FeatureType::DISCRETE, - 4 => FeatureType::STRING, - 5 => FeatureType::SPARSE_BINARY, - 6 => FeatureType::SPARSE_CONTINUOUS, - 7 => FeatureType::UNKNOWN, - 8 => FeatureType::BLOB, - 9 => FeatureType::TENSOR, - 10 => FeatureType::SPARSE_TENSOR, - 11 => FeatureType::FEATURE_TYPE11, - 12 => FeatureType::FEATURE_TYPE12, - _ => FeatureType(i) - } - } -} - -impl From<&i32> for FeatureType { - fn from(i: &i32) -> Self { - FeatureType::from(*i) - } -} - -impl From for i32 { - fn from(e: FeatureType) -> i32 { - e.0 - } -} - -impl From<&FeatureType> for i32 { - fn from(e: &FeatureType) -> i32 { - e.0 - } -} - -// -// DataRecord -// - -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct DataRecord { - pub binary_features: Option>, - pub continuous_features: Option>>, - pub discrete_features: Option>, - pub string_features: Option>, - pub sparse_binary_features: Option>>, - pub sparse_continuous_features: Option>>>, - pub blob_features: Option>>, - pub tensors: Option>, - pub sparse_tensors: Option>, -} - -impl DataRecord { - pub fn new(binary_features: F1, continuous_features: F2, discrete_features: F3, string_features: F4, sparse_binary_features: F5, sparse_continuous_features: F6, blob_features: F7, tensors: F8, sparse_tensors: F9) -> DataRecord where F1: Into>>, F2: Into>>>, F3: Into>>, F4: Into>>, F5: Into>>>, F6: Into>>>>, F7: Into>>>, F8: Into>>, F9: Into>> { - DataRecord { - binary_features: binary_features.into(), - continuous_features: continuous_features.into(), - discrete_features: discrete_features.into(), - string_features: string_features.into(), - sparse_binary_features: sparse_binary_features.into(), - sparse_continuous_features: sparse_continuous_features.into(), - blob_features: blob_features.into(), - tensors: tensors.into(), - sparse_tensors: sparse_tensors.into(), - } - } -} - -impl TSerializable for DataRecord { - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - i_prot.read_struct_begin()?; - let mut f_1: Option> = None; - let mut f_2: Option>> = None; - let mut f_3: Option> = None; - let mut f_4: Option> = None; - let mut f_5: Option>> = None; - let mut f_6: Option>>> = None; - let mut f_7: Option>> = None; - let mut f_8: Option> = None; - let mut f_9: Option> = None; - loop { - let field_ident = i_prot.read_field_begin()?; - if field_ident.field_type == TType::Stop { - break; - } - let field_id = field_id(&field_ident)?; - match field_id { - 1 => { - let set_ident = i_prot.read_set_begin()?; - let mut val: BTreeSet = BTreeSet::new(); - for _ in 0..set_ident.size { - let set_elem_0 = i_prot.read_i64()?; - val.insert(set_elem_0); - } - i_prot.read_set_end()?; - f_1 = Some(val); - }, - 2 => { - let map_ident = i_prot.read_map_begin()?; - let mut val: BTreeMap> = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_1 = i_prot.read_i64()?; - let map_val_2 = OrderedFloat::from(i_prot.read_double()?); - val.insert(map_key_1, map_val_2); - } - i_prot.read_map_end()?; - f_2 = Some(val); - }, - 3 => { - let map_ident = i_prot.read_map_begin()?; - let mut val: BTreeMap = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_3 = i_prot.read_i64()?; - let map_val_4 = i_prot.read_i64()?; - val.insert(map_key_3, map_val_4); - } - i_prot.read_map_end()?; - f_3 = Some(val); - }, - 4 => { - let map_ident = i_prot.read_map_begin()?; - let mut val: BTreeMap = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_5 = i_prot.read_i64()?; - let map_val_6 = i_prot.read_string()?; - val.insert(map_key_5, map_val_6); - } - i_prot.read_map_end()?; - f_4 = Some(val); - }, - 5 => { - let map_ident = i_prot.read_map_begin()?; - let mut val: BTreeMap> = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_7 = i_prot.read_i64()?; - let set_ident = i_prot.read_set_begin()?; - let mut map_val_8: BTreeSet = BTreeSet::new(); - for _ in 0..set_ident.size { - let set_elem_9 = i_prot.read_string()?; - map_val_8.insert(set_elem_9); - } - i_prot.read_set_end()?; - val.insert(map_key_7, map_val_8); - } - i_prot.read_map_end()?; - f_5 = Some(val); - }, - 6 => { - let map_ident = i_prot.read_map_begin()?; - let mut val: BTreeMap>> = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_10 = i_prot.read_i64()?; - let map_ident = i_prot.read_map_begin()?; - let mut map_val_11: BTreeMap> = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_12 = i_prot.read_string()?; - let map_val_13 = OrderedFloat::from(i_prot.read_double()?); - map_val_11.insert(map_key_12, map_val_13); - } - i_prot.read_map_end()?; - val.insert(map_key_10, map_val_11); - } - i_prot.read_map_end()?; - f_6 = Some(val); - }, - 7 => { - let map_ident = i_prot.read_map_begin()?; - let mut val: BTreeMap> = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_14 = i_prot.read_i64()?; - let map_val_15 = i_prot.read_bytes()?; - val.insert(map_key_14, map_val_15); - } - i_prot.read_map_end()?; - f_7 = Some(val); - }, - 8 => { - let map_ident = i_prot.read_map_begin()?; - let mut val: BTreeMap = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_16 = i_prot.read_i64()?; - let map_val_17 = tensor::GeneralTensor::read_from_in_protocol(i_prot)?; - val.insert(map_key_16, map_val_17); - } - i_prot.read_map_end()?; - f_8 = Some(val); - }, - 9 => { - let map_ident = i_prot.read_map_begin()?; - let mut val: BTreeMap = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_18 = i_prot.read_i64()?; - let map_val_19 = tensor::SparseTensor::read_from_in_protocol(i_prot)?; - val.insert(map_key_18, map_val_19); - } - i_prot.read_map_end()?; - f_9 = Some(val); - }, - _ => { - i_prot.skip(field_ident.field_type)?; - }, - }; - i_prot.read_field_end()?; - } - i_prot.read_struct_end()?; - let ret = DataRecord { - binary_features: f_1, - continuous_features: f_2, - discrete_features: f_3, - string_features: f_4, - sparse_binary_features: f_5, - sparse_continuous_features: f_6, - blob_features: f_7, - tensors: f_8, - sparse_tensors: f_9, - }; - Ok(ret) - } - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let struct_ident = TStructIdentifier::new("DataRecord"); - o_prot.write_struct_begin(&struct_ident)?; - if let Some(ref fld_var) = self.binary_features { - o_prot.write_field_begin(&TFieldIdentifier::new("binaryFeatures", TType::Set, 1))?; - o_prot.write_set_begin(&TSetIdentifier::new(TType::I64, fld_var.len() as i32))?; - for e in fld_var { - o_prot.write_i64(*e)?; - } - o_prot.write_set_end()?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.continuous_features { - o_prot.write_field_begin(&TFieldIdentifier::new("continuousFeatures", TType::Map, 2))?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::I64, TType::Double, fld_var.len() as i32))?; - for (k, v) in fld_var { - o_prot.write_i64(*k)?; - o_prot.write_double((*v).into())?; - } - o_prot.write_map_end()?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.discrete_features { - o_prot.write_field_begin(&TFieldIdentifier::new("discreteFeatures", TType::Map, 3))?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::I64, TType::I64, fld_var.len() as i32))?; - for (k, v) in fld_var { - o_prot.write_i64(*k)?; - o_prot.write_i64(*v)?; - } - o_prot.write_map_end()?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.string_features { - o_prot.write_field_begin(&TFieldIdentifier::new("stringFeatures", TType::Map, 4))?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::I64, TType::String, fld_var.len() as i32))?; - for (k, v) in fld_var { - o_prot.write_i64(*k)?; - o_prot.write_string(v)?; - } - o_prot.write_map_end()?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.sparse_binary_features { - o_prot.write_field_begin(&TFieldIdentifier::new("sparseBinaryFeatures", TType::Map, 5))?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::I64, TType::Set, fld_var.len() as i32))?; - for (k, v) in fld_var { - o_prot.write_i64(*k)?; - o_prot.write_set_begin(&TSetIdentifier::new(TType::String, v.len() as i32))?; - for e in v { - o_prot.write_string(e)?; - } - o_prot.write_set_end()?; - } - o_prot.write_map_end()?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.sparse_continuous_features { - o_prot.write_field_begin(&TFieldIdentifier::new("sparseContinuousFeatures", TType::Map, 6))?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::I64, TType::Map, fld_var.len() as i32))?; - for (k, v) in fld_var { - o_prot.write_i64(*k)?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::String, TType::Double, v.len() as i32))?; - for (k, v) in v { - o_prot.write_string(k)?; - o_prot.write_double((*v).into())?; - } - o_prot.write_map_end()?; - } - o_prot.write_map_end()?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.blob_features { - o_prot.write_field_begin(&TFieldIdentifier::new("blobFeatures", TType::Map, 7))?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::I64, TType::String, fld_var.len() as i32))?; - for (k, v) in fld_var { - o_prot.write_i64(*k)?; - o_prot.write_bytes(v)?; - } - o_prot.write_map_end()?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.tensors { - o_prot.write_field_begin(&TFieldIdentifier::new("tensors", TType::Map, 8))?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::I64, TType::Struct, fld_var.len() as i32))?; - for (k, v) in fld_var { - o_prot.write_i64(*k)?; - v.write_to_out_protocol(o_prot)?; - } - o_prot.write_map_end()?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.sparse_tensors { - o_prot.write_field_begin(&TFieldIdentifier::new("sparseTensors", TType::Map, 9))?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::I64, TType::Struct, fld_var.len() as i32))?; - for (k, v) in fld_var { - o_prot.write_i64(*k)?; - v.write_to_out_protocol(o_prot)?; - } - o_prot.write_map_end()?; - o_prot.write_field_end()? - } - o_prot.write_field_stop()?; - o_prot.write_struct_end() - } -} - -impl Default for DataRecord { - fn default() -> Self { - DataRecord{ - binary_features: Some(BTreeSet::new()), - continuous_features: Some(BTreeMap::new()), - discrete_features: Some(BTreeMap::new()), - string_features: Some(BTreeMap::new()), - sparse_binary_features: Some(BTreeMap::new()), - sparse_continuous_features: Some(BTreeMap::new()), - blob_features: Some(BTreeMap::new()), - tensors: Some(BTreeMap::new()), - sparse_tensors: Some(BTreeMap::new()), - } - } -} - -// -// CompactDataRecord -// - -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct CompactDataRecord { - pub binary_features: Option>, - pub continuous_features: Option>, - pub discrete_features: Option>, - pub string_features: Option>, - pub sparse_binary_features: Option>>, - pub sparse_binary_features_with16b_sparse_key: Option>>, - pub sparse_binary_features_with32b_sparse_key: Option>>, - pub sparse_binary_features_with64b_sparse_key: Option>>, - pub sparse_continuous_features: Option>>, - pub sparse_continuous_features_with16b_sparse_key: Option>>, - pub sparse_continuous_features_with32b_sparse_key: Option>>, - pub sparse_continuous_features_with64b_sparse_key: Option>>, - pub blob_features: Option>>, - pub tensors: Option>, - pub sparse_tensors: Option>, -} - -impl CompactDataRecord { - pub fn new(binary_features: F1, continuous_features: F2, discrete_features: F3, string_features: F4, sparse_binary_features: F5, sparse_binary_features_with16b_sparse_key: F6, sparse_binary_features_with32b_sparse_key: F7, sparse_binary_features_with64b_sparse_key: F8, sparse_continuous_features: F9, sparse_continuous_features_with16b_sparse_key: F10, sparse_continuous_features_with32b_sparse_key: F11, sparse_continuous_features_with64b_sparse_key: F12, blob_features: F13, tensors: F14, sparse_tensors: F15) -> CompactDataRecord where F1: Into>>, F2: Into>>, F3: Into>>, F4: Into>>, F5: Into>>>, F6: Into>>>, F7: Into>>>, F8: Into>>>, F9: Into>>>, F10: Into>>>, F11: Into>>>, F12: Into>>>, F13: Into>>>, F14: Into>>, F15: Into>> { - CompactDataRecord { - binary_features: binary_features.into(), - continuous_features: continuous_features.into(), - discrete_features: discrete_features.into(), - string_features: string_features.into(), - sparse_binary_features: sparse_binary_features.into(), - sparse_binary_features_with16b_sparse_key: sparse_binary_features_with16b_sparse_key.into(), - sparse_binary_features_with32b_sparse_key: sparse_binary_features_with32b_sparse_key.into(), - sparse_binary_features_with64b_sparse_key: sparse_binary_features_with64b_sparse_key.into(), - sparse_continuous_features: sparse_continuous_features.into(), - sparse_continuous_features_with16b_sparse_key: sparse_continuous_features_with16b_sparse_key.into(), - sparse_continuous_features_with32b_sparse_key: sparse_continuous_features_with32b_sparse_key.into(), - sparse_continuous_features_with64b_sparse_key: sparse_continuous_features_with64b_sparse_key.into(), - blob_features: blob_features.into(), - tensors: tensors.into(), - sparse_tensors: sparse_tensors.into(), - } - } -} - -impl TSerializable for CompactDataRecord { - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - i_prot.read_struct_begin()?; - let mut f_1: Option> = None; - let mut f_2: Option> = None; - let mut f_3: Option> = None; - let mut f_4: Option> = None; - let mut f_5: Option>> = None; - let mut f_6: Option>> = None; - let mut f_7: Option>> = None; - let mut f_8: Option>> = None; - let mut f_9: Option>> = None; - let mut f_10: Option>> = None; - let mut f_11: Option>> = None; - let mut f_12: Option>> = None; - let mut f_13: Option>> = None; - let mut f_14: Option> = None; - let mut f_15: Option> = None; - loop { - let field_ident = i_prot.read_field_begin()?; - if field_ident.field_type == TType::Stop { - break; - } - let field_id = field_id(&field_ident)?; - match field_id { - 1 => { - let set_ident = i_prot.read_set_begin()?; - let mut val: BTreeSet = BTreeSet::new(); - for _ in 0..set_ident.size { - let set_elem_20 = i_prot.read_i64()?; - val.insert(set_elem_20); - } - i_prot.read_set_end()?; - f_1 = Some(val); - }, - 2 => { - let map_ident = i_prot.read_map_begin()?; - let mut val: BTreeMap = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_21 = i_prot.read_i64()?; - let map_val_22 = i_prot.read_i32()?; - val.insert(map_key_21, map_val_22); - } - i_prot.read_map_end()?; - f_2 = Some(val); - }, - 3 => { - let map_ident = i_prot.read_map_begin()?; - let mut val: BTreeMap = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_23 = i_prot.read_i64()?; - let map_val_24 = i_prot.read_i64()?; - val.insert(map_key_23, map_val_24); - } - i_prot.read_map_end()?; - f_3 = Some(val); - }, - 4 => { - let map_ident = i_prot.read_map_begin()?; - let mut val: BTreeMap = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_25 = i_prot.read_i64()?; - let map_val_26 = i_prot.read_string()?; - val.insert(map_key_25, map_val_26); - } - i_prot.read_map_end()?; - f_4 = Some(val); - }, - 5 => { - let map_ident = i_prot.read_map_begin()?; - let mut val: BTreeMap> = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_27 = i_prot.read_i64()?; - let set_ident = i_prot.read_set_begin()?; - let mut map_val_28: BTreeSet = BTreeSet::new(); - for _ in 0..set_ident.size { - let set_elem_29 = i_prot.read_string()?; - map_val_28.insert(set_elem_29); - } - i_prot.read_set_end()?; - val.insert(map_key_27, map_val_28); - } - i_prot.read_map_end()?; - f_5 = Some(val); - }, - 6 => { - let map_ident = i_prot.read_map_begin()?; - let mut val: BTreeMap> = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_30 = i_prot.read_i64()?; - let set_ident = i_prot.read_set_begin()?; - let mut map_val_31: BTreeSet = BTreeSet::new(); - for _ in 0..set_ident.size { - let set_elem_32 = i_prot.read_i16()?; - map_val_31.insert(set_elem_32); - } - i_prot.read_set_end()?; - val.insert(map_key_30, map_val_31); - } - i_prot.read_map_end()?; - f_6 = Some(val); - }, - 7 => { - let map_ident = i_prot.read_map_begin()?; - let mut val: BTreeMap> = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_33 = i_prot.read_i64()?; - let set_ident = i_prot.read_set_begin()?; - let mut map_val_34: BTreeSet = BTreeSet::new(); - for _ in 0..set_ident.size { - let set_elem_35 = i_prot.read_i32()?; - map_val_34.insert(set_elem_35); - } - i_prot.read_set_end()?; - val.insert(map_key_33, map_val_34); - } - i_prot.read_map_end()?; - f_7 = Some(val); - }, - 8 => { - let map_ident = i_prot.read_map_begin()?; - let mut val: BTreeMap> = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_36 = i_prot.read_i64()?; - let set_ident = i_prot.read_set_begin()?; - let mut map_val_37: BTreeSet = BTreeSet::new(); - for _ in 0..set_ident.size { - let set_elem_38 = i_prot.read_i64()?; - map_val_37.insert(set_elem_38); - } - i_prot.read_set_end()?; - val.insert(map_key_36, map_val_37); - } - i_prot.read_map_end()?; - f_8 = Some(val); - }, - 9 => { - let map_ident = i_prot.read_map_begin()?; - let mut val: BTreeMap> = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_39 = i_prot.read_i64()?; - let map_ident = i_prot.read_map_begin()?; - let mut map_val_40: BTreeMap = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_41 = i_prot.read_string()?; - let map_val_42 = i_prot.read_i32()?; - map_val_40.insert(map_key_41, map_val_42); - } - i_prot.read_map_end()?; - val.insert(map_key_39, map_val_40); - } - i_prot.read_map_end()?; - f_9 = Some(val); - }, - 10 => { - let map_ident = i_prot.read_map_begin()?; - let mut val: BTreeMap> = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_43 = i_prot.read_i64()?; - let map_ident = i_prot.read_map_begin()?; - let mut map_val_44: BTreeMap = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_45 = i_prot.read_i16()?; - let map_val_46 = i_prot.read_i32()?; - map_val_44.insert(map_key_45, map_val_46); - } - i_prot.read_map_end()?; - val.insert(map_key_43, map_val_44); - } - i_prot.read_map_end()?; - f_10 = Some(val); - }, - 11 => { - let map_ident = i_prot.read_map_begin()?; - let mut val: BTreeMap> = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_47 = i_prot.read_i64()?; - let map_ident = i_prot.read_map_begin()?; - let mut map_val_48: BTreeMap = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_49 = i_prot.read_i32()?; - let map_val_50 = i_prot.read_i32()?; - map_val_48.insert(map_key_49, map_val_50); - } - i_prot.read_map_end()?; - val.insert(map_key_47, map_val_48); - } - i_prot.read_map_end()?; - f_11 = Some(val); - }, - 12 => { - let map_ident = i_prot.read_map_begin()?; - let mut val: BTreeMap> = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_51 = i_prot.read_i64()?; - let map_ident = i_prot.read_map_begin()?; - let mut map_val_52: BTreeMap = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_53 = i_prot.read_i64()?; - let map_val_54 = i_prot.read_i32()?; - map_val_52.insert(map_key_53, map_val_54); - } - i_prot.read_map_end()?; - val.insert(map_key_51, map_val_52); - } - i_prot.read_map_end()?; - f_12 = Some(val); - }, - 13 => { - let map_ident = i_prot.read_map_begin()?; - let mut val: BTreeMap> = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_55 = i_prot.read_i64()?; - let map_val_56 = i_prot.read_bytes()?; - val.insert(map_key_55, map_val_56); - } - i_prot.read_map_end()?; - f_13 = Some(val); - }, - 14 => { - let map_ident = i_prot.read_map_begin()?; - let mut val: BTreeMap = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_57 = i_prot.read_i64()?; - let map_val_58 = tensor::GeneralTensor::read_from_in_protocol(i_prot)?; - val.insert(map_key_57, map_val_58); - } - i_prot.read_map_end()?; - f_14 = Some(val); - }, - 15 => { - let map_ident = i_prot.read_map_begin()?; - let mut val: BTreeMap = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_59 = i_prot.read_i64()?; - let map_val_60 = tensor::SparseTensor::read_from_in_protocol(i_prot)?; - val.insert(map_key_59, map_val_60); - } - i_prot.read_map_end()?; - f_15 = Some(val); - }, - _ => { - i_prot.skip(field_ident.field_type)?; - }, - }; - i_prot.read_field_end()?; - } - i_prot.read_struct_end()?; - let ret = CompactDataRecord { - binary_features: f_1, - continuous_features: f_2, - discrete_features: f_3, - string_features: f_4, - sparse_binary_features: f_5, - sparse_binary_features_with16b_sparse_key: f_6, - sparse_binary_features_with32b_sparse_key: f_7, - sparse_binary_features_with64b_sparse_key: f_8, - sparse_continuous_features: f_9, - sparse_continuous_features_with16b_sparse_key: f_10, - sparse_continuous_features_with32b_sparse_key: f_11, - sparse_continuous_features_with64b_sparse_key: f_12, - blob_features: f_13, - tensors: f_14, - sparse_tensors: f_15, - }; - Ok(ret) - } - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let struct_ident = TStructIdentifier::new("CompactDataRecord"); - o_prot.write_struct_begin(&struct_ident)?; - if let Some(ref fld_var) = self.binary_features { - o_prot.write_field_begin(&TFieldIdentifier::new("binaryFeatures", TType::Set, 1))?; - o_prot.write_set_begin(&TSetIdentifier::new(TType::I64, fld_var.len() as i32))?; - for e in fld_var { - o_prot.write_i64(*e)?; - } - o_prot.write_set_end()?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.continuous_features { - o_prot.write_field_begin(&TFieldIdentifier::new("continuousFeatures", TType::Map, 2))?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::I64, TType::I32, fld_var.len() as i32))?; - for (k, v) in fld_var { - o_prot.write_i64(*k)?; - o_prot.write_i32(*v)?; - } - o_prot.write_map_end()?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.discrete_features { - o_prot.write_field_begin(&TFieldIdentifier::new("discreteFeatures", TType::Map, 3))?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::I64, TType::I64, fld_var.len() as i32))?; - for (k, v) in fld_var { - o_prot.write_i64(*k)?; - o_prot.write_i64(*v)?; - } - o_prot.write_map_end()?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.string_features { - o_prot.write_field_begin(&TFieldIdentifier::new("stringFeatures", TType::Map, 4))?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::I64, TType::String, fld_var.len() as i32))?; - for (k, v) in fld_var { - o_prot.write_i64(*k)?; - o_prot.write_string(v)?; - } - o_prot.write_map_end()?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.sparse_binary_features { - o_prot.write_field_begin(&TFieldIdentifier::new("sparseBinaryFeatures", TType::Map, 5))?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::I64, TType::Set, fld_var.len() as i32))?; - for (k, v) in fld_var { - o_prot.write_i64(*k)?; - o_prot.write_set_begin(&TSetIdentifier::new(TType::String, v.len() as i32))?; - for e in v { - o_prot.write_string(e)?; - } - o_prot.write_set_end()?; - } - o_prot.write_map_end()?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.sparse_binary_features_with16b_sparse_key { - o_prot.write_field_begin(&TFieldIdentifier::new("sparseBinaryFeaturesWith16bSparseKey", TType::Map, 6))?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::I64, TType::Set, fld_var.len() as i32))?; - for (k, v) in fld_var { - o_prot.write_i64(*k)?; - o_prot.write_set_begin(&TSetIdentifier::new(TType::I16, v.len() as i32))?; - for e in v { - o_prot.write_i16(*e)?; - } - o_prot.write_set_end()?; - } - o_prot.write_map_end()?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.sparse_binary_features_with32b_sparse_key { - o_prot.write_field_begin(&TFieldIdentifier::new("sparseBinaryFeaturesWith32bSparseKey", TType::Map, 7))?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::I64, TType::Set, fld_var.len() as i32))?; - for (k, v) in fld_var { - o_prot.write_i64(*k)?; - o_prot.write_set_begin(&TSetIdentifier::new(TType::I32, v.len() as i32))?; - for e in v { - o_prot.write_i32(*e)?; - } - o_prot.write_set_end()?; - } - o_prot.write_map_end()?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.sparse_binary_features_with64b_sparse_key { - o_prot.write_field_begin(&TFieldIdentifier::new("sparseBinaryFeaturesWith64bSparseKey", TType::Map, 8))?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::I64, TType::Set, fld_var.len() as i32))?; - for (k, v) in fld_var { - o_prot.write_i64(*k)?; - o_prot.write_set_begin(&TSetIdentifier::new(TType::I64, v.len() as i32))?; - for e in v { - o_prot.write_i64(*e)?; - } - o_prot.write_set_end()?; - } - o_prot.write_map_end()?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.sparse_continuous_features { - o_prot.write_field_begin(&TFieldIdentifier::new("sparseContinuousFeatures", TType::Map, 9))?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::I64, TType::Map, fld_var.len() as i32))?; - for (k, v) in fld_var { - o_prot.write_i64(*k)?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::String, TType::I32, v.len() as i32))?; - for (k, v) in v { - o_prot.write_string(k)?; - o_prot.write_i32(*v)?; - } - o_prot.write_map_end()?; - } - o_prot.write_map_end()?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.sparse_continuous_features_with16b_sparse_key { - o_prot.write_field_begin(&TFieldIdentifier::new("sparseContinuousFeaturesWith16bSparseKey", TType::Map, 10))?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::I64, TType::Map, fld_var.len() as i32))?; - for (k, v) in fld_var { - o_prot.write_i64(*k)?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::I16, TType::I32, v.len() as i32))?; - for (k, v) in v { - o_prot.write_i16(*k)?; - o_prot.write_i32(*v)?; - } - o_prot.write_map_end()?; - } - o_prot.write_map_end()?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.sparse_continuous_features_with32b_sparse_key { - o_prot.write_field_begin(&TFieldIdentifier::new("sparseContinuousFeaturesWith32bSparseKey", TType::Map, 11))?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::I64, TType::Map, fld_var.len() as i32))?; - for (k, v) in fld_var { - o_prot.write_i64(*k)?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::I32, TType::I32, v.len() as i32))?; - for (k, v) in v { - o_prot.write_i32(*k)?; - o_prot.write_i32(*v)?; - } - o_prot.write_map_end()?; - } - o_prot.write_map_end()?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.sparse_continuous_features_with64b_sparse_key { - o_prot.write_field_begin(&TFieldIdentifier::new("sparseContinuousFeaturesWith64bSparseKey", TType::Map, 12))?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::I64, TType::Map, fld_var.len() as i32))?; - for (k, v) in fld_var { - o_prot.write_i64(*k)?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::I64, TType::I32, v.len() as i32))?; - for (k, v) in v { - o_prot.write_i64(*k)?; - o_prot.write_i32(*v)?; - } - o_prot.write_map_end()?; - } - o_prot.write_map_end()?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.blob_features { - o_prot.write_field_begin(&TFieldIdentifier::new("blobFeatures", TType::Map, 13))?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::I64, TType::String, fld_var.len() as i32))?; - for (k, v) in fld_var { - o_prot.write_i64(*k)?; - o_prot.write_bytes(v)?; - } - o_prot.write_map_end()?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.tensors { - o_prot.write_field_begin(&TFieldIdentifier::new("tensors", TType::Map, 14))?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::I64, TType::Struct, fld_var.len() as i32))?; - for (k, v) in fld_var { - o_prot.write_i64(*k)?; - v.write_to_out_protocol(o_prot)?; - } - o_prot.write_map_end()?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.sparse_tensors { - o_prot.write_field_begin(&TFieldIdentifier::new("sparseTensors", TType::Map, 15))?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::I64, TType::Struct, fld_var.len() as i32))?; - for (k, v) in fld_var { - o_prot.write_i64(*k)?; - v.write_to_out_protocol(o_prot)?; - } - o_prot.write_map_end()?; - o_prot.write_field_end()? - } - o_prot.write_field_stop()?; - o_prot.write_struct_end() - } -} - -impl Default for CompactDataRecord { - fn default() -> Self { - CompactDataRecord{ - binary_features: Some(BTreeSet::new()), - continuous_features: Some(BTreeMap::new()), - discrete_features: Some(BTreeMap::new()), - string_features: Some(BTreeMap::new()), - sparse_binary_features: Some(BTreeMap::new()), - sparse_binary_features_with16b_sparse_key: Some(BTreeMap::new()), - sparse_binary_features_with32b_sparse_key: Some(BTreeMap::new()), - sparse_binary_features_with64b_sparse_key: Some(BTreeMap::new()), - sparse_continuous_features: Some(BTreeMap::new()), - sparse_continuous_features_with16b_sparse_key: Some(BTreeMap::new()), - sparse_continuous_features_with32b_sparse_key: Some(BTreeMap::new()), - sparse_continuous_features_with64b_sparse_key: Some(BTreeMap::new()), - blob_features: Some(BTreeMap::new()), - tensors: Some(BTreeMap::new()), - sparse_tensors: Some(BTreeMap::new()), - } - } -} - -// -// TensorRecord -// - -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct TensorRecord { - pub tensors: Option>, - pub sparse_tensors: Option>, -} - -impl TensorRecord { - pub fn new(tensors: F1, sparse_tensors: F2) -> TensorRecord where F1: Into>>, F2: Into>> { - TensorRecord { - tensors: tensors.into(), - sparse_tensors: sparse_tensors.into(), - } - } -} - -impl TSerializable for TensorRecord { - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - i_prot.read_struct_begin()?; - let mut f_1: Option> = None; - let mut f_2: Option> = None; - loop { - let field_ident = i_prot.read_field_begin()?; - if field_ident.field_type == TType::Stop { - break; - } - let field_id = field_id(&field_ident)?; - match field_id { - 1 => { - let map_ident = i_prot.read_map_begin()?; - let mut val: BTreeMap = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_61 = i_prot.read_i64()?; - let map_val_62 = tensor::GeneralTensor::read_from_in_protocol(i_prot)?; - val.insert(map_key_61, map_val_62); - } - i_prot.read_map_end()?; - f_1 = Some(val); - }, - 2 => { - let map_ident = i_prot.read_map_begin()?; - let mut val: BTreeMap = BTreeMap::new(); - for _ in 0..map_ident.size { - let map_key_63 = i_prot.read_i64()?; - let map_val_64 = tensor::SparseTensor::read_from_in_protocol(i_prot)?; - val.insert(map_key_63, map_val_64); - } - i_prot.read_map_end()?; - f_2 = Some(val); - }, - _ => { - i_prot.skip(field_ident.field_type)?; - }, - }; - i_prot.read_field_end()?; - } - i_prot.read_struct_end()?; - let ret = TensorRecord { - tensors: f_1, - sparse_tensors: f_2, - }; - Ok(ret) - } - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let struct_ident = TStructIdentifier::new("TensorRecord"); - o_prot.write_struct_begin(&struct_ident)?; - if let Some(ref fld_var) = self.tensors { - o_prot.write_field_begin(&TFieldIdentifier::new("tensors", TType::Map, 1))?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::I64, TType::Struct, fld_var.len() as i32))?; - for (k, v) in fld_var { - o_prot.write_i64(*k)?; - v.write_to_out_protocol(o_prot)?; - } - o_prot.write_map_end()?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.sparse_tensors { - o_prot.write_field_begin(&TFieldIdentifier::new("sparseTensors", TType::Map, 2))?; - o_prot.write_map_begin(&TMapIdentifier::new(TType::I64, TType::Struct, fld_var.len() as i32))?; - for (k, v) in fld_var { - o_prot.write_i64(*k)?; - v.write_to_out_protocol(o_prot)?; - } - o_prot.write_map_end()?; - o_prot.write_field_end()? - } - o_prot.write_field_stop()?; - o_prot.write_struct_end() - } -} - -impl Default for TensorRecord { - fn default() -> Self { - TensorRecord{ - tensors: Some(BTreeMap::new()), - sparse_tensors: Some(BTreeMap::new()), - } - } -} - -// -// FeatureMetaInfo -// - -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct FeatureMetaInfo { - pub feature_id: Option, - pub full_feature_name: Option, - pub feature_type: Option, -} - -impl FeatureMetaInfo { - pub fn new(feature_id: F1, full_feature_name: F2, feature_type: F3) -> FeatureMetaInfo where F1: Into>, F2: Into>, F3: Into> { - FeatureMetaInfo { - feature_id: feature_id.into(), - full_feature_name: full_feature_name.into(), - feature_type: feature_type.into(), - } - } -} - -impl TSerializable for FeatureMetaInfo { - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - i_prot.read_struct_begin()?; - let mut f_1: Option = None; - let mut f_2: Option = None; - let mut f_3: Option = None; - loop { - let field_ident = i_prot.read_field_begin()?; - if field_ident.field_type == TType::Stop { - break; - } - let field_id = field_id(&field_ident)?; - match field_id { - 1 => { - let val = i_prot.read_i64()?; - f_1 = Some(val); - }, - 2 => { - let val = i_prot.read_string()?; - f_2 = Some(val); - }, - 3 => { - let val = FeatureType::read_from_in_protocol(i_prot)?; - f_3 = Some(val); - }, - _ => { - i_prot.skip(field_ident.field_type)?; - }, - }; - i_prot.read_field_end()?; - } - i_prot.read_struct_end()?; - let ret = FeatureMetaInfo { - feature_id: f_1, - full_feature_name: f_2, - feature_type: f_3, - }; - Ok(ret) - } - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let struct_ident = TStructIdentifier::new("FeatureMetaInfo"); - o_prot.write_struct_begin(&struct_ident)?; - if let Some(fld_var) = self.feature_id { - o_prot.write_field_begin(&TFieldIdentifier::new("featureId", TType::I64, 1))?; - o_prot.write_i64(fld_var)?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.full_feature_name { - o_prot.write_field_begin(&TFieldIdentifier::new("fullFeatureName", TType::String, 2))?; - o_prot.write_string(fld_var)?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.feature_type { - o_prot.write_field_begin(&TFieldIdentifier::new("featureType", TType::I32, 3))?; - fld_var.write_to_out_protocol(o_prot)?; - o_prot.write_field_end()? - } - o_prot.write_field_stop()?; - o_prot.write_struct_end() - } -} - -impl Default for FeatureMetaInfo { - fn default() -> Self { - FeatureMetaInfo{ - feature_id: Some(0), - full_feature_name: Some("".to_owned()), - feature_type: None, - } - } -} - -// -// FeatureMetaInfoList -// - -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct FeatureMetaInfoList { - pub contents: Option>, -} - -impl FeatureMetaInfoList { - pub fn new(contents: F1) -> FeatureMetaInfoList where F1: Into>> { - FeatureMetaInfoList { - contents: contents.into(), - } - } -} - -impl TSerializable for FeatureMetaInfoList { - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - i_prot.read_struct_begin()?; - let mut f_1: Option> = None; - loop { - let field_ident = i_prot.read_field_begin()?; - if field_ident.field_type == TType::Stop { - break; - } - let field_id = field_id(&field_ident)?; - match field_id { - 1 => { - let list_ident = i_prot.read_list_begin()?; - let mut val: Vec = Vec::with_capacity(list_ident.size as usize); - for _ in 0..list_ident.size { - let list_elem_65 = FeatureMetaInfo::read_from_in_protocol(i_prot)?; - val.push(list_elem_65); - } - i_prot.read_list_end()?; - f_1 = Some(val); - }, - _ => { - i_prot.skip(field_ident.field_type)?; - }, - }; - i_prot.read_field_end()?; - } - i_prot.read_struct_end()?; - let ret = FeatureMetaInfoList { - contents: f_1, - }; - Ok(ret) - } - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let struct_ident = TStructIdentifier::new("FeatureMetaInfoList"); - o_prot.write_struct_begin(&struct_ident)?; - if let Some(ref fld_var) = self.contents { - o_prot.write_field_begin(&TFieldIdentifier::new("contents", TType::List, 1))?; - o_prot.write_list_begin(&TListIdentifier::new(TType::Struct, fld_var.len() as i32))?; - for e in fld_var { - e.write_to_out_protocol(o_prot)?; - } - o_prot.write_list_end()?; - o_prot.write_field_end()? - } - o_prot.write_field_stop()?; - o_prot.write_struct_end() - } -} - -impl Default for FeatureMetaInfoList { - fn default() -> Self { - FeatureMetaInfoList{ - contents: Some(Vec::new()), - } - } -} - diff --git a/navi/thrift_bpr_adapter/thrift/src/decoder.docx b/navi/thrift_bpr_adapter/thrift/src/decoder.docx new file mode 100644 index 000000000..20ba0722c Binary files /dev/null and b/navi/thrift_bpr_adapter/thrift/src/decoder.docx differ diff --git a/navi/thrift_bpr_adapter/thrift/src/decoder.rs b/navi/thrift_bpr_adapter/thrift/src/decoder.rs deleted file mode 100644 index 835d7d5d7..000000000 --- a/navi/thrift_bpr_adapter/thrift/src/decoder.rs +++ /dev/null @@ -1,78 +0,0 @@ - -// A feature value can be one of these -enum FeatureVal { - Empty, - U8Vector(Vec), - FloatVector(Vec), -} - -// A Feture has a name and a value -// The name for now is 'id' of type string -// Eventually this needs to be flexible - example to accomodate feature-id -struct Feature { - id: String, - val: FeatureVal, -} - -impl Feature { - fn new() -> Feature { - Feature { - id: String::new(), - val: FeatureVal::Empty - } - } -} - -// A single inference record will have multiple features -struct Record { - fields: Vec, -} - -impl Record { - fn new() -> Record { - Record { fields: vec![] } - } -} - -// This is the main API used by external components -// Given a serialized input, decode it into Records -fn decode(input: Vec) -> Vec { - // For helping define the interface - vec![get_random_record(), get_random_record()] -} - -// Used for testing the API, will be eventually removed -fn get_random_record() -> Record { - let mut record: Record = Record::new(); - - let f1: Feature = Feature { - id: String::from("continuous_features"), - val: FeatureVal::FloatVector(vec![1.0f32; 2134]), - }; - - record.fields.push(f1); - - let f2: Feature = Feature { - id: String::from("user_embedding"), - val: FeatureVal::FloatVector(vec![2.0f32; 200]), - }; - - record.fields.push(f2); - - let f3: Feature = Feature { - id: String::from("author_embedding"), - val: FeatureVal::FloatVector(vec![3.0f32; 200]), - }; - - record.fields.push(f3); - - let f4: Feature = Feature { - id: String::from("binary_features"), - val: FeatureVal::U8Vector(vec![4u8; 43]), - }; - - record.fields.push(f4); - - record -} - diff --git a/navi/thrift_bpr_adapter/thrift/src/lib.docx b/navi/thrift_bpr_adapter/thrift/src/lib.docx new file mode 100644 index 000000000..77508c1b4 Binary files /dev/null and b/navi/thrift_bpr_adapter/thrift/src/lib.docx differ diff --git a/navi/thrift_bpr_adapter/thrift/src/lib.rs b/navi/thrift_bpr_adapter/thrift/src/lib.rs deleted file mode 100644 index bd549b494..000000000 --- a/navi/thrift_bpr_adapter/thrift/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub mod prediction_service; -pub mod data; -pub mod tensor; - diff --git a/navi/thrift_bpr_adapter/thrift/src/main.docx b/navi/thrift_bpr_adapter/thrift/src/main.docx new file mode 100644 index 000000000..030a7df29 Binary files /dev/null and b/navi/thrift_bpr_adapter/thrift/src/main.docx differ diff --git a/navi/thrift_bpr_adapter/thrift/src/main.rs b/navi/thrift_bpr_adapter/thrift/src/main.rs deleted file mode 100644 index 45d3c70e1..000000000 --- a/navi/thrift_bpr_adapter/thrift/src/main.rs +++ /dev/null @@ -1,81 +0,0 @@ -use std::collections::BTreeSet; -use std::collections::BTreeMap; - -use bpr_thrift::data::DataRecord; -use bpr_thrift::prediction_service::BatchPredictionRequest; -use thrift::OrderedFloat; - -use thrift::protocol::TBinaryInputProtocol; -use thrift::protocol::TSerializable; -use thrift::transport::TBufferChannel; -use thrift::Result; - -fn main() { - let data_path = "/tmp/current/timelines/output-1"; - let bin_data: Vec = std::fs::read(data_path).expect("Could not read file!"); - - println!("Length : {}", bin_data.len()); - - let mut bc = TBufferChannel::with_capacity(bin_data.len(), 0); - - bc.set_readable_bytes(&bin_data); - - let mut protocol = TBinaryInputProtocol::new(bc, true); - - let result: Result = - BatchPredictionRequest::read_from_in_protocol(&mut protocol); - - match result { - Ok(bpr) => logBP(bpr), - Err(err) => println!("Error {}", err), - } -} - -fn logBP(bpr: BatchPredictionRequest) { - println!("-------[OUTPUT]---------------"); - println!("data {:?}", bpr); - println!("------------------------------"); - - /* - let common = bpr.common_features; - let recs = bpr.individual_features_list; - - println!("--------[Len : {}]------------------", recs.len()); - - println!("-------[COMMON]---------------"); - match common { - Some(dr) => logDR(dr), - None => println!("None"), - } - println!("------------------------------"); - for rec in recs { - logDR(rec); - } - println!("------------------------------"); - */ -} - -fn logDR(dr: DataRecord) { - println!("--------[DR]------------------"); - - match dr.binary_features { - Some(bf) => logBin(bf), - _ => (), - } - - match dr.continuous_features { - Some(cf) => logCF(cf), - _ => (), - } - println!("------------------------------"); -} - -fn logBin(bin: BTreeSet) { - println!("B: {:?}", bin) -} - -fn logCF(cf: BTreeMap>) { - for (id, fs) in cf { - println!("C: {} -> [{}]", id, fs); - } -} diff --git a/navi/thrift_bpr_adapter/thrift/src/prediction_service.docx b/navi/thrift_bpr_adapter/thrift/src/prediction_service.docx new file mode 100644 index 000000000..904029d90 Binary files /dev/null and b/navi/thrift_bpr_adapter/thrift/src/prediction_service.docx differ diff --git a/navi/thrift_bpr_adapter/thrift/src/prediction_service.rs b/navi/thrift_bpr_adapter/thrift/src/prediction_service.rs deleted file mode 100644 index 8e2f71d3b..000000000 --- a/navi/thrift_bpr_adapter/thrift/src/prediction_service.rs +++ /dev/null @@ -1,1067 +0,0 @@ -// Autogenerated by Thrift Compiler (0.17.0) -// DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING - -#![allow(unused_imports)] -#![allow(unused_extern_crates)] -#![allow(clippy::too_many_arguments, clippy::type_complexity, clippy::vec_box)] -#![cfg_attr(rustfmt, rustfmt_skip)] - -use std::cell::RefCell; -use std::collections::{BTreeMap, BTreeSet}; -use std::convert::{From, TryFrom}; -use std::default::Default; -use std::error::Error; -use std::fmt; -use std::fmt::{Display, Formatter}; -use std::rc::Rc; - -use thrift::OrderedFloat; -use thrift::{ApplicationError, ApplicationErrorKind, ProtocolError, ProtocolErrorKind, TThriftClient}; -use thrift::protocol::{TFieldIdentifier, TListIdentifier, TMapIdentifier, TMessageIdentifier, TMessageType, TInputProtocol, TOutputProtocol, TSerializable, TSetIdentifier, TStructIdentifier, TType}; -use thrift::protocol::field_id; -use thrift::protocol::verify_expected_message_type; -use thrift::protocol::verify_expected_sequence_number; -use thrift::protocol::verify_expected_service_call; -use thrift::protocol::verify_required_field_exists; -use thrift::server::TProcessor; - -use crate::data; - -// -// PredictionServiceException -// - -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct PredictionServiceException { - pub description: Option, -} - -impl PredictionServiceException { - pub fn new(description: F1) -> PredictionServiceException where F1: Into> { - PredictionServiceException { - description: description.into(), - } - } -} - -impl TSerializable for PredictionServiceException { - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - i_prot.read_struct_begin()?; - let mut f_1: Option = Some("".to_owned()); - loop { - let field_ident = i_prot.read_field_begin()?; - if field_ident.field_type == TType::Stop { - break; - } - let field_id = field_id(&field_ident)?; - match field_id { - 1 => { - let val = i_prot.read_string()?; - f_1 = Some(val); - }, - _ => { - i_prot.skip(field_ident.field_type)?; - }, - }; - i_prot.read_field_end()?; - } - i_prot.read_struct_end()?; - let ret = PredictionServiceException { - description: f_1, - }; - Ok(ret) - } - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let struct_ident = TStructIdentifier::new("PredictionServiceException"); - o_prot.write_struct_begin(&struct_ident)?; - if let Some(ref fld_var) = self.description { - o_prot.write_field_begin(&TFieldIdentifier::new("description", TType::String, 1))?; - o_prot.write_string(fld_var)?; - o_prot.write_field_end()? - } - o_prot.write_field_stop()?; - o_prot.write_struct_end() - } -} - -impl Default for PredictionServiceException { - fn default() -> Self { - PredictionServiceException{ - description: Some("".to_owned()), - } - } -} - -impl Error for PredictionServiceException {} - -impl From for thrift::Error { - fn from(e: PredictionServiceException) -> Self { - thrift::Error::User(Box::new(e)) - } -} - -impl Display for PredictionServiceException { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - write!(f, "remote service threw PredictionServiceException") - } -} - -// -// PredictionRequest -// - -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct PredictionRequest { - pub features: data::DataRecord, -} - -impl PredictionRequest { - pub fn new(features: data::DataRecord) -> PredictionRequest { - PredictionRequest { - features, - } - } -} - -impl TSerializable for PredictionRequest { - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - i_prot.read_struct_begin()?; - let mut f_1: Option = None; - loop { - let field_ident = i_prot.read_field_begin()?; - if field_ident.field_type == TType::Stop { - break; - } - let field_id = field_id(&field_ident)?; - match field_id { - 1 => { - let val = data::DataRecord::read_from_in_protocol(i_prot)?; - f_1 = Some(val); - }, - _ => { - i_prot.skip(field_ident.field_type)?; - }, - }; - i_prot.read_field_end()?; - } - i_prot.read_struct_end()?; - verify_required_field_exists("PredictionRequest.features", &f_1)?; - let ret = PredictionRequest { - features: f_1.expect("auto-generated code should have checked for presence of required fields"), - }; - Ok(ret) - } - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let struct_ident = TStructIdentifier::new("PredictionRequest"); - o_prot.write_struct_begin(&struct_ident)?; - o_prot.write_field_begin(&TFieldIdentifier::new("features", TType::Struct, 1))?; - self.features.write_to_out_protocol(o_prot)?; - o_prot.write_field_end()?; - o_prot.write_field_stop()?; - o_prot.write_struct_end() - } -} - -// -// PredictionResponse -// - -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct PredictionResponse { - pub prediction: data::DataRecord, -} - -impl PredictionResponse { - pub fn new(prediction: data::DataRecord) -> PredictionResponse { - PredictionResponse { - prediction, - } - } -} - -impl TSerializable for PredictionResponse { - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - i_prot.read_struct_begin()?; - let mut f_1: Option = None; - loop { - let field_ident = i_prot.read_field_begin()?; - if field_ident.field_type == TType::Stop { - break; - } - let field_id = field_id(&field_ident)?; - match field_id { - 1 => { - let val = data::DataRecord::read_from_in_protocol(i_prot)?; - f_1 = Some(val); - }, - _ => { - i_prot.skip(field_ident.field_type)?; - }, - }; - i_prot.read_field_end()?; - } - i_prot.read_struct_end()?; - verify_required_field_exists("PredictionResponse.prediction", &f_1)?; - let ret = PredictionResponse { - prediction: f_1.expect("auto-generated code should have checked for presence of required fields"), - }; - Ok(ret) - } - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let struct_ident = TStructIdentifier::new("PredictionResponse"); - o_prot.write_struct_begin(&struct_ident)?; - o_prot.write_field_begin(&TFieldIdentifier::new("prediction", TType::Struct, 1))?; - self.prediction.write_to_out_protocol(o_prot)?; - o_prot.write_field_end()?; - o_prot.write_field_stop()?; - o_prot.write_struct_end() - } -} - -// -// BatchPredictionRequest -// - -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct BatchPredictionRequest { - pub individual_features_list: Vec, - pub common_features: Option, -} - -impl BatchPredictionRequest { - pub fn new(individual_features_list: Vec, common_features: F2) -> BatchPredictionRequest where F2: Into> { - BatchPredictionRequest { - individual_features_list, - common_features: common_features.into(), - } - } -} - -impl TSerializable for BatchPredictionRequest { - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - i_prot.read_struct_begin()?; - let mut f_1: Option> = None; - let mut f_2: Option = None; - loop { - let field_ident = i_prot.read_field_begin()?; - if field_ident.field_type == TType::Stop { - break; - } - let field_id = field_id(&field_ident)?; - match field_id { - 1 => { - let list_ident = i_prot.read_list_begin()?; - let mut val: Vec = Vec::with_capacity(list_ident.size as usize); - for _ in 0..list_ident.size { - let list_elem_0 = data::DataRecord::read_from_in_protocol(i_prot)?; - val.push(list_elem_0); - } - i_prot.read_list_end()?; - f_1 = Some(val); - }, - 2 => { - let val = data::DataRecord::read_from_in_protocol(i_prot)?; - f_2 = Some(val); - }, - _ => { - i_prot.skip(field_ident.field_type)?; - }, - }; - i_prot.read_field_end()?; - } - i_prot.read_struct_end()?; - verify_required_field_exists("BatchPredictionRequest.individual_features_list", &f_1)?; - let ret = BatchPredictionRequest { - individual_features_list: f_1.expect("auto-generated code should have checked for presence of required fields"), - common_features: f_2, - }; - Ok(ret) - } - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let struct_ident = TStructIdentifier::new("BatchPredictionRequest"); - o_prot.write_struct_begin(&struct_ident)?; - o_prot.write_field_begin(&TFieldIdentifier::new("individualFeaturesList", TType::List, 1))?; - o_prot.write_list_begin(&TListIdentifier::new(TType::Struct, self.individual_features_list.len() as i32))?; - for e in &self.individual_features_list { - e.write_to_out_protocol(o_prot)?; - } - o_prot.write_list_end()?; - o_prot.write_field_end()?; - if let Some(ref fld_var) = self.common_features { - o_prot.write_field_begin(&TFieldIdentifier::new("commonFeatures", TType::Struct, 2))?; - fld_var.write_to_out_protocol(o_prot)?; - o_prot.write_field_end()? - } - o_prot.write_field_stop()?; - o_prot.write_struct_end() - } -} - -// -// BatchPredictionResponse -// - -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct BatchPredictionResponse { - pub predictions: Vec, -} - -impl BatchPredictionResponse { - pub fn new(predictions: Vec) -> BatchPredictionResponse { - BatchPredictionResponse { - predictions, - } - } -} - -impl TSerializable for BatchPredictionResponse { - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - i_prot.read_struct_begin()?; - let mut f_1: Option> = None; - loop { - let field_ident = i_prot.read_field_begin()?; - if field_ident.field_type == TType::Stop { - break; - } - let field_id = field_id(&field_ident)?; - match field_id { - 1 => { - let list_ident = i_prot.read_list_begin()?; - let mut val: Vec = Vec::with_capacity(list_ident.size as usize); - for _ in 0..list_ident.size { - let list_elem_1 = data::DataRecord::read_from_in_protocol(i_prot)?; - val.push(list_elem_1); - } - i_prot.read_list_end()?; - f_1 = Some(val); - }, - _ => { - i_prot.skip(field_ident.field_type)?; - }, - }; - i_prot.read_field_end()?; - } - i_prot.read_struct_end()?; - verify_required_field_exists("BatchPredictionResponse.predictions", &f_1)?; - let ret = BatchPredictionResponse { - predictions: f_1.expect("auto-generated code should have checked for presence of required fields"), - }; - Ok(ret) - } - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let struct_ident = TStructIdentifier::new("BatchPredictionResponse"); - o_prot.write_struct_begin(&struct_ident)?; - o_prot.write_field_begin(&TFieldIdentifier::new("predictions", TType::List, 1))?; - o_prot.write_list_begin(&TListIdentifier::new(TType::Struct, self.predictions.len() as i32))?; - for e in &self.predictions { - e.write_to_out_protocol(o_prot)?; - } - o_prot.write_list_end()?; - o_prot.write_field_end()?; - o_prot.write_field_stop()?; - o_prot.write_struct_end() - } -} - -// -// DataRecordPair -// - -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct DataRecordPair { - pub first: Option, - pub second: Option, -} - -impl DataRecordPair { - pub fn new(first: F1, second: F2) -> DataRecordPair where F1: Into>, F2: Into> { - DataRecordPair { - first: first.into(), - second: second.into(), - } - } -} - -impl TSerializable for DataRecordPair { - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - i_prot.read_struct_begin()?; - let mut f_1: Option = None; - let mut f_2: Option = None; - loop { - let field_ident = i_prot.read_field_begin()?; - if field_ident.field_type == TType::Stop { - break; - } - let field_id = field_id(&field_ident)?; - match field_id { - 1 => { - let val = data::DataRecord::read_from_in_protocol(i_prot)?; - f_1 = Some(val); - }, - 2 => { - let val = data::DataRecord::read_from_in_protocol(i_prot)?; - f_2 = Some(val); - }, - _ => { - i_prot.skip(field_ident.field_type)?; - }, - }; - i_prot.read_field_end()?; - } - i_prot.read_struct_end()?; - let ret = DataRecordPair { - first: f_1, - second: f_2, - }; - Ok(ret) - } - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let struct_ident = TStructIdentifier::new("DataRecordPair"); - o_prot.write_struct_begin(&struct_ident)?; - if let Some(ref fld_var) = self.first { - o_prot.write_field_begin(&TFieldIdentifier::new("first", TType::Struct, 1))?; - fld_var.write_to_out_protocol(o_prot)?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.second { - o_prot.write_field_begin(&TFieldIdentifier::new("second", TType::Struct, 2))?; - fld_var.write_to_out_protocol(o_prot)?; - o_prot.write_field_end()? - } - o_prot.write_field_stop()?; - o_prot.write_struct_end() - } -} - -impl Default for DataRecordPair { - fn default() -> Self { - DataRecordPair{ - first: None, - second: None, - } - } -} - -// -// PredictionTrainingExample -// - -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct PredictionTrainingExample { - pub features: Option, - pub features_for_pairwise_learning: Option, - pub compact_features: Option, - pub compressed_data_record: Option>, -} - -impl PredictionTrainingExample { - pub fn new(features: F1, features_for_pairwise_learning: F2, compact_features: F3, compressed_data_record: F4) -> PredictionTrainingExample where F1: Into>, F2: Into>, F3: Into>, F4: Into>> { - PredictionTrainingExample { - features: features.into(), - features_for_pairwise_learning: features_for_pairwise_learning.into(), - compact_features: compact_features.into(), - compressed_data_record: compressed_data_record.into(), - } - } -} - -impl TSerializable for PredictionTrainingExample { - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - i_prot.read_struct_begin()?; - let mut f_1: Option = None; - let mut f_2: Option = None; - let mut f_3: Option = None; - let mut f_4: Option> = None; - loop { - let field_ident = i_prot.read_field_begin()?; - if field_ident.field_type == TType::Stop { - break; - } - let field_id = field_id(&field_ident)?; - match field_id { - 1 => { - let val = data::DataRecord::read_from_in_protocol(i_prot)?; - f_1 = Some(val); - }, - 2 => { - let val = DataRecordPair::read_from_in_protocol(i_prot)?; - f_2 = Some(val); - }, - 3 => { - let val = data::CompactDataRecord::read_from_in_protocol(i_prot)?; - f_3 = Some(val); - }, - 4 => { - let val = i_prot.read_bytes()?; - f_4 = Some(val); - }, - _ => { - i_prot.skip(field_ident.field_type)?; - }, - }; - i_prot.read_field_end()?; - } - i_prot.read_struct_end()?; - let ret = PredictionTrainingExample { - features: f_1, - features_for_pairwise_learning: f_2, - compact_features: f_3, - compressed_data_record: f_4, - }; - Ok(ret) - } - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let struct_ident = TStructIdentifier::new("PredictionTrainingExample"); - o_prot.write_struct_begin(&struct_ident)?; - if let Some(ref fld_var) = self.features { - o_prot.write_field_begin(&TFieldIdentifier::new("features", TType::Struct, 1))?; - fld_var.write_to_out_protocol(o_prot)?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.features_for_pairwise_learning { - o_prot.write_field_begin(&TFieldIdentifier::new("featuresForPairwiseLearning", TType::Struct, 2))?; - fld_var.write_to_out_protocol(o_prot)?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.compact_features { - o_prot.write_field_begin(&TFieldIdentifier::new("compactFeatures", TType::Struct, 3))?; - fld_var.write_to_out_protocol(o_prot)?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.compressed_data_record { - o_prot.write_field_begin(&TFieldIdentifier::new("compressedDataRecord", TType::String, 4))?; - o_prot.write_bytes(fld_var)?; - o_prot.write_field_end()? - } - o_prot.write_field_stop()?; - o_prot.write_struct_end() - } -} - -impl Default for PredictionTrainingExample { - fn default() -> Self { - PredictionTrainingExample{ - features: None, - features_for_pairwise_learning: None, - compact_features: None, - compressed_data_record: Some(Vec::new()), - } - } -} - -// -// PredictionService service client -// - -pub trait TPredictionServiceSyncClient { - fn get_prediction(&mut self, request: PredictionRequest) -> thrift::Result; - fn get_batch_prediction(&mut self, batch_request: BatchPredictionRequest) -> thrift::Result; -} - -pub trait TPredictionServiceSyncClientMarker {} - -pub struct PredictionServiceSyncClient where IP: TInputProtocol, OP: TOutputProtocol { - _i_prot: IP, - _o_prot: OP, - _sequence_number: i32, -} - -impl PredictionServiceSyncClient where IP: TInputProtocol, OP: TOutputProtocol { - pub fn new(input_protocol: IP, output_protocol: OP) -> PredictionServiceSyncClient { - PredictionServiceSyncClient { _i_prot: input_protocol, _o_prot: output_protocol, _sequence_number: 0 } - } -} - -impl TThriftClient for PredictionServiceSyncClient where IP: TInputProtocol, OP: TOutputProtocol { - fn i_prot_mut(&mut self) -> &mut dyn TInputProtocol { &mut self._i_prot } - fn o_prot_mut(&mut self) -> &mut dyn TOutputProtocol { &mut self._o_prot } - fn sequence_number(&self) -> i32 { self._sequence_number } - fn increment_sequence_number(&mut self) -> i32 { self._sequence_number += 1; self._sequence_number } -} - -impl TPredictionServiceSyncClientMarker for PredictionServiceSyncClient where IP: TInputProtocol, OP: TOutputProtocol {} - -impl TPredictionServiceSyncClient for C { - fn get_prediction(&mut self, request: PredictionRequest) -> thrift::Result { - ( - { - self.increment_sequence_number(); - let message_ident = TMessageIdentifier::new("getPrediction", TMessageType::Call, self.sequence_number()); - let call_args = PredictionServiceGetPredictionArgs { request }; - self.o_prot_mut().write_message_begin(&message_ident)?; - call_args.write_to_out_protocol(self.o_prot_mut())?; - self.o_prot_mut().write_message_end()?; - self.o_prot_mut().flush() - } - )?; - { - let message_ident = self.i_prot_mut().read_message_begin()?; - verify_expected_sequence_number(self.sequence_number(), message_ident.sequence_number)?; - verify_expected_service_call("getPrediction", &message_ident.name)?; - if message_ident.message_type == TMessageType::Exception { - let remote_error = thrift::Error::read_application_error_from_in_protocol(self.i_prot_mut())?; - self.i_prot_mut().read_message_end()?; - return Err(thrift::Error::Application(remote_error)) - } - verify_expected_message_type(TMessageType::Reply, message_ident.message_type)?; - let result = PredictionServiceGetPredictionResult::read_from_in_protocol(self.i_prot_mut())?; - self.i_prot_mut().read_message_end()?; - result.ok_or() - } - } - fn get_batch_prediction(&mut self, batch_request: BatchPredictionRequest) -> thrift::Result { - ( - { - self.increment_sequence_number(); - let message_ident = TMessageIdentifier::new("getBatchPrediction", TMessageType::Call, self.sequence_number()); - let call_args = PredictionServiceGetBatchPredictionArgs { batch_request }; - self.o_prot_mut().write_message_begin(&message_ident)?; - call_args.write_to_out_protocol(self.o_prot_mut())?; - self.o_prot_mut().write_message_end()?; - self.o_prot_mut().flush() - } - )?; - { - let message_ident = self.i_prot_mut().read_message_begin()?; - verify_expected_sequence_number(self.sequence_number(), message_ident.sequence_number)?; - verify_expected_service_call("getBatchPrediction", &message_ident.name)?; - if message_ident.message_type == TMessageType::Exception { - let remote_error = thrift::Error::read_application_error_from_in_protocol(self.i_prot_mut())?; - self.i_prot_mut().read_message_end()?; - return Err(thrift::Error::Application(remote_error)) - } - verify_expected_message_type(TMessageType::Reply, message_ident.message_type)?; - let result = PredictionServiceGetBatchPredictionResult::read_from_in_protocol(self.i_prot_mut())?; - self.i_prot_mut().read_message_end()?; - result.ok_or() - } - } -} - -// -// PredictionService service processor -// - -pub trait PredictionServiceSyncHandler { - fn handle_get_prediction(&self, request: PredictionRequest) -> thrift::Result; - fn handle_get_batch_prediction(&self, batch_request: BatchPredictionRequest) -> thrift::Result; -} - -pub struct PredictionServiceSyncProcessor { - handler: H, -} - -impl PredictionServiceSyncProcessor { - pub fn new(handler: H) -> PredictionServiceSyncProcessor { - PredictionServiceSyncProcessor { - handler, - } - } - fn process_get_prediction(&self, incoming_sequence_number: i32, i_prot: &mut dyn TInputProtocol, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - TPredictionServiceProcessFunctions::process_get_prediction(&self.handler, incoming_sequence_number, i_prot, o_prot) - } - fn process_get_batch_prediction(&self, incoming_sequence_number: i32, i_prot: &mut dyn TInputProtocol, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - TPredictionServiceProcessFunctions::process_get_batch_prediction(&self.handler, incoming_sequence_number, i_prot, o_prot) - } -} - -pub struct TPredictionServiceProcessFunctions; - -impl TPredictionServiceProcessFunctions { - pub fn process_get_prediction(handler: &H, incoming_sequence_number: i32, i_prot: &mut dyn TInputProtocol, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let args = PredictionServiceGetPredictionArgs::read_from_in_protocol(i_prot)?; - match handler.handle_get_prediction(args.request) { - Ok(handler_return) => { - let message_ident = TMessageIdentifier::new("getPrediction", TMessageType::Reply, incoming_sequence_number); - o_prot.write_message_begin(&message_ident)?; - let ret = PredictionServiceGetPredictionResult { result_value: Some(handler_return), prediction_service_exception: None }; - ret.write_to_out_protocol(o_prot)?; - o_prot.write_message_end()?; - o_prot.flush() - }, - Err(e) => { - match e { - thrift::Error::User(usr_err) => { - if usr_err.downcast_ref::().is_some() { - let err = usr_err.downcast::().expect("downcast already checked"); - let ret_err = PredictionServiceGetPredictionResult{ result_value: None, prediction_service_exception: Some(*err) }; - let message_ident = TMessageIdentifier::new("getPrediction", TMessageType::Reply, incoming_sequence_number); - o_prot.write_message_begin(&message_ident)?; - ret_err.write_to_out_protocol(o_prot)?; - o_prot.write_message_end()?; - o_prot.flush() - } else { - let ret_err = { - ApplicationError::new( - ApplicationErrorKind::Unknown, - usr_err.to_string() - ) - }; - let message_ident = TMessageIdentifier::new("getPrediction", TMessageType::Exception, incoming_sequence_number); - o_prot.write_message_begin(&message_ident)?; - thrift::Error::write_application_error_to_out_protocol(&ret_err, o_prot)?; - o_prot.write_message_end()?; - o_prot.flush() - } - }, - thrift::Error::Application(app_err) => { - let message_ident = TMessageIdentifier::new("getPrediction", TMessageType::Exception, incoming_sequence_number); - o_prot.write_message_begin(&message_ident)?; - thrift::Error::write_application_error_to_out_protocol(&app_err, o_prot)?; - o_prot.write_message_end()?; - o_prot.flush() - }, - _ => { - let ret_err = { - ApplicationError::new( - ApplicationErrorKind::Unknown, - e.to_string() - ) - }; - let message_ident = TMessageIdentifier::new("getPrediction", TMessageType::Exception, incoming_sequence_number); - o_prot.write_message_begin(&message_ident)?; - thrift::Error::write_application_error_to_out_protocol(&ret_err, o_prot)?; - o_prot.write_message_end()?; - o_prot.flush() - }, - } - }, - } - } - pub fn process_get_batch_prediction(handler: &H, incoming_sequence_number: i32, i_prot: &mut dyn TInputProtocol, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let args = PredictionServiceGetBatchPredictionArgs::read_from_in_protocol(i_prot)?; - match handler.handle_get_batch_prediction(args.batch_request) { - Ok(handler_return) => { - let message_ident = TMessageIdentifier::new("getBatchPrediction", TMessageType::Reply, incoming_sequence_number); - o_prot.write_message_begin(&message_ident)?; - let ret = PredictionServiceGetBatchPredictionResult { result_value: Some(handler_return), prediction_service_exception: None }; - ret.write_to_out_protocol(o_prot)?; - o_prot.write_message_end()?; - o_prot.flush() - }, - Err(e) => { - match e { - thrift::Error::User(usr_err) => { - if usr_err.downcast_ref::().is_some() { - let err = usr_err.downcast::().expect("downcast already checked"); - let ret_err = PredictionServiceGetBatchPredictionResult{ result_value: None, prediction_service_exception: Some(*err) }; - let message_ident = TMessageIdentifier::new("getBatchPrediction", TMessageType::Reply, incoming_sequence_number); - o_prot.write_message_begin(&message_ident)?; - ret_err.write_to_out_protocol(o_prot)?; - o_prot.write_message_end()?; - o_prot.flush() - } else { - let ret_err = { - ApplicationError::new( - ApplicationErrorKind::Unknown, - usr_err.to_string() - ) - }; - let message_ident = TMessageIdentifier::new("getBatchPrediction", TMessageType::Exception, incoming_sequence_number); - o_prot.write_message_begin(&message_ident)?; - thrift::Error::write_application_error_to_out_protocol(&ret_err, o_prot)?; - o_prot.write_message_end()?; - o_prot.flush() - } - }, - thrift::Error::Application(app_err) => { - let message_ident = TMessageIdentifier::new("getBatchPrediction", TMessageType::Exception, incoming_sequence_number); - o_prot.write_message_begin(&message_ident)?; - thrift::Error::write_application_error_to_out_protocol(&app_err, o_prot)?; - o_prot.write_message_end()?; - o_prot.flush() - }, - _ => { - let ret_err = { - ApplicationError::new( - ApplicationErrorKind::Unknown, - e.to_string() - ) - }; - let message_ident = TMessageIdentifier::new("getBatchPrediction", TMessageType::Exception, incoming_sequence_number); - o_prot.write_message_begin(&message_ident)?; - thrift::Error::write_application_error_to_out_protocol(&ret_err, o_prot)?; - o_prot.write_message_end()?; - o_prot.flush() - }, - } - }, - } - } -} - -impl TProcessor for PredictionServiceSyncProcessor { - fn process(&self, i_prot: &mut dyn TInputProtocol, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let message_ident = i_prot.read_message_begin()?; - let res = match &*message_ident.name { - "getPrediction" => { - self.process_get_prediction(message_ident.sequence_number, i_prot, o_prot) - }, - "getBatchPrediction" => { - self.process_get_batch_prediction(message_ident.sequence_number, i_prot, o_prot) - }, - method => { - Err( - thrift::Error::Application( - ApplicationError::new( - ApplicationErrorKind::UnknownMethod, - format!("unknown method {}", method) - ) - ) - ) - }, - }; - thrift::server::handle_process_result(&message_ident, res, o_prot) - } -} - -// -// PredictionServiceGetPredictionArgs -// - -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -struct PredictionServiceGetPredictionArgs { - request: PredictionRequest, -} - -impl PredictionServiceGetPredictionArgs { - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - i_prot.read_struct_begin()?; - let mut f_1: Option = None; - loop { - let field_ident = i_prot.read_field_begin()?; - if field_ident.field_type == TType::Stop { - break; - } - let field_id = field_id(&field_ident)?; - match field_id { - 1 => { - let val = PredictionRequest::read_from_in_protocol(i_prot)?; - f_1 = Some(val); - }, - _ => { - i_prot.skip(field_ident.field_type)?; - }, - }; - i_prot.read_field_end()?; - } - i_prot.read_struct_end()?; - verify_required_field_exists("PredictionServiceGetPredictionArgs.request", &f_1)?; - let ret = PredictionServiceGetPredictionArgs { - request: f_1.expect("auto-generated code should have checked for presence of required fields"), - }; - Ok(ret) - } - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let struct_ident = TStructIdentifier::new("getPrediction_args"); - o_prot.write_struct_begin(&struct_ident)?; - o_prot.write_field_begin(&TFieldIdentifier::new("request", TType::Struct, 1))?; - self.request.write_to_out_protocol(o_prot)?; - o_prot.write_field_end()?; - o_prot.write_field_stop()?; - o_prot.write_struct_end() - } -} - -// -// PredictionServiceGetPredictionResult -// - -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -struct PredictionServiceGetPredictionResult { - result_value: Option, - prediction_service_exception: Option, -} - -impl PredictionServiceGetPredictionResult { - fn ok_or(self) -> thrift::Result { - if self.prediction_service_exception.is_some() { - Err(thrift::Error::User(Box::new(self.prediction_service_exception.unwrap()))) - } else if self.result_value.is_some() { - Ok(self.result_value.unwrap()) - } else { - Err( - thrift::Error::Application( - ApplicationError::new( - ApplicationErrorKind::MissingResult, - "no result received for PredictionServiceGetPrediction" - ) - ) - ) - } - } - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - i_prot.read_struct_begin()?; - let mut f_0: Option = None; - let mut f_1: Option = None; - loop { - let field_ident = i_prot.read_field_begin()?; - if field_ident.field_type == TType::Stop { - break; - } - let field_id = field_id(&field_ident)?; - match field_id { - 0 => { - let val = PredictionResponse::read_from_in_protocol(i_prot)?; - f_0 = Some(val); - }, - 1 => { - let val = PredictionServiceException::read_from_in_protocol(i_prot)?; - f_1 = Some(val); - }, - _ => { - i_prot.skip(field_ident.field_type)?; - }, - }; - i_prot.read_field_end()?; - } - i_prot.read_struct_end()?; - let ret = PredictionServiceGetPredictionResult { - result_value: f_0, - prediction_service_exception: f_1, - }; - Ok(ret) - } - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let struct_ident = TStructIdentifier::new("PredictionServiceGetPredictionResult"); - o_prot.write_struct_begin(&struct_ident)?; - if let Some(ref fld_var) = self.result_value { - o_prot.write_field_begin(&TFieldIdentifier::new("result_value", TType::Struct, 0))?; - fld_var.write_to_out_protocol(o_prot)?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.prediction_service_exception { - o_prot.write_field_begin(&TFieldIdentifier::new("predictionServiceException", TType::Struct, 1))?; - fld_var.write_to_out_protocol(o_prot)?; - o_prot.write_field_end()? - } - o_prot.write_field_stop()?; - o_prot.write_struct_end() - } -} - -// -// PredictionServiceGetBatchPredictionArgs -// - -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -struct PredictionServiceGetBatchPredictionArgs { - batch_request: BatchPredictionRequest, -} - -impl PredictionServiceGetBatchPredictionArgs { - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - i_prot.read_struct_begin()?; - let mut f_1: Option = None; - loop { - let field_ident = i_prot.read_field_begin()?; - if field_ident.field_type == TType::Stop { - break; - } - let field_id = field_id(&field_ident)?; - match field_id { - 1 => { - let val = BatchPredictionRequest::read_from_in_protocol(i_prot)?; - f_1 = Some(val); - }, - _ => { - i_prot.skip(field_ident.field_type)?; - }, - }; - i_prot.read_field_end()?; - } - i_prot.read_struct_end()?; - verify_required_field_exists("PredictionServiceGetBatchPredictionArgs.batch_request", &f_1)?; - let ret = PredictionServiceGetBatchPredictionArgs { - batch_request: f_1.expect("auto-generated code should have checked for presence of required fields"), - }; - Ok(ret) - } - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let struct_ident = TStructIdentifier::new("getBatchPrediction_args"); - o_prot.write_struct_begin(&struct_ident)?; - o_prot.write_field_begin(&TFieldIdentifier::new("batchRequest", TType::Struct, 1))?; - self.batch_request.write_to_out_protocol(o_prot)?; - o_prot.write_field_end()?; - o_prot.write_field_stop()?; - o_prot.write_struct_end() - } -} - -// -// PredictionServiceGetBatchPredictionResult -// - -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -struct PredictionServiceGetBatchPredictionResult { - result_value: Option, - prediction_service_exception: Option, -} - -impl PredictionServiceGetBatchPredictionResult { - fn ok_or(self) -> thrift::Result { - if self.prediction_service_exception.is_some() { - Err(thrift::Error::User(Box::new(self.prediction_service_exception.unwrap()))) - } else if self.result_value.is_some() { - Ok(self.result_value.unwrap()) - } else { - Err( - thrift::Error::Application( - ApplicationError::new( - ApplicationErrorKind::MissingResult, - "no result received for PredictionServiceGetBatchPrediction" - ) - ) - ) - } - } - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - i_prot.read_struct_begin()?; - let mut f_0: Option = None; - let mut f_1: Option = None; - loop { - let field_ident = i_prot.read_field_begin()?; - if field_ident.field_type == TType::Stop { - break; - } - let field_id = field_id(&field_ident)?; - match field_id { - 0 => { - let val = BatchPredictionResponse::read_from_in_protocol(i_prot)?; - f_0 = Some(val); - }, - 1 => { - let val = PredictionServiceException::read_from_in_protocol(i_prot)?; - f_1 = Some(val); - }, - _ => { - i_prot.skip(field_ident.field_type)?; - }, - }; - i_prot.read_field_end()?; - } - i_prot.read_struct_end()?; - let ret = PredictionServiceGetBatchPredictionResult { - result_value: f_0, - prediction_service_exception: f_1, - }; - Ok(ret) - } - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let struct_ident = TStructIdentifier::new("PredictionServiceGetBatchPredictionResult"); - o_prot.write_struct_begin(&struct_ident)?; - if let Some(ref fld_var) = self.result_value { - o_prot.write_field_begin(&TFieldIdentifier::new("result_value", TType::Struct, 0))?; - fld_var.write_to_out_protocol(o_prot)?; - o_prot.write_field_end()? - } - if let Some(ref fld_var) = self.prediction_service_exception { - o_prot.write_field_begin(&TFieldIdentifier::new("predictionServiceException", TType::Struct, 1))?; - fld_var.write_to_out_protocol(o_prot)?; - o_prot.write_field_end()? - } - o_prot.write_field_stop()?; - o_prot.write_struct_end() - } -} - diff --git a/navi/thrift_bpr_adapter/thrift/src/tensor.docx b/navi/thrift_bpr_adapter/thrift/src/tensor.docx new file mode 100644 index 000000000..25b419732 Binary files /dev/null and b/navi/thrift_bpr_adapter/thrift/src/tensor.docx differ diff --git a/navi/thrift_bpr_adapter/thrift/src/tensor.rs b/navi/thrift_bpr_adapter/thrift/src/tensor.rs deleted file mode 100644 index dd2a3ec89..000000000 --- a/navi/thrift_bpr_adapter/thrift/src/tensor.rs +++ /dev/null @@ -1,1146 +0,0 @@ -// Autogenerated by Thrift Compiler (0.17.0) -// DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING - -#![allow(unused_imports)] -#![allow(unused_extern_crates)] -#![allow(clippy::too_many_arguments, clippy::type_complexity, clippy::vec_box)] -#![cfg_attr(rustfmt, rustfmt_skip)] - -use std::cell::RefCell; -use std::collections::{BTreeMap, BTreeSet}; -use std::convert::{From, TryFrom}; -use std::default::Default; -use std::error::Error; -use std::fmt; -use std::fmt::{Display, Formatter}; -use std::rc::Rc; - -use thrift::OrderedFloat; -use thrift::{ApplicationError, ApplicationErrorKind, ProtocolError, ProtocolErrorKind, TThriftClient}; -use thrift::protocol::{TFieldIdentifier, TListIdentifier, TMapIdentifier, TMessageIdentifier, TMessageType, TInputProtocol, TOutputProtocol, TSerializable, TSetIdentifier, TStructIdentifier, TType}; -use thrift::protocol::field_id; -use thrift::protocol::verify_expected_message_type; -use thrift::protocol::verify_expected_sequence_number; -use thrift::protocol::verify_expected_service_call; -use thrift::protocol::verify_required_field_exists; -use thrift::server::TProcessor; - -#[derive(Copy, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct DataType(pub i32); - -impl DataType { - pub const FLOAT: DataType = DataType(0); - pub const DOUBLE: DataType = DataType(1); - pub const INT32: DataType = DataType(2); - pub const INT64: DataType = DataType(3); - pub const UINT8: DataType = DataType(4); - pub const STRING: DataType = DataType(5); - pub const BYTE: DataType = DataType(6); - pub const BOOL: DataType = DataType(7); - pub const RESERVED_1: DataType = DataType(8); - pub const RESERVED_2: DataType = DataType(9); - pub const RESERVED_3: DataType = DataType(10); - pub const ENUM_VALUES: &'static [Self] = &[ - Self::FLOAT, - Self::DOUBLE, - Self::INT32, - Self::INT64, - Self::UINT8, - Self::STRING, - Self::BYTE, - Self::BOOL, - Self::RESERVED_1, - Self::RESERVED_2, - Self::RESERVED_3, - ]; -} - -impl TSerializable for DataType { - #[allow(clippy::trivially_copy_pass_by_ref)] - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - o_prot.write_i32(self.0) - } - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - let enum_value = i_prot.read_i32()?; - Ok(DataType::from(enum_value)) - } -} - -impl From for DataType { - fn from(i: i32) -> Self { - match i { - 0 => DataType::FLOAT, - 1 => DataType::DOUBLE, - 2 => DataType::INT32, - 3 => DataType::INT64, - 4 => DataType::UINT8, - 5 => DataType::STRING, - 6 => DataType::BYTE, - 7 => DataType::BOOL, - 8 => DataType::RESERVED_1, - 9 => DataType::RESERVED_2, - 10 => DataType::RESERVED_3, - _ => DataType(i) - } - } -} - -impl From<&i32> for DataType { - fn from(i: &i32) -> Self { - DataType::from(*i) - } -} - -impl From for i32 { - fn from(e: DataType) -> i32 { - e.0 - } -} - -impl From<&DataType> for i32 { - fn from(e: &DataType) -> i32 { - e.0 - } -} - -// -// StringTensor -// - -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct StringTensor { - pub strings: Vec, - pub shape: Option>, -} - -impl StringTensor { - pub fn new(strings: Vec, shape: F2) -> StringTensor where F2: Into>> { - StringTensor { - strings, - shape: shape.into(), - } - } -} - -impl TSerializable for StringTensor { - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - i_prot.read_struct_begin()?; - let mut f_1: Option> = None; - let mut f_2: Option> = None; - loop { - let field_ident = i_prot.read_field_begin()?; - if field_ident.field_type == TType::Stop { - break; - } - let field_id = field_id(&field_ident)?; - match field_id { - 1 => { - let list_ident = i_prot.read_list_begin()?; - let mut val: Vec = Vec::with_capacity(list_ident.size as usize); - for _ in 0..list_ident.size { - let list_elem_0 = i_prot.read_string()?; - val.push(list_elem_0); - } - i_prot.read_list_end()?; - f_1 = Some(val); - }, - 2 => { - let list_ident = i_prot.read_list_begin()?; - let mut val: Vec = Vec::with_capacity(list_ident.size as usize); - for _ in 0..list_ident.size { - let list_elem_1 = i_prot.read_i64()?; - val.push(list_elem_1); - } - i_prot.read_list_end()?; - f_2 = Some(val); - }, - _ => { - i_prot.skip(field_ident.field_type)?; - }, - }; - i_prot.read_field_end()?; - } - i_prot.read_struct_end()?; - verify_required_field_exists("StringTensor.strings", &f_1)?; - let ret = StringTensor { - strings: f_1.expect("auto-generated code should have checked for presence of required fields"), - shape: f_2, - }; - Ok(ret) - } - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let struct_ident = TStructIdentifier::new("StringTensor"); - o_prot.write_struct_begin(&struct_ident)?; - o_prot.write_field_begin(&TFieldIdentifier::new("strings", TType::List, 1))?; - o_prot.write_list_begin(&TListIdentifier::new(TType::String, self.strings.len() as i32))?; - for e in &self.strings { - o_prot.write_string(e)?; - } - o_prot.write_list_end()?; - o_prot.write_field_end()?; - if let Some(ref fld_var) = self.shape { - o_prot.write_field_begin(&TFieldIdentifier::new("shape", TType::List, 2))?; - o_prot.write_list_begin(&TListIdentifier::new(TType::I64, fld_var.len() as i32))?; - for e in fld_var { - o_prot.write_i64(*e)?; - } - o_prot.write_list_end()?; - o_prot.write_field_end()? - } - o_prot.write_field_stop()?; - o_prot.write_struct_end() - } -} - -// -// Int32Tensor -// - -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct Int32Tensor { - pub ints: Vec, - pub shape: Option>, -} - -impl Int32Tensor { - pub fn new(ints: Vec, shape: F2) -> Int32Tensor where F2: Into>> { - Int32Tensor { - ints, - shape: shape.into(), - } - } -} - -impl TSerializable for Int32Tensor { - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - i_prot.read_struct_begin()?; - let mut f_1: Option> = None; - let mut f_2: Option> = None; - loop { - let field_ident = i_prot.read_field_begin()?; - if field_ident.field_type == TType::Stop { - break; - } - let field_id = field_id(&field_ident)?; - match field_id { - 1 => { - let list_ident = i_prot.read_list_begin()?; - let mut val: Vec = Vec::with_capacity(list_ident.size as usize); - for _ in 0..list_ident.size { - let list_elem_2 = i_prot.read_i32()?; - val.push(list_elem_2); - } - i_prot.read_list_end()?; - f_1 = Some(val); - }, - 2 => { - let list_ident = i_prot.read_list_begin()?; - let mut val: Vec = Vec::with_capacity(list_ident.size as usize); - for _ in 0..list_ident.size { - let list_elem_3 = i_prot.read_i64()?; - val.push(list_elem_3); - } - i_prot.read_list_end()?; - f_2 = Some(val); - }, - _ => { - i_prot.skip(field_ident.field_type)?; - }, - }; - i_prot.read_field_end()?; - } - i_prot.read_struct_end()?; - verify_required_field_exists("Int32Tensor.ints", &f_1)?; - let ret = Int32Tensor { - ints: f_1.expect("auto-generated code should have checked for presence of required fields"), - shape: f_2, - }; - Ok(ret) - } - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let struct_ident = TStructIdentifier::new("Int32Tensor"); - o_prot.write_struct_begin(&struct_ident)?; - o_prot.write_field_begin(&TFieldIdentifier::new("ints", TType::List, 1))?; - o_prot.write_list_begin(&TListIdentifier::new(TType::I32, self.ints.len() as i32))?; - for e in &self.ints { - o_prot.write_i32(*e)?; - } - o_prot.write_list_end()?; - o_prot.write_field_end()?; - if let Some(ref fld_var) = self.shape { - o_prot.write_field_begin(&TFieldIdentifier::new("shape", TType::List, 2))?; - o_prot.write_list_begin(&TListIdentifier::new(TType::I64, fld_var.len() as i32))?; - for e in fld_var { - o_prot.write_i64(*e)?; - } - o_prot.write_list_end()?; - o_prot.write_field_end()? - } - o_prot.write_field_stop()?; - o_prot.write_struct_end() - } -} - -// -// Int64Tensor -// - -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct Int64Tensor { - pub longs: Vec, - pub shape: Option>, -} - -impl Int64Tensor { - pub fn new(longs: Vec, shape: F2) -> Int64Tensor where F2: Into>> { - Int64Tensor { - longs, - shape: shape.into(), - } - } -} - -impl TSerializable for Int64Tensor { - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - i_prot.read_struct_begin()?; - let mut f_1: Option> = None; - let mut f_2: Option> = None; - loop { - let field_ident = i_prot.read_field_begin()?; - if field_ident.field_type == TType::Stop { - break; - } - let field_id = field_id(&field_ident)?; - match field_id { - 1 => { - let list_ident = i_prot.read_list_begin()?; - let mut val: Vec = Vec::with_capacity(list_ident.size as usize); - for _ in 0..list_ident.size { - let list_elem_4 = i_prot.read_i64()?; - val.push(list_elem_4); - } - i_prot.read_list_end()?; - f_1 = Some(val); - }, - 2 => { - let list_ident = i_prot.read_list_begin()?; - let mut val: Vec = Vec::with_capacity(list_ident.size as usize); - for _ in 0..list_ident.size { - let list_elem_5 = i_prot.read_i64()?; - val.push(list_elem_5); - } - i_prot.read_list_end()?; - f_2 = Some(val); - }, - _ => { - i_prot.skip(field_ident.field_type)?; - }, - }; - i_prot.read_field_end()?; - } - i_prot.read_struct_end()?; - verify_required_field_exists("Int64Tensor.longs", &f_1)?; - let ret = Int64Tensor { - longs: f_1.expect("auto-generated code should have checked for presence of required fields"), - shape: f_2, - }; - Ok(ret) - } - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let struct_ident = TStructIdentifier::new("Int64Tensor"); - o_prot.write_struct_begin(&struct_ident)?; - o_prot.write_field_begin(&TFieldIdentifier::new("longs", TType::List, 1))?; - o_prot.write_list_begin(&TListIdentifier::new(TType::I64, self.longs.len() as i32))?; - for e in &self.longs { - o_prot.write_i64(*e)?; - } - o_prot.write_list_end()?; - o_prot.write_field_end()?; - if let Some(ref fld_var) = self.shape { - o_prot.write_field_begin(&TFieldIdentifier::new("shape", TType::List, 2))?; - o_prot.write_list_begin(&TListIdentifier::new(TType::I64, fld_var.len() as i32))?; - for e in fld_var { - o_prot.write_i64(*e)?; - } - o_prot.write_list_end()?; - o_prot.write_field_end()? - } - o_prot.write_field_stop()?; - o_prot.write_struct_end() - } -} - -// -// FloatTensor -// - -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct FloatTensor { - pub floats: Vec>, - pub shape: Option>, -} - -impl FloatTensor { - pub fn new(floats: Vec>, shape: F2) -> FloatTensor where F2: Into>> { - FloatTensor { - floats, - shape: shape.into(), - } - } -} - -impl TSerializable for FloatTensor { - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - i_prot.read_struct_begin()?; - let mut f_1: Option>> = None; - let mut f_2: Option> = None; - loop { - let field_ident = i_prot.read_field_begin()?; - if field_ident.field_type == TType::Stop { - break; - } - let field_id = field_id(&field_ident)?; - match field_id { - 1 => { - let list_ident = i_prot.read_list_begin()?; - let mut val: Vec> = Vec::with_capacity(list_ident.size as usize); - for _ in 0..list_ident.size { - let list_elem_6 = OrderedFloat::from(i_prot.read_double()?); - val.push(list_elem_6); - } - i_prot.read_list_end()?; - f_1 = Some(val); - }, - 2 => { - let list_ident = i_prot.read_list_begin()?; - let mut val: Vec = Vec::with_capacity(list_ident.size as usize); - for _ in 0..list_ident.size { - let list_elem_7 = i_prot.read_i64()?; - val.push(list_elem_7); - } - i_prot.read_list_end()?; - f_2 = Some(val); - }, - _ => { - i_prot.skip(field_ident.field_type)?; - }, - }; - i_prot.read_field_end()?; - } - i_prot.read_struct_end()?; - verify_required_field_exists("FloatTensor.floats", &f_1)?; - let ret = FloatTensor { - floats: f_1.expect("auto-generated code should have checked for presence of required fields"), - shape: f_2, - }; - Ok(ret) - } - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let struct_ident = TStructIdentifier::new("FloatTensor"); - o_prot.write_struct_begin(&struct_ident)?; - o_prot.write_field_begin(&TFieldIdentifier::new("floats", TType::List, 1))?; - o_prot.write_list_begin(&TListIdentifier::new(TType::Double, self.floats.len() as i32))?; - for e in &self.floats { - o_prot.write_double((*e).into())?; - } - o_prot.write_list_end()?; - o_prot.write_field_end()?; - if let Some(ref fld_var) = self.shape { - o_prot.write_field_begin(&TFieldIdentifier::new("shape", TType::List, 2))?; - o_prot.write_list_begin(&TListIdentifier::new(TType::I64, fld_var.len() as i32))?; - for e in fld_var { - o_prot.write_i64(*e)?; - } - o_prot.write_list_end()?; - o_prot.write_field_end()? - } - o_prot.write_field_stop()?; - o_prot.write_struct_end() - } -} - -// -// DoubleTensor -// - -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct DoubleTensor { - pub doubles: Vec>, - pub shape: Option>, -} - -impl DoubleTensor { - pub fn new(doubles: Vec>, shape: F2) -> DoubleTensor where F2: Into>> { - DoubleTensor { - doubles, - shape: shape.into(), - } - } -} - -impl TSerializable for DoubleTensor { - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - i_prot.read_struct_begin()?; - let mut f_1: Option>> = None; - let mut f_2: Option> = None; - loop { - let field_ident = i_prot.read_field_begin()?; - if field_ident.field_type == TType::Stop { - break; - } - let field_id = field_id(&field_ident)?; - match field_id { - 1 => { - let list_ident = i_prot.read_list_begin()?; - let mut val: Vec> = Vec::with_capacity(list_ident.size as usize); - for _ in 0..list_ident.size { - let list_elem_8 = OrderedFloat::from(i_prot.read_double()?); - val.push(list_elem_8); - } - i_prot.read_list_end()?; - f_1 = Some(val); - }, - 2 => { - let list_ident = i_prot.read_list_begin()?; - let mut val: Vec = Vec::with_capacity(list_ident.size as usize); - for _ in 0..list_ident.size { - let list_elem_9 = i_prot.read_i64()?; - val.push(list_elem_9); - } - i_prot.read_list_end()?; - f_2 = Some(val); - }, - _ => { - i_prot.skip(field_ident.field_type)?; - }, - }; - i_prot.read_field_end()?; - } - i_prot.read_struct_end()?; - verify_required_field_exists("DoubleTensor.doubles", &f_1)?; - let ret = DoubleTensor { - doubles: f_1.expect("auto-generated code should have checked for presence of required fields"), - shape: f_2, - }; - Ok(ret) - } - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let struct_ident = TStructIdentifier::new("DoubleTensor"); - o_prot.write_struct_begin(&struct_ident)?; - o_prot.write_field_begin(&TFieldIdentifier::new("doubles", TType::List, 1))?; - o_prot.write_list_begin(&TListIdentifier::new(TType::Double, self.doubles.len() as i32))?; - for e in &self.doubles { - o_prot.write_double((*e).into())?; - } - o_prot.write_list_end()?; - o_prot.write_field_end()?; - if let Some(ref fld_var) = self.shape { - o_prot.write_field_begin(&TFieldIdentifier::new("shape", TType::List, 2))?; - o_prot.write_list_begin(&TListIdentifier::new(TType::I64, fld_var.len() as i32))?; - for e in fld_var { - o_prot.write_i64(*e)?; - } - o_prot.write_list_end()?; - o_prot.write_field_end()? - } - o_prot.write_field_stop()?; - o_prot.write_struct_end() - } -} - -// -// BoolTensor -// - -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct BoolTensor { - pub booleans: Vec, - pub shape: Option>, -} - -impl BoolTensor { - pub fn new(booleans: Vec, shape: F2) -> BoolTensor where F2: Into>> { - BoolTensor { - booleans, - shape: shape.into(), - } - } -} - -impl TSerializable for BoolTensor { - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - i_prot.read_struct_begin()?; - let mut f_1: Option> = None; - let mut f_2: Option> = None; - loop { - let field_ident = i_prot.read_field_begin()?; - if field_ident.field_type == TType::Stop { - break; - } - let field_id = field_id(&field_ident)?; - match field_id { - 1 => { - let list_ident = i_prot.read_list_begin()?; - let mut val: Vec = Vec::with_capacity(list_ident.size as usize); - for _ in 0..list_ident.size { - let list_elem_10 = i_prot.read_bool()?; - val.push(list_elem_10); - } - i_prot.read_list_end()?; - f_1 = Some(val); - }, - 2 => { - let list_ident = i_prot.read_list_begin()?; - let mut val: Vec = Vec::with_capacity(list_ident.size as usize); - for _ in 0..list_ident.size { - let list_elem_11 = i_prot.read_i64()?; - val.push(list_elem_11); - } - i_prot.read_list_end()?; - f_2 = Some(val); - }, - _ => { - i_prot.skip(field_ident.field_type)?; - }, - }; - i_prot.read_field_end()?; - } - i_prot.read_struct_end()?; - verify_required_field_exists("BoolTensor.booleans", &f_1)?; - let ret = BoolTensor { - booleans: f_1.expect("auto-generated code should have checked for presence of required fields"), - shape: f_2, - }; - Ok(ret) - } - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let struct_ident = TStructIdentifier::new("BoolTensor"); - o_prot.write_struct_begin(&struct_ident)?; - o_prot.write_field_begin(&TFieldIdentifier::new("booleans", TType::List, 1))?; - o_prot.write_list_begin(&TListIdentifier::new(TType::Bool, self.booleans.len() as i32))?; - for e in &self.booleans { - o_prot.write_bool(*e)?; - } - o_prot.write_list_end()?; - o_prot.write_field_end()?; - if let Some(ref fld_var) = self.shape { - o_prot.write_field_begin(&TFieldIdentifier::new("shape", TType::List, 2))?; - o_prot.write_list_begin(&TListIdentifier::new(TType::I64, fld_var.len() as i32))?; - for e in fld_var { - o_prot.write_i64(*e)?; - } - o_prot.write_list_end()?; - o_prot.write_field_end()? - } - o_prot.write_field_stop()?; - o_prot.write_struct_end() - } -} - -// -// RawTypedTensor -// - -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct RawTypedTensor { - pub data_type: DataType, - pub content: Vec, - pub shape: Option>, -} - -impl RawTypedTensor { - pub fn new(data_type: DataType, content: Vec, shape: F3) -> RawTypedTensor where F3: Into>> { - RawTypedTensor { - data_type, - content, - shape: shape.into(), - } - } -} - -impl TSerializable for RawTypedTensor { - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - i_prot.read_struct_begin()?; - let mut f_1: Option = None; - let mut f_2: Option> = None; - let mut f_3: Option> = None; - loop { - let field_ident = i_prot.read_field_begin()?; - if field_ident.field_type == TType::Stop { - break; - } - let field_id = field_id(&field_ident)?; - match field_id { - 1 => { - let val = DataType::read_from_in_protocol(i_prot)?; - f_1 = Some(val); - }, - 2 => { - let val = i_prot.read_bytes()?; - f_2 = Some(val); - }, - 3 => { - let list_ident = i_prot.read_list_begin()?; - let mut val: Vec = Vec::with_capacity(list_ident.size as usize); - for _ in 0..list_ident.size { - let list_elem_12 = i_prot.read_i64()?; - val.push(list_elem_12); - } - i_prot.read_list_end()?; - f_3 = Some(val); - }, - _ => { - i_prot.skip(field_ident.field_type)?; - }, - }; - i_prot.read_field_end()?; - } - i_prot.read_struct_end()?; - verify_required_field_exists("RawTypedTensor.data_type", &f_1)?; - verify_required_field_exists("RawTypedTensor.content", &f_2)?; - let ret = RawTypedTensor { - data_type: f_1.expect("auto-generated code should have checked for presence of required fields"), - content: f_2.expect("auto-generated code should have checked for presence of required fields"), - shape: f_3, - }; - Ok(ret) - } - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let struct_ident = TStructIdentifier::new("RawTypedTensor"); - o_prot.write_struct_begin(&struct_ident)?; - o_prot.write_field_begin(&TFieldIdentifier::new("dataType", TType::I32, 1))?; - self.data_type.write_to_out_protocol(o_prot)?; - o_prot.write_field_end()?; - o_prot.write_field_begin(&TFieldIdentifier::new("content", TType::String, 2))?; - o_prot.write_bytes(&self.content)?; - o_prot.write_field_end()?; - if let Some(ref fld_var) = self.shape { - o_prot.write_field_begin(&TFieldIdentifier::new("shape", TType::List, 3))?; - o_prot.write_list_begin(&TListIdentifier::new(TType::I64, fld_var.len() as i32))?; - for e in fld_var { - o_prot.write_i64(*e)?; - } - o_prot.write_list_end()?; - o_prot.write_field_end()? - } - o_prot.write_field_stop()?; - o_prot.write_struct_end() - } -} - -// -// BinaryTensor -// - -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct BinaryTensor { - pub binaries: Vec>, - pub shape: Option>, -} - -impl BinaryTensor { - pub fn new(binaries: Vec>, shape: F2) -> BinaryTensor where F2: Into>> { - BinaryTensor { - binaries, - shape: shape.into(), - } - } -} - -impl TSerializable for BinaryTensor { - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - i_prot.read_struct_begin()?; - let mut f_1: Option>> = None; - let mut f_2: Option> = None; - loop { - let field_ident = i_prot.read_field_begin()?; - if field_ident.field_type == TType::Stop { - break; - } - let field_id = field_id(&field_ident)?; - match field_id { - 1 => { - let list_ident = i_prot.read_list_begin()?; - let mut val: Vec> = Vec::with_capacity(list_ident.size as usize); - for _ in 0..list_ident.size { - let list_elem_13 = i_prot.read_bytes()?; - val.push(list_elem_13); - } - i_prot.read_list_end()?; - f_1 = Some(val); - }, - 2 => { - let list_ident = i_prot.read_list_begin()?; - let mut val: Vec = Vec::with_capacity(list_ident.size as usize); - for _ in 0..list_ident.size { - let list_elem_14 = i_prot.read_i64()?; - val.push(list_elem_14); - } - i_prot.read_list_end()?; - f_2 = Some(val); - }, - _ => { - i_prot.skip(field_ident.field_type)?; - }, - }; - i_prot.read_field_end()?; - } - i_prot.read_struct_end()?; - verify_required_field_exists("BinaryTensor.binaries", &f_1)?; - let ret = BinaryTensor { - binaries: f_1.expect("auto-generated code should have checked for presence of required fields"), - shape: f_2, - }; - Ok(ret) - } - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let struct_ident = TStructIdentifier::new("BinaryTensor"); - o_prot.write_struct_begin(&struct_ident)?; - o_prot.write_field_begin(&TFieldIdentifier::new("binaries", TType::List, 1))?; - o_prot.write_list_begin(&TListIdentifier::new(TType::String, self.binaries.len() as i32))?; - for e in &self.binaries { - o_prot.write_bytes(e)?; - } - o_prot.write_list_end()?; - o_prot.write_field_end()?; - if let Some(ref fld_var) = self.shape { - o_prot.write_field_begin(&TFieldIdentifier::new("shape", TType::List, 2))?; - o_prot.write_list_begin(&TListIdentifier::new(TType::I64, fld_var.len() as i32))?; - for e in fld_var { - o_prot.write_i64(*e)?; - } - o_prot.write_list_end()?; - o_prot.write_field_end()? - } - o_prot.write_field_stop()?; - o_prot.write_struct_end() - } -} - -// -// GeneralTensor -// - -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub enum GeneralTensor { - RawTypedTensor(RawTypedTensor), - StringTensor(StringTensor), - Int32Tensor(Int32Tensor), - Int64Tensor(Int64Tensor), - FloatTensor(FloatTensor), - DoubleTensor(DoubleTensor), - BoolTensor(BoolTensor), - BinaryTensor(BinaryTensor), -} - -impl TSerializable for GeneralTensor { - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - let mut ret: Option = None; - let mut received_field_count = 0; - i_prot.read_struct_begin()?; - loop { - let field_ident = i_prot.read_field_begin()?; - if field_ident.field_type == TType::Stop { - break; - } - let field_id = field_id(&field_ident)?; - match field_id { - 1 => { - let val = RawTypedTensor::read_from_in_protocol(i_prot)?; - if ret.is_none() { - ret = Some(GeneralTensor::RawTypedTensor(val)); - } - received_field_count += 1; - }, - 2 => { - let val = StringTensor::read_from_in_protocol(i_prot)?; - if ret.is_none() { - ret = Some(GeneralTensor::StringTensor(val)); - } - received_field_count += 1; - }, - 3 => { - let val = Int32Tensor::read_from_in_protocol(i_prot)?; - if ret.is_none() { - ret = Some(GeneralTensor::Int32Tensor(val)); - } - received_field_count += 1; - }, - 4 => { - let val = Int64Tensor::read_from_in_protocol(i_prot)?; - if ret.is_none() { - ret = Some(GeneralTensor::Int64Tensor(val)); - } - received_field_count += 1; - }, - 5 => { - let val = FloatTensor::read_from_in_protocol(i_prot)?; - if ret.is_none() { - ret = Some(GeneralTensor::FloatTensor(val)); - } - received_field_count += 1; - }, - 6 => { - let val = DoubleTensor::read_from_in_protocol(i_prot)?; - if ret.is_none() { - ret = Some(GeneralTensor::DoubleTensor(val)); - } - received_field_count += 1; - }, - 7 => { - let val = BoolTensor::read_from_in_protocol(i_prot)?; - if ret.is_none() { - ret = Some(GeneralTensor::BoolTensor(val)); - } - received_field_count += 1; - }, - 8 => { - let val = BinaryTensor::read_from_in_protocol(i_prot)?; - if ret.is_none() { - ret = Some(GeneralTensor::BinaryTensor(val)); - } - received_field_count += 1; - }, - _ => { - i_prot.skip(field_ident.field_type)?; - received_field_count += 1; - }, - }; - i_prot.read_field_end()?; - } - i_prot.read_struct_end()?; - if received_field_count == 0 { - Err( - thrift::Error::Protocol( - ProtocolError::new( - ProtocolErrorKind::InvalidData, - "received empty union from remote GeneralTensor" - ) - ) - ) - } else if received_field_count > 1 { - Err( - thrift::Error::Protocol( - ProtocolError::new( - ProtocolErrorKind::InvalidData, - "received multiple fields for union from remote GeneralTensor" - ) - ) - ) - } else { - Ok(ret.expect("return value should have been constructed")) - } - } - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let struct_ident = TStructIdentifier::new("GeneralTensor"); - o_prot.write_struct_begin(&struct_ident)?; - match *self { - GeneralTensor::RawTypedTensor(ref f) => { - o_prot.write_field_begin(&TFieldIdentifier::new("rawTypedTensor", TType::Struct, 1))?; - f.write_to_out_protocol(o_prot)?; - o_prot.write_field_end()?; - }, - GeneralTensor::StringTensor(ref f) => { - o_prot.write_field_begin(&TFieldIdentifier::new("stringTensor", TType::Struct, 2))?; - f.write_to_out_protocol(o_prot)?; - o_prot.write_field_end()?; - }, - GeneralTensor::Int32Tensor(ref f) => { - o_prot.write_field_begin(&TFieldIdentifier::new("int32Tensor", TType::Struct, 3))?; - f.write_to_out_protocol(o_prot)?; - o_prot.write_field_end()?; - }, - GeneralTensor::Int64Tensor(ref f) => { - o_prot.write_field_begin(&TFieldIdentifier::new("int64Tensor", TType::Struct, 4))?; - f.write_to_out_protocol(o_prot)?; - o_prot.write_field_end()?; - }, - GeneralTensor::FloatTensor(ref f) => { - o_prot.write_field_begin(&TFieldIdentifier::new("floatTensor", TType::Struct, 5))?; - f.write_to_out_protocol(o_prot)?; - o_prot.write_field_end()?; - }, - GeneralTensor::DoubleTensor(ref f) => { - o_prot.write_field_begin(&TFieldIdentifier::new("doubleTensor", TType::Struct, 6))?; - f.write_to_out_protocol(o_prot)?; - o_prot.write_field_end()?; - }, - GeneralTensor::BoolTensor(ref f) => { - o_prot.write_field_begin(&TFieldIdentifier::new("boolTensor", TType::Struct, 7))?; - f.write_to_out_protocol(o_prot)?; - o_prot.write_field_end()?; - }, - GeneralTensor::BinaryTensor(ref f) => { - o_prot.write_field_begin(&TFieldIdentifier::new("binaryTensor", TType::Struct, 8))?; - f.write_to_out_protocol(o_prot)?; - o_prot.write_field_end()?; - }, - } - o_prot.write_field_stop()?; - o_prot.write_struct_end() - } -} - -// -// COOSparseTensor -// - -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct COOSparseTensor { - pub dense_shape: Vec, - pub indices: Int64Tensor, - pub values: GeneralTensor, -} - -impl COOSparseTensor { - pub fn new(dense_shape: Vec, indices: Int64Tensor, values: GeneralTensor) -> COOSparseTensor { - COOSparseTensor { - dense_shape, - indices, - values, - } - } -} - -impl TSerializable for COOSparseTensor { - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - i_prot.read_struct_begin()?; - let mut f_1: Option> = None; - let mut f_2: Option = None; - let mut f_3: Option = None; - loop { - let field_ident = i_prot.read_field_begin()?; - if field_ident.field_type == TType::Stop { - break; - } - let field_id = field_id(&field_ident)?; - match field_id { - 1 => { - let list_ident = i_prot.read_list_begin()?; - let mut val: Vec = Vec::with_capacity(list_ident.size as usize); - for _ in 0..list_ident.size { - let list_elem_15 = i_prot.read_i64()?; - val.push(list_elem_15); - } - i_prot.read_list_end()?; - f_1 = Some(val); - }, - 2 => { - let val = Int64Tensor::read_from_in_protocol(i_prot)?; - f_2 = Some(val); - }, - 3 => { - let val = GeneralTensor::read_from_in_protocol(i_prot)?; - f_3 = Some(val); - }, - _ => { - i_prot.skip(field_ident.field_type)?; - }, - }; - i_prot.read_field_end()?; - } - i_prot.read_struct_end()?; - verify_required_field_exists("COOSparseTensor.dense_shape", &f_1)?; - verify_required_field_exists("COOSparseTensor.indices", &f_2)?; - verify_required_field_exists("COOSparseTensor.values", &f_3)?; - let ret = COOSparseTensor { - dense_shape: f_1.expect("auto-generated code should have checked for presence of required fields"), - indices: f_2.expect("auto-generated code should have checked for presence of required fields"), - values: f_3.expect("auto-generated code should have checked for presence of required fields"), - }; - Ok(ret) - } - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let struct_ident = TStructIdentifier::new("COOSparseTensor"); - o_prot.write_struct_begin(&struct_ident)?; - o_prot.write_field_begin(&TFieldIdentifier::new("denseShape", TType::List, 1))?; - o_prot.write_list_begin(&TListIdentifier::new(TType::I64, self.dense_shape.len() as i32))?; - for e in &self.dense_shape { - o_prot.write_i64(*e)?; - } - o_prot.write_list_end()?; - o_prot.write_field_end()?; - o_prot.write_field_begin(&TFieldIdentifier::new("indices", TType::Struct, 2))?; - self.indices.write_to_out_protocol(o_prot)?; - o_prot.write_field_end()?; - o_prot.write_field_begin(&TFieldIdentifier::new("values", TType::Struct, 3))?; - self.values.write_to_out_protocol(o_prot)?; - o_prot.write_field_end()?; - o_prot.write_field_stop()?; - o_prot.write_struct_end() - } -} - -// -// SparseTensor -// - -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub enum SparseTensor { - CooSparseTensor(COOSparseTensor), -} - -impl TSerializable for SparseTensor { - fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol) -> thrift::Result { - let mut ret: Option = None; - let mut received_field_count = 0; - i_prot.read_struct_begin()?; - loop { - let field_ident = i_prot.read_field_begin()?; - if field_ident.field_type == TType::Stop { - break; - } - let field_id = field_id(&field_ident)?; - match field_id { - 1 => { - let val = COOSparseTensor::read_from_in_protocol(i_prot)?; - if ret.is_none() { - ret = Some(SparseTensor::CooSparseTensor(val)); - } - received_field_count += 1; - }, - _ => { - i_prot.skip(field_ident.field_type)?; - received_field_count += 1; - }, - }; - i_prot.read_field_end()?; - } - i_prot.read_struct_end()?; - if received_field_count == 0 { - Err( - thrift::Error::Protocol( - ProtocolError::new( - ProtocolErrorKind::InvalidData, - "received empty union from remote SparseTensor" - ) - ) - ) - } else if received_field_count > 1 { - Err( - thrift::Error::Protocol( - ProtocolError::new( - ProtocolErrorKind::InvalidData, - "received multiple fields for union from remote SparseTensor" - ) - ) - ) - } else { - Ok(ret.expect("return value should have been constructed")) - } - } - fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol) -> thrift::Result<()> { - let struct_ident = TStructIdentifier::new("SparseTensor"); - o_prot.write_struct_begin(&struct_ident)?; - match *self { - SparseTensor::CooSparseTensor(ref f) => { - o_prot.write_field_begin(&TFieldIdentifier::new("cooSparseTensor", TType::Struct, 1))?; - f.write_to_out_protocol(o_prot)?; - o_prot.write_field_end()?; - }, - } - o_prot.write_field_stop()?; - o_prot.write_struct_end() - } -} - diff --git a/product-mixer/README.docx b/product-mixer/README.docx new file mode 100644 index 000000000..e2f5592bb Binary files /dev/null and b/product-mixer/README.docx differ diff --git a/product-mixer/README.md b/product-mixer/README.md deleted file mode 100644 index 1852d96c0..000000000 --- a/product-mixer/README.md +++ /dev/null @@ -1,41 +0,0 @@ -Product Mixer -============= - -## Overview - -Product Mixer is a common service framework and set of libraries that make it easy to build, -iterate on, and own product surface areas. It consists of: - -- **Core Libraries:** A set of libraries that enable you to build execution pipelines out of - reusable components. You define your logic in small, well-defined, reusable components and focus - on expressing the business logic you want to have. Then you can define easy to understand pipelines - that compose your components. Product Mixer handles the execution and monitoring of your pipelines - allowing you to focus on what really matters, your business logic. - -- **Service Framework:** A common service skeleton for teams to host their Product Mixer products. - -- **Component Library:** A shared library of components made by the Product Mixer Team, or - contributed by users. This enables you to both easily share the reusable components you make as well - as benefit from the work other teams have done by utilizing their shared components in the library. - -## Architecture - -The bulk of a Product Mixer can be broken down into Pipelines and Components. Components allow you -to break business logic into separate, standardized, reusable, testable, and easily composable -pieces, where each component has a well defined abstraction. Pipelines are essentially configuration -files specifying which Components should be used and when. This makes it easy to understand how your -code will execute while keeping it organized and structured in a maintainable way. - -Requests first go to Product Pipelines, which are used to select which Mixer Pipeline or -Recommendation Pipeline to run for a given request. Each Mixer or Recommendation -Pipeline may run multiple Candidate Pipelines to fetch candidates to include in the response. - -Mixer Pipelines combine the results of multiple heterogeneous Candidate Pipelines together -(e.g. ads, tweets, users) while Recommendation Pipelines are used to score (via Scoring Pipelines) -and rank the results of homogenous Candidate Pipelines so that the top ranked ones can be returned. -These pipelines also marshall candidates into a domain object and then into a transport object -to return to the caller. - -Candidate Pipelines fetch candidates from underlying Candidate Sources and perform some basic -operations on the Candidates, such as filtering out unwanted candidates, applying decorations, -and hydrating features. diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/account_recommendations_mixer/AccountRecommendationsMixerCandidateSource.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/account_recommendations_mixer/AccountRecommendationsMixerCandidateSource.docx new file mode 100644 index 000000000..eb7d86ec4 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/account_recommendations_mixer/AccountRecommendationsMixerCandidateSource.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/account_recommendations_mixer/AccountRecommendationsMixerCandidateSource.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/account_recommendations_mixer/AccountRecommendationsMixerCandidateSource.scala deleted file mode 100644 index 72698e1c7..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/account_recommendations_mixer/AccountRecommendationsMixerCandidateSource.scala +++ /dev/null @@ -1,57 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.account_recommendations_mixer - -import com.twitter.account_recommendations_mixer.{thriftscala => t} -import com.twitter.product_mixer.component_library.model.candidate.UserCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSourceWithExtractedFeatures -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidatesWithSourceFeatures -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -object WhoToFollowModuleHeaderFeature extends Feature[UserCandidate, t.Header] -object WhoToFollowModuleFooterFeature extends Feature[UserCandidate, Option[t.Footer]] -object WhoToFollowModuleDisplayOptionsFeature - extends Feature[UserCandidate, Option[t.DisplayOptions]] - -@Singleton -class AccountRecommendationsMixerCandidateSource @Inject() ( - accountRecommendationsMixer: t.AccountRecommendationsMixer.MethodPerEndpoint) - extends CandidateSourceWithExtractedFeatures[ - t.AccountRecommendationsMixerRequest, - t.RecommendedUser - ] { - - override val identifier: CandidateSourceIdentifier = - CandidateSourceIdentifier(name = "AccountRecommendationsMixer") - - override def apply( - request: t.AccountRecommendationsMixerRequest - ): Stitch[CandidatesWithSourceFeatures[t.RecommendedUser]] = { - Stitch - .callFuture(accountRecommendationsMixer.getWtfRecommendations(request)) - .map { response: t.WhoToFollowResponse => - responseToCandidatesWithSourceFeatures( - response.userRecommendations, - response.header, - response.footer, - response.displayOptions) - } - } - - private def responseToCandidatesWithSourceFeatures( - userRecommendations: Seq[t.RecommendedUser], - header: t.Header, - footer: Option[t.Footer], - displayOptions: Option[t.DisplayOptions], - ): CandidatesWithSourceFeatures[t.RecommendedUser] = { - val features = FeatureMapBuilder() - .add(WhoToFollowModuleHeaderFeature, header) - .add(WhoToFollowModuleFooterFeature, footer) - .add(WhoToFollowModuleDisplayOptionsFeature, displayOptions) - .build() - CandidatesWithSourceFeatures(userRecommendations, features) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/account_recommendations_mixer/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/account_recommendations_mixer/BUILD deleted file mode 100644 index d5eaef320..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/account_recommendations_mixer/BUILD +++ /dev/null @@ -1,22 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "account-recommendations-mixer/thrift/src/main/thrift:thrift-scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "src/thrift/com/twitter/ads/adserver:adserver_common-scala", - "stitch/stitch-core", - ], - exports = [ - "account-recommendations-mixer/thrift/src/main/thrift:thrift-scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/account_recommendations_mixer/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/account_recommendations_mixer/BUILD.docx new file mode 100644 index 000000000..d0d5de299 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/account_recommendations_mixer/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads/AdsProdStratoCandidateSource.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads/AdsProdStratoCandidateSource.docx new file mode 100644 index 000000000..e9fca4dc2 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads/AdsProdStratoCandidateSource.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads/AdsProdStratoCandidateSource.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads/AdsProdStratoCandidateSource.scala deleted file mode 100644 index 0509738a0..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads/AdsProdStratoCandidateSource.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.ads - -import com.twitter.adserver.thriftscala.AdImpression -import com.twitter.adserver.thriftscala.AdRequestParams -import com.twitter.adserver.thriftscala.AdRequestResponse -import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyFetcherSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.strato.client.Fetcher -import com.twitter.strato.generated.client.ads.admixer.MakeAdRequestClientColumn -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class AdsProdStratoCandidateSource @Inject() (adsClient: MakeAdRequestClientColumn) - extends StratoKeyFetcherSource[ - AdRequestParams, - AdRequestResponse, - AdImpression - ] { - - override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("AdsProdStrato") - - override val fetcher: Fetcher[AdRequestParams, Unit, AdRequestResponse] = adsClient.fetcher - - override protected def stratoResultTransformer( - stratoResult: AdRequestResponse - ): Seq[AdImpression] = - stratoResult.impressions -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads/AdsProdThriftCandidateSource.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads/AdsProdThriftCandidateSource.docx new file mode 100644 index 000000000..7c990ec90 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads/AdsProdThriftCandidateSource.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads/AdsProdThriftCandidateSource.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads/AdsProdThriftCandidateSource.scala deleted file mode 100644 index 08df2eb0d..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads/AdsProdThriftCandidateSource.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.ads - -import com.twitter.adserver.thriftscala.AdImpression -import com.twitter.adserver.thriftscala.AdRequestParams -import com.twitter.adserver.thriftscala.NewAdServer -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class AdsProdThriftCandidateSource @Inject() ( - adServerClient: NewAdServer.MethodPerEndpoint) - extends CandidateSource[AdRequestParams, AdImpression] { - - override val identifier: CandidateSourceIdentifier = - CandidateSourceIdentifier("AdsProdThrift") - - override def apply(request: AdRequestParams): Stitch[Seq[AdImpression]] = - Stitch.callFuture(adServerClient.makeAdRequest(request)).map(_.impressions) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads/AdsStagingCandidateSource.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads/AdsStagingCandidateSource.docx new file mode 100644 index 000000000..2928c5145 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads/AdsStagingCandidateSource.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads/AdsStagingCandidateSource.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads/AdsStagingCandidateSource.scala deleted file mode 100644 index d719f7608..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads/AdsStagingCandidateSource.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.ads - -import com.twitter.adserver.thriftscala.AdImpression -import com.twitter.adserver.thriftscala.AdRequestParams -import com.twitter.adserver.thriftscala.AdRequestResponse -import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyFetcherSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.strato.client.Fetcher -import com.twitter.strato.generated.client.ads.admixer.MakeAdRequestStagingClientColumn -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class AdsStagingCandidateSource @Inject() (adsClient: MakeAdRequestStagingClientColumn) - extends StratoKeyFetcherSource[ - AdRequestParams, - AdRequestResponse, - AdImpression - ] { - override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("AdsStaging") - - override val fetcher: Fetcher[AdRequestParams, Unit, AdRequestResponse] = adsClient.fetcher - - override protected def stratoResultTransformer( - stratoResult: AdRequestResponse - ): Seq[AdImpression] = - stratoResult.impressions -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads/BUILD deleted file mode 100644 index f3b49ec31..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", - "src/thrift/com/twitter/ads/adserver:adserver_common-scala", - "src/thrift/com/twitter/ads/adserver:adserver_rpc-scala", - "strato/config/columns/ads/admixer:admixer-strato-client", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", - "src/thrift/com/twitter/ads/adserver:adserver_common-scala", - "src/thrift/com/twitter/ads/adserver:adserver_rpc-scala", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads/BUILD.docx new file mode 100644 index 000000000..f379876ae Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ann/AnnCandidateSource.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ann/AnnCandidateSource.docx new file mode 100644 index 000000000..4d74bed1b Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ann/AnnCandidateSource.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ann/AnnCandidateSource.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ann/AnnCandidateSource.scala deleted file mode 100644 index 53bbb8d4f..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ann/AnnCandidateSource.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.ann - -import com.twitter.ann.common._ -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import com.twitter.util.{Time => _, _} -import com.twitter.finagle.util.DefaultTimer - -/** - * @param annQueryableById Ann Queryable by Id client that returns nearest neighbors for a sequence of queries - * @param identifier Candidate Source Identifier - * @tparam T1 type of the query. - * @tparam T2 type of the result. - * @tparam P runtime parameters supported by the index. - * @tparam D distance function used in the index. - */ -class AnnCandidateSource[T1, T2, P <: RuntimeParams, D <: Distance[D]]( - val annQueryableById: QueryableById[T1, T2, P, D], - val batchSize: Int, - val timeoutPerRequest: Duration, - override val identifier: CandidateSourceIdentifier) - extends CandidateSource[AnnIdQuery[T1, P], NeighborWithDistanceWithSeed[T1, T2, D]] { - - implicit val timer = DefaultTimer - - override def apply( - request: AnnIdQuery[T1, P] - ): Stitch[Seq[NeighborWithDistanceWithSeed[T1, T2, D]]] = { - val ids = request.ids - val numOfNeighbors = request.numOfNeighbors - val runtimeParams = request.runtimeParams - Stitch - .collect( - ids - .grouped(batchSize).map { batchedIds => - annQueryableById - .batchQueryWithDistanceById(batchedIds, numOfNeighbors, runtimeParams).map { - annResult => annResult.toSeq - }.within(timeoutPerRequest).handle { case _ => Seq.empty } - }.toSeq).map(_.flatten) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ann/AnnIdQuery.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ann/AnnIdQuery.docx new file mode 100644 index 000000000..71aa2f739 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ann/AnnIdQuery.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ann/AnnIdQuery.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ann/AnnIdQuery.scala deleted file mode 100644 index b262f6ac4..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ann/AnnIdQuery.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.ann - -import com.twitter.ann.common._ - -/** - * A [[AnnIdQuery]] is a query class which defines the ann entities with runtime params and number of neighbors requested - * - * @param ids Sequence of queries - * @param numOfNeighbors Number of neighbors requested - * @param runtimeParams ANN Runtime Params - * @param batchSize Batch size to the stitch client - * @tparam T type of query. - * @tparam P runtime parameters supported by the index. - */ -case class AnnIdQuery[T, P <: RuntimeParams]( - ids: Seq[T], - numOfNeighbors: Int, - runtimeParams: P) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ann/BUILD.bazel b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ann/BUILD.bazel deleted file mode 100644 index 46b1efdbb..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ann/BUILD.bazel +++ /dev/null @@ -1,17 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "ann/src/main/scala/com/twitter/ann/common", - "ann/src/main/scala/com/twitter/ann/hnsw", - "ann/src/main/thrift/com/twitter/ann/common:ann-common-scala", - "product-mixer/component-library/src/main/thrift/com/twitter/product_mixer/component_library:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "servo/manhattan/src/main/scala", - "servo/repo/src/main/scala", - "servo/util/src/main/scala", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ann/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ann/BUILD.docx new file mode 100644 index 000000000..7fa79e842 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ann/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/audiospace/BUILD.bazel b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/audiospace/BUILD.bazel deleted file mode 100644 index 8b93dea9f..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/audiospace/BUILD.bazel +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", - "src/thrift/com/twitter/periscope/audio_space:audio_space-scala", - "strato/config/columns/periscope:periscope-strato-client", - "strato/config/src/thrift/com/twitter/strato/graphql:graphql-scala", - "strato/src/main/scala/com/twitter/strato/client", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/audiospace/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/audiospace/BUILD.docx new file mode 100644 index 000000000..7c5f33d18 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/audiospace/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/audiospace/CreatedSpacesCandidateSource.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/audiospace/CreatedSpacesCandidateSource.docx new file mode 100644 index 000000000..976f234cf Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/audiospace/CreatedSpacesCandidateSource.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/audiospace/CreatedSpacesCandidateSource.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/audiospace/CreatedSpacesCandidateSource.scala deleted file mode 100644 index c5b820065..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/audiospace/CreatedSpacesCandidateSource.scala +++ /dev/null @@ -1,49 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.audiospace - -import com.twitter.periscope.audio_space.thriftscala.CreatedSpacesView -import com.twitter.periscope.audio_space.thriftscala.SpaceSlice -import com.twitter.product_mixer.component_library.model.cursor.NextCursorFeature -import com.twitter.product_mixer.component_library.model.cursor.PreviousCursorFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyViewFetcherWithSourceFeaturesSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.strato.client.Fetcher -import com.twitter.strato.generated.client.periscope.CreatedSpacesSliceOnUserClientColumn -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class CreatedSpacesCandidateSource @Inject() ( - column: CreatedSpacesSliceOnUserClientColumn) - extends StratoKeyViewFetcherWithSourceFeaturesSource[ - Long, - CreatedSpacesView, - SpaceSlice, - String - ] { - - override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("CreatedSpaces") - - override val fetcher: Fetcher[Long, CreatedSpacesView, SpaceSlice] = column.fetcher - - override def stratoResultTransformer( - stratoKey: Long, - stratoResult: SpaceSlice - ): Seq[String] = - stratoResult.items - - override protected def extractFeaturesFromStratoResult( - stratoKey: Long, - stratoResult: SpaceSlice - ): FeatureMap = { - val featureMapBuilder = FeatureMapBuilder() - stratoResult.sliceInfo.previousCursor.foreach { cursor => - featureMapBuilder.add(PreviousCursorFeature, cursor) - } - stratoResult.sliceInfo.nextCursor.foreach { cursor => - featureMapBuilder.add(NextCursorFeature, cursor) - } - featureMapBuilder.build() - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/business_profiles/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/business_profiles/BUILD deleted file mode 100644 index 0f5552bb5..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/business_profiles/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", - "strato/config/columns/consumer-identity/business-profiles:business-profiles-strato-client", - "strato/config/src/thrift/com/twitter/strato/graphql:graphql-scala", - "strato/src/main/scala/com/twitter/strato/client", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/business_profiles/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/business_profiles/BUILD.docx new file mode 100644 index 000000000..5840592e9 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/business_profiles/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/business_profiles/TeamMembersCandidateSource.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/business_profiles/TeamMembersCandidateSource.docx new file mode 100644 index 000000000..44748785f Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/business_profiles/TeamMembersCandidateSource.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/business_profiles/TeamMembersCandidateSource.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/business_profiles/TeamMembersCandidateSource.scala deleted file mode 100644 index c728194ab..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/business_profiles/TeamMembersCandidateSource.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.business_profiles - -import com.twitter.product_mixer.component_library.model.cursor.NextCursorFeature -import com.twitter.product_mixer.component_library.model.cursor.PreviousCursorFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyViewFetcherWithSourceFeaturesSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.strato.client.Fetcher -import com.twitter.strato.generated.client.consumer_identity.business_profiles.BusinessProfileTeamMembersOnUserClientColumn -import com.twitter.strato.generated.client.consumer_identity.business_profiles.BusinessProfileTeamMembersOnUserClientColumn.{ - Value => TeamMembersSlice -} -import com.twitter.strato.generated.client.consumer_identity.business_profiles.BusinessProfileTeamMembersOnUserClientColumn.{ - View => TeamMembersView -} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class TeamMembersCandidateSource @Inject() ( - column: BusinessProfileTeamMembersOnUserClientColumn) - extends StratoKeyViewFetcherWithSourceFeaturesSource[ - Long, - TeamMembersView, - TeamMembersSlice, - Long - ] { - override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( - "BusinessProfileTeamMembers") - - override val fetcher: Fetcher[Long, TeamMembersView, TeamMembersSlice] = column.fetcher - - override def stratoResultTransformer( - stratoKey: Long, - stratoResult: TeamMembersSlice - ): Seq[Long] = - stratoResult.members - - override protected def extractFeaturesFromStratoResult( - stratoKey: Long, - stratoResult: TeamMembersSlice - ): FeatureMap = { - val featureMapBuilder = FeatureMapBuilder() - stratoResult.previousCursor.foreach { cursor => - featureMapBuilder.add(PreviousCursorFeature, cursor.toString) - } - stratoResult.nextCursor.foreach { cursor => - featureMapBuilder.add(NextCursorFeature, cursor.toString) - } - featureMapBuilder.build() - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/cr_mixer/BUILD.bazel b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/cr_mixer/BUILD.bazel deleted file mode 100644 index 1eef70e77..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/cr_mixer/BUILD.bazel +++ /dev/null @@ -1,12 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "cr-mixer/thrift/src/main/thrift:thrift-scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/cr_mixer/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/cr_mixer/BUILD.docx new file mode 100644 index 000000000..e9e45cbd4 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/cr_mixer/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/cr_mixer/CrMixerFrsBasedTweetRecommendationsCandidateSource.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/cr_mixer/CrMixerFrsBasedTweetRecommendationsCandidateSource.docx new file mode 100644 index 000000000..53269cd0f Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/cr_mixer/CrMixerFrsBasedTweetRecommendationsCandidateSource.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/cr_mixer/CrMixerFrsBasedTweetRecommendationsCandidateSource.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/cr_mixer/CrMixerFrsBasedTweetRecommendationsCandidateSource.scala deleted file mode 100644 index c1e79f238..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/cr_mixer/CrMixerFrsBasedTweetRecommendationsCandidateSource.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.cr_mixer - -import com.twitter.cr_mixer.{thriftscala => t} -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Returns out-of-network Tweet recommendations by using user recommendations - * from FollowRecommendationService as an input seed-set to Earlybird - */ -@Singleton -class CrMixerFrsBasedTweetRecommendationsCandidateSource @Inject() ( - crMixerClient: t.CrMixer.MethodPerEndpoint) - extends CandidateSource[t.FrsTweetRequest, t.FrsTweet] { - - override val identifier: CandidateSourceIdentifier = - CandidateSourceIdentifier("CrMixerFrsBasedTweetRecommendations") - - override def apply(request: t.FrsTweetRequest): Stitch[Seq[t.FrsTweet]] = Stitch - .callFuture(crMixerClient.getFrsBasedTweetRecommendations(request)) - .map(_.tweets) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/cr_mixer/CrMixerTweetRecommendationsCandidateSource.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/cr_mixer/CrMixerTweetRecommendationsCandidateSource.docx new file mode 100644 index 000000000..dbd712f28 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/cr_mixer/CrMixerTweetRecommendationsCandidateSource.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/cr_mixer/CrMixerTweetRecommendationsCandidateSource.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/cr_mixer/CrMixerTweetRecommendationsCandidateSource.scala deleted file mode 100644 index a0cbd666c..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/cr_mixer/CrMixerTweetRecommendationsCandidateSource.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.cr_mixer - -import com.twitter.cr_mixer.{thriftscala => t} -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class CrMixerTweetRecommendationsCandidateSource @Inject() ( - crMixerClient: t.CrMixer.MethodPerEndpoint) - extends CandidateSource[t.CrMixerTweetRequest, t.TweetRecommendation] { - - override val identifier: CandidateSourceIdentifier = - CandidateSourceIdentifier("CrMixerTweetRecommendations") - - override def apply(request: t.CrMixerTweetRequest): Stitch[Seq[t.TweetRecommendation]] = Stitch - .callFuture(crMixerClient.getTweetRecommendations(request)) - .map(_.tweets) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/earlybird/BUILD.bazel b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/earlybird/BUILD.bazel deleted file mode 100644 index 638f549d3..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/earlybird/BUILD.bazel +++ /dev/null @@ -1,12 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "src/thrift/com/twitter/search:earlybird-scala", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/earlybird/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/earlybird/BUILD.docx new file mode 100644 index 000000000..a8b309e03 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/earlybird/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/earlybird/EarlybirdTweetCandidateSource.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/earlybird/EarlybirdTweetCandidateSource.docx new file mode 100644 index 000000000..6070570db Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/earlybird/EarlybirdTweetCandidateSource.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/earlybird/EarlybirdTweetCandidateSource.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/earlybird/EarlybirdTweetCandidateSource.scala deleted file mode 100644 index 9049849be..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/earlybird/EarlybirdTweetCandidateSource.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.earlybird - -import com.twitter.search.earlybird.{thriftscala => t} -import com.twitter.inject.Logging -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class EarlybirdTweetCandidateSource @Inject() ( - earlybirdService: t.EarlybirdService.MethodPerEndpoint) - extends CandidateSource[t.EarlybirdRequest, t.ThriftSearchResult] - with Logging { - - override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("EarlybirdTweets") - - override def apply(request: t.EarlybirdRequest): Stitch[Seq[t.ThriftSearchResult]] = { - Stitch - .callFuture(earlybirdService.search(request)) - .map { response: t.EarlybirdResponse => - response.searchResults.map(_.results).getOrElse(Seq.empty) - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/explore_ranker/BUILD.bazel b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/explore_ranker/BUILD.bazel deleted file mode 100644 index 6633be199..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/explore_ranker/BUILD.bazel +++ /dev/null @@ -1,12 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "explore/explore-ranker/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/explore_ranker/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/explore_ranker/BUILD.docx new file mode 100644 index 000000000..946954ca7 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/explore_ranker/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/explore_ranker/ExploreRankerCandidateSource.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/explore_ranker/ExploreRankerCandidateSource.docx new file mode 100644 index 000000000..c5d3f3e1f Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/explore_ranker/ExploreRankerCandidateSource.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/explore_ranker/ExploreRankerCandidateSource.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/explore_ranker/ExploreRankerCandidateSource.scala deleted file mode 100644 index bd57b785a..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/explore_ranker/ExploreRankerCandidateSource.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.explore_ranker - -import com.twitter.explore_ranker.{thriftscala => t} -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ExploreRankerCandidateSource @Inject() ( - exploreRankerService: t.ExploreRanker.MethodPerEndpoint) - extends CandidateSource[t.ExploreRankerRequest, t.ImmersiveRecsResult] { - - override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("ExploreRanker") - - override def apply( - request: t.ExploreRankerRequest - ): Stitch[Seq[t.ImmersiveRecsResult]] = { - Stitch - .callFuture(exploreRankerService.getRankedResults(request)) - .map { - case t.ExploreRankerResponse( - t.ExploreRankerProductResponse - .ImmersiveRecsResponse(t.ImmersiveRecsResponse(immersiveRecsResults))) => - immersiveRecsResults - case response => - throw new UnsupportedOperationException(s"Unknown response type: $response") - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/flexible_injection_pipeline/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/flexible_injection_pipeline/BUILD deleted file mode 100644 index b456956db..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/flexible_injection_pipeline/BUILD +++ /dev/null @@ -1,17 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "onboarding/service/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "stitch/stitch-core", - ], - exports = [ - "onboarding/service/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/flexible_injection_pipeline/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/flexible_injection_pipeline/BUILD.docx new file mode 100644 index 000000000..a68ed7f55 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/flexible_injection_pipeline/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/flexible_injection_pipeline/PromptCandidateSource.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/flexible_injection_pipeline/PromptCandidateSource.docx new file mode 100644 index 000000000..bde8e25a0 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/flexible_injection_pipeline/PromptCandidateSource.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/flexible_injection_pipeline/PromptCandidateSource.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/flexible_injection_pipeline/PromptCandidateSource.scala deleted file mode 100644 index 48891d7ec..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/flexible_injection_pipeline/PromptCandidateSource.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.flexible_injection_pipeline - -import com.twitter.inject.Logging -import com.twitter.onboarding.injections.{thriftscala => injectionsthrift} -import com.twitter.onboarding.task.service.{thriftscala => servicethrift} -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Returns a list of prompts to insert into a user's timeline (inline prompt, cover modals, etc) - * from go/flip (the prompting platform for Twitter). - */ -@Singleton -class PromptCandidateSource @Inject() (taskService: servicethrift.TaskService.MethodPerEndpoint) - extends CandidateSource[servicethrift.GetInjectionsRequest, IntermediatePrompt] - with Logging { - - override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( - "InjectionPipelinePrompts") - - override def apply( - request: servicethrift.GetInjectionsRequest - ): Stitch[Seq[IntermediatePrompt]] = { - Stitch - .callFuture(taskService.getInjections(request)).map { - _.injections.flatMap { - // The entire carousel is getting added to each IntermediatePrompt item with a - // corresponding index to be unpacked later on to populate its TimelineEntry counterpart. - case injection: injectionsthrift.Injection.TilesCarousel => - injection.tilesCarousel.tiles.zipWithIndex.map { - case (tile: injectionsthrift.Tile, index: Int) => - IntermediatePrompt(injection, Some(index), Some(tile)) - } - case injection => Seq(IntermediatePrompt(injection, None, None)) - } - } - } -} - -/** - * Gives an intermediate step to help 'explosion' of tile carousel tiles due to TimelineModule - * not being an extension of TimelineItem - */ -case class IntermediatePrompt( - injection: injectionsthrift.Injection, - offsetInModule: Option[Int], - carouselTile: Option[injectionsthrift.Tile]) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/hermit/BUILD.bazel b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/hermit/BUILD.bazel deleted file mode 100644 index c6bcf152a..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/hermit/BUILD.bazel +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", - "src/thrift/com/twitter/hermit:hermit-scala", - "strato/config/columns/onboarding:onboarding-strato-client", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", - "src/thrift/com/twitter/hermit:hermit-scala", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/hermit/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/hermit/BUILD.docx new file mode 100644 index 000000000..3213a4f10 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/hermit/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/hermit/UsersSimilarToMeCandidateSource.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/hermit/UsersSimilarToMeCandidateSource.docx new file mode 100644 index 000000000..291b5f0f5 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/hermit/UsersSimilarToMeCandidateSource.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/hermit/UsersSimilarToMeCandidateSource.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/hermit/UsersSimilarToMeCandidateSource.scala deleted file mode 100644 index 9bbfe234a..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/hermit/UsersSimilarToMeCandidateSource.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.hermit - -import com.twitter.hermit.thriftscala.RecommendationRequest -import com.twitter.hermit.thriftscala.RecommendationResponse -import com.twitter.hermit.thriftscala.RelatedUser -import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyViewFetcherSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.strato.client.Fetcher -import com.twitter.strato.generated.client.onboarding.HermitRecommendUsersClientColumn -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class UsersSimilarToMeCandidateSource @Inject() ( - column: HermitRecommendUsersClientColumn) - extends StratoKeyViewFetcherSource[ - Long, - RecommendationRequest, - RecommendationResponse, - RelatedUser - ] { - - override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("UsersSimilarToMe") - - override val fetcher: Fetcher[Long, RecommendationRequest, RecommendationResponse] = - column.fetcher - - override def stratoResultTransformer( - stratoKey: Long, - result: RecommendationResponse - ): Seq[RelatedUser] = result.suggestions.getOrElse(Seq.empty).filter(_.id.isDefined) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/interest_discovery/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/interest_discovery/BUILD deleted file mode 100644 index da4c6d794..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/interest_discovery/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "interests-service/thrift/src/main/thrift:thrift-scala", - "interests_discovery/thrift/src/main/thrift:service-thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "stitch/stitch-core", - ], - exports = [ - "interests-service/thrift/src/main/thrift:thrift-scala", - "interests_discovery/thrift/src/main/thrift:service-thrift-scala", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/interest_discovery/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/interest_discovery/BUILD.docx new file mode 100644 index 000000000..3b9ac5b80 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/interest_discovery/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/interest_discovery/RelatedTopicsCandidateSource.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/interest_discovery/RelatedTopicsCandidateSource.docx new file mode 100644 index 000000000..5c532e124 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/interest_discovery/RelatedTopicsCandidateSource.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/interest_discovery/RelatedTopicsCandidateSource.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/interest_discovery/RelatedTopicsCandidateSource.scala deleted file mode 100644 index 30465089c..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/interest_discovery/RelatedTopicsCandidateSource.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.interest_discovery - -import com.google.inject.Inject -import com.google.inject.Singleton -import com.twitter.inject.Logging -import com.twitter.interests_discovery.{thriftscala => t} -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch - -/** - * Generate a list of related topics results from IDS getRelatedTopics (thrift) endpoint. - * Returns related topics, given a topic, whereas [[RecommendedTopicsCandidateSource]] returns - * recommended topics, given a user. - */ -@Singleton -class RelatedTopicsCandidateSource @Inject() ( - interestDiscoveryService: t.InterestsDiscoveryService.MethodPerEndpoint) - extends CandidateSource[t.RelatedTopicsRequest, t.RelatedTopic] - with Logging { - - override val identifier: CandidateSourceIdentifier = - CandidateSourceIdentifier(name = "RelatedTopics") - - override def apply( - request: t.RelatedTopicsRequest - ): Stitch[Seq[t.RelatedTopic]] = { - Stitch - .callFuture(interestDiscoveryService.getRelatedTopics(request)) - .map { response: t.RelatedTopicsResponse => - response.topics - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/lists/BUILD.bazel b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/lists/BUILD.bazel deleted file mode 100644 index c62b67377..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/lists/BUILD.bazel +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "interests_discovery/thrift/src/main/thrift:service-thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", - "strato/config/columns/recommendations/interests_discovery/recommendations_mh:recommendations_mh-strato-client", - ], - exports = [ - "strato/config/columns/recommendations/interests_discovery/recommendations_mh:recommendations_mh-strato-client", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/lists/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/lists/BUILD.docx new file mode 100644 index 000000000..2b7d65a21 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/lists/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/lists/OrganicPopGeoListsCandidateSource.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/lists/OrganicPopGeoListsCandidateSource.docx new file mode 100644 index 000000000..bcf8cf5df Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/lists/OrganicPopGeoListsCandidateSource.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/lists/OrganicPopGeoListsCandidateSource.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/lists/OrganicPopGeoListsCandidateSource.scala deleted file mode 100644 index a8b6f214c..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/lists/OrganicPopGeoListsCandidateSource.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.lists - -import com.twitter.product_mixer.component_library.model.candidate.TwitterListCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyFetcherSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.strato.client.Fetcher -import com.twitter.strato.generated.client.recommendations.interests_discovery.recommendations_mh.OrganicPopgeoListsClientColumn -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class OrganicPopGeoListsCandidateSource @Inject() ( - organicPopgeoListsClientColumn: OrganicPopgeoListsClientColumn) - extends StratoKeyFetcherSource[ - OrganicPopgeoListsClientColumn.Key, - OrganicPopgeoListsClientColumn.Value, - TwitterListCandidate - ] { - - override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( - "OrganicPopGeoLists") - - override val fetcher: Fetcher[ - OrganicPopgeoListsClientColumn.Key, - Unit, - OrganicPopgeoListsClientColumn.Value - ] = - organicPopgeoListsClientColumn.fetcher - - override def stratoResultTransformer( - stratoResult: OrganicPopgeoListsClientColumn.Value - ): Seq[TwitterListCandidate] = { - stratoResult.recommendedListsByAlgo.flatMap { topLists => - topLists.lists.map { list => - TwitterListCandidate(list.listId) - } - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/people_discovery/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/people_discovery/BUILD deleted file mode 100644 index a54812ce9..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/people_discovery/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "people-discovery/api/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "src/thrift/com/twitter/ads/adserver:adserver_common-scala", - "src/thrift/com/twitter/hermit:hermit-scala", - "stitch/stitch-core", - ], - exports = [ - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "people-discovery/api/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/people_discovery/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/people_discovery/BUILD.docx new file mode 100644 index 000000000..7a4a2ff9c Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/people_discovery/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/people_discovery/PeopleDiscoveryCandidateSource.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/people_discovery/PeopleDiscoveryCandidateSource.docx new file mode 100644 index 000000000..607b95180 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/people_discovery/PeopleDiscoveryCandidateSource.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/people_discovery/PeopleDiscoveryCandidateSource.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/people_discovery/PeopleDiscoveryCandidateSource.scala deleted file mode 100644 index 0b3dacccb..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/people_discovery/PeopleDiscoveryCandidateSource.scala +++ /dev/null @@ -1,71 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.people_discovery - -import com.twitter.peoplediscovery.api.{thriftscala => t} -import com.twitter.product_mixer.component_library.model.candidate.UserCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSourceWithExtractedFeatures -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidatesWithSourceFeatures -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.pipeline_failure.UnexpectedCandidateResult -import com.twitter.stitch.Stitch -import com.twitter.util.logging.Logging -import javax.inject.Inject -import javax.inject.Singleton - -object WhoToFollowModuleHeaderFeature extends Feature[UserCandidate, t.Header] -object WhoToFollowModuleDisplayOptionsFeature - extends Feature[UserCandidate, Option[t.DisplayOptions]] -object WhoToFollowModuleShowMoreFeature extends Feature[UserCandidate, Option[t.ShowMore]] - -@Singleton -class PeopleDiscoveryCandidateSource @Inject() ( - peopleDiscoveryService: t.ThriftPeopleDiscoveryService.MethodPerEndpoint) - extends CandidateSourceWithExtractedFeatures[t.GetModuleRequest, t.RecommendedUser] - with Logging { - - override val identifier: CandidateSourceIdentifier = - CandidateSourceIdentifier(name = "PeopleDiscovery") - - override def apply( - request: t.GetModuleRequest - ): Stitch[CandidatesWithSourceFeatures[t.RecommendedUser]] = { - Stitch - .callFuture(peopleDiscoveryService.getModules(request)) - .map { response: t.GetModuleResponse => - // under the assumption getModules returns a maximum of one module - response.modules - .collectFirst { module => - module.layout match { - case t.Layout.UserBioList(layout) => - layoutToCandidatesWithSourceFeatures( - layout.userRecommendations, - layout.header, - layout.displayOptions, - layout.showMore) - case t.Layout.UserTweetCarousel(layout) => - layoutToCandidatesWithSourceFeatures( - layout.userRecommendations, - layout.header, - layout.displayOptions, - layout.showMore) - } - }.getOrElse(throw PipelineFailure(UnexpectedCandidateResult, "unexpected missing module")) - } - } - - private def layoutToCandidatesWithSourceFeatures( - userRecommendations: Seq[t.RecommendedUser], - header: t.Header, - displayOptions: Option[t.DisplayOptions], - showMore: Option[t.ShowMore], - ): CandidatesWithSourceFeatures[t.RecommendedUser] = { - val features = FeatureMapBuilder() - .add(WhoToFollowModuleHeaderFeature, header) - .add(WhoToFollowModuleDisplayOptionsFeature, displayOptions) - .add(WhoToFollowModuleShowMoreFeature, showMore) - .build() - CandidatesWithSourceFeatures(userRecommendations, features) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/recommendations/BUILD.bazel b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/recommendations/BUILD.bazel deleted file mode 100644 index cb69d82ec..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/recommendations/BUILD.bazel +++ /dev/null @@ -1,21 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "strato/config/columns/onboarding/follow-recommendations-service:follow-recommendations-service-strato-client", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/recommendations/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/recommendations/BUILD.docx new file mode 100644 index 000000000..392b6f0fb Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/recommendations/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/recommendations/UserFollowRecommendationsCandidateSource.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/recommendations/UserFollowRecommendationsCandidateSource.docx new file mode 100644 index 000000000..0b1e01541 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/recommendations/UserFollowRecommendationsCandidateSource.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/recommendations/UserFollowRecommendationsCandidateSource.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/recommendations/UserFollowRecommendationsCandidateSource.scala deleted file mode 100644 index dcfed2ec9..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/recommendations/UserFollowRecommendationsCandidateSource.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.recommendations - -import com.twitter.follow_recommendations.{thriftscala => fr} -import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyViewFetcherSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.strato.client.Fetcher -import com.twitter.strato.generated.client.onboarding.follow_recommendations_service.GetRecommendationsClientColumn -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Returns a list of FollowRecommendations as [[fr.UserRecommendation]]s fetched from Strato - */ -@Singleton -class UserFollowRecommendationsCandidateSource @Inject() ( - getRecommendationsClientColumn: GetRecommendationsClientColumn) - extends StratoKeyViewFetcherSource[ - fr.RecommendationRequest, - Unit, - fr.RecommendationResponse, - fr.UserRecommendation - ] { - - override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( - "FollowRecommendationsService") - - override val fetcher: Fetcher[fr.RecommendationRequest, Unit, fr.RecommendationResponse] = - getRecommendationsClientColumn.fetcher - - override def stratoResultTransformer( - stratoKey: fr.RecommendationRequest, - stratoResult: fr.RecommendationResponse - ): Seq[fr.UserRecommendation] = { - stratoResult.recommendations.map { - case fr.Recommendation.User(userRec: fr.UserRecommendation) => - userRec - case _ => - throw new Exception("Invalid recommendation type returned from FRS") - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/social_graph/BUILD.bazel b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/social_graph/BUILD.bazel deleted file mode 100644 index dbc8cba53..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/social_graph/BUILD.bazel +++ /dev/null @@ -1,22 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "socialgraph/server/src/main/scala/com/twitter/socialgraph/util", - "src/thrift/com/twitter/socialgraph:thrift-scala", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", - "src/thrift/com/twitter/socialgraph:thrift-scala", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/social_graph/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/social_graph/BUILD.docx new file mode 100644 index 000000000..b2d6fa6fd Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/social_graph/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/social_graph/SocialgraphCandidateSource.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/social_graph/SocialgraphCandidateSource.docx new file mode 100644 index 000000000..c32b99b69 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/social_graph/SocialgraphCandidateSource.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/social_graph/SocialgraphCandidateSource.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/social_graph/SocialgraphCandidateSource.scala deleted file mode 100644 index b3cf838f2..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/social_graph/SocialgraphCandidateSource.scala +++ /dev/null @@ -1,57 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.social_graph - -import com.twitter.product_mixer.component_library.model.candidate.CursorType -import com.twitter.product_mixer.component_library.model.candidate.NextCursor -import com.twitter.product_mixer.component_library.model.candidate.PreviousCursor -import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyViewFetcherSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.socialgraph.thriftscala -import com.twitter.socialgraph.thriftscala.IdsRequest -import com.twitter.socialgraph.thriftscala.IdsResult -import com.twitter.socialgraph.util.ByteBufferUtil -import com.twitter.strato.client.Fetcher -import javax.inject.Inject -import javax.inject.Singleton - -sealed trait SocialgraphResponse -case class SocialgraphResult(id: Long) extends SocialgraphResponse -case class SocialgraphCursor(cursor: Long, cursorType: CursorType) extends SocialgraphResponse - -@Singleton -class SocialgraphCandidateSource @Inject() ( - override val fetcher: Fetcher[thriftscala.IdsRequest, Option[ - thriftscala.RequestContext - ], thriftscala.IdsResult]) - extends StratoKeyViewFetcherSource[ - thriftscala.IdsRequest, - Option[thriftscala.RequestContext], - thriftscala.IdsResult, - SocialgraphResponse - ] { - - override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("Socialgraph") - - override def stratoResultTransformer( - stratoKey: IdsRequest, - stratoResult: IdsResult - ): Seq[SocialgraphResponse] = { - val prevCursor = - SocialgraphCursor(ByteBufferUtil.toLong(stratoResult.pageResult.prevCursor), PreviousCursor) - /* When an end cursor is passed to Socialgraph, - * Socialgraph returns the start cursor. To prevent - * clients from circularly fetching the timeline again, - * if we see a start cursor returned from Socialgraph, - * we replace it with an end cursor. - */ - val nextCursor = ByteBufferUtil.toLong(stratoResult.pageResult.nextCursor) match { - case SocialgraphCursorConstants.StartCursor => - SocialgraphCursor(SocialgraphCursorConstants.EndCursor, NextCursor) - case cursor => SocialgraphCursor(cursor, NextCursor) - } - - stratoResult.ids - .map { id => - SocialgraphResult(id) - } ++ Seq(nextCursor, prevCursor) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/social_graph/SocialgraphCursorConstants.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/social_graph/SocialgraphCursorConstants.docx new file mode 100644 index 000000000..130d2ee11 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/social_graph/SocialgraphCursorConstants.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/social_graph/SocialgraphCursorConstants.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/social_graph/SocialgraphCursorConstants.scala deleted file mode 100644 index 8f187d0cb..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/social_graph/SocialgraphCursorConstants.scala +++ /dev/null @@ -1,7 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.social_graph - -object SocialgraphCursorConstants { - val EndCursor: Long = 0L - val StartCursor: Long = -1L - val LastSortIndex: Long = 0L -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker/BUILD.bazel b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker/BUILD.bazel deleted file mode 100644 index c83297fd8..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "src/thrift/com/twitter/timelineranker:thrift-scala", - "src/thrift/com/twitter/timelineranker/server/model:thrift-scala", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker/BUILD.docx new file mode 100644 index 000000000..ffd2d3b41 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker/TimelineRankerInNetworkCandidateSource.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker/TimelineRankerInNetworkCandidateSource.docx new file mode 100644 index 000000000..43e5bbd97 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker/TimelineRankerInNetworkCandidateSource.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker/TimelineRankerInNetworkCandidateSource.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker/TimelineRankerInNetworkCandidateSource.scala deleted file mode 100644 index 9f1262e53..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker/TimelineRankerInNetworkCandidateSource.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.timeline_ranker - -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSourceWithExtractedFeatures -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidatesWithSourceFeatures -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelineranker.{thriftscala => t} -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Map of tweetId -> sourceTweet of retweets present in Timeline Ranker candidates list. - * These tweets are used only for further ranking. They are not returned to the end user. - */ -object TimelineRankerInNetworkSourceTweetsByTweetIdMapFeature - extends Feature[PipelineQuery, Map[Long, t.CandidateTweet]] - -@Singleton -class TimelineRankerInNetworkCandidateSource @Inject() ( - timelineRankerClient: t.TimelineRanker.MethodPerEndpoint) - extends CandidateSourceWithExtractedFeatures[t.RecapQuery, t.CandidateTweet] { - - override val identifier: CandidateSourceIdentifier = - CandidateSourceIdentifier("TimelineRankerInNetwork") - - override def apply( - request: t.RecapQuery - ): Stitch[CandidatesWithSourceFeatures[t.CandidateTweet]] = { - Stitch - .callFuture(timelineRankerClient.getRecycledTweetCandidates(Seq(request))) - .map { response: Seq[t.GetCandidateTweetsResponse] => - val candidates = - response.headOption.flatMap(_.candidates).getOrElse(Seq.empty).filter(_.tweet.nonEmpty) - val sourceTweetsByTweetId = - response.headOption - .flatMap(_.sourceTweets).getOrElse(Seq.empty).filter(_.tweet.nonEmpty) - .map { candidate => - (candidate.tweet.get.id, candidate) - }.toMap - val sourceTweetsByTweetIdMapFeature = FeatureMapBuilder() - .add(TimelineRankerInNetworkSourceTweetsByTweetIdMapFeature, sourceTweetsByTweetId) - .build() - CandidatesWithSourceFeatures( - candidates = candidates, - features = sourceTweetsByTweetIdMapFeature) - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker/TimelineRankerRecapCandidateSource.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker/TimelineRankerRecapCandidateSource.docx new file mode 100644 index 000000000..81b906ee1 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker/TimelineRankerRecapCandidateSource.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker/TimelineRankerRecapCandidateSource.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker/TimelineRankerRecapCandidateSource.scala deleted file mode 100644 index 7cd36fb92..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker/TimelineRankerRecapCandidateSource.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.timeline_ranker - -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import com.twitter.timelineranker.{thriftscala => t} - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class TimelineRankerRecapCandidateSource @Inject() ( - timelineRankerClient: t.TimelineRanker.MethodPerEndpoint) - extends CandidateSource[t.RecapQuery, t.CandidateTweet] { - - override val identifier: CandidateSourceIdentifier = - CandidateSourceIdentifier("TimelineRankerRecap") - - override def apply( - request: t.RecapQuery - ): Stitch[Seq[t.CandidateTweet]] = { - Stitch - .callFuture(timelineRankerClient.getRecapCandidatesFromAuthors(Seq(request))) - .map { response: Seq[t.GetCandidateTweetsResponse] => - response.headOption.flatMap(_.candidates).getOrElse(Seq.empty).filter(_.tweet.nonEmpty) - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker/TimelineRankerUtegCandidateSource.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker/TimelineRankerUtegCandidateSource.docx new file mode 100644 index 000000000..e410113fa Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker/TimelineRankerUtegCandidateSource.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker/TimelineRankerUtegCandidateSource.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker/TimelineRankerUtegCandidateSource.scala deleted file mode 100644 index bee6fe128..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker/TimelineRankerUtegCandidateSource.scala +++ /dev/null @@ -1,49 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.timeline_ranker - -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSourceWithExtractedFeatures -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidatesWithSourceFeatures -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.pipeline_failure.UnexpectedCandidateResult -import com.twitter.stitch.Stitch -import com.twitter.timelineranker.{thriftscala => t} -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Source tweets of retweets present in Timeline Ranker candidates list. - * These tweets are used only for further ranking. They are not returned to the end user. - */ -case object TimelineRankerUtegSourceTweetsFeature - extends Feature[PipelineQuery, Seq[t.CandidateTweet]] - -@Singleton -class TimelineRankerUtegCandidateSource @Inject() ( - timelineRankerClient: t.TimelineRanker.MethodPerEndpoint) - extends CandidateSourceWithExtractedFeatures[t.UtegLikedByTweetsQuery, t.CandidateTweet] { - - override val identifier: CandidateSourceIdentifier = - CandidateSourceIdentifier("TimelineRankerUteg") - - override def apply( - request: t.UtegLikedByTweetsQuery - ): Stitch[CandidatesWithSourceFeatures[t.CandidateTweet]] = { - Stitch - .callFuture(timelineRankerClient.getUtegLikedByTweetCandidates(Seq(request))) - .map { response => - val result = response.headOption.getOrElse( - throw PipelineFailure(UnexpectedCandidateResult, "Empty Timeline Ranker response")) - val candidates = result.candidates.toSeq.flatten - val sourceTweets = result.sourceTweets.toSeq.flatten - - val candidateSourceFeatures = FeatureMapBuilder() - .add(TimelineRankerUtegSourceTweetsFeature, sourceTweets) - .build() - - CandidatesWithSourceFeatures(candidates = candidates, features = candidateSourceFeatures) - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_scorer/BUILD.bazel b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_scorer/BUILD.bazel deleted file mode 100644 index e4435bc0a..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_scorer/BUILD.bazel +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "src/thrift/com/twitter/timelinescorer:thrift-scala", - "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", - "src/thrift/com/twitter/timelinescorer/server/internal:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/suggests/features:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_scorer/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_scorer/BUILD.docx new file mode 100644 index 000000000..68232a3b2 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_scorer/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_scorer/TimelineScorerCandidateSource.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_scorer/TimelineScorerCandidateSource.docx new file mode 100644 index 000000000..71e544d68 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_scorer/TimelineScorerCandidateSource.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_scorer/TimelineScorerCandidateSource.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_scorer/TimelineScorerCandidateSource.scala deleted file mode 100644 index b2d6f3912..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_scorer/TimelineScorerCandidateSource.scala +++ /dev/null @@ -1,156 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.timeline_scorer - -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSourceWithExtractedFeatures -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidatesWithSourceFeatures -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelinescorer.common.scoredtweetcandidate.thriftscala.v1 -import com.twitter.timelinescorer.common.scoredtweetcandidate.thriftscala.v1.Ancestor -import com.twitter.timelinescorer.common.scoredtweetcandidate.{thriftscala => ct} -import com.twitter.timelinescorer.{thriftscala => t} -import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.thriftscala.CandidateTweetSourceId -import javax.inject.Inject -import javax.inject.Singleton - -case class ScoredTweetCandidateWithFocalTweet( - candidate: v1.ScoredTweetCandidate, - focalTweetIdOpt: Option[Long]) - -case object TimelineScorerCandidateSourceSucceededFeature extends Feature[PipelineQuery, Boolean] - -@Singleton -class TimelineScorerCandidateSource @Inject() ( - timelineScorerClient: t.TimelineScorer.MethodPerEndpoint) - extends CandidateSourceWithExtractedFeatures[ - t.ScoredTweetsRequest, - ScoredTweetCandidateWithFocalTweet - ] { - - override val identifier: CandidateSourceIdentifier = - CandidateSourceIdentifier("TimelineScorer") - - private val MaxConversationAncestors = 2 - - override def apply( - request: t.ScoredTweetsRequest - ): Stitch[CandidatesWithSourceFeatures[ScoredTweetCandidateWithFocalTweet]] = { - Stitch - .callFuture(timelineScorerClient.getScoredTweets(request)) - .map { response => - val scoredTweetsOpt = response match { - case t.ScoredTweetsResponse.V1(v1) => v1.scoredTweets - case t.ScoredTweetsResponse.UnknownUnionField(field) => - throw new UnsupportedOperationException(s"Unknown response type: ${field.field.name}") - } - val scoredTweets = scoredTweetsOpt.getOrElse(Seq.empty) - - val allAncestors = scoredTweets.flatMap { - case ct.ScoredTweetCandidate.V1(v1) if isEligibleReply(v1) => - v1.ancestors.get.map(_.tweetId) - case _ => Seq.empty - }.toSet - - // Remove tweets within ancestor list of other tweets to avoid serving duplicates - val keptTweets = scoredTweets.collect { - case ct.ScoredTweetCandidate.V1(v1) if !allAncestors.contains(originalTweetId(v1)) => v1 - } - - // Add parent and root tweet for eligible reply focal tweets - val candidates = keptTweets - .flatMap { - case v1 if isEligibleReply(v1) => - val ancestors = v1.ancestors.get - val focalTweetId = v1.tweetId - - // Include root tweet if the conversation has atleast 2 ancestors - val optionallyIncludedRootTweet = if (ancestors.size >= MaxConversationAncestors) { - val rootTweet = toScoredTweetCandidateFromAncestor( - ancestor = ancestors.last, - inReplyToTweetId = None, - conversationId = v1.conversationId, - ancestors = None, - candidateTweetSourceId = v1.candidateTweetSourceId - ) - Seq((rootTweet, Some(v1))) - } else Seq.empty - - /** - * Setting the in-reply-to tweet id on the immediate parent, if one exists, - * helps ensure tweet type metrics correctly distinguish roots from non-roots. - */ - val inReplyToTweetId = ancestors.tail.headOption.map(_.tweetId) - val parentAncestor = toScoredTweetCandidateFromAncestor( - ancestor = ancestors.head, - inReplyToTweetId = inReplyToTweetId, - conversationId = v1.conversationId, - ancestors = Some(ancestors.tail), - candidateTweetSourceId = v1.candidateTweetSourceId - ) - - optionallyIncludedRootTweet ++ - Seq((parentAncestor, Some(v1)), (v1, Some(v1))) - - case any => Seq((any, None)) // Set focalTweetId to None if not eligible for convo - } - - /** - * Dedup each tweet keeping the one with highest scored Focal Tweet - * Focal Tweet ID != the Conversation ID, which is set to the root of the conversation - * Focal Tweet ID will be defined for tweets with ancestors that should be - * in conversation modules and None for standalone tweets. - */ - val sortedDedupedCandidates = candidates - .groupBy { case (v1, _) => v1.tweetId } - .mapValues { group => - val (candidate, focalTweetOpt) = group.maxBy { - case (_, Some(focal)) => focal.score - case (_, None) => 0 - } - ScoredTweetCandidateWithFocalTweet(candidate, focalTweetOpt.map(focal => focal.tweetId)) - }.values.toSeq.sortBy(_.candidate.tweetId) - - CandidatesWithSourceFeatures( - candidates = sortedDedupedCandidates, - features = FeatureMapBuilder() - .add(TimelineScorerCandidateSourceSucceededFeature, true) - .build() - ) - } - } - - private def isEligibleReply(candidate: ct.ScoredTweetCandidateAliases.V1Alias): Boolean = { - candidate.inReplyToTweetId.nonEmpty && - !candidate.isRetweet.getOrElse(false) && - candidate.ancestors.exists(_.nonEmpty) - } - - /** - * If we have a retweet, get the source tweet id. - * If it is not a retweet, get the regular tweet id. - */ - private def originalTweetId(candidate: ct.ScoredTweetCandidateAliases.V1Alias): Long = { - candidate.sourceTweetId.getOrElse(candidate.tweetId) - } - - private def toScoredTweetCandidateFromAncestor( - ancestor: Ancestor, - inReplyToTweetId: Option[Long], - conversationId: Option[Long], - ancestors: Option[Seq[Ancestor]], - candidateTweetSourceId: Option[CandidateTweetSourceId] - ): ct.ScoredTweetCandidateAliases.V1Alias = { - ct.v1.ScoredTweetCandidate( - tweetId = ancestor.tweetId, - authorId = ancestor.userId.getOrElse(0L), - score = 0.0, - isAncestorCandidate = Some(true), - inReplyToTweetId = inReplyToTweetId, - conversationId = conversationId, - ancestors = ancestors, - candidateTweetSourceId = candidateTweetSourceId - ) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_service/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_service/BUILD deleted file mode 100644 index f9453b9a1..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_service/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", - "stitch/stitch-timelineservice/src/main/scala", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "stitch/stitch-timelineservice/src/main/scala", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_service/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_service/BUILD.docx new file mode 100644 index 000000000..761895711 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_service/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_service/TimelineServiceTweetCandidateSource.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_service/TimelineServiceTweetCandidateSource.docx new file mode 100644 index 000000000..2fe292328 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_service/TimelineServiceTweetCandidateSource.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_service/TimelineServiceTweetCandidateSource.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_service/TimelineServiceTweetCandidateSource.scala deleted file mode 100644 index 053c16a3e..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_service/TimelineServiceTweetCandidateSource.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.timeline_service - -import com.twitter.product_mixer.component_library.model.cursor.NextCursorFeature -import com.twitter.product_mixer.component_library.model.cursor.PreviousCursorFeature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSourceWithExtractedFeatures -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidatesWithSourceFeatures -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.stitch.timelineservice.TimelineService -import com.twitter.timelineservice.{thriftscala => t} -import javax.inject.Inject -import javax.inject.Singleton - -case object TimelineServiceResponseWasTruncatedFeature - extends FeatureWithDefaultOnFailure[PipelineQuery, Boolean] { - override val defaultValue: Boolean = false -} - -@Singleton -class TimelineServiceTweetCandidateSource @Inject() ( - timelineService: TimelineService) - extends CandidateSourceWithExtractedFeatures[t.TimelineQuery, t.Tweet] { - - override val identifier: CandidateSourceIdentifier = - CandidateSourceIdentifier("TimelineServiceTweet") - - override def apply(request: t.TimelineQuery): Stitch[CandidatesWithSourceFeatures[t.Tweet]] = { - timelineService - .getTimeline(request).map { timeline => - val candidates = timeline.entries.collect { - case t.TimelineEntry.Tweet(tweet) => tweet - } - - val candidateSourceFeatures = - FeatureMapBuilder() - .add(TimelineServiceResponseWasTruncatedFeature, timeline.wasTruncated.getOrElse(false)) - .add(PreviousCursorFeature, timeline.responseCursor.flatMap(_.top).getOrElse("")) - .add(NextCursorFeature, timeline.responseCursor.flatMap(_.bottom).getOrElse("")) - .build() - - CandidatesWithSourceFeatures(candidates = candidates, features = candidateSourceFeatures) - } - } - -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store/BUILD deleted file mode 100644 index 60d486676..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", - "src/thrift/com/twitter/timelines/impression:thrift-scala", - "strato/config/columns/timelines/impression-store:impression-store-strato-client", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", - "strato/config/columns/timelines/impression-store:impression-store-strato-client", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store/BUILD.docx new file mode 100644 index 000000000..a7733f4e7 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store/TimelinesImpressionStoreCandidateSourceV2.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store/TimelinesImpressionStoreCandidateSourceV2.docx new file mode 100644 index 000000000..0ce5221bc Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store/TimelinesImpressionStoreCandidateSourceV2.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store/TimelinesImpressionStoreCandidateSourceV2.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store/TimelinesImpressionStoreCandidateSourceV2.scala deleted file mode 100644 index 3c4b615bb..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store/TimelinesImpressionStoreCandidateSourceV2.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.timelines_impression_store - -import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyFetcherSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.strato.client.Fetcher -import com.twitter.strato.generated.client.timelines.impression_store.TweetImpressionStoreManhattanV2OnUserClientColumn -import com.twitter.timelines.impression.thriftscala.TweetImpressionsEntries -import com.twitter.timelines.impression.{thriftscala => t} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class TimelinesImpressionStoreCandidateSourceV2 @Inject() ( - client: TweetImpressionStoreManhattanV2OnUserClientColumn) - extends StratoKeyFetcherSource[ - Long, - t.TweetImpressionsEntries, - t.TweetImpressionsEntry - ] { - - override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( - "TimelinesImpressionStore") - - override val fetcher: Fetcher[Long, Unit, TweetImpressionsEntries] = client.fetcher - - override def stratoResultTransformer( - stratoResult: t.TweetImpressionsEntries - ): Seq[t.TweetImpressionsEntry] = - stratoResult.entries -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/topics/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/topics/BUILD deleted file mode 100644 index d145edc44..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/topics/BUILD +++ /dev/null @@ -1,11 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", - "strato/config/columns/interests:interests-strato-client", - "strato/src/main/scala/com/twitter/strato/client", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/topics/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/topics/BUILD.docx new file mode 100644 index 000000000..b60d67dae Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/topics/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/topics/FollowedTopicsCandidateSource.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/topics/FollowedTopicsCandidateSource.docx new file mode 100644 index 000000000..bad6d90d8 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/topics/FollowedTopicsCandidateSource.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/topics/FollowedTopicsCandidateSource.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/topics/FollowedTopicsCandidateSource.scala deleted file mode 100644 index 6b46e1298..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/topics/FollowedTopicsCandidateSource.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.topics - -import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyViewFetcherSeqSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.strato.client.Fetcher -import com.twitter.strato.generated.client.interests.FollowedTopicsGetterClientColumn -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class FollowedTopicsCandidateSource @Inject() ( - column: FollowedTopicsGetterClientColumn) - extends StratoKeyViewFetcherSeqSource[ - Long, - Unit, - Long - ] { - override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("FollowedTopics") - - override val fetcher: Fetcher[Long, Unit, Seq[Long]] = column.fetcher -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc/BUILD.bazel b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc/BUILD.bazel deleted file mode 100644 index acfc37015..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc/BUILD.bazel +++ /dev/null @@ -1,26 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", - "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", - "strato/config/columns/tweetconvosvc:tweetconvosvc-strato-client", - "tweetconvosvc/common/src/main/thrift/com/twitter/tweetconvosvc/tweet_ancestor:thrift-scala", - "tweetconvosvc/thrift/src/main/thrift:thrift-scala", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", - "strato/config/columns/tweetconvosvc:tweetconvosvc-strato-client", - "tweetconvosvc/thrift/src/main/thrift:thrift-scala", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc/BUILD.docx new file mode 100644 index 000000000..7ea8f0d6a Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc/ConversationServiceCandidateSource.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc/ConversationServiceCandidateSource.docx new file mode 100644 index 000000000..eef2e9b09 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc/ConversationServiceCandidateSource.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc/ConversationServiceCandidateSource.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc/ConversationServiceCandidateSource.scala deleted file mode 100644 index 889a0164d..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc/ConversationServiceCandidateSource.scala +++ /dev/null @@ -1,173 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc - -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSourceWithExtractedFeatures -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidatesWithSourceFeatures -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import com.twitter.tweetconvosvc.tweet_ancestor.{thriftscala => ta} -import com.twitter.tweetconvosvc.{thriftscala => tcs} -import com.twitter.util.Return -import com.twitter.util.Throw -import javax.inject.Inject -import javax.inject.Singleton - -case class ConversationServiceCandidateSourceRequest( - tweetsWithConversationMetadata: Seq[TweetWithConversationMetadata]) - -case class TweetWithConversationMetadata( - tweetId: Long, - userId: Option[Long], - sourceTweetId: Option[Long], - sourceUserId: Option[Long], - inReplyToTweetId: Option[Long], - conversationId: Option[Long], - ancestors: Seq[ta.TweetAncestor]) - -/** - * Candidate source that fetches ancestors of input candidates from Tweetconvosvc and - * returns a flattened list of input and ancestor candidates. - */ -@Singleton -class ConversationServiceCandidateSource @Inject() ( - conversationServiceClient: tcs.ConversationService.MethodPerEndpoint) - extends CandidateSourceWithExtractedFeatures[ - ConversationServiceCandidateSourceRequest, - TweetWithConversationMetadata - ] { - - override val identifier: CandidateSourceIdentifier = - CandidateSourceIdentifier("ConversationService") - - private val maxModuleSize = 3 - private val maxAncestorsInConversation = 2 - private val numberOfRootTweets = 1 - private val maxTweetsInConversationWithSameId = 1 - - override def apply( - request: ConversationServiceCandidateSourceRequest - ): Stitch[CandidatesWithSourceFeatures[TweetWithConversationMetadata]] = { - val inputTweetsWithConversationMetadata: Seq[TweetWithConversationMetadata] = - request.tweetsWithConversationMetadata - val ancestorsRequest = - tcs.GetAncestorsRequest(inputTweetsWithConversationMetadata.map(_.tweetId)) - - // build the tweets with conversation metadata by calling the conversation service with reduced - // ancestors to limit to maxModuleSize - val tweetsWithConversationMetadataFromAncestors: Stitch[Seq[TweetWithConversationMetadata]] = - Stitch - .callFuture(conversationServiceClient.getAncestors(ancestorsRequest)) - .map { getAncestorsResponse: tcs.GetAncestorsResponse => - inputTweetsWithConversationMetadata - .zip(getAncestorsResponse.ancestors).collect { - case (focalTweet, tcs.TweetAncestorsResult.TweetAncestors(ancestorsResult)) - if ancestorsResult.nonEmpty => - getTweetsInThread(focalTweet, ancestorsResult.head) - }.flatten - } - - // dedupe the tweets in the list and transform the calling error to - // return the requested tweets with conversation metadata - val transformedTweetsWithConversationMetadata: Stitch[Seq[TweetWithConversationMetadata]] = - tweetsWithConversationMetadataFromAncestors.transform { - case Return(ancestors) => - Stitch.value(dedupeCandidates(inputTweetsWithConversationMetadata, ancestors)) - case Throw(_) => - Stitch.value(inputTweetsWithConversationMetadata) - } - - // return the candidates with empty source features from transformed tweetsWithConversationMetadata - transformedTweetsWithConversationMetadata.map { - responseTweetsWithConversationMetadata: Seq[TweetWithConversationMetadata] => - CandidatesWithSourceFeatures( - responseTweetsWithConversationMetadata, - FeatureMap.empty - ) - } - } - - private def getTweetsInThread( - focalTweet: TweetWithConversationMetadata, - ancestors: ta.TweetAncestors - ): Seq[TweetWithConversationMetadata] = { - // Re-add the focal tweet so we can easily build modules and dedupe later. - // Note, TweetConvoSVC returns the bottom of the thread first, so we - // reverse them for easy rendering. - val focalTweetWithConversationMetadata = TweetWithConversationMetadata( - tweetId = focalTweet.tweetId, - userId = focalTweet.userId, - sourceTweetId = focalTweet.sourceTweetId, - sourceUserId = focalTweet.sourceUserId, - inReplyToTweetId = focalTweet.inReplyToTweetId, - conversationId = Some(focalTweet.tweetId), - ancestors = ancestors.ancestors - ) - - val parentTweets = ancestors.ancestors.map { ancestor => - TweetWithConversationMetadata( - tweetId = ancestor.tweetId, - userId = Some(ancestor.userId), - sourceTweetId = None, - sourceUserId = None, - inReplyToTweetId = None, - conversationId = Some(focalTweet.tweetId), - ancestors = Seq.empty - ) - } ++ getTruncatedRootTweet(ancestors, focalTweet.tweetId) - - val (intermediates, root) = parentTweets.splitAt(parentTweets.size - numberOfRootTweets) - val truncatedIntermediates = - intermediates.take(maxModuleSize - maxAncestorsInConversation).reverse - root ++ truncatedIntermediates :+ focalTweetWithConversationMetadata - } - - /** - * Ancestor store truncates at 256 ancestors. For very large reply threads, we try best effort - * to append the root tweet to the ancestor list based on the conversationId and - * conversationRootAuthorId. When rendering conversation modules, we can display the root tweet - * instead of the 256th highest ancestor. - */ - private def getTruncatedRootTweet( - ancestors: ta.TweetAncestors, - focalTweetId: Long - ): Option[TweetWithConversationMetadata] = { - ancestors.conversationRootAuthorId.collect { - case rootAuthorId - if ancestors.state == ta.ReplyState.Partial && - ancestors.ancestors.last.tweetId != ancestors.conversationId => - TweetWithConversationMetadata( - tweetId = ancestors.conversationId, - userId = Some(rootAuthorId), - sourceTweetId = None, - sourceUserId = None, - inReplyToTweetId = None, - conversationId = Some(focalTweetId), - ancestors = Seq.empty - ) - } - } - - private def dedupeCandidates( - inputTweetsWithConversationMetadata: Seq[TweetWithConversationMetadata], - ancestors: Seq[TweetWithConversationMetadata] - ): Seq[TweetWithConversationMetadata] = { - val dedupedAncestors: Iterable[TweetWithConversationMetadata] = ancestors - .groupBy(_.tweetId).map { - case (_, duplicateAncestors) - if duplicateAncestors.size > maxTweetsInConversationWithSameId => - duplicateAncestors.maxBy(_.conversationId.getOrElse(0L)) - case (_, nonDuplicateAncestors) => nonDuplicateAncestors.head - } - // Sort by tweet id to prevent issues with future assumptions of the root being the first - // tweet and the focal being the last tweet in a module. The tweets as a whole do not need - // to be sorted overall, only the relative order within modules must be kept. - val sortedDedupedAncestors: Seq[TweetWithConversationMetadata] = - dedupedAncestors.toSeq.sortBy(_.tweetId) - - val ancestorIds = sortedDedupedAncestors.map(_.tweetId).toSet - val updatedCandidates = inputTweetsWithConversationMetadata.filterNot { candidate => - ancestorIds.contains(candidate.tweetId) - } - sortedDedupedAncestors ++ updatedCandidates - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc/ConversationServiceResponseFeatureTransformer.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc/ConversationServiceResponseFeatureTransformer.docx new file mode 100644 index 000000000..98d05d6d0 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc/ConversationServiceResponseFeatureTransformer.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc/ConversationServiceResponseFeatureTransformer.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc/ConversationServiceResponseFeatureTransformer.scala deleted file mode 100644 index 59a0c725f..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc/ConversationServiceResponseFeatureTransformer.scala +++ /dev/null @@ -1,49 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc - -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.timelineservice.suggests.thriftscala.SuggestType - -object AuthorIdFeature extends Feature[TweetCandidate, Option[Long]] -object AncestorIdsFeature extends Feature[TweetCandidate, Seq[Long]] -object ConversationModuleFocalTweetIdFeature extends Feature[TweetCandidate, Option[Long]] -object InReplyToFeature extends Feature[TweetCandidate, Option[Long]] -object IsRetweetFeature extends Feature[TweetCandidate, Boolean] -object SourceTweetIdFeature extends Feature[TweetCandidate, Option[Long]] -object SourceUserIdFeature extends Feature[TweetCandidate, Option[Long]] -object SuggestTypeFeature extends Feature[TweetCandidate, Option[SuggestType]] - -object ConversationServiceResponseFeatureTransformer - extends CandidateFeatureTransformer[TweetWithConversationMetadata] { - override val identifier: TransformerIdentifier = - TransformerIdentifier("ConversationServiceResponse") - - override val features: Set[Feature[_, _]] = - Set( - AuthorIdFeature, - InReplyToFeature, - IsRetweetFeature, - SourceTweetIdFeature, - SourceUserIdFeature, - ConversationModuleFocalTweetIdFeature, - AncestorIdsFeature, - SuggestTypeFeature - ) - - override def transform(candidate: TweetWithConversationMetadata): FeatureMap = { - FeatureMapBuilder() - .add(AuthorIdFeature, candidate.userId) - .add(InReplyToFeature, candidate.inReplyToTweetId) - .add(IsRetweetFeature, candidate.sourceTweetId.isDefined) - .add(SourceTweetIdFeature, candidate.sourceTweetId) - .add(SourceUserIdFeature, candidate.sourceUserId) - .add(ConversationModuleFocalTweetIdFeature, candidate.conversationId) - .add(AncestorIdsFeature, candidate.ancestors.map(_.tweetId)) - .add(SuggestTypeFeature, Some(SuggestType.OrganicConversation)) - .build() - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc/DropMaxConversationModuleItemCandidates.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc/DropMaxConversationModuleItemCandidates.docx new file mode 100644 index 000000000..c5b07f8e8 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc/DropMaxConversationModuleItemCandidates.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc/DropMaxConversationModuleItemCandidates.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc/DropMaxConversationModuleItemCandidates.scala deleted file mode 100644 index 426adb98c..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc/DropMaxConversationModuleItemCandidates.scala +++ /dev/null @@ -1,55 +0,0 @@ -package com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc - -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * Takes a conversation module item and truncates it to be at most the focal tweet, the focal tweet's - * in reply to tweet and optionally, the root conversation tweet if desired. - * @param pipelineScope What pipeline scopes to include in this. - * @param includeRootTweet Whether to include the root tweet at the top of the conversation or not. - * @tparam Query - */ -case class DropMaxConversationModuleItemCandidates[-Query <: PipelineQuery]( - override val pipelineScope: CandidateScope, - includeRootTweet: Boolean) - extends Selector[Query] { - override def apply( - query: Query, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val updatedCandidates = remainingCandidates.collect { - case moduleCandidate: ModuleCandidateWithDetails if pipelineScope.contains(moduleCandidate) => - updateConversationModule(moduleCandidate, includeRootTweet) - case candidates => candidates - } - SelectorResult(remainingCandidates = updatedCandidates, result = result) - } - - private def updateConversationModule( - module: ModuleCandidateWithDetails, - includeRootTweet: Boolean - ): ModuleCandidateWithDetails = { - // If the thread is only the root tweet & a focal tweet replying to it, no truncation can be done. - if (module.candidates.length <= 2) { - module - } else { - // If a thread is more 3 or more tweets, we optionally keep the root tweet if desired, and take - // the focal tweet tweet and its direct ancestor (the one it would have replied to) and return - // those. - val tweetCandidates = module.candidates - val replyAndFocalTweet = tweetCandidates.takeRight(2) - val updatedConversation = if (includeRootTweet) { - tweetCandidates.headOption ++ replyAndFocalTweet - } else { - replyAndFocalTweet - } - module.copy(candidates = updatedConversation.toSeq) - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/slice/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/slice/BUILD deleted file mode 100644 index 2ed1a6316..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/slice/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/slice/builder", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/slice", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/slice/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/slice/builder", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/slice", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/slice/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/slice/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/slice/BUILD.docx new file mode 100644 index 000000000..3dc59cf37 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/slice/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/slice/SliceItemCandidateDecorator.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/slice/SliceItemCandidateDecorator.docx new file mode 100644 index 000000000..b04584131 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/slice/SliceItemCandidateDecorator.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/slice/SliceItemCandidateDecorator.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/slice/SliceItemCandidateDecorator.scala deleted file mode 100644 index abe128926..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/slice/SliceItemCandidateDecorator.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.slice - -import com.twitter.product_mixer.component_library.model.candidate.CursorCandidate -import com.twitter.product_mixer.component_library.model.presentation.slice.SliceItemPresentation -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.DecoratorIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.slice.CursorItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.decorator.Decoration -import com.twitter.product_mixer.core.functional_component.decorator.slice.builder.CandidateSliceItemBuilder -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.stitch.Stitch - -/** - * Adds a [[Decoration]] for all `candidates` that are [[CursorCandidate]]s - * - * @note Only [[CursorCandidate]]s get decorated in [[SliceItemCandidateDecorator]] - * because the [[com.twitter.product_mixer.component_library.premarshaller.slice.SliceDomainMarshaller]] - * handles the undecorated non-[[CursorCandidate]] `candidates` directly. - */ -case class SliceItemCandidateDecorator[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]( - cursorBuilder: CandidateSliceItemBuilder[Query, CursorCandidate, CursorItem], - override val identifier: DecoratorIdentifier = DecoratorIdentifier("SliceItemCandidate")) - extends CandidateDecorator[Query, Candidate] { - - override def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[Candidate]] - ): Stitch[Seq[Decoration]] = { - val cursorPresentations = candidates.collect { - case CandidateWithFeatures(candidate: CursorCandidate, features) => - val cursorItem = cursorBuilder(query, candidate, features) - val presentation = SliceItemPresentation(sliceItem = cursorItem) - - Decoration(candidate, presentation) - } - - Stitch.value(cursorPresentations) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/slice/builder/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/slice/builder/BUILD deleted file mode 100644 index bbbde90f6..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/slice/builder/BUILD +++ /dev/null @@ -1,17 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/slice/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/slice/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/slice/builder/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/slice/builder/BUILD.docx new file mode 100644 index 000000000..a933a8e0e Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/slice/builder/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/slice/builder/CursorCandidateSliceItemBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/slice/builder/CursorCandidateSliceItemBuilder.docx new file mode 100644 index 000000000..7281a99a9 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/slice/builder/CursorCandidateSliceItemBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/slice/builder/CursorCandidateSliceItemBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/slice/builder/CursorCandidateSliceItemBuilder.scala deleted file mode 100644 index 307911c85..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/slice/builder/CursorCandidateSliceItemBuilder.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.slice.builder - -import com.twitter.product_mixer.component_library.model.candidate.CursorCandidate -import com.twitter.product_mixer.component_library.model.candidate.{ - NextCursor => CursorCandidateNextCursor -} -import com.twitter.product_mixer.component_library.model.candidate.{ - PreviousCursor => CursorCandidatePreviousCursor -} -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.response.slice.CursorItem -import com.twitter.product_mixer.core.model.marshalling.response.slice.NextCursor -import com.twitter.product_mixer.core.model.marshalling.response.slice.PreviousCursor -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.functional_component.decorator.slice.builder.CandidateSliceItemBuilder - -case class CursorCandidateSliceItemBuilder() - extends CandidateSliceItemBuilder[PipelineQuery, CursorCandidate, CursorItem] { - - override def apply( - query: PipelineQuery, - candidate: CursorCandidate, - featureMap: FeatureMap - ): CursorItem = - candidate.cursorType match { - case CursorCandidateNextCursor => CursorItem(candidate.value, NextCursor) - case CursorCandidatePreviousCursor => CursorItem(candidate.value, PreviousCursor) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/BUILD deleted file mode 100644 index ea3b7b298..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/BUILD +++ /dev/null @@ -1,108 +0,0 @@ -scala_library( - name = "urt", - sources = ["**/*.scala"] + exclude_globs(["builder/richtext/*.scala"]), - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - scalac_plugins = ["no-roomba"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - ":richtext", - "3rdparty/jvm/com/twitter/bijection:json", - "3rdparty/jvm/com/twitter/bijection:scrooge", - "explore/explore-mixer/server/src/main/scala/com/twitter/explore_mixer/model/request", - "interests-mixer/server/src/main/scala/com/twitter/interests_mixer/model/request", - "onboarding/service/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/people_discovery", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/suggestion", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/trends_events", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline/transformer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/alert", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/cover", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/icon", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/item", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/metadata", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/operation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/timeline_module", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/thrift/com/twitter/ads/adserver:ad_metadata_container-scala", - "src/thrift/com/twitter/ads/adserver:adserver_common-scala", - "src/thrift/com/twitter/hermit:hermit-scala", - "src/thrift/com/twitter/suggests/controller_data:controller_data-scala", - "src/thrift/com/twitter/timelines/service:thrift-scala", - "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", - "stringcenter/client", - "stringcenter/client/src/main/java", - "stringcenter/client/src/main/scala/com/twitter/stringcenter/client", - "timelines/src/main/scala/com/twitter/timelines/util", - "trends/trending_content/src/main/scala/com/twitter/trends/trending_content/util:compacting-number-localizer", - "tweetconvosvc/common/src/main/thrift/com/twitter/tweetconvosvc/tweet_ancestor:thrift-scala", - "twitter-text/lib/java/src/main/java/com/twitter/twittertext", - ], - exports = [ - ":richtext", - "3rdparty/jvm/com/twitter/bijection:json", - "3rdparty/jvm/com/twitter/bijection:scrooge", - "explore/explore-mixer/server/src/main/scala/com/twitter/explore_mixer/model/request", - "interests-mixer/server/src/main/scala/com/twitter/interests_mixer/model/request", - "onboarding/service/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/suggestion", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline/transformer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/alert", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/cover", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/icon", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/item", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/metadata", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/operation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/timeline_module", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/thrift/com/twitter/ads/adserver:ad_metadata_container-scala", - "src/thrift/com/twitter/ads/adserver:adserver_common-scala", - "src/thrift/com/twitter/suggests/controller_data:controller_data-scala", - "src/thrift/com/twitter/timelines/service:thrift-scala", - "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", - "stringcenter/client", - "stringcenter/client/src/main/java", - "stringcenter/client/src/main/scala/com/twitter/stringcenter/client", - "timelines/src/main/scala/com/twitter/timelines/util", - "tweetconvosvc/common/src/main/thrift/com/twitter/tweetconvosvc/tweet_ancestor:thrift-scala", - "twitter-text/lib/java/src/main/java/com/twitter/twittertext", - ], -) - -scala_library( - name = "richtext", - sources = ["builder/richtext/*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/metadata", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/richtext", - "twitter-text/lib/java/src/main/java/com/twitter/twittertext", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/metadata", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/richtext", - "twitter-text/lib/java/src/main/java/com/twitter/twittertext", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/BUILD.docx new file mode 100644 index 000000000..36866f788 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/UrtConversationItemCandidateDecorator.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/UrtConversationItemCandidateDecorator.docx new file mode 100644 index 000000000..f15427c96 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/UrtConversationItemCandidateDecorator.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/UrtConversationItemCandidateDecorator.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/UrtConversationItemCandidateDecorator.scala deleted file mode 100644 index d0a517b2d..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/UrtConversationItemCandidateDecorator.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt - -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate -import com.twitter.product_mixer.component_library.model.presentation.urt.ConversationModuleItem -import com.twitter.product_mixer.component_library.model.presentation.urt.UrtItemPresentation -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.decorator.Decoration -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.DecoratorIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.ModuleItemTreeDisplay -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -case class UrtConversationItemCandidateDecorator[ - Query <: PipelineQuery, - Candidate <: BaseTweetCandidate -]( - tweetCandidateUrtItemBuilder: TweetCandidateUrtItemBuilder[Query, Candidate], - override val identifier: DecoratorIdentifier = DecoratorIdentifier("UrtConversationItem")) - extends CandidateDecorator[Query, Candidate] { - - override def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[Candidate]] - ): Stitch[Seq[Decoration]] = { - val candidatePresentations = candidates.view.zipWithIndex.map { - case (candidate, index) => - val itemPresentation = new UrtItemPresentation( - timelineItem = tweetCandidateUrtItemBuilder( - pipelineQuery = query, - tweetCandidate = candidate.candidate, - candidateFeatures = candidate.features) - ) with ConversationModuleItem { - override val treeDisplay: Option[ModuleItemTreeDisplay] = None - override val dispensable: Boolean = index < candidates.length - 1 - } - - Decoration(candidate.candidate, itemPresentation) - } - - Stitch.value(candidatePresentations) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/UrtItemCandidateDecorator.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/UrtItemCandidateDecorator.docx new file mode 100644 index 000000000..ed94e3cbf Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/UrtItemCandidateDecorator.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/UrtItemCandidateDecorator.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/UrtItemCandidateDecorator.scala deleted file mode 100644 index 81398cd3d..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/UrtItemCandidateDecorator.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt - -import com.twitter.product_mixer.component_library.model.presentation.urt.UrtItemPresentation -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.DecoratorIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.decorator.Decoration -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.stitch.Stitch - -/** - * Decorator that will apply the provided [[CandidateUrtEntryBuilder]] to each candidate independently to make a [[TimelineItem]] - */ -case class UrtItemCandidateDecorator[ - Query <: PipelineQuery, - BuilderInput <: UniversalNoun[Any], - BuilderOutput <: TimelineItem -]( - builder: CandidateUrtEntryBuilder[Query, BuilderInput, BuilderOutput], - override val identifier: DecoratorIdentifier = DecoratorIdentifier("UrtItemCandidate")) - extends CandidateDecorator[Query, BuilderInput] { - - override def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[BuilderInput]] - ): Stitch[Seq[Decoration]] = { - val candidatePresentations = candidates.map { candidate => - val itemPresentation = UrtItemPresentation( - timelineItem = builder(query, candidate.candidate, candidate.features) - ) - - Decoration(candidate.candidate, itemPresentation) - } - - Stitch.value(candidatePresentations) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/UrtItemInModuleDecorator.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/UrtItemInModuleDecorator.docx new file mode 100644 index 000000000..8c4b2d485 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/UrtItemInModuleDecorator.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/UrtItemInModuleDecorator.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/UrtItemInModuleDecorator.scala deleted file mode 100644 index 928b506b9..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/UrtItemInModuleDecorator.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt - -import com.twitter.product_mixer.component_library.model.presentation.urt.UrtItemPresentation -import com.twitter.product_mixer.component_library.model.presentation.urt.UrtModulePresentation -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.DecoratorIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.decorator.Decoration -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseTimelineModuleBuilder -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.stitch.Stitch - -/** - * Decorator that will apply the provided [[urtItemCandidateDecorator]] to all the `candidates` and apply - * the same [[UrtModulePresentation]] from [[moduleBuilder]] to each Candidate. - */ -case class UrtItemInModuleDecorator[ - Query <: PipelineQuery, - BuilderInput <: UniversalNoun[Any], - BuilderOutput <: TimelineItem -]( - urtItemCandidateDecorator: CandidateDecorator[Query, BuilderInput], - moduleBuilder: BaseTimelineModuleBuilder[Query, BuilderInput], - override val identifier: DecoratorIdentifier = DecoratorIdentifier("UrtItemInModule")) - extends CandidateDecorator[Query, BuilderInput] { - - override def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[BuilderInput]] - ): Stitch[Seq[Decoration]] = { - if (candidates.nonEmpty) { - val urtItemCandidatesWithDecoration = urtItemCandidateDecorator(query, candidates) - - // Pass candidates to support when the module is constructed dynamically based on the list - val modulePresentation = - UrtModulePresentation(moduleBuilder(query, candidates)) - - urtItemCandidatesWithDecoration.map { candidates => - candidates.collect { - case Decoration(candidate, urtItemPresentation: UrtItemPresentation) => - Decoration( - candidate, - urtItemPresentation.copy(modulePresentation = Some(modulePresentation))) - } - } - } else { - Stitch.Nil - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/UrtMultipleModulesDecorator.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/UrtMultipleModulesDecorator.docx new file mode 100644 index 000000000..41ba2c221 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/UrtMultipleModulesDecorator.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/UrtMultipleModulesDecorator.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/UrtMultipleModulesDecorator.scala deleted file mode 100644 index 3c13056b3..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/UrtMultipleModulesDecorator.scala +++ /dev/null @@ -1,108 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt - -import com.twitter.product_mixer.component_library.model.presentation.urt.UrtItemPresentation -import com.twitter.product_mixer.component_library.model.presentation.urt.UrtModulePresentation -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.DecoratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.decorator.Decoration -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.stitch.Stitch -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ModuleIdGeneration -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.AutomaticUniqueModuleId -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseTimelineModuleBuilder - -/** - * Given a [[CandidateWithFeatures]] return the corresponding group with which it should be - * associated. Returning none will result in the candidate not being assigned to any module. - */ -trait GroupByKey[-Query <: PipelineQuery, -BuilderInput <: UniversalNoun[Any], Key] { - def apply(query: Query, candidate: BuilderInput, candidateFeatures: FeatureMap): Option[Key] -} - -/** - * Similar to [[UrtItemInModuleDecorator]] except that this decorator can assign items to different - * modules based on the provided [[GroupByKey]]. - * - * @param urtItemCandidateDecorator decorates individual item candidates - * @param moduleBuilder builds a module from a particular candidate group - * @param groupByKey assigns each candidate a module group. Returning [[None]] will result in the - * candidate not being assigned to a module - */ -case class UrtMultipleModulesDecorator[ - -Query <: PipelineQuery, - -BuilderInput <: UniversalNoun[Any], - GroupKey -]( - urtItemCandidateDecorator: CandidateDecorator[Query, BuilderInput], - moduleBuilder: BaseTimelineModuleBuilder[Query, BuilderInput], - groupByKey: GroupByKey[Query, BuilderInput, GroupKey], - override val identifier: DecoratorIdentifier = DecoratorIdentifier("UrtMultipleModules")) - extends CandidateDecorator[Query, BuilderInput] { - - override def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[BuilderInput]] - ): Stitch[Seq[Decoration]] = { - if (candidates.nonEmpty) { - - /** Individual candidates with [[UrtItemPresentation]]s */ - val decoratedCandidatesStitch: Stitch[ - Seq[(CandidateWithFeatures[BuilderInput], Decoration)] - ] = urtItemCandidateDecorator(query, candidates).map(candidates.zip(_)) - - decoratedCandidatesStitch.map { decoratedCandidates => - // Group candidates into modules - val candidatesByModule: Map[Option[GroupKey], Seq[ - (CandidateWithFeatures[BuilderInput], Decoration) - ]] = - decoratedCandidates.groupBy { - case (CandidateWithFeatures(candidate, features), _) => - groupByKey(query, candidate, features) - } - - candidatesByModule.iterator.zipWithIndex.flatMap { - - // A None group key indicates these candidates should not be put into a module. Return - // the decorated candidates. - case ((None, candidateGroup), _) => - candidateGroup.map { - case (_, decoration) => decoration - } - - // Build a UrtModulePresentation and add it to each candidate's decoration. - case ((_, candidateGroup), index) => - val (candidatesWithFeatures, decorations) = candidateGroup.unzip - - /** - * Build the module and update its ID if [[AutomaticUniqueModuleId]]s are being used. - * Forcing IDs to be different ensures that modules are never accidentally grouped - * together, since all other fields might otherwise be equal (candidates aren't added - * to modules until the domain marshalling phase). - */ - val timelineModule = { - val module = moduleBuilder(query, candidatesWithFeatures) - - ModuleIdGeneration(module.id) match { - case id: AutomaticUniqueModuleId => module.copy(id = id.withOffset(index).moduleId) - case _ => module - } - } - - val modulePresentation = UrtModulePresentation(timelineModule) - - decorations.collect { - case Decoration(candidate, urtItemPresentation: UrtItemPresentation) => - Decoration( - candidate, - urtItemPresentation.copy(modulePresentation = Some(modulePresentation))) - } - }.toSeq - } - } else { - Stitch.Nil - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/contextual_ref/ContextualTweetRefBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/contextual_ref/ContextualTweetRefBuilder.docx new file mode 100644 index 000000000..d644fa87b Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/contextual_ref/ContextualTweetRefBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/contextual_ref/ContextualTweetRefBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/contextual_ref/ContextualTweetRefBuilder.scala deleted file mode 100644 index 0dd1234fc..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/contextual_ref/ContextualTweetRefBuilder.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref - -import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate -import com.twitter.product_mixer.core.model.marshalling.response.urt.contextual_ref.ContextualTweetRef -import com.twitter.product_mixer.core.model.marshalling.response.urt.contextual_ref.TweetHydrationContext - -case class ContextualTweetRefBuilder[-Candidate <: BaseTweetCandidate]( - tweetHydrationContext: TweetHydrationContext) { - - def apply(candidate: Candidate): Option[ContextualTweetRef] = - Some(ContextualTweetRef(candidate.id, Some(tweetHydrationContext))) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/conversations/ConversationModuleMetadataBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/conversations/ConversationModuleMetadataBuilder.docx new file mode 100644 index 000000000..f99cb8ca3 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/conversations/ConversationModuleMetadataBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/conversations/ConversationModuleMetadataBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/conversations/ConversationModuleMetadataBuilder.scala deleted file mode 100644 index fdb5c4e48..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/conversations/ConversationModuleMetadataBuilder.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.conversations - -import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseModuleMetadataBuilder -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleConversationMetadata -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleMetadata -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class ConversationModuleMetadataBuilder[ - Query <: PipelineQuery, - Candidate <: BaseTweetCandidate -]( - ancestorIdsFeature: Feature[_, Seq[Long]], - allIdsOrdering: Ordering[Long]) - extends BaseModuleMetadataBuilder[Query, Candidate] { - - override def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[Candidate]] - ): ModuleMetadata = { - - val ancestors = candidates.last.features.getOrElse(ancestorIdsFeature, Seq.empty) - val sortedAllTweetIds = (candidates.last.candidate.id +: ancestors).sorted(allIdsOrdering) - - ModuleMetadata( - adsMetadata = None, - conversationMetadata = Some( - ModuleConversationMetadata( - allTweetIds = Some(sortedAllTweetIds), - socialContext = None, - enableDeduplication = Some(true) - )), - gridCarouselMetadata = None - ) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/FlipPromptCandidateUrtItemBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/FlipPromptCandidateUrtItemBuilder.docx new file mode 100644 index 000000000..8db45f525 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/FlipPromptCandidateUrtItemBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/FlipPromptCandidateUrtItemBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/FlipPromptCandidateUrtItemBuilder.scala deleted file mode 100644 index 2387271da..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/FlipPromptCandidateUrtItemBuilder.scala +++ /dev/null @@ -1,205 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.flexible_injection_pipeline - -import com.twitter.onboarding.injections.thriftscala.Injection -import com.twitter.onboarding.injections.{thriftscala => onboardingthrift} -import com.twitter.product_mixer.component_library.decorator.urt.builder.flexible_injection_pipeline.OnboardingInjectionConversions._ -import com.twitter.product_mixer.component_library.model.candidate.BasePromptCandidate -import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.transformer.FlipPromptCarouselTileFeature -import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.transformer.FlipPromptInjectionsFeature -import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.transformer.FlipPromptOffsetInModuleFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder -import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineItem -import com.twitter.product_mixer.core.model.marshalling.response.urt.cover.CoverFullCoverDisplayType -import com.twitter.product_mixer.core.model.marshalling.response.urt.cover.CoverHalfCoverDisplayType -import com.twitter.product_mixer.core.model.marshalling.response.urt.cover.FullCover -import com.twitter.product_mixer.core.model.marshalling.response.urt.cover.FullCoverContent -import com.twitter.product_mixer.core.model.marshalling.response.urt.cover.HalfCover -import com.twitter.product_mixer.core.model.marshalling.response.urt.cover.HalfCoverContent -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.message.HeaderImagePromptMessageContent -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.message.InlinePromptMessageContent -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.message.MessageContent -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.message.MessagePromptItem -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.prompt.PromptItem -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventInfo -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TimelinesDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object FlipPromptCandidateUrtItemBuilder { - val FlipPromptClientEventInfoElement: String = "flip-prompt-message" -} - -case class FlipPromptCandidateUrtItemBuilder[-Query <: PipelineQuery]() - extends CandidateUrtEntryBuilder[Query, BasePromptCandidate[Any], TimelineItem] { - - override def apply( - query: Query, - promptCandidate: BasePromptCandidate[Any], - candidateFeatures: FeatureMap - ): TimelineItem = { - val injection = candidateFeatures.get(FlipPromptInjectionsFeature) - - injection match { - case onboardingthrift.Injection.InlinePrompt(candidate) => - MessagePromptItem( - id = promptCandidate.id.toString, - sortIndex = None, // Sort indexes are automatically set in the domain marshaller phase - clientEventInfo = buildClientEventInfo(injection), - feedbackActionInfo = candidate.feedbackInfo.map(convertFeedbackInfo), - isPinned = Some(candidate.isPinnedEntry), - content = getInlinePromptMessageContent(candidate), - impressionCallbacks = candidate.impressionCallbacks.map(_.map(convertCallback).toList) - ) - case onboardingthrift.Injection.FullCover(candidate) => - FullCover( - id = promptCandidate.id.toString, - // Note that sort index is not used for Covers, as they are not TimelineEntry and do not have entryId - sortIndex = None, - clientEventInfo = - Some(OnboardingInjectionConversions.convertClientEventInfo(candidate.clientEventInfo)), - content = getFullCoverContent(candidate) - ) - case onboardingthrift.Injection.HalfCover(candidate) => - HalfCover( - id = promptCandidate.id.toString, - // Note that sort index is not used for Covers, as they are not TimelineEntry and do not have entryId - sortIndex = None, - clientEventInfo = - Some(OnboardingInjectionConversions.convertClientEventInfo(candidate.clientEventInfo)), - content = getHalfCoverContent(candidate) - ) - case Injection.TilesCarousel(_) => - val offsetInModuleOption = - candidateFeatures.get(FlipPromptOffsetInModuleFeature) - val offsetInModule = - offsetInModuleOption.getOrElse(throw FlipPromptOffsetInModuleMissing) - val tileOption = - candidateFeatures.get(FlipPromptCarouselTileFeature) - val tile = tileOption.getOrElse(throw FlipPromptCarouselTileMissing) - TilesCarouselConversions.convertTile(tile, offsetInModule) - case onboardingthrift.Injection.RelevancePrompt(candidate) => - PromptItem( - id = promptCandidate.id.toString, - sortIndex = None, // Sort indexes are automatically set in the domain marshaller phase - clientEventInfo = buildClientEventInfo(injection), - content = RelevancePromptConversions.convertContent(candidate), - impressionCallbacks = Some(candidate.impressionCallbacks.map(convertCallback).toList) - ) - case _ => throw new UnsupportedFlipPromptException(injection) - } - } - - private def getInlinePromptMessageContent( - candidate: onboardingthrift.InlinePrompt - ): MessageContent = { - candidate.image match { - case Some(image) => - HeaderImagePromptMessageContent( - headerImage = convertImage(image), - headerText = Some(candidate.headerText.text), - bodyText = candidate.bodyText.map(_.text), - primaryButtonAction = candidate.primaryAction.map(convertButtonAction), - secondaryButtonAction = candidate.secondaryAction.map(convertButtonAction), - headerRichText = Some(convertRichText(candidate.headerText)), - bodyRichText = candidate.bodyText.map(convertRichText), - action = - None - ) - case None => - InlinePromptMessageContent( - headerText = candidate.headerText.text, - bodyText = candidate.bodyText.map(_.text), - primaryButtonAction = candidate.primaryAction.map(convertButtonAction), - secondaryButtonAction = candidate.secondaryAction.map(convertButtonAction), - headerRichText = Some(convertRichText(candidate.headerText)), - bodyRichText = candidate.bodyText.map(convertRichText), - socialContext = candidate.socialContext.map(convertSocialContext), - userFacepile = candidate.promptUserFacepile.map(convertUserFacePile) - ) - } - } - - private def getFullCoverContent( - candidate: onboardingthrift.FullCover - ): FullCoverContent = - FullCoverContent( - displayType = CoverFullCoverDisplayType, - primaryText = convertRichText(candidate.primaryText), - primaryCoverCta = convertCoverCta(candidate.primaryButtonAction), - secondaryCoverCta = candidate.secondaryButtonAction.map(convertCoverCta), - secondaryText = candidate.secondaryText.map(convertRichText), - imageVariant = candidate.image.map(img => convertImageVariant(img.image)), - details = candidate.detailText.map(convertRichText), - dismissInfo = candidate.dismissInfo.map(convertDismissInfo), - imageDisplayType = candidate.image.map(img => convertImageDisplayType(img.imageDisplayType)), - impressionCallbacks = candidate.impressionCallbacks.map(_.map(convertCallback).toList) - ) - - private def getHalfCoverContent( - candidate: onboardingthrift.HalfCover - ): HalfCoverContent = - HalfCoverContent( - displayType = - candidate.displayType.map(convertHalfCoverDisplayType).getOrElse(CoverHalfCoverDisplayType), - primaryText = convertRichText(candidate.primaryText), - primaryCoverCta = convertCoverCta(candidate.primaryButtonAction), - secondaryCoverCta = candidate.secondaryButtonAction.map(convertCoverCta), - secondaryText = candidate.secondaryText.map(convertRichText), - coverImage = candidate.image.map(convertCoverImage), - dismissible = candidate.dismissible, - dismissInfo = candidate.dismissInfo.map(convertDismissInfo), - impressionCallbacks = candidate.impressionCallbacks.map(_.map(convertCallback).toList) - ) - - private def buildClientEventInfo( - injection: Injection - ): Option[ClientEventInfo] = { - injection match { - //To keep parity between TimelineMixer and Product Mixer, inline prompt switches sets the prompt product identifier as the component and no element. Also includes clientEventDetails - case onboardingthrift.Injection.InlinePrompt(candidate) => - val clientEventDetails: ClientEventDetails = - ClientEventDetails( - conversationDetails = None, - timelinesDetails = Some(TimelinesDetails(injectionType = Some("Message"), None, None)), - articleDetails = None, - liveEventDetails = None, - commerceDetails = None - ) - Some( - ClientEventInfo( - component = candidate.injectionIdentifier, - element = None, - details = Some(clientEventDetails), - action = None, - entityToken = None)) - // To keep parity between TLM and PM we swap component and elements. - case onboardingthrift.Injection.RelevancePrompt(candidate) => - Some( - ClientEventInfo( - // Identifier is prefixed with onboarding per TLM - component = Some("onboarding_" + candidate.injectionIdentifier), - element = Some("relevance_prompt"), - details = None, - action = None, - entityToken = None - )) - - case _ => None - } - } - -} - -class UnsupportedFlipPromptException(injection: onboardingthrift.Injection) - extends UnsupportedOperationException( - "Unsupported timeline item " + TransportMarshaller.getSimpleName(injection.getClass)) - -object FlipPromptOffsetInModuleMissing - extends NoSuchElementException( - "FlipPromptOffsetInModuleFeature must be set for the TilesCarousel FLIP injection in PromptCandidateSource") - -object FlipPromptCarouselTileMissing - extends NoSuchElementException( - "FlipPromptCarouselTileFeature must be set for the TilesCarousel FLIP injection in PromptCandidateSource") diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/FlipPromptModuleGrouping.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/FlipPromptModuleGrouping.docx new file mode 100644 index 000000000..4350ee462 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/FlipPromptModuleGrouping.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/FlipPromptModuleGrouping.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/FlipPromptModuleGrouping.scala deleted file mode 100644 index ab6d21057..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/FlipPromptModuleGrouping.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.flexible_injection_pipeline - -import com.twitter.product_mixer.component_library.decorator.urt.GroupByKey -import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.transformer.FlipPromptInjectionsFeature -import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.transformer.FlipPromptOffsetInModuleFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object FlipPromptModuleGrouping extends GroupByKey[PipelineQuery, UniversalNoun[Any], Int] { - override def apply( - query: PipelineQuery, - candidate: UniversalNoun[Any], - candidateFeatures: FeatureMap - ): Option[Int] = { - val injection = candidateFeatures.get(FlipPromptInjectionsFeature) - val offsetInModule = candidateFeatures.getOrElse(FlipPromptOffsetInModuleFeature, None) - - // We return None for any candidate that doesn't have an offsetInModule, so that they are left as independent items. - // Otherwise, we return a hash of the injection instance which will be used to aggregate candidates with matching values into a module. - offsetInModule.map(_ => injection.hashCode()) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/FlipPromptUrtModuleBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/FlipPromptUrtModuleBuilder.docx new file mode 100644 index 000000000..d39365b6d Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/FlipPromptUrtModuleBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/FlipPromptUrtModuleBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/FlipPromptUrtModuleBuilder.scala deleted file mode 100644 index 00383f6ce..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/FlipPromptUrtModuleBuilder.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.flexible_injection_pipeline - -import com.twitter.onboarding.injections.thriftscala.Injection -import com.twitter.onboarding.injections.{thriftscala => onboardingthrift} -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.AutomaticUniqueModuleId -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ModuleIdGeneration -import com.twitter.product_mixer.component_library.model.candidate.BasePromptCandidate -import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.transformer.FlipPromptInjectionsFeature -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseTimelineModuleBuilder -import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.Carousel -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class FlipPromptUrtModuleBuilder[-Query <: PipelineQuery]( - moduleIdGeneration: ModuleIdGeneration = AutomaticUniqueModuleId()) - extends BaseTimelineModuleBuilder[Query, BasePromptCandidate[Any]] { - - override def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[BasePromptCandidate[Any]]] - ): TimelineModule = { - val firstCandidate = candidates.head - val injection = firstCandidate.features.get(FlipPromptInjectionsFeature) - injection match { - case Injection.TilesCarousel(candidate) => - TimelineModule( - id = moduleIdGeneration.moduleId, - sortIndex = None, - entryNamespace = EntryNamespace("flip-timeline-module"), - clientEventInfo = - Some(OnboardingInjectionConversions.convertClientEventInfo(candidate.clientEventInfo)), - feedbackActionInfo = - candidate.feedbackInfo.map(OnboardingInjectionConversions.convertFeedbackInfo), - isPinned = Some(candidate.isPinnedEntry), - // Items are automatically set in the domain marshaller phase - items = Seq.empty, - displayType = Carousel, - header = candidate.header.map(TilesCarouselConversions.convertModuleHeader), - footer = None, - metadata = None, - showMoreBehavior = None - ) - case _ => throw new UnsupportedFlipPromptInModuleException(injection) - } - } -} - -class UnsupportedFlipPromptInModuleException(injection: onboardingthrift.Injection) - extends UnsupportedOperationException( - "Unsupported timeline item in a Flip prompt module " + TransportMarshaller.getSimpleName( - injection.getClass)) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/OnboardingInjectionConversions.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/OnboardingInjectionConversions.docx new file mode 100644 index 000000000..0b32f3a8b Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/OnboardingInjectionConversions.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/OnboardingInjectionConversions.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/OnboardingInjectionConversions.scala deleted file mode 100644 index 92c8d97d7..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/OnboardingInjectionConversions.scala +++ /dev/null @@ -1,361 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.flexible_injection_pipeline - -import com.twitter.onboarding.injections.{thriftscala => onboardingthrift} -import com.twitter.product_mixer.core.model.marshalling.response.urt.cover.CenterCoverHalfCoverDisplayType -import com.twitter.product_mixer.core.model.marshalling.response.urt.cover.CoverBehaviorDismiss -import com.twitter.product_mixer.core.model.marshalling.response.urt.cover.CoverBehaviorNavigate -import com.twitter.product_mixer.core.model.marshalling.response.urt.cover.CoverCta -import com.twitter.product_mixer.core.model.marshalling.response.urt.cover.CoverCtaBehavior -import com.twitter.product_mixer.core.model.marshalling.response.urt.cover.CoverHalfCoverDisplayType -import com.twitter.product_mixer.core.model.marshalling.response.urt.cover.CoverImage -import com.twitter.product_mixer.core.model.marshalling.response.urt.cover.HalfCoverDisplayType -import com.twitter.product_mixer.core.model.marshalling.response.urt.icon._ -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.message.FollowAllMessageActionType -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.message.LargeUserFacepileDisplayType -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.message.MessageAction -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.message.MessageActionType -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.message.MessageImage -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.message.MessageTextAction -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.message.UserFacepile -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Bounce -import com.twitter.product_mixer.core.model.marshalling.response.urt.button.ButtonStyle -import com.twitter.product_mixer.core.model.marshalling.response.urt.button.Default -import com.twitter.product_mixer.core.model.marshalling.response.urt.button.Primary -import com.twitter.product_mixer.core.model.marshalling.response.urt.button.Secondary -import com.twitter.product_mixer.core.model.marshalling.response.urt.button.Text -import com.twitter.product_mixer.core.model.marshalling.response.urt.button.Destructive -import com.twitter.product_mixer.core.model.marshalling.response.urt.button.Neutral -import com.twitter.product_mixer.core.model.marshalling.response.urt.button.DestructiveSecondary -import com.twitter.product_mixer.core.model.marshalling.response.urt.button.DestructiveText -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Callback -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventInfo -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.DeepLink -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Dismiss -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.DismissInfo -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ExternalUrl -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackAction -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackActionInfo -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FollowGeneralContextType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.GeneralContext -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ImageAnimationType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ImageDisplayType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ImageVariant -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FullWidth -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Icon -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.IconSmall -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Url -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.UrtEndpoint -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.UrtEndpointOptions -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.Center -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.Natural -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.Plain -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.ReferenceObject -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichText -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichTextAlignment -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichTextEntity -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichTextFormat -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichTextCashtag -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichTextHashtag -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichTextList -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichTextMention -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichTextUser -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.Strong - -/*** - * Helper class to convert onboarding thrift to product-mixer models - */ -object OnboardingInjectionConversions { - - def convertFeedbackInfo( - feedbackInfo: onboardingthrift.FeedbackInfo - ): FeedbackActionInfo = { - val actions = feedbackInfo.actions.map { - case onboardingthrift.FeedbackAction.DismissAction(dismissAction) => - FeedbackAction( - Dismiss, - prompt = dismissAction.prompt, - confirmation = dismissAction.confirmation, - hasUndoAction = dismissAction.hasUndoAction, - feedbackUrl = dismissAction.feedbackUrl, - childFeedbackActions = - None, - confirmationDisplayType = None, - clientEventInfo = None, - icon = None, - richBehavior = None, - subprompt = None, - encodedFeedbackRequest = None - ) - case onboardingthrift.FeedbackAction.UnknownUnionField(value) => - throw new UnsupportedOperationException(s"Unknown product: $value") - } - - FeedbackActionInfo( - feedbackActions = actions, - feedbackMetadata = None, - displayContext = None, - clientEventInfo = None) - } - - def convertClientEventInfo(input: onboardingthrift.ClientEventInfo): ClientEventInfo = - ClientEventInfo( - component = input.component, - element = input.element, - details = None, - action = input.action, - entityToken = None) - - def convertCallback(callback: onboardingthrift.Callback): Callback = - Callback(callback.endpoint) - - def convertImage(image: onboardingthrift.Image): MessageImage = - MessageImage( - Set(convertImageVariant(image.image)), - backgroundColor = - None - ) - - def convertCoverImage(image: onboardingthrift.Image): CoverImage = - CoverImage( - convertImageVariant(image.image), - imageDisplayType = convertImageDisplayType(image.imageDisplayType), - imageAnimationType = image.imageAnimationType.map(convertImageAnimationType), - ) - - def convertImageDisplayType( - imageDisplayType: onboardingthrift.ImageDisplayType - ): ImageDisplayType = - imageDisplayType match { - case onboardingthrift.ImageDisplayType.Icon => Icon - case onboardingthrift.ImageDisplayType.FullWidth => FullWidth - case onboardingthrift.ImageDisplayType.IconSmall => IconSmall - case onboardingthrift.ImageDisplayType.EnumUnknownImageDisplayType(value) => - throw new UnsupportedOperationException(s"Unknown product: $value") - } - - private def convertImageAnimationType( - imageAnimationType: onboardingthrift.ImageAnimationType - ): ImageAnimationType = - imageAnimationType match { - case onboardingthrift.ImageAnimationType.Bounce => Bounce - case onboardingthrift.ImageAnimationType.EnumUnknownImageAnimationType(value) => - throw new UnsupportedOperationException(s"Unknown product: $value") - } - - def convertImageVariant(imageVariant: onboardingthrift.ImageVariant): ImageVariant = - ImageVariant( - url = imageVariant.url, - width = imageVariant.width, - height = imageVariant.height, - palette = None) - - def convertButtonAction( - buttonAction: onboardingthrift.ButtonAction - ): MessageTextAction = - MessageTextAction( - buttonAction.text, - MessageAction( - dismissOnClick = buttonAction.dismissOnClick.getOrElse(true), - url = getActionUrl(buttonAction), - clientEventInfo = Some(convertClientEventInfo(buttonAction.clientEventInfo)), - onClickCallbacks = buttonAction.callbacks.map(_.map(convertCallback).toList) - ) - ) - - private def getActionUrl(buttonAction: onboardingthrift.ButtonAction) = - buttonAction.buttonBehavior match { - case onboardingthrift.ButtonBehavior.Navigate(navigate) => Some(navigate.url.url) - case onboardingthrift.ButtonBehavior.Dismiss(_) => None - case onboardingthrift.ButtonBehavior.UnknownUnionField(value) => - throw new UnsupportedOperationException(s"Unknown product: $value") - } - - def convertRichText( - richText: com.twitter.onboarding.injections.thriftscala.RichText - ): RichText = { - val entities = richText.entities.map(entity => - RichTextEntity( - entity.fromIndex, - entity.toIndex, - entity.ref.map(convertRef), - entity.format.map(convertFormat))) - RichText( - text = richText.text, - entities = entities.toList, - rtl = richText.rtl, - alignment = richText.alignment.map(convertAlignment)) - } - - private def convertAlignment(alignment: onboardingthrift.RichTextAlignment): RichTextAlignment = - alignment match { - case onboardingthrift.RichTextAlignment.Natural => Natural - case onboardingthrift.RichTextAlignment.Center => Center - case onboardingthrift.RichTextAlignment.EnumUnknownRichTextAlignment(value) => - throw new UnsupportedOperationException(s"Unknown product: $value") - } - - private def convertRef(ref: onboardingthrift.ReferenceObject): ReferenceObject = - ref match { - case onboardingthrift.ReferenceObject.User(user) => RichTextUser(user.id) - case onboardingthrift.ReferenceObject.Mention(mention) => - RichTextMention(mention.id, mention.screenName) - case onboardingthrift.ReferenceObject.Hashtag(hashtag) => RichTextHashtag(hashtag.text) - - case onboardingthrift.ReferenceObject.Cashtag(cashtag) => RichTextCashtag(cashtag.text) - case onboardingthrift.ReferenceObject.TwitterList(twList) => - RichTextList(twList.id, twList.url) - case onboardingthrift.ReferenceObject.Url(url) => RichTextHashtag(url.url) - case onboardingthrift.ReferenceObject.UnknownUnionField(value) => - throw new UnsupportedOperationException(s"Unknown product: $value") - } - - private def convertFormat(format: onboardingthrift.RichTextFormat): RichTextFormat = - format match { - case onboardingthrift.RichTextFormat.Plain => Plain - case onboardingthrift.RichTextFormat.Strong => Strong - case onboardingthrift.RichTextFormat.EnumUnknownRichTextFormat(value) => - throw new UnsupportedOperationException(s"Unknown product: $value") - } - - // Specific to Message prompt - def convertSocialContext(socialContext: onboardingthrift.RichText): SocialContext = - GeneralContext( - contextType = FollowGeneralContextType, - text = socialContext.text, - url = None, - contextImageUrls = None, - landingUrl = None) - - def convertUserFacePile( - userFacepile: onboardingthrift.PromptUserFacepile - ): UserFacepile = - UserFacepile( - userIds = userFacepile.userIds.toList, - featuredUserIds = userFacepile.featuredUserIds.toList, - action = userFacepile.action.map(convertButtonAction), - actionType = userFacepile.actionType.map(convertUserFacePileActionType), - displaysFeaturingText = userFacepile.displaysFeaturingText, - displayType = Some(LargeUserFacepileDisplayType) - ) - - private def convertUserFacePileActionType( - actionType: onboardingthrift.FacepileActionType - ): MessageActionType = - actionType match { - case onboardingthrift.FacepileActionType.FollowAll => FollowAllMessageActionType - case onboardingthrift.FacepileActionType.EnumUnknownFacepileActionType(value) => - throw new UnsupportedOperationException(s"Unknown product: $value") - } - - // Specific to Cover - - def convertHalfCoverDisplayType( - displayType: onboardingthrift.HalfCoverDisplayType - ): HalfCoverDisplayType = - displayType match { - case onboardingthrift.HalfCoverDisplayType.Cover => CoverHalfCoverDisplayType - case onboardingthrift.HalfCoverDisplayType.CenterCover => - CenterCoverHalfCoverDisplayType - case onboardingthrift.HalfCoverDisplayType.EnumUnknownHalfCoverDisplayType(value) => - throw new UnsupportedOperationException(s"Unknown product: $value") - } - - def convertDismissInfo(dismissInfo: onboardingthrift.DismissInfo): DismissInfo = - DismissInfo(dismissInfo.callbacks.map(_.map(convertCallback))) - - def convertCoverCta( - buttonAction: onboardingthrift.ButtonAction - ): CoverCta = - CoverCta( - buttonAction.text, - ctaBehavior = convertCoverCtaBehavior(buttonAction.buttonBehavior), - callbacks = buttonAction.callbacks.map(_.map(convertCallback).toList), - clientEventInfo = Some(convertClientEventInfo(buttonAction.clientEventInfo)), - icon = buttonAction.icon.map(covertHorizonIcon), - buttonStyle = buttonAction.buttonStyle.map(covertButtonStyle) - ) - - private def convertCoverCtaBehavior( - behavior: onboardingthrift.ButtonBehavior - ): CoverCtaBehavior = - behavior match { - case onboardingthrift.ButtonBehavior.Navigate(navigate) => - CoverBehaviorNavigate(convertUrl(navigate.url)) - case onboardingthrift.ButtonBehavior.Dismiss(dismiss) => - CoverBehaviorDismiss(dismiss.feedbackMessage.map(convertRichText)) - case onboardingthrift.ButtonBehavior.UnknownUnionField(value) => - throw new UnsupportedOperationException(s"Unknown product: $value") - } - - private def covertButtonStyle(bStyle: onboardingthrift.CtaButtonStyle): ButtonStyle = - bStyle match { - case onboardingthrift.CtaButtonStyle.Default => Default - case onboardingthrift.CtaButtonStyle.Primary => Primary - case onboardingthrift.CtaButtonStyle.Secondary => Secondary - case onboardingthrift.CtaButtonStyle.Text => Text - case onboardingthrift.CtaButtonStyle.Destructive => Destructive - case onboardingthrift.CtaButtonStyle.Neutral => Neutral - case onboardingthrift.CtaButtonStyle.DestructiveSecondary => DestructiveSecondary - case onboardingthrift.CtaButtonStyle.DestructiveText => DestructiveText - case onboardingthrift.CtaButtonStyle.EnumUnknownCtaButtonStyle(value) => - throw new UnsupportedOperationException(s"Unknown product: $value") - } - - private def covertHorizonIcon(icon: onboardingthrift.HorizonIcon): HorizonIcon = - icon match { - case onboardingthrift.HorizonIcon.Bookmark => Bookmark - case onboardingthrift.HorizonIcon.Moment => Moment - case onboardingthrift.HorizonIcon.Debug => Debug - case onboardingthrift.HorizonIcon.Error => Error - case onboardingthrift.HorizonIcon.Follow => Follow - case onboardingthrift.HorizonIcon.Unfollow => Unfollow - case onboardingthrift.HorizonIcon.Smile => Smile - case onboardingthrift.HorizonIcon.Frown => Frown - case onboardingthrift.HorizonIcon.Help => Help - case onboardingthrift.HorizonIcon.Link => Link - case onboardingthrift.HorizonIcon.Message => Message - case onboardingthrift.HorizonIcon.No => No - case onboardingthrift.HorizonIcon.Outgoing => Outgoing - case onboardingthrift.HorizonIcon.Pin => Pin - case onboardingthrift.HorizonIcon.Retweet => Retweet - case onboardingthrift.HorizonIcon.Speaker => Speaker - case onboardingthrift.HorizonIcon.Trashcan => Trashcan - case onboardingthrift.HorizonIcon.Feedback => Feedback - case onboardingthrift.HorizonIcon.FeedbackClose => FeedbackClose - case onboardingthrift.HorizonIcon.EyeOff => EyeOff - case onboardingthrift.HorizonIcon.Moderation => Moderation - case onboardingthrift.HorizonIcon.Topic => Topic - case onboardingthrift.HorizonIcon.TopicClose => TopicClose - case onboardingthrift.HorizonIcon.Flag => Flag - case onboardingthrift.HorizonIcon.TopicFilled => TopicFilled - case onboardingthrift.HorizonIcon.NotificationsFollow => NotificationsFollow - case onboardingthrift.HorizonIcon.Person => Person - case onboardingthrift.HorizonIcon.Logo => Logo - case onboardingthrift.HorizonIcon.EnumUnknownHorizonIcon(value) => - throw new UnsupportedOperationException(s"Unknown product: $value") - - } - - def convertUrl(url: onboardingthrift.Url): Url = { - val urlType = url.urlType match { - case onboardingthrift.UrlType.ExternalUrl => ExternalUrl - case onboardingthrift.UrlType.DeepLink => DeepLink - case onboardingthrift.UrlType.UrtEndpoint => UrtEndpoint - case onboardingthrift.UrlType.EnumUnknownUrlType(value) => - throw new UnsupportedOperationException(s"Unknown product: $value") - } - Url(urlType, url.url, url.urtEndpointOptions.map(convertUrtEndpointOptions)) - } - - private def convertUrtEndpointOptions( - urtEndpointOptions: onboardingthrift.UrtEndpointOptions - ): UrtEndpointOptions = - UrtEndpointOptions( - requestParams = urtEndpointOptions.requestParams.map(_.toMap), - title = urtEndpointOptions.title, - cacheId = urtEndpointOptions.cacheId, - subtitle = urtEndpointOptions.subtitle - ) - -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/RelevancePromptConversions.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/RelevancePromptConversions.docx new file mode 100644 index 000000000..79551b7cd Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/RelevancePromptConversions.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/RelevancePromptConversions.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/RelevancePromptConversions.scala deleted file mode 100644 index 78e69ee3e..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/RelevancePromptConversions.scala +++ /dev/null @@ -1,76 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.flexible_injection_pipeline - -import com.twitter.onboarding.injections.{thriftscala => onboardingthrift} -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.prompt.Compact -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.prompt.Large -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.prompt.Normal -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.prompt.RelevancePromptContent -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.prompt.RelevancePromptDisplayType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Callback - -/*** - * Helper class to convert Relevance Prompt related onboarding thrift to product-mixer models - */ -object RelevancePromptConversions { - def convertContent( - candidate: onboardingthrift.RelevancePrompt - ): RelevancePromptContent = - RelevancePromptContent( - displayType = convertDisplayType(candidate.displayType), - title = candidate.title.text, - confirmation = buildConfirmation(candidate), - isRelevantText = candidate.isRelevantButton.text, - notRelevantText = candidate.notRelevantButton.text, - isRelevantCallback = convertCallbacks(candidate.isRelevantButton.callbacks), - notRelevantCallback = convertCallbacks(candidate.notRelevantButton.callbacks), - isRelevantFollowUp = None, - notRelevantFollowUp = None - ) - - // Based on com.twitter.timelinemixer.injection.model.candidate.OnboardingRelevancePromptDisplayType#fromThrift - def convertDisplayType( - displayType: onboardingthrift.RelevancePromptDisplayType - ): RelevancePromptDisplayType = - displayType match { - case onboardingthrift.RelevancePromptDisplayType.Normal => Normal - case onboardingthrift.RelevancePromptDisplayType.Compact => Compact - case onboardingthrift.RelevancePromptDisplayType.Large => Large - case onboardingthrift.RelevancePromptDisplayType - .EnumUnknownRelevancePromptDisplayType(value) => - throw new UnsupportedOperationException(s"Unknown display type: $value") - } - - // Based on com.twitter.timelinemixer.injection.model.injection.OnboardingRelevancePromptInjection#buildConfirmation - def buildConfirmation(candidate: onboardingthrift.RelevancePrompt): String = { - val isRelevantTextConfirmation = - buttonToDismissFeedbackText(candidate.isRelevantButton).getOrElse("") - val notRelevantTextConfirmation = - buttonToDismissFeedbackText(candidate.notRelevantButton).getOrElse("") - if (isRelevantTextConfirmation != notRelevantTextConfirmation) - throw new IllegalArgumentException( - s"""confirmation text not consistent for two buttons, : - isRelevantConfirmation: ${isRelevantTextConfirmation} - notRelevantConfirmation: ${notRelevantTextConfirmation} - """ - ) - isRelevantTextConfirmation - } - - // Based on com.twitter.timelinemixer.injection.model.candidate.OnboardingInjectionAction#fromThrift - def buttonToDismissFeedbackText(button: onboardingthrift.ButtonAction): Option[String] = { - button.buttonBehavior match { - case onboardingthrift.ButtonBehavior.Dismiss(d) => d.feedbackMessage.map(_.text) - case _ => None - } - } - - // Based on com.twitter.timelinemixer.injection.model.injection.OnboardingRelevancePromptInjection#buildCallback - def convertCallbacks(onboardingCallbacks: Option[Seq[onboardingthrift.Callback]]): Callback = { - OnboardingInjectionConversions.convertCallback( - onboardingCallbacks - .flatMap(_.headOption) - .getOrElse( - throw new NoSuchElementException(s"Callback must be provided for the Relevance Prompt") - )) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/TilesCarouselConversions.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/TilesCarouselConversions.docx new file mode 100644 index 000000000..f699a8c41 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/TilesCarouselConversions.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/TilesCarouselConversions.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/TilesCarouselConversions.scala deleted file mode 100644 index dd4ce7c75..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/flexible_injection_pipeline/TilesCarouselConversions.scala +++ /dev/null @@ -1,154 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.flexible_injection_pipeline - -import com.twitter.onboarding.injections.{thriftscala => onboardingthrift} -import com.twitter.product_mixer.component_library.decorator.urt.builder.flexible_injection_pipeline.OnboardingInjectionConversions.convertImageVariant -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.Classic -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.BlackRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.ClearRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.DeepBlueRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.DeepGrayRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.DeepGreenRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.DeepOrangeRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.DeepPurpleRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.DeepRedRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.DeepYellowRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.FadedBlueRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.FadedGrayRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.FadedGreenRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.FadedOrangeRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.FadedPurpleRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.FadedRedRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.FadedYellowRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.FaintBlueRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.FaintGrayRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.LightBlueRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.LightGrayRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.LightGreenRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.LightOrangeRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.LightPurpleRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.LightRedRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.LightYellowRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.MediumGrayRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.MediumGreenRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.MediumOrangeRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.MediumPurpleRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.MediumRedRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.MediumYellowRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.RosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.TextBlackRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.TextBlueRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.TwitterBlueRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.WhiteRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tile.CallToActionTileContent -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tile.StandardTileContent -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tile.TileItem -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Badge -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleHeader - -object TilesCarouselConversions { - // Tiles Carousel Conversions - def convertTile(tile: onboardingthrift.Tile, id: Long): TileItem = { - tile.content match { - case standard: onboardingthrift.TileContent.Standard => - TileItem( - id = id, - sortIndex = None, - clientEventInfo = - Some(OnboardingInjectionConversions.convertClientEventInfo(tile.clientEventInfo)), - feedbackActionInfo = None, - title = standard.standard.title, - supportingText = "", - url = tile.url.map(OnboardingInjectionConversions.convertUrl), - image = tile.image.map(img => convertImageVariant(img.image)), - content = StandardTileContent( - title = standard.standard.title, - supportingText = "", - badge = standard.standard.badge.map(convertTileBadge) - ) - ) - case cta: onboardingthrift.TileContent.CallToAction => - TileItem( - id = id, - sortIndex = None, - clientEventInfo = - Some(OnboardingInjectionConversions.convertClientEventInfo(tile.clientEventInfo)), - feedbackActionInfo = None, - title = cta.callToAction.text, - supportingText = "", - url = tile.url.map(OnboardingInjectionConversions.convertUrl), - image = None, - content = CallToActionTileContent( - text = cta.callToAction.text, - richText = None, - ctaButton = None - ) - ) - case _ => - throw new UnsupportedTileCarouselConversionException(s"Tile Content: ${tile.content}") - } - } - - private def convertTileBadge(badge: onboardingthrift.Badge): Badge = { - Badge( - text = badge.text, - textColorName = badge.textColor.map(convertRosettaColor), - backgroundColorName = badge.backgroundColor.map(convertRosettaColor)) - } - - def convertModuleHeader(header: onboardingthrift.TilesCarouselHeader): ModuleHeader = { - ModuleHeader(header.header, None, None, None, None, Classic) - } - - private def convertRosettaColor(color: onboardingthrift.RosettaColor): RosettaColor = - color match { - case onboardingthrift.RosettaColor.White => WhiteRosettaColor - case onboardingthrift.RosettaColor.Black => BlackRosettaColor - case onboardingthrift.RosettaColor.Clear => ClearRosettaColor - case onboardingthrift.RosettaColor.TextBlack => TextBlackRosettaColor - case onboardingthrift.RosettaColor.TextBlue => TextBlueRosettaColor - - case onboardingthrift.RosettaColor.DeepGray => DeepGrayRosettaColor - case onboardingthrift.RosettaColor.MediumGray => MediumGrayRosettaColor - case onboardingthrift.RosettaColor.LightGray => LightGrayRosettaColor - case onboardingthrift.RosettaColor.FadedGray => FadedGrayRosettaColor - case onboardingthrift.RosettaColor.FaintGray => FaintGrayRosettaColor - - case onboardingthrift.RosettaColor.DeepOrange => DeepOrangeRosettaColor - case onboardingthrift.RosettaColor.MediumOrange => MediumOrangeRosettaColor - case onboardingthrift.RosettaColor.LightOrange => LightOrangeRosettaColor - case onboardingthrift.RosettaColor.FadedOrange => FadedOrangeRosettaColor - - case onboardingthrift.RosettaColor.DeepYellow => DeepYellowRosettaColor - case onboardingthrift.RosettaColor.MediumYellow => MediumYellowRosettaColor - case onboardingthrift.RosettaColor.LightYellow => LightYellowRosettaColor - case onboardingthrift.RosettaColor.FadedYellow => FadedYellowRosettaColor - - case onboardingthrift.RosettaColor.DeepGreen => DeepGreenRosettaColor - case onboardingthrift.RosettaColor.MediumGreen => MediumGreenRosettaColor - case onboardingthrift.RosettaColor.LightGreen => LightGreenRosettaColor - case onboardingthrift.RosettaColor.FadedGreen => FadedGreenRosettaColor - - case onboardingthrift.RosettaColor.DeepBlue => DeepBlueRosettaColor - case onboardingthrift.RosettaColor.TwitterBlue => TwitterBlueRosettaColor - case onboardingthrift.RosettaColor.LightBlue => LightBlueRosettaColor - case onboardingthrift.RosettaColor.FadedBlue => FadedBlueRosettaColor - case onboardingthrift.RosettaColor.FaintBlue => FaintBlueRosettaColor - - case onboardingthrift.RosettaColor.DeepPurple => DeepPurpleRosettaColor - case onboardingthrift.RosettaColor.MediumPurple => MediumPurpleRosettaColor - case onboardingthrift.RosettaColor.LightPurple => LightPurpleRosettaColor - case onboardingthrift.RosettaColor.FadedPurple => FadedPurpleRosettaColor - - case onboardingthrift.RosettaColor.DeepRed => DeepRedRosettaColor - case onboardingthrift.RosettaColor.MediumRed => MediumRedRosettaColor - case onboardingthrift.RosettaColor.LightRed => LightRedRosettaColor - case onboardingthrift.RosettaColor.FadedRed => FadedRedRosettaColor - case onboardingthrift.RosettaColor.EnumUnknownRosettaColor(i) => - throw new UnknownThriftEnumException("RosettaColor") - } - class UnknownThriftEnumException(enumName: String) - extends Exception(s"Unknown Thrift Enum Found: ${enumName}") - - class UnsupportedTileCarouselConversionException(UnsupportedTileType: String) - extends Exception(s"Unsupported Tile Type Found: ${UnsupportedTileType}") -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/icon/HorizonIconBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/icon/HorizonIconBuilder.docx new file mode 100644 index 000000000..9b06f3fee Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/icon/HorizonIconBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/icon/HorizonIconBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/icon/HorizonIconBuilder.scala deleted file mode 100644 index dd9a59e6c..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/icon/HorizonIconBuilder.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.icon - -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.icon.BaseHorizonIconBuilder -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.icon.HorizonIcon -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class HorizonIconBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( - icon: HorizonIcon) - extends BaseHorizonIconBuilder[Query, Candidate] { - - override def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[Candidate]] - ): Option[HorizonIcon] = Some(icon) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/ad/AdsCandidateUrtItemBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/ad/AdsCandidateUrtItemBuilder.docx new file mode 100644 index 000000000..1b1a5be76 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/ad/AdsCandidateUrtItemBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/ad/AdsCandidateUrtItemBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/ad/AdsCandidateUrtItemBuilder.scala deleted file mode 100644 index 1593485fc..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/ad/AdsCandidateUrtItemBuilder.scala +++ /dev/null @@ -1,274 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.ad - -import com.twitter.ads.adserver.{thriftscala => ads} -import com.twitter.adserver.{thriftscala => adserver} -import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder.TweetClientEventInfoElement -import com.twitter.product_mixer.component_library.model.candidate.ads.AdsCandidate -import com.twitter.product_mixer.component_library.model.candidate.ads.AdsTweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineItem -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.Tweet -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetDisplayType -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.AdMetadataContainer -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.Amplify -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.CallToAction -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.ClickTrackingInfo -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.DcmUrlOverrideType -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.DirectSponsorshipType -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.DisclaimerIssue -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.DisclaimerPolitical -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.DisclaimerType -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.DisclosureType -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.DynamicPrerollType -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.Earned -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.IndirectSponsorshipType -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.Issue -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.LiveTvEvent -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.Marketplace -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.MediaInfo -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.NoDisclosure -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.NoSponsorshipSponsorshipType -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.Political -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.Preroll -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.PrerollMetadata -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.PromotedMetadata -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.SkAdNetworkData -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.SponsorshipType -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.UnknownUrlOverrideType -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.VideoVariant -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.util.AdMetadataContainerSerializer -import com.twitter.timelines.util.PrerollMetadataSerializer - -/** - * [[AdsCandidateUrtItemBuilder]] takes a [[AdsCandidate]] (with a [[Query]] as additional context) - * and converts it into the Product Mixer URT representation, or throws an error. - * - * Currently, the only supported form for URT representation of the [[AdsCandidate]] is a [[Tweet]], - * but in the future it could be expanded to handle other forms of ads. - * - * @param tweetClientEventInfoBuilder Optionally, provide a ClientEventInfoBuilder for Tweets - * that given an AdsTweetCandidate and element of "tweet". - * @param tweetDisplayType Should be [[EmphasizedPromotedTweet]] on Profile timelines, - * otherwise [[Tweet]] - */ -case class AdsCandidateUrtItemBuilder[Query <: PipelineQuery]( - tweetClientEventInfoBuilder: Option[BaseClientEventInfoBuilder[Query, AdsTweetCandidate]] = None, - contextualTweetRefBuilder: Option[ContextualTweetRefBuilder[AdsTweetCandidate]] = None, - tweetDisplayType: TweetDisplayType = Tweet) - extends CandidateUrtEntryBuilder[Query, AdsCandidate, TimelineItem] { - - override def apply( - pipelineQuery: Query, - candidate: AdsCandidate, - candidateFeatures: FeatureMap - ): TimelineItem = { - candidate match { - case tweetCandidate: AdsTweetCandidate => - TweetItem( - id = tweetCandidate.id, - entryNamespace = TweetItem.PromotedTweetEntryNamespace, - sortIndex = None, // Sort indexes are automatically set in the domain marshaller phase - clientEventInfo = tweetClientEventInfoBuilder.flatMap( - _.apply( - pipelineQuery, - tweetCandidate, - candidateFeatures, - Some(TweetClientEventInfoElement))), - feedbackActionInfo = None, - isPinned = None, - entryIdToReplace = None, - socialContext = None, - highlights = None, - innerTombstoneInfo = None, - timelinesScoreInfo = None, - hasModeratedReplies = None, - forwardPivot = None, - innerForwardPivot = None, - conversationAnnotation = None, - promotedMetadata = Some(promotedMetadata(tweetCandidate.adImpression)), - displayType = tweetDisplayType, - contextualTweetRef = contextualTweetRefBuilder.flatMap(_.apply(tweetCandidate)), - prerollMetadata = prerollMetadata(tweetCandidate.adImpression), - replyBadge = None, - destination = None - ) - } - } - - private def promotedMetadata(impression: adserver.AdImpression) = { - PromotedMetadata( - advertiserId = impression.advertiserId, - impressionString = impression.impressionString, - disclosureType = impression.disclosureType.map(convertDisclosureType), - experimentValues = impression.experimentValues.map(_.toMap), - promotedTrendId = impression.promotedTrendId.map(_.toLong), - promotedTrendName = impression.promotedTrendName, - promotedTrendQueryTerm = impression.promotedTrendQueryTerm, - promotedTrendDescription = impression.promotedTrendDescription, - clickTrackingInfo = impression.clickTrackingInfo.map(convertClickTrackingInfo), - adMetadataContainer = adMetadataContainer(impression) - ) - } - - private def convertDisclosureType( - disclosureType: adserver.DisclosureType - ): DisclosureType = disclosureType match { - case adserver.DisclosureType.None => NoDisclosure - case adserver.DisclosureType.Political => Political - case adserver.DisclosureType.Earned => Earned - case adserver.DisclosureType.Issue => Issue - case _ => throw new UnsupportedDisclosureTypeException(disclosureType) - } - - private def convertClickTrackingInfo( - clickTracking: adserver.ClickTrackingInfo - ): ClickTrackingInfo = ClickTrackingInfo( - urlParams = clickTracking.urlParams.getOrElse(Map.empty), - urlOverride = clickTracking.urlOverride, - urlOverrideType = clickTracking.urlOverrideType.map { - case adserver.UrlOverrideType.Unknown => UnknownUrlOverrideType - case adserver.UrlOverrideType.Dcm => DcmUrlOverrideType - case _ => throw new UnsupportedClickTrackingInfoException(clickTracking) - } - ) - - private def prerollMetadata(adImpression: adserver.AdImpression): Option[PrerollMetadata] = { - adImpression.serializedPrerollMetadata - .flatMap(PrerollMetadataSerializer.deserialize).map { metadata => - PrerollMetadata( - metadata.preroll.map(convertPreroll), - metadata.videoAnalyticsScribePassthrough - ) - } - } - - private def adMetadataContainer( - adImpression: adserver.AdImpression - ): Option[AdMetadataContainer] = { - adImpression.serializedAdMetadataContainer - .flatMap(AdMetadataContainerSerializer.deserialize).map { container => - AdMetadataContainer( - removePromotedAttributionForPreroll = container.removePromotedAttributionForPreroll, - sponsorshipCandidate = container.sponsorshipCandidate, - sponsorshipOrganization = container.sponsorshipOrganization, - sponsorshipOrganizationWebsite = container.sponsorshipOrganizationWebsite, - sponsorshipType = container.sponsorshipType.map(convertSponsorshipType), - disclaimerType = container.disclaimerType.map(convertDisclaimerType), - skAdNetworkDataList = container.skAdNetworkDataList.map(convertSkAdNetworkDataList), - unifiedCardOverride = container.unifiedCardOverride - ) - } - } - - private def convertSponsorshipType( - sponsorshipType: ads.SponsorshipType - ): SponsorshipType = sponsorshipType match { - case ads.SponsorshipType.Direct => DirectSponsorshipType - case ads.SponsorshipType.Indirect => IndirectSponsorshipType - case ads.SponsorshipType.NoSponsorship => NoSponsorshipSponsorshipType - // Thrift has extras (e.g. Sponsorship4) that are not used in practice - case _ => throw new UnsupportedSponsorshipTypeException(sponsorshipType) - } - - private def convertDisclaimerType( - disclaimerType: ads.DisclaimerType - ): DisclaimerType = disclaimerType match { - case ads.DisclaimerType.Political => DisclaimerPolitical - case ads.DisclaimerType.Issue => DisclaimerIssue - case _ => throw new UnsupportedDisclaimerTypeException(disclaimerType) - } - - private def convertDynamicPrerollType( - dynamicPrerollType: ads.DynamicPrerollType - ): DynamicPrerollType = - dynamicPrerollType match { - case ads.DynamicPrerollType.Amplify => Amplify - case ads.DynamicPrerollType.Marketplace => Marketplace - case ads.DynamicPrerollType.LiveTvEvent => LiveTvEvent - case _ => throw new UnsupportedDynamicPrerollTypeException(dynamicPrerollType) - } - - private def convertMediaInfo(mediaInfo: ads.MediaInfo): MediaInfo = { - MediaInfo( - uuid = mediaInfo.uuid, - publisherId = mediaInfo.publisherId, - callToAction = mediaInfo.callToAction.map(convertCallToAction), - durationMillis = mediaInfo.durationMillis, - videoVariants = mediaInfo.videoVariants.map(convertVideoVariants), - advertiserName = mediaInfo.advertiserName, - renderAdByAdvertiserName = mediaInfo.renderAdByAdvertiserName, - advertiserProfileImageUrl = mediaInfo.advertiserProfileImageUrl - ) - } - - private def convertVideoVariants(videoVariants: Seq[ads.VideoVariant]): Seq[VideoVariant] = { - videoVariants.map(videoVariant => - VideoVariant( - url = videoVariant.url, - contentType = videoVariant.contentType, - bitrate = videoVariant.bitrate - )) - } - - private def convertCallToAction(callToAction: ads.CallToAction): CallToAction = { - CallToAction( - callToActionType = callToAction.callToActionType, - url = callToAction.url - ) - } - - private def convertPreroll( - preroll: ads.Preroll - ): Preroll = { - Preroll( - preroll.prerollId, - preroll.dynamicPrerollType.map(convertDynamicPrerollType), - preroll.mediaInfo.map(convertMediaInfo) - ) - } - - private def convertSkAdNetworkDataList( - skAdNetworkDataList: Seq[ads.SkAdNetworkData] - ): Seq[SkAdNetworkData] = skAdNetworkDataList.map(sdAdNetwork => - SkAdNetworkData( - version = sdAdNetwork.version, - srcAppId = sdAdNetwork.srcAppId, - dstAppId = sdAdNetwork.dstAppId, - adNetworkId = sdAdNetwork.adNetworkId, - campaignId = sdAdNetwork.campaignId, - impressionTimeInMillis = sdAdNetwork.impressionTimeInMillis, - nonce = sdAdNetwork.nonce, - signature = sdAdNetwork.signature, - fidelityType = sdAdNetwork.fidelityType - )) -} - -class UnsupportedClickTrackingInfoException(clickTrackingInfo: adserver.ClickTrackingInfo) - extends UnsupportedOperationException( - s"Unsupported ClickTrackingInfo: $clickTrackingInfo" - ) - -class UnsupportedDisclaimerTypeException(disclaimerType: ads.DisclaimerType) - extends UnsupportedOperationException( - s"Unsupported DisclaimerType: $disclaimerType" - ) - -class UnsupportedDisclosureTypeException(disclosureType: adserver.DisclosureType) - extends UnsupportedOperationException( - s"Unsupported DisclosureType: $disclosureType" - ) - -class UnsupportedDynamicPrerollTypeException(dynamicPrerollType: ads.DynamicPrerollType) - extends UnsupportedOperationException( - s"Unsupported DynamicPrerollType: $dynamicPrerollType" - ) - -class UnsupportedSponsorshipTypeException(sponsorshipType: ads.SponsorshipType) - extends UnsupportedOperationException( - s"Unsupported SponsorshipType: $sponsorshipType" - ) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/DurationParamBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/DurationParamBuilder.docx new file mode 100644 index 000000000..b3cd6b70c Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/DurationParamBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/DurationParamBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/DurationParamBuilder.scala deleted file mode 100644 index 0a4e76319..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/DurationParamBuilder.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.alert - -import com.twitter.product_mixer.component_library.model.candidate.ShowAlertCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.item.alert.BaseDurationBuilder -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration - -case class DurationParamBuilder( - durationParam: Param[Duration]) - extends BaseDurationBuilder[PipelineQuery] { - - def apply( - query: PipelineQuery, - candidate: ShowAlertCandidate, - features: FeatureMap - ): Option[Duration] = - Some(query.params(durationParam)) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/ShowAlertCandidateUrtItemBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/ShowAlertCandidateUrtItemBuilder.docx new file mode 100644 index 000000000..9fd462d61 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/ShowAlertCandidateUrtItemBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/ShowAlertCandidateUrtItemBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/ShowAlertCandidateUrtItemBuilder.scala deleted file mode 100644 index d2028181c..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/ShowAlertCandidateUrtItemBuilder.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.alert - -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.alert.ShowAlertCandidateUrtItemBuilder.ShowAlertClientEventInfoElement -import com.twitter.product_mixer.component_library.model.candidate.ShowAlertCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.item.alert.BaseDurationBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.item.alert.BaseShowAlertColorConfigurationBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.item.alert.BaseShowAlertDisplayLocationBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.item.alert.BaseShowAlertIconDisplayInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.item.alert.BaseShowAlertNavigationMetadataBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.item.alert.BaseShowAlertUserIdsBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.ShowAlert -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.richtext.BaseRichTextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.alert.ShowAlertType - -object ShowAlertCandidateUrtItemBuilder { - val ShowAlertClientEventInfoElement: String = "showAlert" -} - -case class ShowAlertCandidateUrtItemBuilder[-Query <: PipelineQuery]( - alertType: ShowAlertType, - displayLocationBuilder: BaseShowAlertDisplayLocationBuilder[Query], - colorConfigBuilder: BaseShowAlertColorConfigurationBuilder[Query], - triggerDelayBuilder: Option[BaseDurationBuilder[Query]] = None, - displayDurationBuilder: Option[BaseDurationBuilder[Query]] = None, - clientEventInfoBuilder: Option[BaseClientEventInfoBuilder[Query, ShowAlertCandidate]] = None, - collapseDelayBuilder: Option[BaseDurationBuilder[Query]] = None, - userIdsBuilder: Option[BaseShowAlertUserIdsBuilder[Query]] = None, - richTextBuilder: Option[BaseRichTextBuilder[Query, ShowAlertCandidate]] = None, - iconDisplayInfoBuilder: Option[BaseShowAlertIconDisplayInfoBuilder[Query]] = None, - navigationMetadataBuilder: Option[BaseShowAlertNavigationMetadataBuilder[Query]] = None) - extends CandidateUrtEntryBuilder[ - Query, - ShowAlertCandidate, - ShowAlert - ] { - - override def apply( - query: Query, - candidate: ShowAlertCandidate, - features: FeatureMap, - ): ShowAlert = ShowAlert( - id = candidate.id, - sortIndex = None, - alertType = alertType, - triggerDelay = triggerDelayBuilder.flatMap(_.apply(query, candidate, features)), - displayDuration = displayDurationBuilder.flatMap(_.apply(query, candidate, features)), - clientEventInfo = clientEventInfoBuilder.flatMap( - _.apply(query, candidate, features, Some(ShowAlertClientEventInfoElement))), - collapseDelay = collapseDelayBuilder.flatMap(_.apply(query, candidate, features)), - userIds = userIdsBuilder.flatMap(_.apply(query, candidate, features)), - richText = richTextBuilder.map(_.apply(query, candidate, features)), - iconDisplayInfo = iconDisplayInfoBuilder.flatMap(_.apply(query, candidate, features)), - displayLocation = displayLocationBuilder(query, candidate, features), - colorConfig = colorConfigBuilder(query, candidate, features), - navigationMetadata = navigationMetadataBuilder.flatMap(_.apply(query, candidate, features)), - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/StaticShowAlertColorConfigurationBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/StaticShowAlertColorConfigurationBuilder.docx new file mode 100644 index 000000000..768c4c06b Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/StaticShowAlertColorConfigurationBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/StaticShowAlertColorConfigurationBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/StaticShowAlertColorConfigurationBuilder.scala deleted file mode 100644 index 322d16401..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/StaticShowAlertColorConfigurationBuilder.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.alert - -import com.twitter.product_mixer.component_library.model.candidate.ShowAlertCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.item.alert.BaseShowAlertColorConfigurationBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.alert.ShowAlertColorConfiguration -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class StaticShowAlertColorConfigurationBuilder[-Query <: PipelineQuery]( - configuration: ShowAlertColorConfiguration) - extends BaseShowAlertColorConfigurationBuilder[Query] { - - def apply( - query: Query, - candidate: ShowAlertCandidate, - features: FeatureMap - ): ShowAlertColorConfiguration = configuration -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/StaticShowAlertDisplayLocationBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/StaticShowAlertDisplayLocationBuilder.docx new file mode 100644 index 000000000..4ea924384 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/StaticShowAlertDisplayLocationBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/StaticShowAlertDisplayLocationBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/StaticShowAlertDisplayLocationBuilder.scala deleted file mode 100644 index 74ea989dd..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/StaticShowAlertDisplayLocationBuilder.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.alert - -import com.twitter.product_mixer.component_library.model.candidate.ShowAlertCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.item.alert.BaseShowAlertDisplayLocationBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.alert.ShowAlertDisplayLocation -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class StaticShowAlertDisplayLocationBuilder[-Query <: PipelineQuery]( - location: ShowAlertDisplayLocation) - extends BaseShowAlertDisplayLocationBuilder[Query] { - - def apply( - query: Query, - candidate: ShowAlertCandidate, - features: FeatureMap - ): ShowAlertDisplayLocation = location -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/StaticShowAlertIconDisplayInfoBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/StaticShowAlertIconDisplayInfoBuilder.docx new file mode 100644 index 000000000..9b5c94781 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/StaticShowAlertIconDisplayInfoBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/StaticShowAlertIconDisplayInfoBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/StaticShowAlertIconDisplayInfoBuilder.scala deleted file mode 100644 index 4c5dcf2a0..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/alert/StaticShowAlertIconDisplayInfoBuilder.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.alert - -import com.twitter.product_mixer.component_library.model.candidate.ShowAlertCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.item.alert.BaseShowAlertIconDisplayInfoBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.alert.ShowAlertIconDisplayInfo -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class StaticShowAlertIconDisplayInfoBuilder[-Query <: PipelineQuery]( - iconDisplayInfo: ShowAlertIconDisplayInfo) - extends BaseShowAlertIconDisplayInfoBuilder[Query] { - - def apply( - query: Query, - candidate: ShowAlertCandidate, - features: FeatureMap - ): Option[ShowAlertIconDisplayInfo] = Some(iconDisplayInfo) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/article/ArticleCandidateUrtItemBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/article/ArticleCandidateUrtItemBuilder.docx new file mode 100644 index 000000000..c889fb830 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/article/ArticleCandidateUrtItemBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/article/ArticleCandidateUrtItemBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/article/ArticleCandidateUrtItemBuilder.scala deleted file mode 100644 index 87d611183..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/article/ArticleCandidateUrtItemBuilder.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.article - -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.article.ArticleCandidateUrtItemBuilder.ArticleClientEventInfoElement -import com.twitter.product_mixer.component_library.model.candidate.BaseArticleCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.article.ArticleDisplayType -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.article.ArticleItem -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.article.ArticleSeedType -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.article.FollowingListSeed -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object ArticleCandidateUrtItemBuilder { - val ArticleClientEventInfoElement: String = "article" -} - -case class ArticleCandidateUrtItemBuilder[ - -Query <: PipelineQuery, - Candidate <: BaseArticleCandidate -]( - clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, Candidate], - articleSeedType: ArticleSeedType = FollowingListSeed, - feedbackActionInfoBuilder: Option[ - BaseFeedbackActionInfoBuilder[Query, Candidate] - ] = None, - displayType: Option[ArticleDisplayType] = None, - socialContextBuilder: Option[BaseSocialContextBuilder[Query, Candidate]] = None, -) extends CandidateUrtEntryBuilder[Query, Candidate, ArticleItem] { - - override def apply( - query: Query, - articleCandidate: Candidate, - candidateFeatures: FeatureMap - ): ArticleItem = ArticleItem( - id = articleCandidate.id, - sortIndex = None, // Sort indexes are automatically set in the domain marshaller phase - clientEventInfo = clientEventInfoBuilder( - query, - articleCandidate, - candidateFeatures, - Some(ArticleClientEventInfoElement)), - feedbackActionInfo = - feedbackActionInfoBuilder.flatMap(_.apply(query, articleCandidate, candidateFeatures)), - displayType = displayType, - socialContext = - socialContextBuilder.flatMap(_.apply(query, articleCandidate, candidateFeatures)), - articleSeedType = articleSeedType - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/audio_space/AudioSpaceCandidateUrtItemBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/audio_space/AudioSpaceCandidateUrtItemBuilder.docx new file mode 100644 index 000000000..a2edf158d Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/audio_space/AudioSpaceCandidateUrtItemBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/audio_space/AudioSpaceCandidateUrtItemBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/audio_space/AudioSpaceCandidateUrtItemBuilder.scala deleted file mode 100644 index 1d1f7c90a..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/audio_space/AudioSpaceCandidateUrtItemBuilder.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.audio_space - -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.audio_space.AudioSpaceCandidateUrtItemBuilder.AudioSpaceClientEventInfoElement -import com.twitter.product_mixer.component_library.model.candidate.AudioSpaceCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.audio_space.AudioSpaceItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object AudioSpaceCandidateUrtItemBuilder { - val AudioSpaceClientEventInfoElement: String = "audiospace" -} - -case class AudioSpaceCandidateUrtItemBuilder[-Query <: PipelineQuery]( - clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, UniversalNoun[Any]], - feedbackActionInfoBuilder: Option[ - BaseFeedbackActionInfoBuilder[Query, UniversalNoun[Any]] - ] = None) - extends CandidateUrtEntryBuilder[Query, AudioSpaceCandidate, AudioSpaceItem] { - - override def apply( - query: Query, - audioSpaceCandidate: AudioSpaceCandidate, - candidateFeatures: FeatureMap - ): AudioSpaceItem = AudioSpaceItem( - id = audioSpaceCandidate.id, - sortIndex = None, // Sort indexes are automatically set in the domain marshaller phase - clientEventInfo = clientEventInfoBuilder( - query, - audioSpaceCandidate, - candidateFeatures, - Some(AudioSpaceClientEventInfoElement)), - feedbackActionInfo = - feedbackActionInfoBuilder.flatMap(_.apply(query, audioSpaceCandidate, candidateFeatures)) - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/card/CardCandidateUtrItemBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/card/CardCandidateUtrItemBuilder.docx new file mode 100644 index 000000000..e7843f5ad Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/card/CardCandidateUtrItemBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/card/CardCandidateUtrItemBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/card/CardCandidateUtrItemBuilder.scala deleted file mode 100644 index e935df9dd..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/card/CardCandidateUtrItemBuilder.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.card - -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.card.CardCandidateUtrItemBuilder.CardClientEventInfoElement -import com.twitter.product_mixer.component_library.model.candidate.CardCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseStr -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseUrlBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.card.CardDisplayType -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.card.CardItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object CardCandidateUtrItemBuilder { - val CardClientEventInfoElement: String = "card" -} - -case class CardCandidateUtrItemBuilder[-Query <: PipelineQuery]( - clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, CardCandidate], - cardUrlBuilder: BaseStr[Query, CardCandidate], - textBuilder: Option[BaseStr[Query, CardCandidate]], - subtextBuilder: Option[BaseStr[Query, CardCandidate]], - urlBuilder: Option[BaseUrlBuilder[Query, CardCandidate]], - cardDisplayType: Option[CardDisplayType], - feedbackActionInfoBuilder: Option[ - BaseFeedbackActionInfoBuilder[Query, CardCandidate], - ] = None) - extends CandidateUrtEntryBuilder[Query, CardCandidate, CardItem] { - - override def apply( - query: Query, - cardCandidate: CardCandidate, - candidateFeatures: FeatureMap - ): CardItem = CardItem( - id = cardCandidate.id, - sortIndex = None, // Sort indexes are automatically set in the domain marshaller phase - clientEventInfo = clientEventInfoBuilder( - query, - cardCandidate, - candidateFeatures, - Some(CardClientEventInfoElement)), - feedbackActionInfo = - feedbackActionInfoBuilder.flatMap(_.apply(query, cardCandidate, candidateFeatures)), - cardUrl = cardUrlBuilder(query, cardCandidate, candidateFeatures), - text = textBuilder.map(_.apply(query, cardCandidate, candidateFeatures)), - subtext = textBuilder.map(_.apply(query, cardCandidate, candidateFeatures)), - url = urlBuilder.map(_.apply(query, cardCandidate, candidateFeatures)), - displayType = cardDisplayType - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/commerce/CommerceProductCandidateUrtItemBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/commerce/CommerceProductCandidateUrtItemBuilder.docx new file mode 100644 index 000000000..f032715aa Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/commerce/CommerceProductCandidateUrtItemBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/commerce/CommerceProductCandidateUrtItemBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/commerce/CommerceProductCandidateUrtItemBuilder.scala deleted file mode 100644 index a56385a7d..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/commerce/CommerceProductCandidateUrtItemBuilder.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.commerce - -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.commerce.CommerceProductCandidateUrtItemBuilder.CommerceProductClientEventInfoElement -import com.twitter.product_mixer.component_library.model.candidate.CommerceProductCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.commerce.CommerceProductItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object CommerceProductCandidateUrtItemBuilder { - val CommerceProductClientEventInfoElement: String = "commerce-product" -} - -case class CommerceProductCandidateUrtItemBuilder[-Query <: PipelineQuery]( - clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, CommerceProductCandidate], - feedbackActionInfoBuilder: Option[BaseFeedbackActionInfoBuilder[Query, CommerceProductCandidate]]) - extends CandidateUrtEntryBuilder[ - Query, - CommerceProductCandidate, - CommerceProductItem - ] { - - override def apply( - query: Query, - candidate: CommerceProductCandidate, - candidateFeatures: FeatureMap - ): CommerceProductItem = - CommerceProductItem( - id = candidate.id, - sortIndex = None, - clientEventInfo = clientEventInfoBuilder( - query, - candidate, - candidateFeatures, - Some(CommerceProductClientEventInfoElement)), - feedbackActionInfo = - feedbackActionInfoBuilder.flatMap(_.apply(query, candidate, candidateFeatures)) - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/commerce/CommerceProductGroupCandidateUrtItemBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/commerce/CommerceProductGroupCandidateUrtItemBuilder.docx new file mode 100644 index 000000000..faed9bf53 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/commerce/CommerceProductGroupCandidateUrtItemBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/commerce/CommerceProductGroupCandidateUrtItemBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/commerce/CommerceProductGroupCandidateUrtItemBuilder.scala deleted file mode 100644 index a48049314..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/commerce/CommerceProductGroupCandidateUrtItemBuilder.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.commerce - -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.commerce.CommerceProductGroupCandidateUrtItemBuilder.CommerceProductGroupClientEventInfoElement -import com.twitter.product_mixer.component_library.model.candidate.CommerceProductGroupCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.commerce.CommerceProductGroupItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object CommerceProductGroupCandidateUrtItemBuilder { - val CommerceProductGroupClientEventInfoElement: String = "commerce-product-group" -} - -case class CommerceProductGroupCandidateUrtItemBuilder[-Query <: PipelineQuery]( - clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, CommerceProductGroupCandidate], - feedbackActionInfoBuilder: Option[ - BaseFeedbackActionInfoBuilder[Query, CommerceProductGroupCandidate] - ]) extends CandidateUrtEntryBuilder[ - Query, - CommerceProductGroupCandidate, - CommerceProductGroupItem - ] { - - override def apply( - query: Query, - candidate: CommerceProductGroupCandidate, - candidateFeatures: FeatureMap - ): CommerceProductGroupItem = - CommerceProductGroupItem( - id = candidate.id, - sortIndex = None, - clientEventInfo = clientEventInfoBuilder( - query, - candidate, - candidateFeatures, - Some(CommerceProductGroupClientEventInfoElement)), - feedbackActionInfo = - feedbackActionInfoBuilder.flatMap(_.apply(query, candidate, candidateFeatures)) - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/event_summary/EventCandidateUrtItemBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/event_summary/EventCandidateUrtItemBuilder.docx new file mode 100644 index 000000000..6a0138cb1 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/event_summary/EventCandidateUrtItemBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/event_summary/EventCandidateUrtItemBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/event_summary/EventCandidateUrtItemBuilder.scala deleted file mode 100644 index 27d7f1e19..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/event_summary/EventCandidateUrtItemBuilder.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.event_summary - -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.event_summary.EventCandidateUrtItemBuilder.EventClientEventInfoElement -import com.twitter.product_mixer.component_library.model.candidate.trends_events.EventDisplayType -import com.twitter.product_mixer.component_library.model.candidate.trends_events.EventImage -import com.twitter.product_mixer.component_library.model.candidate.trends_events.EventTimeString -import com.twitter.product_mixer.component_library.model.candidate.trends_events.EventTitleFeature -import com.twitter.product_mixer.component_library.model.candidate.trends_events.EventUrl -import com.twitter.product_mixer.component_library.model.candidate.trends_events.UnifiedEventCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineItem -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.event.EventSummaryItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object EventCandidateUrtItemBuilder { - val EventClientEventInfoElement = "event" -} - -case class EventCandidateUrtItemBuilder[Query <: PipelineQuery]( - clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, UnifiedEventCandidate], - feedbackActionInfoBuilder: Option[BaseFeedbackActionInfoBuilder[Query, UnifiedEventCandidate]] = - None) - extends CandidateUrtEntryBuilder[Query, UnifiedEventCandidate, TimelineItem] { - - override def apply( - query: Query, - candidate: UnifiedEventCandidate, - candidateFeatures: FeatureMap - ): TimelineItem = { - EventSummaryItem( - id = candidate.id, - sortIndex = None, // Sort indexes are automatically set in the domain marshaller phase - clientEventInfo = clientEventInfoBuilder( - query = query, - candidate = candidate, - candidateFeatures = candidateFeatures, - element = Some(EventClientEventInfoElement) - ), - feedbackActionInfo = - feedbackActionInfoBuilder.flatMap(_.apply(query, candidate, candidateFeatures)), - title = candidateFeatures.get(EventTitleFeature), - displayType = candidateFeatures.get(EventDisplayType), - url = candidateFeatures.get(EventUrl), - image = candidateFeatures.getOrElse(EventImage, None), - timeString = candidateFeatures.getOrElse(EventTimeString, None) - ) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/generic_summary/GenericSummaryActionBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/generic_summary/GenericSummaryActionBuilder.docx new file mode 100644 index 000000000..033b52461 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/generic_summary/GenericSummaryActionBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/generic_summary/GenericSummaryActionBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/generic_summary/GenericSummaryActionBuilder.scala deleted file mode 100644 index 8cdbb065d..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/generic_summary/GenericSummaryActionBuilder.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.generic_summary - -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.generic_summary.GenericSummaryActionBuilder.GenericSummaryActionClientEventInfoElement -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseUrlBuilder -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.generic_summary.GenericSummaryAction -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object GenericSummaryActionBuilder { - val GenericSummaryActionClientEventInfoElement: String = "genericsummary-action" -} - -case class GenericSummaryActionBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( - urlBuilder: BaseUrlBuilder[Query, Candidate], - clientEventInfoBuilder: Option[BaseClientEventInfoBuilder[Query, Candidate]] = None) { - - def apply( - query: Query, - candidate: Candidate, - candidateFeatures: FeatureMap - ): GenericSummaryAction = GenericSummaryAction( - url = urlBuilder.apply(query, candidate, candidateFeatures), - clientEventInfo = clientEventInfoBuilder.flatMap( - _.apply( - query, - candidate, - candidateFeatures, - Some(GenericSummaryActionClientEventInfoElement))) - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/generic_summary/GenericSummaryCandidateUrtItemBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/generic_summary/GenericSummaryCandidateUrtItemBuilder.docx new file mode 100644 index 000000000..4b8a4143b Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/generic_summary/GenericSummaryCandidateUrtItemBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/generic_summary/GenericSummaryCandidateUrtItemBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/generic_summary/GenericSummaryCandidateUrtItemBuilder.scala deleted file mode 100644 index 759dd48c2..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/generic_summary/GenericSummaryCandidateUrtItemBuilder.scala +++ /dev/null @@ -1,64 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.generic_summary - -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.generic_summary.GenericSummaryCandidateUrtItemBuilder.GenericSummaryClientEventInfoElement -import com.twitter.product_mixer.component_library.model.candidate.GenericSummaryCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.richtext.BaseRichTextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.generic_summary.GenericSummaryItem -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.generic_summary.GenericSummaryItemDisplayType -import com.twitter.product_mixer.core.model.marshalling.response.urt.media.Media -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.PromotedMetadata -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.util.Time - -object GenericSummaryCandidateUrtItemBuilder { - val GenericSummaryClientEventInfoElement: String = "genericsummary" -} - -case class GenericSummaryCandidateUrtItemBuilder[-Query <: PipelineQuery]( - clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, GenericSummaryCandidate], - headlineRichTextBuilder: BaseRichTextBuilder[Query, GenericSummaryCandidate], - displayType: GenericSummaryItemDisplayType, - genericSummaryContextCandidateUrtItemBuilder: Option[ - GenericSummaryContextBuilder[Query, GenericSummaryCandidate] - ] = None, - genericSummaryActionCandidateUrtItemBuilder: Option[ - GenericSummaryActionBuilder[Query, GenericSummaryCandidate] - ] = None, - timestamp: Option[Time] = None, - userAttributionIds: Option[Seq[Long]] = None, - media: Option[Media] = None, - promotedMetadata: Option[PromotedMetadata] = None, - feedbackActionInfoBuilder: Option[BaseFeedbackActionInfoBuilder[Query, GenericSummaryCandidate]] = - None) - extends CandidateUrtEntryBuilder[Query, GenericSummaryCandidate, GenericSummaryItem] { - - override def apply( - query: Query, - genericSummaryCandidate: GenericSummaryCandidate, - candidateFeatures: FeatureMap - ): GenericSummaryItem = GenericSummaryItem( - id = genericSummaryCandidate.id, - sortIndex = None, // Sort indexes are automatically set in the domain marshaller phase - clientEventInfo = clientEventInfoBuilder( - query, - genericSummaryCandidate, - candidateFeatures, - Some(GenericSummaryClientEventInfoElement)), - feedbackActionInfo = - feedbackActionInfoBuilder.flatMap(_.apply(query, genericSummaryCandidate, candidateFeatures)), - headline = headlineRichTextBuilder.apply(query, genericSummaryCandidate, candidateFeatures), - displayType = displayType, - userAttributionIds = userAttributionIds.getOrElse(Seq.empty), - media = media, - context = genericSummaryContextCandidateUrtItemBuilder.map( - _.apply(query, genericSummaryCandidate, candidateFeatures)), - timestamp = timestamp, - onClickAction = genericSummaryActionCandidateUrtItemBuilder.map( - _.apply(query, genericSummaryCandidate, candidateFeatures)), - promotedMetadata = promotedMetadata - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/generic_summary/GenericSummaryContextBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/generic_summary/GenericSummaryContextBuilder.docx new file mode 100644 index 000000000..d3fcbd6c8 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/generic_summary/GenericSummaryContextBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/generic_summary/GenericSummaryContextBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/generic_summary/GenericSummaryContextBuilder.scala deleted file mode 100644 index e78476280..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/generic_summary/GenericSummaryContextBuilder.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.generic_summary - -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.richtext.BaseRichTextBuilder -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.icon.HorizonIcon -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.generic_summary.GenericSummaryContext -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class GenericSummaryContextBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( - richTextBuilder: BaseRichTextBuilder[Query, Candidate], - icon: Option[HorizonIcon] = None) { - - def apply( - query: Query, - candidate: Candidate, - candidateFeatures: FeatureMap - ): GenericSummaryContext = GenericSummaryContext( - richTextBuilder.apply(query, candidate, candidateFeatures), - icon - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/icon_label/IconLabelCandidateUrtItemBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/icon_label/IconLabelCandidateUrtItemBuilder.docx new file mode 100644 index 000000000..9567db578 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/icon_label/IconLabelCandidateUrtItemBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/icon_label/IconLabelCandidateUrtItemBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/icon_label/IconLabelCandidateUrtItemBuilder.scala deleted file mode 100644 index 698b64f3b..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/icon_label/IconLabelCandidateUrtItemBuilder.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.icon_label - -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.icon_label.IconLabelCandidateUrtItemBuilder.IconLabelClientEventInfoElement -import com.twitter.product_mixer.component_library.model.candidate.LabelCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.richtext.BaseRichTextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.icon.HorizonIcon -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.icon_label.IconLabelItem -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichTextEntity -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object IconLabelCandidateUrtItemBuilder { - val IconLabelClientEventInfoElement: String = "iconlabel" -} - -case class IconLabelCandidateUrtItemBuilder[-Query <: PipelineQuery, Candidate <: LabelCandidate]( - richTextBuilder: BaseRichTextBuilder[Query, Candidate], - icon: Option[HorizonIcon] = None, - entities: Option[List[RichTextEntity]] = None, - clientEventInfoBuilder: Option[BaseClientEventInfoBuilder[Query, Candidate]] = None, - feedbackActionInfoBuilder: Option[BaseFeedbackActionInfoBuilder[Query, Candidate]] = None) - extends CandidateUrtEntryBuilder[Query, Candidate, IconLabelItem] { - - override def apply( - query: Query, - labelCandidate: Candidate, - candidateFeatures: FeatureMap - ): IconLabelItem = - IconLabelItem( - id = labelCandidate.id.toString, - sortIndex = None, // Sort indexes are automatically set in the domain marshaller phase - clientEventInfo = clientEventInfoBuilder.flatMap( - _.apply(query, labelCandidate, candidateFeatures, Some(IconLabelClientEventInfoElement))), - feedbackActionInfo = - feedbackActionInfoBuilder.flatMap(_.apply(query, labelCandidate, candidateFeatures)), - text = richTextBuilder(query, labelCandidate, candidateFeatures), - icon = icon, - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/message/CompactPromptCandidateUrtItemStringCenterBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/message/CompactPromptCandidateUrtItemStringCenterBuilder.docx new file mode 100644 index 000000000..6665defda Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/message/CompactPromptCandidateUrtItemStringCenterBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/message/CompactPromptCandidateUrtItemStringCenterBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/message/CompactPromptCandidateUrtItemStringCenterBuilder.scala deleted file mode 100644 index 8f3d7826b..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/message/CompactPromptCandidateUrtItemStringCenterBuilder.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.message - -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.message.CompactPromptCandidateUrtItemStringCenterBuilder.CompactPromptClientEventInfoElement -import com.twitter.product_mixer.component_library.model.candidate.CompactPromptCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseStr -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.richtext.BaseRichTextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.message.CompactPromptMessageContent -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.message.MessagePromptItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object CompactPromptCandidateUrtItemStringCenterBuilder { - val CompactPromptClientEventInfoElement: String = "message" -} - -case class CompactPromptCandidateUrtItemStringCenterBuilder[-Query <: PipelineQuery]( - clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, CompactPromptCandidate], - feedbackActionInfoBuilder: Option[ - BaseFeedbackActionInfoBuilder[Query, CompactPromptCandidate] - ] = None, - headerTextBuilder: BaseStr[Query, CompactPromptCandidate], - bodyTextBuilder: Option[BaseStr[Query, CompactPromptCandidate]] = None, - headerRichTextBuilder: Option[BaseRichTextBuilder[Query, CompactPromptCandidate]] = None, - bodyRichTextBuilder: Option[BaseRichTextBuilder[Query, CompactPromptCandidate]] = None) - extends CandidateUrtEntryBuilder[Query, CompactPromptCandidate, MessagePromptItem] { - - override def apply( - query: Query, - compactPromptCandidate: CompactPromptCandidate, - candidateFeatures: FeatureMap - ): MessagePromptItem = - MessagePromptItem( - id = compactPromptCandidate.id.toString, - sortIndex = None, // Sort indexes are automatically set in the domain marshaller phase - clientEventInfo = clientEventInfoBuilder( - query, - compactPromptCandidate, - candidateFeatures, - Some(CompactPromptClientEventInfoElement)), - feedbackActionInfo = feedbackActionInfoBuilder.flatMap( - _.apply(query, compactPromptCandidate, candidateFeatures)), - isPinned = None, - content = CompactPromptMessageContent( - headerText = headerTextBuilder.apply(query, compactPromptCandidate, candidateFeatures), - bodyText = bodyTextBuilder.map(_.apply(query, compactPromptCandidate, candidateFeatures)), - primaryButtonAction = None, - secondaryButtonAction = None, - action = None, - headerRichText = - headerRichTextBuilder.map(_.apply(query, compactPromptCandidate, candidateFeatures)), - bodyRichText = - bodyRichTextBuilder.map(_.apply(query, compactPromptCandidate, candidateFeatures)) - ), - impressionCallbacks = None - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/message/InlinePromptCandidateUrtItemStringCenterBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/message/InlinePromptCandidateUrtItemStringCenterBuilder.docx new file mode 100644 index 000000000..193885eef Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/message/InlinePromptCandidateUrtItemStringCenterBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/message/InlinePromptCandidateUrtItemStringCenterBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/message/InlinePromptCandidateUrtItemStringCenterBuilder.scala deleted file mode 100644 index 0ebee8e8c..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/message/InlinePromptCandidateUrtItemStringCenterBuilder.scala +++ /dev/null @@ -1,74 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.message - -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.message.InlinePromptCandidateUrtItemStringCenterBuilder.InlinePromptClientEventInfoElement -import com.twitter.product_mixer.component_library.model.candidate.InlinePromptCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseStr -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.richtext.BaseRichTextBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.message.InlinePromptMessageContent -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.message.MessagePromptItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object InlinePromptCandidateUrtItemStringCenterBuilder { - val InlinePromptClientEventInfoElement: String = "message" -} - -case class InlinePromptCandidateUrtItemStringCenterBuilder[-Query <: PipelineQuery]( - clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, InlinePromptCandidate], - feedbackActionInfoBuilder: Option[ - BaseFeedbackActionInfoBuilder[Query, InlinePromptCandidate] - ] = None, - headerTextBuilder: BaseStr[Query, InlinePromptCandidate], - bodyTextBuilder: Option[BaseStr[Query, InlinePromptCandidate]] = None, - headerRichTextBuilder: Option[BaseRichTextBuilder[Query, InlinePromptCandidate]] = None, - bodyRichTextBuilder: Option[BaseRichTextBuilder[Query, InlinePromptCandidate]] = None, - primaryMessageTextActionBuilder: Option[ - MessageTextActionBuilder[Query, InlinePromptCandidate] - ] = None, - secondaryMessageTextActionBuilder: Option[ - MessageTextActionBuilder[Query, InlinePromptCandidate] - ] = None, - socialContextBuilder: Option[BaseSocialContextBuilder[Query, InlinePromptCandidate]] = None, - userFacePileBuilder: Option[ - UserFacePileBuilder - ] = None) - extends CandidateUrtEntryBuilder[Query, InlinePromptCandidate, MessagePromptItem] { - - override def apply( - query: Query, - inlinePromptCandidate: InlinePromptCandidate, - candidateFeatures: FeatureMap - ): MessagePromptItem = - MessagePromptItem( - id = inlinePromptCandidate.id, - sortIndex = None, // Sort indexes are automatically set in the domain marshaller phase - clientEventInfo = clientEventInfoBuilder( - query, - inlinePromptCandidate, - candidateFeatures, - Some(InlinePromptClientEventInfoElement)), - feedbackActionInfo = - feedbackActionInfoBuilder.flatMap(_.apply(query, inlinePromptCandidate, candidateFeatures)), - isPinned = None, - content = InlinePromptMessageContent( - headerText = headerTextBuilder.apply(query, inlinePromptCandidate, candidateFeatures), - bodyText = bodyTextBuilder.map(_.apply(query, inlinePromptCandidate, candidateFeatures)), - primaryButtonAction = primaryMessageTextActionBuilder.map( - _.apply(query, inlinePromptCandidate, candidateFeatures)), - secondaryButtonAction = secondaryMessageTextActionBuilder.map( - _.apply(query, inlinePromptCandidate, candidateFeatures)), - headerRichText = - headerRichTextBuilder.map(_.apply(query, inlinePromptCandidate, candidateFeatures)), - bodyRichText = - bodyRichTextBuilder.map(_.apply(query, inlinePromptCandidate, candidateFeatures)), - socialContext = - socialContextBuilder.flatMap(_.apply(query, inlinePromptCandidate, candidateFeatures)), - userFacepile = userFacePileBuilder.map(_.apply()) - ), - impressionCallbacks = None - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/message/MessageTextActionBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/message/MessageTextActionBuilder.docx new file mode 100644 index 000000000..15537a0af Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/message/MessageTextActionBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/message/MessageTextActionBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/message/MessageTextActionBuilder.scala deleted file mode 100644 index 2871ac682..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/message/MessageTextActionBuilder.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.message - -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseStr -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.message.MessageAction -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.message.MessageTextAction -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Callback -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventInfo -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object MessageTextActionBuilder { - val MessageTextActionClientEventInfoElement: String = "message-text-action" -} - -case class MessageTextActionBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( - textBuilder: BaseStr[Query, Candidate], - dismissOnClick: Boolean, - url: Option[String] = None, - clientEventInfo: Option[ClientEventInfo] = None, - onClickCallbacks: Option[List[Callback]] = None) { - - def apply( - query: Query, - candidate: Candidate, - candidateFeatures: FeatureMap - ): MessageTextAction = MessageTextAction( - text = textBuilder(query, candidate, candidateFeatures), - action = MessageAction( - dismissOnClick, - url, - clientEventInfo, - onClickCallbacks - ) - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/message/UserFacePileBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/message/UserFacePileBuilder.docx new file mode 100644 index 000000000..f45c9560e Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/message/UserFacePileBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/message/UserFacePileBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/message/UserFacePileBuilder.scala deleted file mode 100644 index ce87e64cb..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/message/UserFacePileBuilder.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.message - -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.message.MessageActionType -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.message.MessageTextAction -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.message.UserFacepile -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.message.UserFacepileDisplayType - -case class UserFacePileBuilder( - userIds: Seq[Long], - featuredUserIds: Seq[Long], - action: Option[MessageTextAction], - actionType: Option[MessageActionType], - displaysFeaturingText: Option[Boolean], - displayType: Option[UserFacepileDisplayType]) { - - def apply(): UserFacepile = UserFacepile( - userIds = userIds, - featuredUserIds = featuredUserIds, - action = action, - actionType = actionType, - displaysFeaturingText = displaysFeaturingText, - displayType = displayType - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/moment/MomentAnnotationCandidateUrtItemBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/moment/MomentAnnotationCandidateUrtItemBuilder.docx new file mode 100644 index 000000000..4735f17af Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/moment/MomentAnnotationCandidateUrtItemBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/moment/MomentAnnotationCandidateUrtItemBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/moment/MomentAnnotationCandidateUrtItemBuilder.scala deleted file mode 100644 index 5d916f424..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/moment/MomentAnnotationCandidateUrtItemBuilder.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.moment - -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.moment.MomentAnnotationCandidateUrtItemBuilder.MomentAnnotationItemClientEventInfoElement -import com.twitter.product_mixer.component_library.model.candidate.MomentAnnotationCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.richtext.BaseRichTextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.moment.MomentAnnotationItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object MomentAnnotationCandidateUrtItemBuilder { - val MomentAnnotationItemClientEventInfoElement = "metadata" -} - -case class MomentAnnotationCandidateUrtItemBuilder[Query <: PipelineQuery]( - clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, MomentAnnotationCandidate], - annotationTextRichTextBuilder: BaseRichTextBuilder[Query, MomentAnnotationCandidate], - annotationHeaderRichTextBuilder: BaseRichTextBuilder[Query, MomentAnnotationCandidate], - feedbackActionInfoBuilder: Option[ - BaseFeedbackActionInfoBuilder[Query, MomentAnnotationCandidate] - ] = None, -) extends CandidateUrtEntryBuilder[Query, MomentAnnotationCandidate, MomentAnnotationItem] { - - override def apply( - query: Query, - candidate: MomentAnnotationCandidate, - candidateFeatures: FeatureMap - ): MomentAnnotationItem = MomentAnnotationItem( - id = candidate.id, - sortIndex = None, // Sort indexes are automatically set in the domain marshaller phase - clientEventInfo = clientEventInfoBuilder( - query, - candidate, - candidateFeatures, - Some(MomentAnnotationItemClientEventInfoElement)), - feedbackActionInfo = - feedbackActionInfoBuilder.flatMap(_.apply(query, candidate, candidateFeatures)), - isPinned = None, - text = - candidate.text.map(_ => annotationTextRichTextBuilder(query, candidate, candidateFeatures)), - header = candidate.header.map(_ => - annotationHeaderRichTextBuilder(query, candidate, candidateFeatures)), - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/relevance_prompt/RelevancePromptCandidateUrtItemStringCenterBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/relevance_prompt/RelevancePromptCandidateUrtItemStringCenterBuilder.docx new file mode 100644 index 000000000..93ef7c49b Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/relevance_prompt/RelevancePromptCandidateUrtItemStringCenterBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/relevance_prompt/RelevancePromptCandidateUrtItemStringCenterBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/relevance_prompt/RelevancePromptCandidateUrtItemStringCenterBuilder.scala deleted file mode 100644 index 7ab7308b2..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/relevance_prompt/RelevancePromptCandidateUrtItemStringCenterBuilder.scala +++ /dev/null @@ -1,62 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.relevance_prompt - -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.relevance_prompt.RelevancePromptCandidateUrtItemStringCenterBuilder.RelevancePromptClientEventInfoElement -import com.twitter.product_mixer.component_library.model.candidate.RelevancePromptCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseStr -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.prompt.PromptItem -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.prompt.RelevancePromptContent -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.prompt.RelevancePromptDisplayType -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.prompt.RelevancePromptFollowUpFeedbackType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Callback -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object RelevancePromptCandidateUrtItemStringCenterBuilder { - val RelevancePromptClientEventInfoElement: String = "relevance_prompt" -} - -case class RelevancePromptCandidateUrtItemStringCenterBuilder[-Query <: PipelineQuery]( - clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, RelevancePromptCandidate], - titleTextBuilder: BaseStr[Query, RelevancePromptCandidate], - confirmationTextBuilder: BaseStr[Query, RelevancePromptCandidate], - isRelevantTextBuilder: BaseStr[Query, RelevancePromptCandidate], - notRelevantTextBuilder: BaseStr[Query, RelevancePromptCandidate], - displayType: RelevancePromptDisplayType, - isRelevantCallback: Callback, - notRelevantCallback: Callback, - isRelevantFollowUp: Option[RelevancePromptFollowUpFeedbackType] = None, - notRelevantFollowUp: Option[RelevancePromptFollowUpFeedbackType] = None, - impressionCallbacks: Option[List[Callback]] = None) - extends CandidateUrtEntryBuilder[Query, RelevancePromptCandidate, PromptItem] { - - override def apply( - query: Query, - relevancePromptCandidate: RelevancePromptCandidate, - candidateFeatures: FeatureMap - ): PromptItem = - PromptItem( - id = relevancePromptCandidate.id, - sortIndex = None, - clientEventInfo = clientEventInfoBuilder( - query, - relevancePromptCandidate, - candidateFeatures, - Some(RelevancePromptClientEventInfoElement)), - feedbackActionInfo = None, - content = RelevancePromptContent( - title = titleTextBuilder(query, relevancePromptCandidate, candidateFeatures), - confirmation = confirmationTextBuilder(query, relevancePromptCandidate, candidateFeatures), - isRelevantText = isRelevantTextBuilder(query, relevancePromptCandidate, candidateFeatures), - notRelevantText = - notRelevantTextBuilder(query, relevancePromptCandidate, candidateFeatures), - isRelevantCallback = isRelevantCallback, - notRelevantCallback = notRelevantCallback, - displayType = displayType, - isRelevantFollowUp = isRelevantFollowUp, - notRelevantFollowUp = notRelevantFollowUp, - ), - impressionCallbacks = impressionCallbacks - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/suggestion/SpellingSuggestionCandidateUrtItemBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/suggestion/SpellingSuggestionCandidateUrtItemBuilder.docx new file mode 100644 index 000000000..6f02cbc6c Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/suggestion/SpellingSuggestionCandidateUrtItemBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/suggestion/SpellingSuggestionCandidateUrtItemBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/suggestion/SpellingSuggestionCandidateUrtItemBuilder.scala deleted file mode 100644 index ce0beeb9e..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/suggestion/SpellingSuggestionCandidateUrtItemBuilder.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.suggestion - -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.suggestion.SpellingSuggestionCandidateUrtItemBuilder.SpellingItemClientEventInfoElement -import com.twitter.product_mixer.component_library.model.candidate.suggestion.SpellingSuggestionCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.suggestion.SpellingItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object SpellingSuggestionCandidateUrtItemBuilder { - val SpellingItemClientEventInfoElement: String = "spelling" -} - -case class SpellingSuggestionCandidateUrtItemBuilder[Query <: PipelineQuery]( - clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, SpellingSuggestionCandidate], - feedbackActionInfoBuilder: Option[ - BaseFeedbackActionInfoBuilder[Query, SpellingSuggestionCandidate] - ] = None, -) extends CandidateUrtEntryBuilder[Query, SpellingSuggestionCandidate, SpellingItem] { - - override def apply( - query: Query, - candidate: SpellingSuggestionCandidate, - candidateFeatures: FeatureMap - ): SpellingItem = SpellingItem( - id = candidate.id, - sortIndex = None, // Sort indexes are automatically set in the domain marshaller phase - clientEventInfo = clientEventInfoBuilder( - query, - candidate, - candidateFeatures, - Some(SpellingItemClientEventInfoElement)), - feedbackActionInfo = - feedbackActionInfoBuilder.flatMap(_.apply(query, candidate, candidateFeatures)), - textResult = candidate.textResult, - spellingActionType = candidate.spellingActionType, - originalQuery = candidate.originalQuery - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/tile/TileCandidateUrtItemBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/tile/TileCandidateUrtItemBuilder.docx new file mode 100644 index 000000000..09a9f30cd Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/tile/TileCandidateUrtItemBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/tile/TileCandidateUrtItemBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/tile/TileCandidateUrtItemBuilder.scala deleted file mode 100644 index 4c449637f..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/tile/TileCandidateUrtItemBuilder.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.tile - -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tile.TileCandidateUrtItemBuilder.TopicTileClientEventInfoElement -import com.twitter.product_mixer.component_library.model.candidate.PromptCarouselTileCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tile.StandardTileContent -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tile.TileItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object TileCandidateUrtItemBuilder { - val TopicTileClientEventInfoElement: String = "tile" -} - -case class TileCandidateUrtItemBuilder[-Query <: PipelineQuery]( - clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, PromptCarouselTileCandidate], - feedbackActionInfoBuilder: Option[ - BaseFeedbackActionInfoBuilder[Query, PromptCarouselTileCandidate] - ] = None) - extends CandidateUrtEntryBuilder[Query, PromptCarouselTileCandidate, TileItem] { - - override def apply( - query: Query, - tileCandidate: PromptCarouselTileCandidate, - candidateFeatures: FeatureMap - ): TileItem = TileItem( - id = tileCandidate.id, - sortIndex = None, // Sort indexes are automatically set in the domain marshaller phase - clientEventInfo = clientEventInfoBuilder( - query, - tileCandidate, - candidateFeatures, - Some(TopicTileClientEventInfoElement)), - title = "", //This data is ignored do - supportingText = "", - feedbackActionInfo = - feedbackActionInfoBuilder.flatMap(_.apply(query, tileCandidate, candidateFeatures)), - image = None, - url = None, - content = StandardTileContent( - title = "", - supportingText = "", - badge = None - ) - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/ParamTopicDisplayTypeBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/ParamTopicDisplayTypeBuilder.docx new file mode 100644 index 000000000..e16fd00f1 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/ParamTopicDisplayTypeBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/ParamTopicDisplayTypeBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/ParamTopicDisplayTypeBuilder.scala deleted file mode 100644 index 20253fea0..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/ParamTopicDisplayTypeBuilder.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.topic - -import com.twitter.product_mixer.component_library.model.candidate.TopicCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.FSEnumParam -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.topic.BasicTopicDisplayType -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.topic.PillTopicDisplayType -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.topic.NoIconTopicDisplayType -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.topic.PillWithoutActionIconDisplayType -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.topic.TopicDisplayType -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.item.topic.BaseTopicDisplayTypeBuilder - -object TopicCandidateDisplayType extends Enumeration { - type TopicDisplayType = Value - - val Basic = Value - val Pill = Value - val NoIcon = Value - val PillWithoutActionIcon = Value -} - -case class ParamTopicDisplayTypeBuilder( - displayTypeParam: FSEnumParam[TopicCandidateDisplayType.type]) - extends BaseTopicDisplayTypeBuilder[PipelineQuery, TopicCandidate] { - - override def apply( - query: PipelineQuery, - candidate: TopicCandidate, - candidateFeatures: FeatureMap - ): Option[TopicDisplayType] = { - val displayType = query.params(displayTypeParam) - displayType match { - case TopicCandidateDisplayType.Basic => Some(BasicTopicDisplayType) - case TopicCandidateDisplayType.Pill => Some(PillTopicDisplayType) - case TopicCandidateDisplayType.NoIcon => - Some(NoIconTopicDisplayType) - case TopicCandidateDisplayType.PillWithoutActionIcon => Some(PillWithoutActionIconDisplayType) - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/ParamTopicFunctionalityTypeBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/ParamTopicFunctionalityTypeBuilder.docx new file mode 100644 index 000000000..01282e865 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/ParamTopicFunctionalityTypeBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/ParamTopicFunctionalityTypeBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/ParamTopicFunctionalityTypeBuilder.scala deleted file mode 100644 index 78d941c19..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/ParamTopicFunctionalityTypeBuilder.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.topic - -import com.twitter.product_mixer.component_library.model.candidate.TopicCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.topic.BasicTopicFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.topic.PivotTopicFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.topic.RecommendationTopicFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.topic.TopicFunctionalityType -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.item.topic.BaseTopicFunctionalityTypeBuilder -import com.twitter.timelines.configapi.FSEnumParam - -object TopicFunctionalityTypeParamValue extends Enumeration { - type TopicFunctionalityType = Value - - val Basic = Value - val Pivot = Value - val Recommendation = Value -} - -case class ParamTopicFunctionalityTypeBuilder( - functionalityTypeParam: FSEnumParam[TopicFunctionalityTypeParamValue.type]) - extends BaseTopicFunctionalityTypeBuilder[PipelineQuery, TopicCandidate] { - - override def apply( - query: PipelineQuery, - candidate: TopicCandidate, - candidateFeatures: FeatureMap - ): Option[TopicFunctionalityType] = { - val functionalityType = query.params(functionalityTypeParam) - functionalityType match { - case TopicFunctionalityTypeParamValue.Basic => Some(BasicTopicFunctionalityType) - case TopicFunctionalityTypeParamValue.Pivot => Some(PivotTopicFunctionalityType) - case TopicFunctionalityTypeParamValue.Recommendation => - Some(RecommendationTopicFunctionalityType) - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/StaticTopicDisplayTypeBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/StaticTopicDisplayTypeBuilder.docx new file mode 100644 index 000000000..12d5a2b50 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/StaticTopicDisplayTypeBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/StaticTopicDisplayTypeBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/StaticTopicDisplayTypeBuilder.scala deleted file mode 100644 index fbe8c2783..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/StaticTopicDisplayTypeBuilder.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.topic - -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.component_library.model.candidate.BaseTopicCandidate -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.item.topic.BaseTopicDisplayTypeBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.topic.TopicDisplayType - -case class StaticTopicDisplayTypeBuilder( - displayType: TopicDisplayType) - extends BaseTopicDisplayTypeBuilder[PipelineQuery, BaseTopicCandidate] { - - override def apply( - query: PipelineQuery, - candidate: BaseTopicCandidate, - candidateFeatures: FeatureMap - ): Option[TopicDisplayType] = Some(displayType) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/StaticTopicFunctionalityTypeBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/StaticTopicFunctionalityTypeBuilder.docx new file mode 100644 index 000000000..7b7d8733f Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/StaticTopicFunctionalityTypeBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/StaticTopicFunctionalityTypeBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/StaticTopicFunctionalityTypeBuilder.scala deleted file mode 100644 index 73e99d0a3..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/StaticTopicFunctionalityTypeBuilder.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.topic - -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.component_library.model.candidate.BaseTopicCandidate -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.item.topic.BaseTopicFunctionalityTypeBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.topic.TopicFunctionalityType -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class StaticTopicFunctionalityTypeBuilder( - functionalityType: TopicFunctionalityType) - extends BaseTopicFunctionalityTypeBuilder[PipelineQuery, BaseTopicCandidate] { - - override def apply( - query: PipelineQuery, - candidate: BaseTopicCandidate, - candidateFeatures: FeatureMap - ): Option[TopicFunctionalityType] = Some(functionalityType) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/TopicCandidateUrtItemBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/TopicCandidateUrtItemBuilder.docx new file mode 100644 index 000000000..9dd694798 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/TopicCandidateUrtItemBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/TopicCandidateUrtItemBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/TopicCandidateUrtItemBuilder.scala deleted file mode 100644 index a049e08fd..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/TopicCandidateUrtItemBuilder.scala +++ /dev/null @@ -1,47 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.topic - -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.topic.TopicCandidateUrtItemBuilder.TopicClientEventInfoElement -import com.twitter.product_mixer.component_library.model.candidate.BaseTopicCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.item.topic.BaseTopicDisplayTypeBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.item.topic.BaseTopicFunctionalityTypeBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.topic.TopicItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object TopicCandidateUrtItemBuilder { - val TopicClientEventInfoElement: String = "topic" -} - -case class TopicCandidateUrtItemBuilder[-Query <: PipelineQuery, Candidate <: BaseTopicCandidate]( - clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, Candidate], - topicFunctionalityTypeBuilder: Option[BaseTopicFunctionalityTypeBuilder[Query, Candidate]] = None, - topicDisplayTypeBuilder: Option[BaseTopicDisplayTypeBuilder[Query, Candidate]] = None, - feedbackActionInfoBuilder: Option[ - BaseFeedbackActionInfoBuilder[Query, Candidate] - ] = None) - extends CandidateUrtEntryBuilder[Query, Candidate, TopicItem] { - - override def apply( - query: Query, - topicCandidate: Candidate, - candidateFeatures: FeatureMap - ): TopicItem = - TopicItem( - id = topicCandidate.id, - sortIndex = None, // Sort indexes are automatically set in the domain marshaller phase - clientEventInfo = clientEventInfoBuilder( - query, - topicCandidate, - candidateFeatures, - Some(TopicClientEventInfoElement)), - feedbackActionInfo = - feedbackActionInfoBuilder.flatMap(_.apply(query, topicCandidate, candidateFeatures)), - topicFunctionalityType = - topicFunctionalityTypeBuilder.flatMap(_.apply(query, topicCandidate, candidateFeatures)), - topicDisplayType = - topicDisplayTypeBuilder.flatMap(_.apply(query, topicCandidate, candidateFeatures)) - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/VerticalGridTopicCandidateUrtItemBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/VerticalGridTopicCandidateUrtItemBuilder.docx new file mode 100644 index 000000000..9b63d0ebf Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/VerticalGridTopicCandidateUrtItemBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/VerticalGridTopicCandidateUrtItemBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/VerticalGridTopicCandidateUrtItemBuilder.scala deleted file mode 100644 index 40f436159..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/topic/VerticalGridTopicCandidateUrtItemBuilder.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.topic - -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.topic.TopicCandidateUrtItemBuilder.TopicClientEventInfoElement -import com.twitter.product_mixer.component_library.model.candidate.TopicCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseUrlBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.vertical_grid_item.VerticalGridItem -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.vertical_grid_item.VerticalGridItemTileStyle -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.vertical_grid_item.VerticalGridItemTopicFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.vertical_grid_item.VerticalGridItemTopicTile -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class VerticalGridTopicCandidateUrtItemBuilder[-Query <: PipelineQuery]( - clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, TopicCandidate], - verticalGridItemTopicFunctionalityType: VerticalGridItemTopicFunctionalityType, - verticalGridItemTileStyle: VerticalGridItemTileStyle, - urlBuilder: Option[BaseUrlBuilder[Query, TopicCandidate]] = None, - feedbackActionInfoBuilder: Option[ - BaseFeedbackActionInfoBuilder[Query, TopicCandidate] - ] = None) - extends CandidateUrtEntryBuilder[Query, TopicCandidate, VerticalGridItem] { - - override def apply( - query: Query, - topicCandidate: TopicCandidate, - candidateFeatures: FeatureMap - ): VerticalGridItem = { - VerticalGridItemTopicTile( - id = topicCandidate.id, - sortIndex = None, // Sort indexes are automatically set in the domain marshaller phase - clientEventInfo = clientEventInfoBuilder( - query, - topicCandidate, - candidateFeatures, - Some(TopicClientEventInfoElement)), - feedbackActionInfo = - feedbackActionInfoBuilder.flatMap(_.apply(query, topicCandidate, candidateFeatures)), - style = Some(verticalGridItemTileStyle), - functionalityType = Some(verticalGridItemTopicFunctionalityType), - url = urlBuilder.map(_.apply(query, topicCandidate, candidateFeatures)) - ) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/trend/TrendCandidateUrtItemBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/trend/TrendCandidateUrtItemBuilder.docx new file mode 100644 index 000000000..64e98ab2f Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/trend/TrendCandidateUrtItemBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/trend/TrendCandidateUrtItemBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/trend/TrendCandidateUrtItemBuilder.scala deleted file mode 100644 index c3f5c45ad..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/trend/TrendCandidateUrtItemBuilder.scala +++ /dev/null @@ -1,63 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.trend - -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.trend.TrendCandidateUrtItemBuilder.TrendsClientEventInfoElement -import com.twitter.product_mixer.component_library.model.candidate.trends_events.TrendDescription -import com.twitter.product_mixer.component_library.model.candidate.trends_events.TrendDomainContext -import com.twitter.product_mixer.component_library.model.candidate.trends_events.TrendGroupedTrends -import com.twitter.product_mixer.component_library.model.candidate.trends_events.TrendNormalizedTrendName -import com.twitter.product_mixer.component_library.model.candidate.trends_events.TrendTrendName -import com.twitter.product_mixer.component_library.model.candidate.trends_events.TrendTweetCount -import com.twitter.product_mixer.component_library.model.candidate.trends_events.TrendUrl -import com.twitter.product_mixer.component_library.model.candidate.trends_events.UnifiedTrendCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.promoted.BasePromotedMetadataBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineItem -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.trend.TrendItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object TrendCandidateUrtItemBuilder { - final val TrendsClientEventInfoElement = "trend" -} - -case class TrendCandidateUrtItemBuilder[Query <: PipelineQuery]( - trendMetaDescriptionBuilder: TrendMetaDescriptionBuilder[Query, UnifiedTrendCandidate], - promotedMetadataBuilder: BasePromotedMetadataBuilder[Query, UnifiedTrendCandidate], - clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, UnifiedTrendCandidate], - feedbackActionInfoBuilder: Option[BaseFeedbackActionInfoBuilder[Query, UnifiedTrendCandidate]] = - None) - extends CandidateUrtEntryBuilder[Query, UnifiedTrendCandidate, TimelineItem] { - - override def apply( - query: Query, - candidate: UnifiedTrendCandidate, - candidateFeatures: FeatureMap - ): TimelineItem = { - TrendItem( - id = candidate.id, - sortIndex = None, // Sort indexes are automatically set in the domain marshaller phase - clientEventInfo = clientEventInfoBuilder( - query = query, - candidate = candidate, - candidateFeatures = candidateFeatures, - element = Some(TrendsClientEventInfoElement) - ), - feedbackActionInfo = None, - normalizedTrendName = candidateFeatures.get(TrendNormalizedTrendName), - trendName = candidateFeatures.get(TrendTrendName), - url = candidateFeatures.get(TrendUrl), - description = candidateFeatures.getOrElse(TrendDescription, None), - metaDescription = trendMetaDescriptionBuilder(query, candidate, candidateFeatures), - tweetCount = candidateFeatures.getOrElse(TrendTweetCount, None), - domainContext = candidateFeatures.getOrElse(TrendDomainContext, None), - promotedMetadata = promotedMetadataBuilder( - query = query, - candidate = candidate, - candidateFeatures = candidateFeatures - ), - groupedTrends = candidateFeatures.getOrElse(TrendGroupedTrends, None) - ) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/trend/TrendMetaDescriptionBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/trend/TrendMetaDescriptionBuilder.docx new file mode 100644 index 000000000..a32e84c62 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/trend/TrendMetaDescriptionBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/trend/TrendMetaDescriptionBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/trend/TrendMetaDescriptionBuilder.scala deleted file mode 100644 index b8cc639a0..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/trend/TrendMetaDescriptionBuilder.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.trend - -import com.twitter.product_mixer.component_library.decorator.urt.builder.stringcenter.Str -import com.twitter.product_mixer.component_library.model.candidate.trends_events.PromotedTrendAdvertiserNameFeature -import com.twitter.product_mixer.component_library.model.candidate.trends_events.TrendTweetCount -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.trends.trending_content.util.CompactingNumberLocalizer - -case class TrendMetaDescriptionBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( - promotedByMetaDescriptionStr: Str[PipelineQuery, UniversalNoun[Any]], - tweetCountMetaDescriptionStr: Str[PipelineQuery, UniversalNoun[Any]], - compactingNumberLocalizer: CompactingNumberLocalizer) { - - def apply( - query: Query, - candidate: Candidate, - candidateFeatures: FeatureMap - ): Option[String] = { - val promotedMetaDescription = - candidateFeatures.getOrElse(PromotedTrendAdvertiserNameFeature, None).map { advertiserName => - promotedByMetaDescriptionStr(query, candidate, candidateFeatures).format(advertiserName) - } - - val organicMetaDescription = candidateFeatures.getOrElse(TrendTweetCount, None).map { - tweetCount => - val compactedTweetCount = compactingNumberLocalizer.localizeAndCompact( - query.getLanguageCode - .getOrElse("en"), - tweetCount) - tweetCountMetaDescriptionStr(query, candidate, candidateFeatures).format( - compactedTweetCount) - } - - promotedMetaDescription.orElse(organicMetaDescription) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/trend/TrendPromotedMetadataBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/trend/TrendPromotedMetadataBuilder.docx new file mode 100644 index 000000000..b53def3ce Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/trend/TrendPromotedMetadataBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/trend/TrendPromotedMetadataBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/trend/TrendPromotedMetadataBuilder.scala deleted file mode 100644 index 9c1c34fa5..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/trend/TrendPromotedMetadataBuilder.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.trend - -import com.twitter.product_mixer.component_library.model.candidate.trends_events.PromotedTrendDescriptionFeature -import com.twitter.product_mixer.component_library.model.candidate.trends_events.PromotedTrendDisclosureTypeFeature -import com.twitter.product_mixer.component_library.model.candidate.trends_events.PromotedTrendIdFeature -import com.twitter.product_mixer.component_library.model.candidate.trends_events.PromotedTrendImpressionIdFeature -import com.twitter.product_mixer.component_library.model.candidate.trends_events.PromotedTrendNameFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.promoted.BasePromotedMetadataBuilder -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.PromotedMetadata -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object TrendPromotedMetadataBuilder - extends BasePromotedMetadataBuilder[PipelineQuery, UniversalNoun[Any]] { - - override def apply( - query: PipelineQuery, - candidate: UniversalNoun[Any], - candidateFeatures: FeatureMap - ): Option[PromotedMetadata] = { - // If a promoted trend name exists, then this is a promoted trend - candidateFeatures.getOrElse(PromotedTrendNameFeature, None).map { promotedTrendName => - PromotedMetadata( - // This is the current product behavior that advertiserId is always set to 0L. - // Correct advertiser name comes from Trend's trendMetadata.metaDescription. - advertiserId = 0L, - disclosureType = candidateFeatures.getOrElse(PromotedTrendDisclosureTypeFeature, None), - experimentValues = None, - promotedTrendId = candidateFeatures.getOrElse(PromotedTrendIdFeature, None), - promotedTrendName = Some(promotedTrendName), - promotedTrendQueryTerm = None, - adMetadataContainer = None, - promotedTrendDescription = - candidateFeatures.getOrElse(PromotedTrendDescriptionFeature, None), - impressionString = candidateFeatures.getOrElse(PromotedTrendImpressionIdFeature, None), - clickTrackingInfo = None - ) - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/tweet/TweetCandidateUrtItemBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/tweet/TweetCandidateUrtItemBuilder.docx new file mode 100644 index 000000000..c66f3886a Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/tweet/TweetCandidateUrtItemBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/tweet/TweetCandidateUrtItemBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/tweet/TweetCandidateUrtItemBuilder.scala deleted file mode 100644 index d02b96f1c..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/tweet/TweetCandidateUrtItemBuilder.scala +++ /dev/null @@ -1,92 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet - -import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder.TweetClientEventInfoElement -import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate -import com.twitter.product_mixer.component_library.model.candidate.IsPinnedFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.item.tweet.BaseEntryIdToReplaceBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.item.tweet.BaseTimelinesScoreInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.item.tweet.BaseTweetHighlightsBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseUrlBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.conversation_annotation.ConversationAnnotation -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.forward_pivot.ForwardPivot -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tombstone.TombstoneInfo -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.Tweet -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetDisplayType -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Badge -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.PrerollMetadata -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.PromotedMetadata -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case object TweetCandidateUrtItemBuilder { - val TweetClientEventInfoElement = "tweet" -} - -case class TweetCandidateUrtItemBuilder[Query <: PipelineQuery, Candidate <: BaseTweetCandidate]( - clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, Candidate], - displayType: TweetDisplayType = Tweet, - entryIdToReplaceBuilder: Option[BaseEntryIdToReplaceBuilder[Query, Candidate]] = None, - socialContextBuilder: Option[BaseSocialContextBuilder[Query, Candidate]] = None, - highlightsBuilder: Option[BaseTweetHighlightsBuilder[Query, Candidate]] = None, - innerTombstoneInfo: Option[TombstoneInfo] = None, - timelinesScoreInfoBuilder: Option[BaseTimelinesScoreInfoBuilder[Query, Candidate]] = None, - hasModeratedReplies: Option[Boolean] = None, - forwardPivot: Option[ForwardPivot] = None, - innerForwardPivot: Option[ForwardPivot] = None, - feedbackActionInfoBuilder: Option[BaseFeedbackActionInfoBuilder[Query, Candidate]] = None, - promotedMetadata: Option[PromotedMetadata] = None, - conversationAnnotation: Option[ConversationAnnotation] = None, - contextualTweetRefBuilder: Option[ContextualTweetRefBuilder[Candidate]] = None, - prerollMetadata: Option[PrerollMetadata] = None, - replyBadge: Option[Badge] = None, - destinationBuilder: Option[BaseUrlBuilder[Query, Candidate]] = None) - extends CandidateUrtEntryBuilder[Query, Candidate, TweetItem] { - - override def apply( - pipelineQuery: Query, - tweetCandidate: Candidate, - candidateFeatures: FeatureMap - ): TweetItem = { - val isPinned = candidateFeatures.getTry(IsPinnedFeature).toOption - - TweetItem( - id = tweetCandidate.id, - entryNamespace = TweetItem.TweetEntryNamespace, - sortIndex = None, // Sort indexes are automatically set in the domain marshaller phase - clientEventInfo = clientEventInfoBuilder( - pipelineQuery, - tweetCandidate, - candidateFeatures, - Some(TweetClientEventInfoElement)), - feedbackActionInfo = feedbackActionInfoBuilder.flatMap( - _.apply(pipelineQuery, tweetCandidate, candidateFeatures)), - isPinned = isPinned, - entryIdToReplace = - entryIdToReplaceBuilder.flatMap(_.apply(pipelineQuery, tweetCandidate, candidateFeatures)), - socialContext = - socialContextBuilder.flatMap(_.apply(pipelineQuery, tweetCandidate, candidateFeatures)), - highlights = - highlightsBuilder.flatMap(_.apply(pipelineQuery, tweetCandidate, candidateFeatures)), - displayType = displayType, - innerTombstoneInfo = innerTombstoneInfo, - timelinesScoreInfo = timelinesScoreInfoBuilder - .flatMap(_.apply(pipelineQuery, tweetCandidate, candidateFeatures)), - hasModeratedReplies = hasModeratedReplies, - forwardPivot = forwardPivot, - innerForwardPivot = innerForwardPivot, - promotedMetadata = promotedMetadata, - conversationAnnotation = conversationAnnotation, - contextualTweetRef = contextualTweetRefBuilder.flatMap(_.apply(tweetCandidate)), - prerollMetadata = prerollMetadata, - replyBadge = replyBadge, - destination = - destinationBuilder.map(_.apply(pipelineQuery, tweetCandidate, candidateFeatures)) - ) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/twitter_list/TwitterListCandidateUrtItemBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/twitter_list/TwitterListCandidateUrtItemBuilder.docx new file mode 100644 index 000000000..ad908e10d Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/twitter_list/TwitterListCandidateUrtItemBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/twitter_list/TwitterListCandidateUrtItemBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/twitter_list/TwitterListCandidateUrtItemBuilder.scala deleted file mode 100644 index 41ed4ed71..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/twitter_list/TwitterListCandidateUrtItemBuilder.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.twitter_list - -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.twitter_list.TwitterListCandidateUrtItemBuilder.ListClientEventInfoElement -import com.twitter.product_mixer.component_library.model.candidate.TwitterListCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.twitter_list.TwitterListDisplayType -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.twitter_list.TwitterListItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object TwitterListCandidateUrtItemBuilder { - val ListClientEventInfoElement: String = "list" -} - -case class TwitterListCandidateUrtItemBuilder[-Query <: PipelineQuery]( - clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, TwitterListCandidate], - feedbackActionInfoBuilder: Option[ - BaseFeedbackActionInfoBuilder[Query, TwitterListCandidate] - ] = None, - displayType: Option[TwitterListDisplayType] = None) - extends CandidateUrtEntryBuilder[Query, TwitterListCandidate, TwitterListItem] { - - override def apply( - query: Query, - twitterListCandidate: TwitterListCandidate, - candidateFeatures: FeatureMap - ): TwitterListItem = TwitterListItem( - id = twitterListCandidate.id, - sortIndex = None, // Sort indexes are automatically set in the domain marshaller phase - clientEventInfo = clientEventInfoBuilder( - query, - twitterListCandidate, - candidateFeatures, - Some(ListClientEventInfoElement)), - feedbackActionInfo = - feedbackActionInfoBuilder.flatMap(_.apply(query, twitterListCandidate, candidateFeatures)), - displayType = displayType - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/unified_trend_event/UnifiedTrendEventCandidateUrtItemBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/unified_trend_event/UnifiedTrendEventCandidateUrtItemBuilder.docx new file mode 100644 index 000000000..4fc4f001f Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/unified_trend_event/UnifiedTrendEventCandidateUrtItemBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/unified_trend_event/UnifiedTrendEventCandidateUrtItemBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/unified_trend_event/UnifiedTrendEventCandidateUrtItemBuilder.scala deleted file mode 100644 index c4b67ce04..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/unified_trend_event/UnifiedTrendEventCandidateUrtItemBuilder.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.unified_trend_event - -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.event_summary.EventCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.trend.TrendCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.model.candidate.trends_events.UnifiedEventCandidate -import com.twitter.product_mixer.component_library.model.candidate.trends_events.UnifiedTrendCandidate -import com.twitter.product_mixer.component_library.model.candidate.trends_events.UnifiedTrendEventCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class UnifiedTrendEventCandidateUrtItemBuilder[Query <: PipelineQuery]( - eventCandidateUrtItemBuilder: EventCandidateUrtItemBuilder[Query], - trendCandidateUrtItemBuilder: TrendCandidateUrtItemBuilder[Query]) - extends CandidateUrtEntryBuilder[Query, UnifiedTrendEventCandidate[Any], TimelineItem] { - - override def apply( - query: Query, - candidate: UnifiedTrendEventCandidate[Any], - candidateFeatures: FeatureMap - ): TimelineItem = { - candidate match { - case event: UnifiedEventCandidate => - eventCandidateUrtItemBuilder( - query = query, - candidate = event, - candidateFeatures = candidateFeatures) - case trend: UnifiedTrendCandidate => - trendCandidateUrtItemBuilder( - query = query, - candidate = trend, - candidateFeatures = candidateFeatures) - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/user/UserCandidateUrtItemBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/user/UserCandidateUrtItemBuilder.docx new file mode 100644 index 000000000..ee1a00969 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/user/UserCandidateUrtItemBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/user/UserCandidateUrtItemBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/user/UserCandidateUrtItemBuilder.scala deleted file mode 100644 index 95a7e6bd9..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/item/user/UserCandidateUrtItemBuilder.scala +++ /dev/null @@ -1,62 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.item.user - -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.user.UserCandidateUrtItemBuilder.UserClientEventInfoElement -import com.twitter.product_mixer.component_library.model.candidate.BaseUserCandidate -import com.twitter.product_mixer.component_library.model.candidate.IsMarkUnreadFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.item.user.BaseUserReactiveTriggersBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.promoted.BasePromotedMetadataBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.User -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserDisplayType -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object UserCandidateUrtItemBuilder { - val UserClientEventInfoElement: String = "user" -} - -case class UserCandidateUrtItemBuilder[Query <: PipelineQuery, UserCandidate <: BaseUserCandidate]( - clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, UserCandidate], - feedbackActionInfoBuilder: Option[ - BaseFeedbackActionInfoBuilder[Query, UserCandidate] - ] = None, - displayType: UserDisplayType = User, - promotedMetadataBuilder: Option[BasePromotedMetadataBuilder[Query, UserCandidate]] = None, - socialContextBuilder: Option[BaseSocialContextBuilder[Query, UserCandidate]] = None, - reactiveTriggersBuilder: Option[BaseUserReactiveTriggersBuilder[Query, UserCandidate]] = None, - enableReactiveBlending: Option[Boolean] = None) - extends CandidateUrtEntryBuilder[Query, UserCandidate, UserItem] { - - override def apply( - query: Query, - userCandidate: UserCandidate, - candidateFeatures: FeatureMap - ): UserItem = { - val isMarkUnread = candidateFeatures.getTry(IsMarkUnreadFeature).toOption - - UserItem( - id = userCandidate.id, - sortIndex = None, // Sort indexes are automatically set in the domain marshaller phase - clientEventInfo = clientEventInfoBuilder( - query, - userCandidate, - candidateFeatures, - Some(UserClientEventInfoElement)), - feedbackActionInfo = - feedbackActionInfoBuilder.flatMap(_.apply(query, userCandidate, candidateFeatures)), - isMarkUnread = isMarkUnread, - displayType = displayType, - promotedMetadata = - promotedMetadataBuilder.flatMap(_.apply(query, userCandidate, candidateFeatures)), - socialContext = - socialContextBuilder.flatMap(_.apply(query, userCandidate, candidateFeatures)), - reactiveTriggers = - reactiveTriggersBuilder.flatMap(_.apply(query, userCandidate, candidateFeatures)), - enableReactiveBlending = enableReactiveBlending - ) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/ClientEventInfoBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/ClientEventInfoBuilder.docx new file mode 100644 index 000000000..d2852ebdf Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/ClientEventInfoBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/ClientEventInfoBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/ClientEventInfoBuilder.scala deleted file mode 100644 index 0c0eca37b..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/ClientEventInfoBuilder.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.metadata - -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventInfo -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventDetailsBuilder - -/** - * Sets the [[ClientEventInfo]] with the `component` field set to [[component]] - * @see [[http://go/client-events]] - */ -case class ClientEventInfoBuilder[-Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]( - component: String, - detailsBuilder: Option[BaseClientEventDetailsBuilder[Query, Candidate]] = None) - extends BaseClientEventInfoBuilder[Query, Candidate] { - - override def apply( - query: Query, - candidate: Candidate, - candidateFeatures: FeatureMap, - element: Option[String] - ): Option[ClientEventInfo] = - Some( - ClientEventInfo( - component = Some(component), - element = element, - details = detailsBuilder.flatMap(_.apply(query, candidate, candidateFeatures)), - action = None, - entityToken = None) - ) -} - -/** - * In rare cases you might not want to send client event info. For - * example, this might be set already on the client for some legacy - * timelines. - */ -object EmptyClientEventInfoBuilder - extends BaseClientEventInfoBuilder[PipelineQuery, UniversalNoun[Any]] { - override def apply( - query: PipelineQuery, - candidate: UniversalNoun[Any], - candidateFeatures: FeatureMap, - element: Option[String] - ): Option[ClientEventInfo] = None -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/ConversationTweetClientEventDetailsBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/ConversationTweetClientEventDetailsBuilder.docx new file mode 100644 index 000000000..363008b7e Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/ConversationTweetClientEventDetailsBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/ConversationTweetClientEventDetailsBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/ConversationTweetClientEventDetailsBuilder.scala deleted file mode 100644 index ad8034dbc..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/ConversationTweetClientEventDetailsBuilder.scala +++ /dev/null @@ -1,63 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.metadata - -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.bijection.Base64String -import com.twitter.bijection.{Injection => Serializer} -import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ConversationTweetClientEventDetailsBuilder.ControllerDataSerializer -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventDetailsBuilder -import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TimelinesDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.suggests.controller_data.home_tweets.thriftscala.HomeTweetsControllerData -import com.twitter.suggests.controller_data.thriftscala.ControllerData -import com.twitter.suggests.controller_data.home_tweets.v1.thriftscala.{ - HomeTweetsControllerData => HomeTweetsControllerDataV1 -} -import com.twitter.suggests.controller_data.v2.thriftscala.{ControllerData => ControllerDataV2} - -object ConversationTweetClientEventDetailsBuilder { - implicit val ByteSerializer: Serializer[ControllerData, Array[Byte]] = - BinaryScalaCodec(ControllerData) - - val ControllerDataSerializer: Serializer[ControllerData, String] = - Serializer.connect[ControllerData, Array[Byte], Base64String, String] -} - -case class ConversationTweetClientEventDetailsBuilder[-Query <: PipelineQuery]( - injectionType: Option[String]) - extends BaseClientEventDetailsBuilder[Query, BaseTweetCandidate] { - - override def apply( - query: Query, - tweetCandidate: BaseTweetCandidate, - candidateFeatures: FeatureMap - ): Option[ClientEventDetails] = - Some( - ClientEventDetails( - conversationDetails = None, - timelinesDetails = Some( - TimelinesDetails( - injectionType = injectionType, - controllerData = Some(buildControllerData(query.getUserOrGuestId)), - sourceData = None)), - articleDetails = None, - liveEventDetails = None, - commerceDetails = None - )) - - private def buildControllerData(traceId: Option[Long]): String = - ControllerDataSerializer( - ControllerData.V2( - ControllerDataV2.HomeTweets( - HomeTweetsControllerData.V1( - HomeTweetsControllerDataV1( - tweetTypesBitmap = 0L, - traceId = traceId, - ) - ) - ) - ) - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/StaticUrlBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/StaticUrlBuilder.docx new file mode 100644 index 000000000..c9b2f8746 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/StaticUrlBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/StaticUrlBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/StaticUrlBuilder.scala deleted file mode 100644 index 7db66990f..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/StaticUrlBuilder.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.metadata - -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseUrlBuilder -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Url -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.UrlType -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class StaticUrlBuilder(url: String, urlType: UrlType) - extends BaseUrlBuilder[PipelineQuery, UniversalNoun[Any]] { - - override def apply( - query: PipelineQuery, - candidate: UniversalNoun[Any], - candidateFeatures: FeatureMap - ): Url = Url(url = url, urlType = urlType) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/TopicClientEventDetailsBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/TopicClientEventDetailsBuilder.docx new file mode 100644 index 000000000..4b744ab4f Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/TopicClientEventDetailsBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/TopicClientEventDetailsBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/TopicClientEventDetailsBuilder.scala deleted file mode 100644 index af28b49f3..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/TopicClientEventDetailsBuilder.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.metadata - -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.bijection.Base64String -import com.twitter.bijection.{Injection => Serializer} -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventDetailsBuilder -import com.twitter.product_mixer.component_library.model.candidate.BaseTopicCandidate -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TimelinesDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.suggests.controller_data.thriftscala.ControllerData -import com.twitter.suggests.controller_data.timelines_topic.thriftscala.TimelinesTopicControllerData -import com.twitter.suggests.controller_data.timelines_topic.v1.thriftscala.{ - TimelinesTopicControllerData => TimelinesTopicControllerDataV1 -} -import com.twitter.suggests.controller_data.v2.thriftscala.{ControllerData => ControllerDataV2} - -object TopicClientEventDetailsBuilder { - implicit val ByteSerializer: Serializer[ControllerData, Array[Byte]] = - BinaryScalaCodec(ControllerData) - - val ControllerDataSerializer: Serializer[ControllerData, String] = - Serializer.connect[ControllerData, Array[Byte], Base64String, String] -} - -case class TopicClientEventDetailsBuilder[-Query <: PipelineQuery]() - extends BaseClientEventDetailsBuilder[Query, BaseTopicCandidate] { - - import TopicClientEventDetailsBuilder._ - - override def apply( - query: Query, - topicCandidate: BaseTopicCandidate, - candidateFeatures: FeatureMap - ): Option[ClientEventDetails] = - Some( - ClientEventDetails( - conversationDetails = None, - timelinesDetails = Some( - TimelinesDetails( - injectionType = None, - controllerData = buildControllerData(topicCandidate.id), - sourceData = None)), - articleDetails = None, - liveEventDetails = None, - commerceDetails = None - )) - - private def buildControllerData(topicId: Long): Option[String] = - Some( - ControllerData - .V2(ControllerDataV2.TimelinesTopic(TimelinesTopicControllerData.V1( - TimelinesTopicControllerDataV1(topicTypesBitmap = 0L, topicId = topicId))))) - .map(ControllerDataSerializer) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/TopicNotInterestedFeedbackActionInfoBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/TopicNotInterestedFeedbackActionInfoBuilder.docx new file mode 100644 index 000000000..1fdfa055a Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/TopicNotInterestedFeedbackActionInfoBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/TopicNotInterestedFeedbackActionInfoBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/TopicNotInterestedFeedbackActionInfoBuilder.scala deleted file mode 100644 index cfb7cb5ff..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/TopicNotInterestedFeedbackActionInfoBuilder.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.metadata - -import com.twitter.product_mixer.component_library.model.candidate.BaseTopicCandidate -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackAction -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackActionInfo -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichBehavior -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorMarkNotInterestedTopic -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class TopicNotInterestedFeedbackActionInfoBuilder[-Query <: PipelineQuery]() - extends BaseFeedbackActionInfoBuilder[Query, BaseTopicCandidate] { - - override def apply( - query: Query, - topicCandidate: BaseTopicCandidate, - candidateFeatures: FeatureMap - ): Option[FeedbackActionInfo] = { - Some( - FeedbackActionInfo( - feedbackActions = Seq( - FeedbackAction( - feedbackType = RichBehavior, - richBehavior = Some( - RichFeedbackBehaviorMarkNotInterestedTopic(topicCandidate.id.toString) - ), - hasUndoAction = Some(true), - prompt = None, - confirmation = None, - feedbackUrl = None, - clientEventInfo = None, - childFeedbackActions = None, - confirmationDisplayType = None, - icon = None, - subprompt = None, - encodedFeedbackRequest = None - ) - ), - feedbackMetadata = None, - displayContext = None, - clientEventInfo = None - )) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/TopicTweetClientEventDetailsBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/TopicTweetClientEventDetailsBuilder.docx new file mode 100644 index 000000000..9bc311f6d Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/TopicTweetClientEventDetailsBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/TopicTweetClientEventDetailsBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/TopicTweetClientEventDetailsBuilder.scala deleted file mode 100644 index 451b347ae..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/TopicTweetClientEventDetailsBuilder.scala +++ /dev/null @@ -1,66 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.metadata - -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.bijection.Base64String -import com.twitter.bijection.{Injection => Serializer} -import com.twitter.interests_mixer.model.request.{HasTopicId => InterestsMixerHasTopicId} -import com.twitter.explore_mixer.model.request.{HasTopicId => ExploreMixerHasTopicId} -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventDetailsBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TimelinesDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.suggests.controller_data.home_tweets.thriftscala.HomeTweetsControllerData -import com.twitter.suggests.controller_data.home_tweets.v1.thriftscala.{ - HomeTweetsControllerData => HomeTweetsControllerDataV1 -} -import com.twitter.suggests.controller_data.thriftscala.ControllerData -import com.twitter.suggests.controller_data.v2.thriftscala.{ControllerData => ControllerDataV2} - -object TopicTweetClientEventDetailsBuilder { - implicit val ByteSerializer: Serializer[ControllerData, Array[Byte]] = - BinaryScalaCodec(ControllerData) - - val ControllerDataSerializer: Serializer[ControllerData, String] = - Serializer.connect[ControllerData, Array[Byte], Base64String, String] -} - -case class TopicTweetClientEventDetailsBuilder[-Query <: PipelineQuery]() - extends BaseClientEventDetailsBuilder[Query, TweetCandidate] { - - import TopicTweetClientEventDetailsBuilder._ - - override def apply( - query: Query, - topicTweetCandidate: TweetCandidate, - candidateFeatures: FeatureMap - ): Option[ClientEventDetails] = - Some( - ClientEventDetails( - conversationDetails = None, - timelinesDetails = Some( - TimelinesDetails( - injectionType = None, - controllerData = buildControllerData(getTopicId(query)), - sourceData = None)), - articleDetails = None, - liveEventDetails = None, - commerceDetails = None - )) - - private def getTopicId(query: Query): Option[Long] = { - query match { - case query: InterestsMixerHasTopicId => query.topicId - case query: ExploreMixerHasTopicId => query.topicId - case _ => None - } - } - - private def buildControllerData(topicId: Option[Long]): Option[String] = - Some( - ControllerData - .V2(ControllerDataV2.HomeTweets(HomeTweetsControllerData.V1( - HomeTweetsControllerDataV1(tweetTypesBitmap = 0L, topicId = topicId))))) - .map(ControllerDataSerializer) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/TopicsToFollowModuleMetadataBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/TopicsToFollowModuleMetadataBuilder.docx new file mode 100644 index 000000000..8c01ad5a9 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/TopicsToFollowModuleMetadataBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/TopicsToFollowModuleMetadataBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/TopicsToFollowModuleMetadataBuilder.scala deleted file mode 100644 index fa32e2127..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/TopicsToFollowModuleMetadataBuilder.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.metadata - -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseModuleMetadataBuilder -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.GridCarouselMetadata -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleMetadata -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.Param - -object TopicsToFollowModuleMetadataBuilder { - - val TopicsPerRow = 7 - - /* - * rows = min(MAX_NUM_ROWS, # topics / TOPICS_PER_ROW) - * where TOPICS_PER_ROW = 7 - */ - def getCarouselRowCount(topicsCount: Int, maxCarouselRows: Int): Int = - Math.min(maxCarouselRows, (topicsCount / TopicsPerRow) + 1) -} - -case class TopicsToFollowModuleMetadataBuilder(maxCarouselRowsParam: Param[Int]) - extends BaseModuleMetadataBuilder[PipelineQuery, UniversalNoun[Any]] { - - import TopicsToFollowModuleMetadataBuilder._ - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[UniversalNoun[Any]]] - ): ModuleMetadata = { - val rowCount = getCarouselRowCount(candidates.size, query.params(maxCarouselRowsParam)) - ModuleMetadata( - adsMetadata = None, - conversationMetadata = None, - gridCarouselMetadata = Some(GridCarouselMetadata(numRows = Some(rowCount))) - ) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/WhoToFollowFeedbackActionInfoBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/WhoToFollowFeedbackActionInfoBuilder.docx new file mode 100644 index 000000000..d118dab46 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/WhoToFollowFeedbackActionInfoBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/WhoToFollowFeedbackActionInfoBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/WhoToFollowFeedbackActionInfoBuilder.scala deleted file mode 100644 index 04d6fae7b..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/metadata/WhoToFollowFeedbackActionInfoBuilder.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.metadata - -import com.twitter.product_mixer.component_library.decorator.urt.builder.stringcenter.Str -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.icon.Frown -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackAction -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackActionInfo -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SeeFewer -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stringcenter.client.ExternalStringRegistry -import com.twitter.stringcenter.client.StringCenter - -case class WhoToFollowFeedbackActionInfoBuilder[ - -Query <: PipelineQuery, - -Candidate <: UniversalNoun[Any] -]( - externalStringRegistry: ExternalStringRegistry, - stringCenter: StringCenter, - encodedFeedbackRequest: Option[String]) - extends BaseFeedbackActionInfoBuilder[Query, Candidate] { - - private val seeLessOftenFeedback = - externalStringRegistry.createProdString("Feedback.seeLessOften") - private val seeLessOftenConfirmationFeedback = - externalStringRegistry.createProdString("Feedback.seeLessOftenConfirmation") - - override def apply( - query: Query, - candidate: Candidate, - candidateFeatures: FeatureMap - ): Option[FeedbackActionInfo] = Some( - FeedbackActionInfo( - feedbackActions = Seq( - FeedbackAction( - feedbackType = SeeFewer, - prompt = Some( - Str(seeLessOftenFeedback, stringCenter, None) - .apply(query, candidate, candidateFeatures)), - confirmation = Some( - Str(seeLessOftenConfirmationFeedback, stringCenter, None) - .apply(query, candidate, candidateFeatures)), - childFeedbackActions = None, - feedbackUrl = None, - confirmationDisplayType = None, - clientEventInfo = None, - richBehavior = None, - subprompt = None, - icon = Some(Frown), // ignored by unsupported clients - hasUndoAction = Some(true), - encodedFeedbackRequest = encodedFeedbackRequest - ) - ), - feedbackMetadata = None, - displayContext = None, - clientEventInfo = None - ) - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/operation/CursorCandidateUrtOperationBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/operation/CursorCandidateUrtOperationBuilder.docx new file mode 100644 index 000000000..17c725670 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/operation/CursorCandidateUrtOperationBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/operation/CursorCandidateUrtOperationBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/operation/CursorCandidateUrtOperationBuilder.scala deleted file mode 100644 index 6cc1ad677..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/operation/CursorCandidateUrtOperationBuilder.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.operation - -import com.twitter.product_mixer.component_library.model.candidate.CursorCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorDisplayTreatment -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorOperation -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorType -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class CursorCandidateUrtOperationBuilder[-Query <: PipelineQuery]( - cursorType: CursorType, - displayTreatment: Option[CursorDisplayTreatment] = None, - idToReplace: Option[Long] = None) - extends CandidateUrtEntryBuilder[Query, CursorCandidate, CursorOperation] { - - override def apply( - query: Query, - cursorCandidate: CursorCandidate, - candidateFeatures: FeatureMap - ): CursorOperation = CursorOperation( - id = cursorCandidate.id, - sortIndex = None, // Sort indexes are automatically set in the domain marshaller phase - value = cursorCandidate.value, - cursorType = cursorType, - displayTreatment = displayTreatment, - idToReplace = idToReplace - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/promoted/FeaturePromotedMetadataBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/promoted/FeaturePromotedMetadataBuilder.docx new file mode 100644 index 000000000..ba2d01832 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/promoted/FeaturePromotedMetadataBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/promoted/FeaturePromotedMetadataBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/promoted/FeaturePromotedMetadataBuilder.scala deleted file mode 100644 index 6edc3b593..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/promoted/FeaturePromotedMetadataBuilder.scala +++ /dev/null @@ -1,106 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.promoted - -import com.twitter.ads.adserver.{thriftscala => ads} -import com.twitter.ads.common.base.{thriftscala => ac} -import com.twitter.adserver.{thriftscala => ad} -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.promoted.BasePromotedMetadataBuilder -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted._ -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.util.AdMetadataContainerSerializer - -case class FeaturePromotedMetadataBuilder(adImpressionFeature: Feature[_, Option[ad.AdImpression]]) - extends BasePromotedMetadataBuilder[PipelineQuery, UniversalNoun[Any]] { - - def apply( - query: PipelineQuery, - candidate: UniversalNoun[Any], - candidateFeatures: FeatureMap - ): Option[PromotedMetadata] = { - candidateFeatures.getOrElse(adImpressionFeature, None).map { impression => - PromotedMetadata( - advertiserId = impression.advertiserId, - disclosureType = impression.disclosureType.map(convertDisclosureType), - experimentValues = impression.experimentValues.map(_.toMap), - promotedTrendId = impression.promotedTrendId.map(_.toLong), - promotedTrendName = impression.promotedTrendName, - promotedTrendQueryTerm = impression.promotedTrendQueryTerm, - adMetadataContainer = - impression.serializedAdMetadataContainer.flatMap(convertAdMetadataContainer), - promotedTrendDescription = impression.promotedTrendDescription, - impressionString = impression.impressionString, - clickTrackingInfo = impression.clickTrackingInfo.map(convertClickTrackingInfo), - ) - } - } - - private def convertAdMetadataContainer( - serializedAdMetadataContainer: ac.SerializedThrift - ): Option[AdMetadataContainer] = - AdMetadataContainerSerializer.deserialize(serializedAdMetadataContainer).map { container => - AdMetadataContainer( - removePromotedAttributionForPreroll = container.removePromotedAttributionForPreroll, - sponsorshipCandidate = container.sponsorshipCandidate, - sponsorshipOrganization = container.sponsorshipOrganization, - sponsorshipOrganizationWebsite = container.sponsorshipOrganizationWebsite, - sponsorshipType = container.sponsorshipType.map(convertSponsorshipType), - disclaimerType = container.disclaimerType.map(convertDisclaimerType), - skAdNetworkDataList = container.skAdNetworkDataList.map(convertSkAdNetworkDataList), - unifiedCardOverride = container.unifiedCardOverride - ) - } - - private def convertDisclosureType(disclosureType: ad.DisclosureType): DisclosureType = - disclosureType match { - case ad.DisclosureType.None => NoDisclosure - case ad.DisclosureType.Political => Political - case ad.DisclosureType.Earned => Earned - case ad.DisclosureType.Issue => Issue - case _ => throw new UnsupportedOperationException(s"Unsupported: $disclosureType") - } - - private def convertSponsorshipType(sponsorshipType: ads.SponsorshipType): SponsorshipType = - sponsorshipType match { - case ads.SponsorshipType.Direct => DirectSponsorshipType - case ads.SponsorshipType.Indirect => IndirectSponsorshipType - case ads.SponsorshipType.NoSponsorship => NoSponsorshipSponsorshipType - case _ => throw new UnsupportedOperationException(s"Unsupported: $sponsorshipType") - } - - private def convertDisclaimerType(disclaimerType: ads.DisclaimerType): DisclaimerType = - disclaimerType match { - case ads.DisclaimerType.Political => DisclaimerPolitical - case ads.DisclaimerType.Issue => DisclaimerIssue - case _ => throw new UnsupportedOperationException(s"Unsupported: $disclaimerType") - } - - private def convertSkAdNetworkDataList( - skAdNetworkDataList: Seq[ads.SkAdNetworkData] - ): Seq[SkAdNetworkData] = skAdNetworkDataList.map { sdAdNetwork => - SkAdNetworkData( - version = sdAdNetwork.version, - srcAppId = sdAdNetwork.srcAppId, - dstAppId = sdAdNetwork.dstAppId, - adNetworkId = sdAdNetwork.adNetworkId, - campaignId = sdAdNetwork.campaignId, - impressionTimeInMillis = sdAdNetwork.impressionTimeInMillis, - nonce = sdAdNetwork.nonce, - signature = sdAdNetwork.signature, - fidelityType = sdAdNetwork.fidelityType - ) - } - - private def convertClickTrackingInfo(clickTracking: ad.ClickTrackingInfo): ClickTrackingInfo = - ClickTrackingInfo( - urlParams = clickTracking.urlParams.getOrElse(Map.empty), - urlOverride = clickTracking.urlOverride, - urlOverrideType = clickTracking.urlOverrideType.map { - case ad.UrlOverrideType.Unknown => UnknownUrlOverrideType - case ad.UrlOverrideType.Dcm => DcmUrlOverrideType - case ad.UrlOverrideType.EnumUnknownUrlOverrideType(value) => - throw new UnsupportedOperationException(s"Unsupported: $value") - } - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/RichTextBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/RichTextBuilder.docx new file mode 100644 index 000000000..c644eebd7 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/RichTextBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/RichTextBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/RichTextBuilder.scala deleted file mode 100644 index cb5ff66df..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/RichTextBuilder.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.richtext - -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseStr -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.richtext.BaseRichTextBuilder -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.UrlType -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichText -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichTextAlignment -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class RichTextBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( - textBuilder: BaseStr[Query, Candidate], - linkMap: Map[String, String], - rtl: Option[Boolean], - alignment: Option[RichTextAlignment], - linkTypeMap: Map[String, UrlType] = Map.empty) - extends BaseRichTextBuilder[Query, Candidate] { - - def apply(query: Query, candidate: Candidate, candidateFeatures: FeatureMap): RichText = { - RichTextMarkupUtil.richTextFromMarkup( - text = textBuilder(query, candidate, candidateFeatures), - linkMap = linkMap, - rtl = rtl, - alignment = alignment, - linkTypeMap = linkTypeMap) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/RichTextMarkupUtil.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/RichTextMarkupUtil.docx new file mode 100644 index 000000000..bdc549369 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/RichTextMarkupUtil.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/RichTextMarkupUtil.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/RichTextMarkupUtil.scala deleted file mode 100644 index dacb72b90..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/RichTextMarkupUtil.scala +++ /dev/null @@ -1,135 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.richtext - -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ExternalUrl -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Url -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.UrlType -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichText -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichTextAlignment -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichTextEntity -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.Strong - -/* - * RichTextMarkupUtil facilitates building a Product Mixer URT RichText object out of - * a string with inline XML markup. - * - * This allows us to use a string like "Our system Product Mixer is the best". Using - * inline markup like this is advantageous since the string can go through translation/localization and the - * translators will move the tags around in each language as appropriate. - * - * This class is derived from the OCF (onboarding/serve)'s RichTextUtil, but they diverge because: - * - We generate ProMix URT structures, not OCF URT structures - * - The OCF supports some internal OCF tags, like - * - The OCF has additional legacy support and processing that we don't need - */ - -object RichTextMarkupUtil { - - // Matches a anchor element, extracting the 'a' tag and the display text. - // First group is the tag - // Second group is the display text - // Allows any character in the display text, but matches reluctantly - private val LinkAnchorRegex = """(?i)(?s)(.*?)""".r - - // Matches a bold text section - private val BoldRegex = """(?i)(?s)(.*?)""".r - - def richTextFromMarkup( - text: String, - linkMap: Map[String, String], - rtl: Option[Boolean] = None, - alignment: Option[RichTextAlignment] = None, - linkTypeMap: Map[String, UrlType] = Map.empty - ): RichText = { - - // Mutable! - var currentText = text - val entities = scala.collection.mutable.ArrayBuffer.empty[RichTextEntity] - - // Using a while loop since we want to execute the regex after each iteration, so our indexes remain consistent - - // Handle Links - var matchOpt = LinkAnchorRegex.findFirstMatchIn(currentText) - while (matchOpt.isDefined) { - matchOpt.foreach { linkMatch => - val tag = linkMatch.group(1) - val displayText = linkMatch.group(2) - - currentText = currentText.substring(0, linkMatch.start) + displayText + currentText - .substring(linkMatch.end) - - adjustEntities( - entities, - linkMatch.start, - linkMatch.end - (linkMatch.start + displayText.length)) - - entities.append( - RichTextEntity( - fromIndex = linkMatch.start, - toIndex = linkMatch.start + displayText.length, - ref = linkMap.get(tag).map { url => - Url( - urlType = linkTypeMap.getOrElse(tag, ExternalUrl), - url = url - ) - }, - format = None - ) - ) - } - matchOpt = LinkAnchorRegex.findFirstMatchIn(currentText) - } - - // Handle Bold - matchOpt = BoldRegex.findFirstMatchIn(currentText) - while (matchOpt.isDefined) { - matchOpt.foreach { boldMatch => - val text = boldMatch.group(1) - - currentText = - currentText.substring(0, boldMatch.start) + text + currentText.substring(boldMatch.end) - - adjustEntities(entities, boldMatch.start, boldMatch.end - (boldMatch.start + text.length)) - - entities.append( - RichTextEntity( - fromIndex = boldMatch.start, - toIndex = boldMatch.start + text.length, - ref = None, - format = Some(Strong), - ) - ) - } - - matchOpt = BoldRegex.findFirstMatchIn(currentText) - } - - RichText( - currentText, - entities.sortBy(_.fromIndex).toList, // always return immutable copies! - rtl, - alignment - ) - } - - /* When we create a new entity, we need to adjust - * any already existing entities that have been moved. - * Entities cannot overlap, so we can just compare start positions. - */ - private def adjustEntities( - entities: scala.collection.mutable.ArrayBuffer[RichTextEntity], - start: Int, - length: Int - ): Unit = { - for (i <- entities.indices) { - if (entities(i).fromIndex > start) { - val old = entities(i) - entities.update( - i, - entities(i).copy( - fromIndex = old.fromIndex - length, - toIndex = old.toIndex - length - )) - } - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/RichTextReferenceObjectBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/RichTextReferenceObjectBuilder.docx new file mode 100644 index 000000000..b55a829ba Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/RichTextReferenceObjectBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/RichTextReferenceObjectBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/RichTextReferenceObjectBuilder.scala deleted file mode 100644 index 503f0841c..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/RichTextReferenceObjectBuilder.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.richtext - -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.ReferenceObject -import com.twitter.twittertext.Extractor - -trait RichTextReferenceObjectBuilder { - def apply(entity: Extractor.Entity): Option[ReferenceObject] -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/RichTextRtlOptionBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/RichTextRtlOptionBuilder.docx new file mode 100644 index 000000000..f9e6bdb66 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/RichTextRtlOptionBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/RichTextRtlOptionBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/RichTextRtlOptionBuilder.scala deleted file mode 100644 index 6fc39d5a4..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/RichTextRtlOptionBuilder.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.richtext - -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -trait RichTextRtlOptionBuilder[-Query <: PipelineQuery] { - def apply(query: Query): Option[Boolean] -} - -case class StaticRichTextRtlOptionBuilder[-Query <: PipelineQuery](rtlOption: Option[Boolean]) - extends RichTextRtlOptionBuilder[Query] { - override def apply(query: Query): Option[Boolean] = rtlOption -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/StaticRichTextBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/StaticRichTextBuilder.docx new file mode 100644 index 000000000..6b8e63dad Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/StaticRichTextBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/StaticRichTextBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/StaticRichTextBuilder.scala deleted file mode 100644 index 64054559a..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/StaticRichTextBuilder.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.richtext - -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.richtext.BaseRichTextBuilder -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichText -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class StaticRichTextBuilder(richText: RichText) - extends BaseRichTextBuilder[PipelineQuery, UniversalNoun[Any]] { - - override def apply( - query: PipelineQuery, - candidate: UniversalNoun[Any], - candidateFeatures: FeatureMap - ): RichText = richText -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextEntityProcessor.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextEntityProcessor.docx new file mode 100644 index 000000000..705a31cbb Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextEntityProcessor.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextEntityProcessor.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextEntityProcessor.scala deleted file mode 100644 index ca27c7e14..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextEntityProcessor.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.richtext.twitter_text - -import com.twitter.product_mixer.component_library.decorator.urt.builder.richtext.RichTextReferenceObjectBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.richtext.twitter_text.TwitterTextEntityProcessor.DefaultReferenceObjectBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ExternalUrl -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Url -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.ReferenceObject -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichTextCashtag -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichTextHashtag -import com.twitter.twittertext.Extractor -import scala.collection.convert.ImplicitConversions._ - -object TwitterTextEntityProcessor { - object DefaultReferenceObjectBuilder extends RichTextReferenceObjectBuilder { - def apply(twitterEntity: Extractor.Entity): Option[ReferenceObject] = { - twitterEntity.getType match { - case Extractor.Entity.Type.URL => - Some(Url(ExternalUrl, twitterEntity.getValue)) - case Extractor.Entity.Type.HASHTAG => - Some(RichTextHashtag(twitterEntity.getValue)) - case Extractor.Entity.Type.CASHTAG => - Some(RichTextCashtag(twitterEntity.getValue)) - case _ => None - } - } - } -} - -/** - * Add the corresponding [[RichTextEntity]] extraction logic into [[TwitterTextRenderer]]. - * The [[TwitterTextRenderer]] after being processed will extract the defined entities. - */ -case class TwitterTextEntityProcessor( - twitterTextReferenceObjectBuilder: RichTextReferenceObjectBuilder = DefaultReferenceObjectBuilder) - extends TwitterTextRendererProcessor { - - private[this] val extractor = new Extractor() - - def process( - twitterTextRenderer: TwitterTextRenderer - ): TwitterTextRenderer = { - val twitterEntities = extractor.extractEntitiesWithIndices(twitterTextRenderer.text) - - twitterEntities.foreach { twitterEntity => - twitterTextReferenceObjectBuilder(twitterEntity).foreach { refObject => - twitterTextRenderer.setRefObject(twitterEntity.getStart, twitterEntity.getEnd, refObject) - } - } - twitterTextRenderer - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextFormatProcessor.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextFormatProcessor.docx new file mode 100644 index 000000000..dcdfabf0b Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextFormatProcessor.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextFormatProcessor.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextFormatProcessor.scala deleted file mode 100644 index cf36eecc6..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextFormatProcessor.scala +++ /dev/null @@ -1,67 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.richtext.twitter_text - -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.Plain -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichText -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichTextFormat -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.Strong -import scala.collection.mutable - -object TwitterTextFormatProcessor { - lazy val defaultFormatProcessor = TwitterTextFormatProcessor() -} - -/** - * Add the corresponding [[RichTextFormat]] extraction logic into [[TwitterTextRenderer]]. - * The [[TwitterTextRenderer]] after being processed will extract the defined entities. - */ -case class TwitterTextFormatProcessor( - formats: Set[RichTextFormat] = Set(Plain, Strong), -) extends TwitterTextRendererProcessor { - - private val formatMap = formats.map { format => format.name.toLowerCase -> format }.toMap - - private[this] val formatMatcher = { - val formatNames = formatMap.keys.toSet - s"<(/?)(${formatNames.mkString("|")})>".r - } - - def renderText(text: String): RichText = { - process(TwitterTextRenderer(text)).build - } - - def process(richTextBuilder: TwitterTextRenderer): TwitterTextRenderer = { - val text = richTextBuilder.text - val nodeStack = mutable.ArrayStack[(RichTextFormat, Int)]() - var offset = 0 - - formatMatcher.findAllMatchIn(text).foreach { m => - formatMap.get(m.group(2)) match { - case Some(format) => { - if (m.group(1).nonEmpty) { - if (!nodeStack.headOption.exists { - case (formatFromStack, _) => formatFromStack == format - }) { - throw UnmatchedFormatTag(format) - } - val (_, startIndex) = nodeStack.pop - richTextBuilder.mergeFormat(startIndex, m.start + offset, format) - } else { - nodeStack.push((format, m.start + offset)) - } - richTextBuilder.remove(m.start + offset, m.end + offset) - offset -= m.end - m.start - } - case _ => // if format is not found, skip this format - } - } - - if (nodeStack.nonEmpty) { - throw UnmatchedFormatTag(nodeStack.head._1) - } - - richTextBuilder - } -} - -case class UnmatchedFormatTag(format: RichTextFormat) - extends Exception(s"Unmatched format start and end tags for $format") diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextRenderer.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextRenderer.docx new file mode 100644 index 000000000..8a649a56a Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextRenderer.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextRenderer.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextRenderer.scala deleted file mode 100644 index 53a534255..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextRenderer.scala +++ /dev/null @@ -1,390 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.richtext.twitter_text - -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.ReferenceObject -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichText -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichTextAlignment -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichTextEntity -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichTextFormat -import scala.annotation.tailrec -import scala.collection.mutable - -object TwitterTextRenderer { - - /** - * Creates a new [[TwitterTextRenderer]] instance. - * @param text The initial text representation - * @param rtl Defines whether this text is in an RTL language - * @param alignment Assigns the [[RichTextAlignment]] of the given text for display - * @return A new [[TwitterTextRenderer]] instance - */ - def apply( - text: String, - rtl: Option[Boolean] = None, - alignment: Option[RichTextAlignment] = None - ): TwitterTextRenderer = { - TwitterTextRenderer(rtl, alignment).append(text) - } - - /** - * Creates a new [[TwitterTextRenderer]] instance from a product-mixer [[RichText]] object. - * Converts Unicode entity indexes into JVM String indexes. - * @param richText The product-mixer [[RichText]] representation - * @return A new [[TwitterTextRenderer]] instance - */ - def fromRichText(richText: RichText): TwitterTextRenderer = { - val builder = TwitterTextRenderer(richText.text, richText.rtl, richText.alignment) - richText.entities.foreach { e => - val startIndex = richText.text.offsetByCodePoints(0, e.fromIndex) - val endIndex = richText.text.offsetByCodePoints(0, e.toIndex) - e.format.foreach { f => - builder.setFormat(startIndex, endIndex, f) - } - e.ref.foreach { r => - builder.setRefObject(startIndex, endIndex, r) - } - } - builder - } - - private def buildRichTextEntity( - text: String, - entity: TwitterTextRendererEntity[_] - ): RichTextEntity = { - val fromIndex = text.codePointCount(0, entity.startIndex) - val toIndex = text.codePointCount(0, entity.endIndex) - - entity.value match { - case format: RichTextFormat => - RichTextEntity(fromIndex, toIndex, ref = None, format = Some(format)) - case ref: ReferenceObject => - RichTextEntity(fromIndex, toIndex, ref = Some(ref), format = None) - } - } -} - -case class TwitterTextRenderer( - rtl: Option[Boolean], - alignment: Option[RichTextAlignment], -) { - private[this] val textBuilder = new mutable.StringBuilder() - - private[richtext] val formatBuffer = - mutable.ArrayBuffer[TwitterTextRendererEntity[RichTextFormat]]() - private[richtext] val refObjectBuffer = - mutable.ArrayBuffer[TwitterTextRendererEntity[ReferenceObject]]() - - /** - * Appends a string with attached [[RichTextFormat]] and [[ReferenceObject]] information. - * @param string The text to append to the end of the existing text - * @param format The [[RichTextFormat]] assigned to the new text - * @param refObject The [[ReferenceObject]] assigned to the new text - * @return this - */ - def append( - string: String, - format: Option[RichTextFormat] = None, - refObject: Option[ReferenceObject] = None - ): TwitterTextRenderer = { - if (string.nonEmpty) { - val start = textBuilder.length - val end = start + string.length - format.foreach { f => - formatBuffer.append(TwitterTextRendererEntity(start, end, f)) - } - refObject.foreach { r => - refObjectBuffer.append(TwitterTextRendererEntity(start, end, r)) - } - textBuilder.append(string) - } - this - } - - /** - * Builds a new [[RichText]] thrift instance with Unicode entity ranges. - */ - def build: RichText = { - val richTextString = this.text - val richTextEntities = this.entities - .map { e => - TwitterTextRenderer.buildRichTextEntity(richTextString, e) - } - - RichText( - text = richTextString, - rtl = rtl, - alignment = alignment, - entities = richTextEntities.toList - ) - } - - /** - * Modifies the TwitterTextRenderer with the provided [[TwitterTextRendererProcessor]] - */ - def transform(twitterTextProcessor: TwitterTextRendererProcessor): TwitterTextRenderer = { - twitterTextProcessor.process(this) - } - - /** - * Builds and returns a sorted list of [[TwitterTextRendererEntity]] with JVM String index entity ranges. - */ - def entities: Seq[TwitterTextRendererEntity[_]] = { - buildEntities(formatBuffer.toList, refObjectBuffer.toList) - } - - /** - * Assigns a [[RichTextFormat]] to the given range while keeping existing formatting information. - * New formatting will only be assigned to unformatted text ranges. - * @param start Start index to apply formatting (inclusive) - * @param end End index to apply formatting (exclusive) - * @param format The format to assign - * @return this - */ - def mergeFormat(start: Int, end: Int, format: RichTextFormat): TwitterTextRenderer = { - validateRange(start, end) - var injectionIndex: Option[Int] = None - var entity = TwitterTextRendererEntity(start, end, format) - - val buffer = mutable.ArrayBuffer[TwitterTextRendererEntity[RichTextFormat]]() - val iterator = formatBuffer.zipWithIndex.reverseIterator - - while (iterator.hasNext && injectionIndex.isEmpty) { - iterator.next match { - case (e, i) if e.startIndex >= end => - buffer.append(e) - - case (e, i) if e.enclosedIn(entity.startIndex, entity.endIndex) => - val endEntity = entity.copy(startIndex = e.endIndex) - if (endEntity.nonEmpty) { buffer.append(endEntity) } - buffer.append(e) - entity = entity.copy(endIndex = e.startIndex) - - case (e, i) if e.encloses(entity.startIndex, entity.endIndex) => - buffer.append(e.copy(startIndex = entity.endIndex)) - buffer.append(e.copy(endIndex = entity.startIndex)) - injectionIndex = Some(i + 1) - - case (e, i) if e.startsBetween(entity.startIndex, entity.endIndex) => - buffer.append(e) - entity = entity.copy(endIndex = e.startIndex) - - case (e, i) if e.endsBetween(entity.startIndex, entity.endIndex) => - buffer.append(e) - entity = entity.copy(startIndex = e.endIndex) - injectionIndex = Some(i + 1) - - case (e, i) if e.endIndex <= entity.startIndex => - buffer.append(e) - injectionIndex = Some(i + 1) - - case _ => // do nothing - } - } - - val index = injectionIndex.map(_ - 1).getOrElse(0) - formatBuffer.remove(index, formatBuffer.length - index) - formatBuffer.appendAll(buffer.reverse) - - if (entity.nonEmpty) { - formatBuffer.insert(injectionIndex.getOrElse(0), entity) - } - - this - } - - /** - * Removes text, formatting, and refObject information from the given range. - * @param start Start index to apply formatting (inclusive) - * @param end End index to apply formatting (exclusive) - * @return this - */ - def remove(start: Int, end: Int): TwitterTextRenderer = replace(start, end, "") - - /** - * Replaces text, formatting, and refObject information in the given range. - * @param start Start index to apply formatting (inclusive) - * @param end End index to apply formatting (exclusive) - * @param string The new text to insert - * @param format The [[RichTextFormat]] assigned to the new text - * @param refObject The [[ReferenceObject]] assigned to the new text - * @return this - */ - def replace( - start: Int, - end: Int, - string: String, - format: Option[RichTextFormat] = None, - refObject: Option[ReferenceObject] = None - ): TwitterTextRenderer = { - validateRange(start, end) - - val newEnd = start + string.length - val formatInjectIndex = removeAndOffsetFormats(start, end, string.length) - val refObjectInjectIndex = removeAndOffsetRefObjects(start, end, string.length) - format.foreach { f => - formatBuffer.insert(formatInjectIndex, TwitterTextRendererEntity(start, newEnd, f)) - } - refObject.foreach { r => - refObjectBuffer.insert(refObjectInjectIndex, TwitterTextRendererEntity(start, newEnd, r)) - } - textBuilder.replace(start, end, string) - - this - } - - /** - * Assigns a [[RichTextFormat]] to the given range. Trims existing format ranges that overlap the - * new format range. Removes format ranges that fall within the new range. - * @param start Start index to apply formatting (inclusive) - * @param end End index to apply formatting (exclusive) - * @param format The format to assign - * @return this - */ - def setFormat(start: Int, end: Int, format: RichTextFormat): TwitterTextRenderer = { - validateRange(start, end) - val bufferIndex = removeAndOffsetFormats(start, end, end - start) - formatBuffer.insert(bufferIndex, TwitterTextRendererEntity(start, end, format)) - - this - } - - private[this] def removeAndOffsetFormats(start: Int, end: Int, newSize: Int): Int = { - val newEnd = start + newSize - val offset = newEnd - end - var injectionIndex: Option[Int] = None - - val buffer = mutable.ArrayBuffer[TwitterTextRendererEntity[RichTextFormat]]() - val iterator = formatBuffer.zipWithIndex.reverseIterator - - while (iterator.hasNext && injectionIndex.isEmpty) { - iterator.next match { - case (e, i) if e.startIndex >= end => - buffer.append(e.offset(offset)) - case (e, i) if e.encloses(start, end) => - buffer.append(e.copy(startIndex = newEnd, endIndex = e.endIndex + offset)) - buffer.append(e.copy(endIndex = e.endIndex + offset)) - injectionIndex = Some(i + 1) - case (e, i) if e.endsBetween(start, end) => - buffer.append(e.copy(endIndex = start)) - injectionIndex = Some(i + 1) - case (e, i) if e.startsBetween(start, end) => - buffer.append(e.copy(startIndex = newEnd, endIndex = e.endIndex + offset)) - case (e, i) if e.endIndex <= start => - buffer.append(e) - injectionIndex = Some(i + 1) - case _ => // do nothing - } - } - val index = injectionIndex.map(_ - 1).getOrElse(0) - formatBuffer.remove(index, formatBuffer.length - index) - formatBuffer.appendAll(buffer.reverse) - - injectionIndex.getOrElse(0) - } - - private[this] def validateRange(start: Int, end: Int): Unit = { - require( - start >= 0 && start < textBuilder.length && end > start && end <= textBuilder.length, - s"The start ($start) and end ($end) indexes must be within the text range (0..${textBuilder.length})" - ) - } - - /** - * Assigns a [[ReferenceObject]] to the given range. Since it makes little sense to trim object - * ranges, existing intersecting or overlapping ranges are removed entirely. - * @param start Start index to apply formatting (inclusive) - * @param end End index to apply formatting (exclusive) - * @param refObject The [[ReferenceObject]] to assign - * @return this - */ - def setRefObject(start: Int, end: Int, refObject: ReferenceObject): TwitterTextRenderer = { - validateRange(start, end) - val bufferIndex = removeAndOffsetRefObjects(start, end, end - start) - refObjectBuffer.insert(bufferIndex, TwitterTextRendererEntity(start, end, refObject)) - - this - } - - private[this] def removeAndOffsetRefObjects(start: Int, end: Int, newSize: Int): Int = { - val newEnd = start + newSize - val offset = newEnd - end - var injectionIndex: Option[Int] = None - - val buffer = mutable.ArrayBuffer[TwitterTextRendererEntity[ReferenceObject]]() - val iterator = refObjectBuffer.zipWithIndex.reverseIterator - - while (iterator.hasNext && injectionIndex.isEmpty) { - iterator.next match { - case (e, i) if e.startIndex >= end => buffer.append(e.offset(offset)) - case (e, i) if e.endIndex <= start => injectionIndex = Some(i + 1) - case _ => // do nothing - } - } - val index = injectionIndex.getOrElse(0) - refObjectBuffer.remove(index, refObjectBuffer.length - index) - refObjectBuffer.appendAll(buffer.reverse) - - index - } - - /** - * Builds and returns the full TwitterTextRenderer text with any changes applied to the builder instance. - */ - def text: String = { - textBuilder.mkString - } - - @tailrec - private def buildEntities( - formats: List[TwitterTextRendererEntity[RichTextFormat]], - refs: List[TwitterTextRendererEntity[ReferenceObject]], - acc: List[TwitterTextRendererEntity[_]] = List() - ): Seq[TwitterTextRendererEntity[_]] = { - (formats, refs) match { - case (Nil, Nil) => acc - case (remainingFormats, Nil) => acc ++ remainingFormats - case (Nil, remainingRefs) => acc ++ remainingRefs - - case (format +: remainingFormats, ref +: remainingRefs) - if format.startIndex < ref.startIndex || (format.startIndex == ref.startIndex && format.endIndex < ref.endIndex) => - buildEntities(remainingFormats, refs, acc :+ format) - - case (format +: remainingFormats, ref +: remainingRefs) - if format.startIndex == ref.startIndex && format.endIndex == ref.endIndex => - buildEntities(remainingFormats, remainingRefs, acc :+ format :+ ref) - - case (_, ref +: remainingRefs) => - buildEntities(formats, remainingRefs, acc :+ ref) - } - } -} - -case class TwitterTextRendererEntity[+T] private[richtext] ( - startIndex: Int, - endIndex: Int, - value: T) { - require(startIndex <= endIndex, "startIndex must be <= than endIndex") - - def nonEmpty: Boolean = !isEmpty - - def isEmpty: Boolean = startIndex == endIndex - - private[richtext] def enclosedIn(start: Int, end: Int): Boolean = { - start <= startIndex && endIndex <= end - } - - private[richtext] def encloses(start: Int, end: Int): Boolean = { - startIndex < start && end < endIndex - } - - private[richtext] def endsBetween(start: Int, end: Int): Boolean = { - start < endIndex && endIndex <= end && startIndex < start - } - - private[richtext] def offset(num: Int): TwitterTextRendererEntity[T] = { - copy(startIndex = startIndex + num, endIndex = endIndex + num) - } - - private[richtext] def startsBetween(start: Int, end: Int): Boolean = { - startIndex >= start && startIndex < end && endIndex > end - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextRendererProcessor.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextRendererProcessor.docx new file mode 100644 index 000000000..a3fc3bb04 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextRendererProcessor.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextRendererProcessor.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextRendererProcessor.scala deleted file mode 100644 index 5e5f45659..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextRendererProcessor.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.richtext.twitter_text - -trait TwitterTextRendererProcessor { - def process(twitterTextRichTextBuilder: TwitterTextRenderer): TwitterTextRenderer -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextRichTextBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextRichTextBuilder.docx new file mode 100644 index 000000000..6a1c8c256 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextRichTextBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextRichTextBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextRichTextBuilder.scala deleted file mode 100644 index 50422a335..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/richtext/twitter_text/TwitterTextRichTextBuilder.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.richtext.twitter_text - -import com.twitter.product_mixer.component_library.decorator.urt.builder.richtext.RichTextReferenceObjectBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.richtext.RichTextRtlOptionBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.richtext.StaticRichTextRtlOptionBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.richtext.twitter_text.TwitterTextEntityProcessor.DefaultReferenceObjectBuilder -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseStr -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.richtext.BaseRichTextBuilder -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.Plain -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichText -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichTextAlignment -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichTextFormat -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.Strong -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class TwitterTextRichTextBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( - stringBuilder: BaseStr[Query, Candidate], - alignment: Option[RichTextAlignment] = None, - formats: Set[RichTextFormat] = Set(Plain, Strong), - twitterTextRtlOptionBuilder: RichTextRtlOptionBuilder[Query] = - StaticRichTextRtlOptionBuilder[Query](None), - twitterTextReferenceObjectBuilder: RichTextReferenceObjectBuilder = DefaultReferenceObjectBuilder) - extends BaseRichTextBuilder[Query, Candidate] { - def apply(query: Query, candidate: Candidate, candidateFeatures: FeatureMap): RichText = { - val twitterTextRenderer = TwitterTextRenderer( - text = stringBuilder(query, candidate, candidateFeatures), - rtl = twitterTextRtlOptionBuilder(query), - alignment = alignment) - - twitterTextRenderer - .transform(TwitterTextFormatProcessor(formats)) - .transform(TwitterTextEntityProcessor(twitterTextReferenceObjectBuilder)) - .build - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/social_context/FeatureSocialContextBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/social_context/FeatureSocialContextBuilder.docx new file mode 100644 index 000000000..bea8124e5 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/social_context/FeatureSocialContextBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/social_context/FeatureSocialContextBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/social_context/FeatureSocialContextBuilder.scala deleted file mode 100644 index d96551bc6..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/social_context/FeatureSocialContextBuilder.scala +++ /dev/null @@ -1,101 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.social_context - -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.service.{thriftscala => t} - -/** - * Use this Builder to create Product Mixer [[SocialContext]] objects when you have a - * Timeline Service Thrift [[SocialContext]] feature that you want to convert - */ -case class FeatureSocialContextBuilder( - socialContextFeature: Feature[_, Option[t.SocialContext]]) - extends BaseSocialContextBuilder[PipelineQuery, UniversalNoun[Any]] { - - override def apply( - query: PipelineQuery, - candidate: UniversalNoun[Any], - candidateFeatures: FeatureMap - ): Option[SocialContext] = { - candidateFeatures.getOrElse(socialContextFeature, None).map { - case t.SocialContext.GeneralContext(context) => - val contextType = context.contextType match { - case t.ContextType.Like => LikeGeneralContextType - case t.ContextType.Follow => FollowGeneralContextType - case t.ContextType.Moment => MomentGeneralContextType - case t.ContextType.Reply => ReplyGeneralContextType - case t.ContextType.Conversation => ConversationGeneralContextType - case t.ContextType.Pin => PinGeneralContextType - case t.ContextType.TextOnly => TextOnlyGeneralContextType - case t.ContextType.Facepile => FacePileGeneralContextType - case t.ContextType.Megaphone => MegaPhoneGeneralContextType - case t.ContextType.Bird => BirdGeneralContextType - case t.ContextType.Feedback => FeedbackGeneralContextType - case t.ContextType.Topic => TopicGeneralContextType - case t.ContextType.List => ListGeneralContextType - case t.ContextType.Retweet => RetweetGeneralContextType - case t.ContextType.Location => LocationGeneralContextType - case t.ContextType.Community => CommunityGeneralContextType - case t.ContextType.SmartBlockExpiration => SmartblockExpirationGeneralContextType - case t.ContextType.Trending => TrendingGeneralContextType - case t.ContextType.Sparkle => SparkleGeneralContextType - case t.ContextType.Spaces => SpacesGeneralContextType - case t.ContextType.ReplyPin => ReplyPinGeneralContextType - case t.ContextType.NewUser => NewUserGeneralContextType - case t.ContextType.EnumUnknownContextType(field) => - throw new UnsupportedOperationException(s"Unknown context type: $field") - } - - val landingUrl = context.landingUrl.map { url => - val endpointOptions = url.urtEndpointOptions.map { options => - UrtEndpointOptions( - requestParams = options.requestParams.map(_.toMap), - title = options.title, - cacheId = options.cacheId, - subtitle = options.subtitle - ) - } - - val urlType = url.urlType match { - case t.UrlType.ExternalUrl => ExternalUrl - case t.UrlType.DeepLink => DeepLink - case t.UrlType.UrtEndpoint => UrtEndpoint - case t.UrlType.EnumUnknownUrlType(field) => - throw new UnsupportedOperationException(s"Unknown url type: $field") - } - - Url(urlType = urlType, url = url.url, urtEndpointOptions = endpointOptions) - } - - GeneralContext( - text = context.text, - contextType = contextType, - url = context.url, - contextImageUrls = context.contextImageUrls.map(_.toList), - landingUrl = landingUrl - ) - case t.SocialContext.TopicContext(context) => - val functionalityType = context.functionalityType match { - case t.TopicContextFunctionalityType.Basic => - BasicTopicContextFunctionalityType - case t.TopicContextFunctionalityType.Recommendation => - RecommendationTopicContextFunctionalityType - case t.TopicContextFunctionalityType.RecWithEducation => - RecWithEducationTopicContextFunctionalityType - case t.TopicContextFunctionalityType.EnumUnknownTopicContextFunctionalityType(field) => - throw new UnsupportedOperationException(s"Unknown functionality type: $field") - } - - TopicContext( - topicId = context.topicId, - functionalityType = Some(functionalityType) - ) - case t.SocialContext.UnknownUnionField(field) => - throw new UnsupportedOperationException(s"Unknown social context: $field") - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/social_context/GeneralModuleSocialContextBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/social_context/GeneralModuleSocialContextBuilder.docx new file mode 100644 index 000000000..236ec69ad Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/social_context/GeneralModuleSocialContextBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/social_context/GeneralModuleSocialContextBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/social_context/GeneralModuleSocialContextBuilder.scala deleted file mode 100644 index c3e9e9919..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/social_context/GeneralModuleSocialContextBuilder.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.social_context - -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseModuleStr -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseModuleSocialContextBuilder -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.GeneralContext -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.GeneralContextType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Url -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * This class works the same as [[GeneralSocialContextBuilder]] but passes a list of candidates - * into [[BaseModuleStr]] when rendering the string. - */ -case class GeneralModuleSocialContextBuilder[ - -Query <: PipelineQuery, - -Candidate <: UniversalNoun[Any] -]( - textBuilder: BaseModuleStr[Query, Candidate], - contextType: GeneralContextType, - url: Option[String] = None, - contextImageUrls: Option[List[String]] = None, - landingUrl: Option[Url] = None) - extends BaseModuleSocialContextBuilder[Query, Candidate] { - - def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[Candidate]] - ): Option[GeneralContext] = - Some( - GeneralContext( - text = textBuilder(query, candidates), - contextType = contextType, - url = url, - contextImageUrls = contextImageUrls, - landingUrl = landingUrl)) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/social_context/GeneralSocialContextBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/social_context/GeneralSocialContextBuilder.docx new file mode 100644 index 000000000..9a7836878 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/social_context/GeneralSocialContextBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/social_context/GeneralSocialContextBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/social_context/GeneralSocialContextBuilder.scala deleted file mode 100644 index ef2221468..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/social_context/GeneralSocialContextBuilder.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.social_context - -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseStr -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.GeneralContext -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.GeneralContextType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Url -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class GeneralSocialContextBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( - textBuilder: BaseStr[Query, Candidate], - contextType: GeneralContextType, - url: Option[String] = None, - contextImageUrls: Option[List[String]] = None, - landingUrl: Option[Url] = None) - extends BaseSocialContextBuilder[Query, Candidate] { - - def apply( - query: Query, - candidate: Candidate, - candidateFeatures: FeatureMap - ): Option[GeneralContext] = - Some( - GeneralContext( - text = textBuilder(query, candidate, candidateFeatures), - contextType = contextType, - url = url, - contextImageUrls = contextImageUrls, - landingUrl = landingUrl)) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/social_context/WhoToFollowSocialContextBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/social_context/WhoToFollowSocialContextBuilder.docx new file mode 100644 index 000000000..52da74d07 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/social_context/WhoToFollowSocialContextBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/social_context/WhoToFollowSocialContextBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/social_context/WhoToFollowSocialContextBuilder.scala deleted file mode 100644 index edcc07fcd..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/social_context/WhoToFollowSocialContextBuilder.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.social_context - -import com.twitter.hermit.{thriftscala => h} -import com.twitter.product_mixer.component_library.model.candidate.UserCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FollowGeneralContextType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.GeneralContext -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.GeneralContextType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.LocationGeneralContextType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.NewUserGeneralContextType -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class WhoToFollowSocialContextBuilder( - socialTextFeature: Feature[_, Option[String]], - contextTypeFeature: Feature[_, Option[h.ContextType]]) - extends BaseSocialContextBuilder[PipelineQuery, UserCandidate] { - - def apply( - query: PipelineQuery, - candidate: UserCandidate, - candidateFeatures: FeatureMap - ): Option[GeneralContext] = { - val socialTextOpt = candidateFeatures.getOrElse(socialTextFeature, None) - val contextTypeOpt = convertContextType(candidateFeatures.getOrElse(contextTypeFeature, None)) - - (socialTextOpt, contextTypeOpt) match { - case (Some(socialText), Some(contextType)) if socialText.nonEmpty => - Some( - GeneralContext( - text = socialText, - contextType = contextType, - url = None, - contextImageUrls = None, - landingUrl = None)) - case _ => None - } - } - - private def convertContextType(contextType: Option[h.ContextType]): Option[GeneralContextType] = - contextType match { - case Some(h.ContextType.Geo) => Some(LocationGeneralContextType) - case Some(h.ContextType.Social) => Some(FollowGeneralContextType) - case Some(h.ContextType.NewUser) => Some(NewUserGeneralContextType) - case _ => None - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/stringcenter/ModuleStr.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/stringcenter/ModuleStr.docx new file mode 100644 index 000000000..ce61bf5a7 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/stringcenter/ModuleStr.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/stringcenter/ModuleStr.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/stringcenter/ModuleStr.scala deleted file mode 100644 index 0abb2ec63..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/stringcenter/ModuleStr.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.stringcenter - -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseModuleStr -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.stringcenter.BaseModuleStringCenterPlaceholderBuilder -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stringcenter.client.StringCenter -import com.twitter.stringcenter.client.core.ExternalString - -/** - * This class works the same as [[Str]] but passes in a list of candidates to the - * [[BaseModuleStringCenterPlaceholderBuilder]] when building the placeholders. - */ -case class ModuleStr[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( - text: ExternalString, - stringCenter: StringCenter, - stringCenterPlaceholderBuilder: Option[ - BaseModuleStringCenterPlaceholderBuilder[Query, Candidate] - ] = None) - extends BaseModuleStr[Query, Candidate] { - - def apply(query: Query, candidates: Seq[CandidateWithFeatures[Candidate]]): String = { - val placeholderMapOpt = - stringCenterPlaceholderBuilder.map(_.apply(query, candidates)) - stringCenter.prepare( - externalString = text, - placeholders = placeholderMapOpt.getOrElse(Map.empty[String, Any]) - ) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/stringcenter/Str.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/stringcenter/Str.docx new file mode 100644 index 000000000..c74efbfa6 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/stringcenter/Str.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/stringcenter/Str.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/stringcenter/Str.scala deleted file mode 100644 index 6f5a6c3b3..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/stringcenter/Str.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.stringcenter - -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseStr -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.stringcenter.BaseStringCenterPlaceholderBuilder -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stringcenter.client.StringCenter -import com.twitter.stringcenter.client.core.ExternalString - -case class StrStatic( - text: String) - extends BaseStr[PipelineQuery, UniversalNoun[Any]] { - def apply( - query: PipelineQuery, - candidate: UniversalNoun[Any], - candidateFeatures: FeatureMap - ): String = text -} - -case class Str[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( - text: ExternalString, - stringCenter: StringCenter, - stringCenterPlaceholderBuilder: Option[BaseStringCenterPlaceholderBuilder[Query, Candidate]] = - None) - extends BaseStr[Query, Candidate] { - - def apply(query: Query, candidate: Candidate, candidateFeatures: FeatureMap): String = { - val placeholderMapOpt = - stringCenterPlaceholderBuilder.map(_.apply(query, candidate, candidateFeatures)) - stringCenter.prepare( - externalString = text, - placeholders = placeholderMapOpt.getOrElse(Map.empty[String, Any]) - ) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/FeatureModuleDisplayTypeBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/FeatureModuleDisplayTypeBuilder.docx new file mode 100644 index 000000000..bcbd83d3d Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/FeatureModuleDisplayTypeBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/FeatureModuleDisplayTypeBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/FeatureModuleDisplayTypeBuilder.scala deleted file mode 100644 index bc9632e5a..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/FeatureModuleDisplayTypeBuilder.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module - -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseModuleDisplayTypeBuilder -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleDisplayType -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.VerticalConversation -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class FeatureModuleDisplayTypeBuilder( - displayTypeFeature: Feature[_, Option[ModuleDisplayType]], - defaultDisplayType: ModuleDisplayType = VerticalConversation) - extends BaseModuleDisplayTypeBuilder[PipelineQuery, UniversalNoun[Any]] { - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[UniversalNoun[Any]]] - ): ModuleDisplayType = candidates.headOption - .flatMap(_.features.getOrElse(displayTypeFeature, None)) - .getOrElse(defaultDisplayType) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleDynamicShowMoreBehaviorRevealByCountBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleDynamicShowMoreBehaviorRevealByCountBuilder.docx new file mode 100644 index 000000000..40a44baeb Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleDynamicShowMoreBehaviorRevealByCountBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleDynamicShowMoreBehaviorRevealByCountBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleDynamicShowMoreBehaviorRevealByCountBuilder.scala deleted file mode 100644 index fd234de7c..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleDynamicShowMoreBehaviorRevealByCountBuilder.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module - -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseModuleShowMoreBehaviorBuilder -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleShowMoreBehavior -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleShowMoreBehaviorRevealByCount -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class ModuleDynamicShowMoreBehaviorRevealByCountBuilder( - initialItemsCount: Int, - showMoreItemsCount: Int) - extends BaseModuleShowMoreBehaviorBuilder[PipelineQuery, UniversalNoun[Any]] { - - override def apply( - query: PipelineQuery, - candidate: Seq[CandidateWithFeatures[UniversalNoun[Any]]] - ): ModuleShowMoreBehavior = ModuleShowMoreBehaviorRevealByCount( - initialItemsCount = initialItemsCount, - showMoreItemsCount = showMoreItemsCount - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleFooterBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleFooterBuilder.docx new file mode 100644 index 000000000..3d930f12c Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleFooterBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleFooterBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleFooterBuilder.scala deleted file mode 100644 index 886825f2d..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleFooterBuilder.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module - -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleFooter -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseStr -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseUrlBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseModuleFooterBuilder -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures - -case class ModuleFooterBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( - textBuilder: BaseStr[Query, Candidate], - urlBuilder: Option[BaseUrlBuilder[Query, Candidate]]) - extends BaseModuleFooterBuilder[Query, Candidate] { - - override def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[Candidate]] - ): Option[ModuleFooter] = { - candidates.headOption.map { candidate => - ModuleFooter( - text = textBuilder(query, candidate.candidate, candidate.features), - landingUrl = urlBuilder.map(_.apply(query, candidate.candidate, candidate.features)) - ) - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleHeaderBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleHeaderBuilder.docx new file mode 100644 index 000000000..6585d1d99 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleHeaderBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleHeaderBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleHeaderBuilder.scala deleted file mode 100644 index fbe256d69..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleHeaderBuilder.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module - -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.icon.BaseHorizonIconBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseStr -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseModuleSocialContextBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseModuleHeaderBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseModuleHeaderDisplayTypeBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.Classic -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ImageVariant -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleHeader -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class ModuleHeaderBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( - textBuilder: BaseStr[Query, Candidate], - isSticky: Option[Boolean] = None, - moduleHeaderIconBuilder: Option[BaseHorizonIconBuilder[Query, Candidate]] = None, - customIcon: Option[ImageVariant] = None, - moduleSocialContextBuilder: Option[BaseModuleSocialContextBuilder[Query, Candidate]] = None, - moduleHeaderDisplayTypeBuilder: BaseModuleHeaderDisplayTypeBuilder[Query, Candidate] = - ModuleHeaderDisplayTypeBuilder(Classic)) - extends BaseModuleHeaderBuilder[Query, Candidate] { - - override def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[Candidate]] - ): Option[ModuleHeader] = { - val firstCandidate = candidates.head - Some( - ModuleHeader( - text = textBuilder(query, firstCandidate.candidate, firstCandidate.features), - sticky = isSticky, - customIcon = customIcon, - socialContext = moduleSocialContextBuilder.flatMap(_.apply(query, candidates)), - icon = moduleHeaderIconBuilder.flatMap(_.apply(query, candidates)), - moduleHeaderDisplayType = moduleHeaderDisplayTypeBuilder(query, candidates), - ) - ) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleHeaderDisplayTypeBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleHeaderDisplayTypeBuilder.docx new file mode 100644 index 000000000..03f741ca6 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleHeaderDisplayTypeBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleHeaderDisplayTypeBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleHeaderDisplayTypeBuilder.scala deleted file mode 100644 index bc0fffa8c..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleHeaderDisplayTypeBuilder.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module - -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseModuleHeaderDisplayTypeBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.Classic -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleHeaderDisplayType -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class ModuleHeaderDisplayTypeBuilder[ - -Query <: PipelineQuery, - -Candidate <: UniversalNoun[Any] -]( - moduleHeaderDisplayType: ModuleHeaderDisplayType = Classic) - extends BaseModuleHeaderDisplayTypeBuilder[Query, Candidate] { - - override def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[Candidate]] - ): ModuleHeaderDisplayType = moduleHeaderDisplayType - -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleIdGeneration.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleIdGeneration.docx new file mode 100644 index 000000000..1d33d00ba Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleIdGeneration.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleIdGeneration.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleIdGeneration.scala deleted file mode 100644 index a885632e0..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleIdGeneration.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module - -/** - * This trait is used for Module ID generation. Clients are safe to ignore this code unless they - * have a specific use case that requires hard-coded, specific, module ids. In that scenario, - * they can use the [[ManualModuleId]] case class. - */ -sealed trait ModuleIdGeneration { - val moduleId: Long -} - -object ModuleIdGeneration { - def apply(moduleId: Long): ModuleIdGeneration = moduleId match { - case moduleId if AutomaticUniqueModuleId.isAutomaticUniqueModuleId(moduleId) => - AutomaticUniqueModuleId(moduleId) - case moduleId => ManualModuleId(moduleId) - } -} - -/** - * Generate unique Ids for each module, which results in unique URT entryIds - * for each module even if they share the same entryNamespace. - * This is the default and recommended use case. - * Note that the module Id value is just a placeholder - */ -case class AutomaticUniqueModuleId private (moduleId: Long = 0L) extends ModuleIdGeneration { - def withOffset(offset: Long): AutomaticUniqueModuleId = copy( - AutomaticUniqueModuleId.idRange.min + offset) -} - -object AutomaticUniqueModuleId { - // We use a specific numeric range to track whether IDs should be automatically generated. - val idRange: Range = Range(-10000, -1000) - - def apply(): AutomaticUniqueModuleId = AutomaticUniqueModuleId(idRange.min) - - def isAutomaticUniqueModuleId(moduleId: Long): Boolean = idRange.contains(moduleId) -} - -/** - * ManualModuleId should normally not be required, but is helpful if the - * entryId of the module must be controlled. A scenario where this may be - * required is if a single candidate source returns multiple modules, and - * each module has the same presentation (e.g. Header, Footer). By setting - * different IDs, we signal to the platform that each module should be separate - * by using a different manual Id. - */ -case class ManualModuleId(override val moduleId: Long) extends ModuleIdGeneration { - // Negative module IDs are reserved for internal usage - if (moduleId < 0) throw new IllegalArgumentException("moduleId must be a positive number") -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleShowMoreBehaviorRevealByCountBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleShowMoreBehaviorRevealByCountBuilder.docx new file mode 100644 index 000000000..c0718bb4f Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleShowMoreBehaviorRevealByCountBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleShowMoreBehaviorRevealByCountBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleShowMoreBehaviorRevealByCountBuilder.scala deleted file mode 100644 index 19bce376a..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ModuleShowMoreBehaviorRevealByCountBuilder.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module - -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleShowMoreBehavior -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleShowMoreBehaviorRevealByCount -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseModuleShowMoreBehaviorBuilder -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.timelines.configapi.Param - -case class ModuleShowMoreBehaviorRevealByCountBuilder( - initialItemsCountParam: Param[Int], - showMoreItemsCountParam: Param[Int]) - extends BaseModuleShowMoreBehaviorBuilder[PipelineQuery, UniversalNoun[Any]] { - - override def apply( - query: PipelineQuery, - candidate: Seq[CandidateWithFeatures[UniversalNoun[Any]]] - ): ModuleShowMoreBehavior = { - ModuleShowMoreBehaviorRevealByCount( - initialItemsCount = query.params(initialItemsCountParam), - showMoreItemsCount = query.params(showMoreItemsCountParam) - ) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ParamGatedModuleFooterBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ParamGatedModuleFooterBuilder.docx new file mode 100644 index 000000000..0193352d5 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ParamGatedModuleFooterBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ParamGatedModuleFooterBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ParamGatedModuleFooterBuilder.scala deleted file mode 100644 index 36654335a..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ParamGatedModuleFooterBuilder.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module - -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseModuleFooterBuilder -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleFooter -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.Param - -case class ParamGatedModuleFooterBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( - enableParam: Param[Boolean], - enabledBuilder: BaseModuleFooterBuilder[Query, Candidate], - defaultBuilder: Option[BaseModuleFooterBuilder[Query, Candidate]] = None) - extends BaseModuleFooterBuilder[Query, Candidate] { - - def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[Candidate]] - ): Option[ModuleFooter] = { - if (query.params(enableParam)) { - enabledBuilder(query, candidates) - } else { - defaultBuilder.flatMap(_.apply(query, candidates)) - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ParamGatedModuleHeaderBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ParamGatedModuleHeaderBuilder.docx new file mode 100644 index 000000000..0412d56ec Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ParamGatedModuleHeaderBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ParamGatedModuleHeaderBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ParamGatedModuleHeaderBuilder.scala deleted file mode 100644 index 44598b093..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ParamGatedModuleHeaderBuilder.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module - -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseModuleHeaderBuilder -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleHeader -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.Param - -case class ParamGatedModuleHeaderBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( - enableParam: Param[Boolean], - enabledBuilder: BaseModuleHeaderBuilder[Query, Candidate], - defaultBuilder: Option[BaseModuleHeaderBuilder[Query, Candidate]] = None) - extends BaseModuleHeaderBuilder[Query, Candidate] { - - def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[Candidate]] - ): Option[ModuleHeader] = { - if (query.params(enableParam)) { - enabledBuilder(query, candidates) - } else { - defaultBuilder.flatMap(_.apply(query, candidates)) - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ParamWhoToFollowModuleDisplayTypeBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ParamWhoToFollowModuleDisplayTypeBuilder.docx new file mode 100644 index 000000000..a76d2a107 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ParamWhoToFollowModuleDisplayTypeBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ParamWhoToFollowModuleDisplayTypeBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ParamWhoToFollowModuleDisplayTypeBuilder.scala deleted file mode 100644 index 1e83a032e..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/ParamWhoToFollowModuleDisplayTypeBuilder.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module - -import com.twitter.product_mixer.core.functional_component.configapi.StaticParam -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseModuleDisplayTypeBuilder -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.Carousel -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.CompactCarousel -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ConversationTree -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.GridCarousel -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleDisplayType -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.Vertical -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.VerticalConversation -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.VerticalGrid -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.VerticalWithContextLine -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.Param - -object WhoToFollowModuleDisplayType extends Enumeration { - type ModuleDisplayType = Value - - val Carousel = Value - val CompactCarousel = Value - val ConversationTree = Value - val GridCarousel = Value - val Vertical = Value - val VerticalConversation = Value - val VerticalGrid = Value - val VerticalWithContextLine = Value -} - -case class ParamWhoToFollowModuleDisplayTypeBuilder( - displayTypeParam: Param[WhoToFollowModuleDisplayType.Value] = - StaticParam(WhoToFollowModuleDisplayType.Vertical)) - extends BaseModuleDisplayTypeBuilder[PipelineQuery, UniversalNoun[Any]] { - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[UniversalNoun[Any]]] - ): ModuleDisplayType = { - val displayType = query.params(displayTypeParam) - displayType match { - case WhoToFollowModuleDisplayType.Carousel => Carousel - case WhoToFollowModuleDisplayType.CompactCarousel => CompactCarousel - case WhoToFollowModuleDisplayType.ConversationTree => ConversationTree - case WhoToFollowModuleDisplayType.GridCarousel => GridCarousel - case WhoToFollowModuleDisplayType.Vertical => Vertical - case WhoToFollowModuleDisplayType.VerticalConversation => VerticalConversation - case WhoToFollowModuleDisplayType.VerticalGrid => VerticalGrid - case WhoToFollowModuleDisplayType.VerticalWithContextLine => VerticalWithContextLine - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/StaticModuleDisplayTypeBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/StaticModuleDisplayTypeBuilder.docx new file mode 100644 index 000000000..3eb348845 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/StaticModuleDisplayTypeBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/StaticModuleDisplayTypeBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/StaticModuleDisplayTypeBuilder.scala deleted file mode 100644 index a7f9c647f..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/StaticModuleDisplayTypeBuilder.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module - -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseModuleDisplayTypeBuilder -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleDisplayType -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class StaticModuleDisplayTypeBuilder(displayType: ModuleDisplayType) - extends BaseModuleDisplayTypeBuilder[PipelineQuery, UniversalNoun[Any]] { - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[UniversalNoun[Any]]] - ): ModuleDisplayType = displayType -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/TimelineModuleBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/TimelineModuleBuilder.docx new file mode 100644 index 000000000..3c9575089 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/TimelineModuleBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/TimelineModuleBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/TimelineModuleBuilder.scala deleted file mode 100644 index 38f1a7657..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt/builder/timeline_module/TimelineModuleBuilder.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module - -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseModuleDisplayTypeBuilder -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseModuleFooterBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseModuleHeaderBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseModuleMetadataBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseModuleShowMoreBehaviorBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseTimelineModuleBuilder -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures - -case class TimelineModuleBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( - entryNamespace: EntryNamespace, - displayTypeBuilder: BaseModuleDisplayTypeBuilder[Query, Candidate], - clientEventInfoBuilder: BaseClientEventInfoBuilder[Query, Candidate], - moduleIdGeneration: ModuleIdGeneration = AutomaticUniqueModuleId(), - feedbackActionInfoBuilder: Option[ - BaseFeedbackActionInfoBuilder[Query, Candidate] - ] = None, - headerBuilder: Option[BaseModuleHeaderBuilder[Query, Candidate]] = None, - footerBuilder: Option[BaseModuleFooterBuilder[Query, Candidate]] = None, - metadataBuilder: Option[BaseModuleMetadataBuilder[Query, Candidate]] = None, - showMoreBehaviorBuilder: Option[BaseModuleShowMoreBehaviorBuilder[Query, Candidate]] = None) - extends BaseTimelineModuleBuilder[Query, Candidate] { - - override def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[Candidate]] - ): TimelineModule = { - val firstCandidate = candidates.head - TimelineModule( - id = moduleIdGeneration.moduleId, - // Sort indexes are automatically set in the domain marshaller phase - sortIndex = None, - entryNamespace = entryNamespace, - // Modules should not need an element by default; only items should - clientEventInfo = - clientEventInfoBuilder(query, firstCandidate.candidate, firstCandidate.features, None), - feedbackActionInfo = feedbackActionInfoBuilder.flatMap( - _.apply(query, firstCandidate.candidate, firstCandidate.features)), - isPinned = None, - // Items are automatically set in the domain marshaller phase - items = Seq.empty, - displayType = displayTypeBuilder(query, candidates), - header = headerBuilder.flatMap(_.apply(query, candidates)), - footer = footerBuilder.flatMap(_.apply(query, candidates)), - metadata = metadataBuilder.map(_.apply(query, candidates)), - showMoreBehavior = showMoreBehaviorBuilder.map(_.apply(query, candidates)) - ) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/BUILD deleted file mode 100644 index 56c339994..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/BUILD +++ /dev/null @@ -1,11 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/github/scopt", - "util/util-core/src/main/java/com/twitter/io", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/BUILD.docx new file mode 100644 index 000000000..03b5dc9a0 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/MetricDefinitions.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/MetricDefinitions.docx new file mode 100644 index 000000000..fe58b79fe Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/MetricDefinitions.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/MetricDefinitions.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/MetricDefinitions.scala deleted file mode 100644 index 5c2a45856..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/MetricDefinitions.scala +++ /dev/null @@ -1,116 +0,0 @@ -package com.twitter.product_mixer.component_library.experiments.metrics - -import com.twitter.util.Return -import com.twitter.util.Throw -import com.twitter.util.Try - -object MetricDefinition { - val SingleQuote = """"""" - val DoubleQuote = """""""" -} - -/** - * Base class for all metric definitions - */ -sealed trait MetricDefinition { - def toCsvField: Seq[String] - val metricDefinitionType: String -} - -/** - * Pattern Metric Definition - * @param pattern the regex pattern for this metric - */ -case class NamedPatternMetricDefinition( - pattern: Seq[String]) - extends MetricDefinition { - override def toCsvField: Seq[String] = pattern - override val metricDefinitionType: String = "NAMED_PATTERN" -} - -/** - * Strainer Metric Definition - * @param strainerExpression a filter on top of client events - */ -case class StrainerMetricDefinition( - strainerExpression: String) - extends MetricDefinition { - import MetricDefinition._ - override def toCsvField: Seq[String] = { - Seq(strainerExpression.replaceAll(SingleQuote, DoubleQuote)) - } - override val metricDefinitionType: String = "STRAINER" -} - -/** - * Lambda Metric Definition - * @param lambdaExpression a scala function mapping client events to a double - */ -case class LambdaMetricDefinition( - lambdaExpression: String) - extends MetricDefinition { - import MetricDefinition._ - override def toCsvField: Seq[String] = { - Seq(lambdaExpression.replaceAll(SingleQuote, DoubleQuote)) - } - override val metricDefinitionType: String = "LAMBDA" -} - -case class BucketRatioMetricDefinition( - numerator: String, - denominator: String) - extends MetricDefinition { - override def toCsvField: Seq[String] = { - Seq(s"(${numerator}) / (${denominator})") - } - override val metricDefinitionType: String = "BUCKET_RATIO" -} - -object Metric { - val bucketRatioPattern = "[(]+(.+)[)]+ / [(]+(.+)[)]+".r - - /** - * Creates a new Metric given a template line. - * @param line semicolon separated line string - * ignore line with comment, represented by hashtag at the beginning of the line - * @throws RuntimeException if the line is invalid - */ - def fromLine(line: String): Metric = { - val splits = line.split(";") - // at least two parts separated by semicolon (third part is optional) - if (splits.lengthCompare(2) >= 0) { - val metricExpression = splits(0) - val metricName = splits(1) - val metricDefinition = Try(splits(2)) match { - case Return("NAMED_PATTERN") => NamedPatternMetricDefinition(Seq(metricExpression)) - case Return("STRAINER") => StrainerMetricDefinition(metricExpression) - case Return("LAMBDA") => LambdaMetricDefinition(metricExpression) - case Return("BUCKET_RATIO") => - metricExpression match { - case bucketRatioPattern(numerator, denominator) => - BucketRatioMetricDefinition(numerator, denominator) - case _ => - throw new RuntimeException( - s"Invalid metric definition for Bucket Ratio. Expected format (numerator)/(denominator) but found $metricExpression") - } - case Return(other) => - throw new RuntimeException(s"Invalid metric definition in line in template file: $line") - // default to named pattern - case Throw(_) => NamedPatternMetricDefinition(List(metricExpression)) - } - - Metric(metricName, metricDefinition) - } else { - throw new RuntimeException(s"Invalid line in template file: $line") - } - } -} - -/** - * - * @param name globally unique metric name (current DDG limitation) - * @param definition the metric definition for this metric - */ -case class Metric( - name: String, - definition: MetricDefinition) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/MetricGroup.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/MetricGroup.docx new file mode 100644 index 000000000..8ede63c7c Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/MetricGroup.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/MetricGroup.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/MetricGroup.scala deleted file mode 100644 index ccd238cd8..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/MetricGroup.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.product_mixer.component_library.experiments.metrics - -import scala.collection.immutable.ListSet - -/** - * - * @param id optional metric group id. If id is None, this means the group - * is being newly created and the id is not provisioned by go/ddg. Otherwise, the metric - * group is present in DDG and has a corresponding id. - * @param name metric group name - * @param description metric group description - * @param metrics set of metrics that belong to this metric group - */ -case class MetricGroup( - id: Option[Long], - name: String, - description: String, - metrics: ListSet[Metric]) { - - /* - * Returns a CSV representation of this metric group that can be imported via DDG's bulk import tool - * The bulk import tool consumes CSV data with the following columns: - * 1. group name - * 2. group description - * 3. metric name - * 4. metric description - * 5. metric pattern - * 6. group id -- numeric id - * 7. (optional) metric type -- `NAMED_PATTERN`, `STRAINER`, or `LAMBDA`. - */ - def toCsv: String = { - val metricCsvLines: ListSet[String] = for { - metric <- metrics - definition <- metric.definition.toCsvField - } yield { - Seq( - name, - description, - metric.name, - metric.name, - // wrap in single quotes so that DDG bulk import tool correctly parses - s""""$definition"""", - id.map(_.toString).getOrElse(""), - metric.definition.metricDefinitionType - ).mkString(",") - } - println(s"Generated metrics in CSV count: ${metricCsvLines.size}") - metricCsvLines.mkString("\n") - } - - // Unique metric names based on globally unique metric name - def uniqueMetricNames: Set[String] = - metrics.groupBy(_.name).keys.toSet -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/MetricTemplateCLIRunner.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/MetricTemplateCLIRunner.docx new file mode 100644 index 000000000..242783d69 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/MetricTemplateCLIRunner.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/MetricTemplateCLIRunner.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/MetricTemplateCLIRunner.scala deleted file mode 100644 index b01af3cc2..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/MetricTemplateCLIRunner.scala +++ /dev/null @@ -1,101 +0,0 @@ -package com.twitter.product_mixer.component_library.experiments.metrics - -import com.twitter.product_mixer.component_library.experiments.metrics.PlaceholderConfig.PlaceholdersMap -import java.io.File -import java.io.PrintWriter -import scala.collection.immutable.ListSet -import scala.io.Source -import scopt.OptionParser - -private case class MetricTemplateCLIConfig( - // default values required for OptionParser - templateFileName: String = null, - outputFileName: String = null, - metricGroupName: String = null, - metricGroupDesc: String = null, - metricGroupId: Option[Long] = None, - absolutePath: Option[String] = None) - -trait MetricTemplateCLIRunner { - def templateDir: String - def placeholders: PlaceholdersMap - private val ProgramName = "Metric Template CLI" - private val VersionNumber = "1.0" - - private def mkPath(fileName: String, absolutePath: Option[String]): String = { - val relativeDir = s"$templateDir/$fileName" - absolutePath match { - case Some(path) => s"$path/$relativeDir" - case _ => relativeDir - } - } - - def main(args: Array[String]): Unit = { - val parser = new OptionParser[MetricTemplateCLIConfig](ProgramName) { - head(ProgramName, VersionNumber) - // option invoked by -o or --output - opt[String]('o', "output") - .required() - .valueName("") - .action((value, config) => config.copy(outputFileName = value)) - .text("output CSV file with interpolated lines") - // option invoked by -t or --template - opt[String]('t', "template") - .required() - .valueName("") - .action((value, config) => config.copy(templateFileName = value)) - .text( - s"input template file (see README.md for template format). Path is relative to $templateDir.") - // option invoked by -n or --name - opt[String]('n', "name") - .required() - .valueName("") - .action((value, config) => config.copy(metricGroupName = value)) - .text("metric group name") - // option invoked by -d or --description - opt[String]('d', "description") - .required() - .valueName("") - .action((value, config) => config.copy(metricGroupDesc = value)) - .text("metric group description") - // option invoked by --id - opt[Long]("id") - .optional() - .valueName("") - .action((value, config) => config.copy(metricGroupId = Some(value))) - .text("metric group ID (metric MUST be created in go/ddg)") - // option invoked by -p or --path - opt[String]('p', "path") - .optional() - .valueName("") - .action((value, config) => config.copy(absolutePath = Some(value))) - .text(s"absolute path pointing to the $templateDir. Required by bazel") - } - - parser.parse(args, MetricTemplateCLIConfig()) match { - case Some(config) => - val templateLines = - Source.fromFile(mkPath(config.templateFileName, config.absolutePath)).getLines.toList - val interpolatedLines = templateLines - .filter(!_.startsWith("#")).flatMap(MetricTemplates.interpolate(_, placeholders)) - val writer = new PrintWriter(new File(mkPath(config.outputFileName, config.absolutePath))) - val metrics = interpolatedLines.map(Metric.fromLine) - println(s"${metrics.size} metric definitions found in template file.") - val dupMetrics = metrics.groupBy(identity).collect { - case (dup, lst) if lst.lengthCompare(1) > 0 => dup - } - println(s"\nWARNING: ${dupMetrics.size} Duplicate metric definition(s)\n$dupMetrics\n") - val metricGroup = MetricGroup( - config.metricGroupId, - config.metricGroupName, - config.metricGroupDesc, - metrics.to[ListSet]) - println(s"${metricGroup.uniqueMetricNames.size} unique DDG metrics with " + - s"${metricGroup.metrics.size} metric definitions in '${metricGroup.name}' metric group.") - writer.write(metricGroup.toCsv) - writer.close() - case _ => - // arguments are bad, error message will have been displayed - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/MetricTemplates.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/MetricTemplates.docx new file mode 100644 index 000000000..78b6e0414 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/MetricTemplates.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/MetricTemplates.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/MetricTemplates.scala deleted file mode 100644 index 9640b1d8b..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/MetricTemplates.scala +++ /dev/null @@ -1,123 +0,0 @@ -package com.twitter.product_mixer.component_library.experiments.metrics - -import com.twitter.product_mixer.component_library.experiments.metrics.PlaceholderConfig.PlaceholdersMap -import reflect.ClassTag -import scala.reflect.runtime.universe._ -import scala.util.matching.Regex - -case class MatchedPlaceholder(outerKey: String, innerKey: Option[String] = None) - -object MetricTemplates { - // Matches "${placeholder}" where `placeholder` is in a matched group - val PlaceholderPattern: Regex = "\\$\\{([^\\}]+)\\}".r.unanchored - // Matches "${placeholder[index]}" where `placeholder` and `index` are in different matched groups - val IndexedPlaceholderPattern: Regex = "\\$\\{([^\\[]+)\\[([^\\]]+)\\]\\}".r.unanchored - val DefaultFieldName = "name" - - def interpolate(inputTemplate: String, placeholders: PlaceholdersMap): Seq[String] = { - val matchedPlaceholders = getMatchedPlaceholders(inputTemplate) - val groupedPlaceholders = matchedPlaceholders.groupBy(_.outerKey) - val placeholderKeyValues = getPlaceholderKeyValues(groupedPlaceholders, placeholders) - val (keys, values) = (placeholderKeyValues.map(_._1), placeholderKeyValues.map(_._2)) - val cross: Seq[List[Named]] = crossProduct(values) - val mirror = runtimeMirror(getClass.getClassLoader) // necessary for reflection - for { - interpolatables <- cross - } yield { - assert( - keys.length == interpolatables.length, - s"Unexpected length mismatch between $keys and $interpolatables") - var replacementStr = inputTemplate - keys.zip(interpolatables).foreach { - case (key, interpolatable) => - val accessors = caseAccessors(mirror, interpolatable) - groupedPlaceholders(key).foreach { placeholder: MatchedPlaceholder => - val templateKey = generateTemplateKey(placeholder) - val fieldName = placeholder.innerKey.getOrElse(DefaultFieldName) - val fieldValue = getFieldValue(mirror, interpolatable, accessors, fieldName) - replacementStr = replacementStr.replaceAll(templateKey, fieldValue) - } - } - replacementStr - } - } - - def getMatchedPlaceholders(inputTemplate: String): Seq[MatchedPlaceholder] = { - for { - matched <- PlaceholderPattern.findAllIn(inputTemplate).toSeq - } yield { - val matchedWithIndexOpt = IndexedPlaceholderPattern.findFirstMatchIn(matched) - val (outer, inner) = matchedWithIndexOpt - .map { matchedWithIndex => - (matchedWithIndex.group(1), Some(matchedWithIndex.group(2))) - }.getOrElse((matched, None)) - val outerKey = unwrap(outer) - val innerKey = inner.map(unwrap(_)) - MatchedPlaceholder(outerKey, innerKey) - } - } - - def unwrap(str: String): String = - str.stripPrefix("${").stripSuffix("}") - - def wrap(str: String): String = - "\\$\\{" + str + "\\}" - - def getPlaceholderKeyValues( - groupedPlaceholders: Map[String, Seq[MatchedPlaceholder]], - placeholders: PlaceholdersMap - ): Seq[(String, Seq[Named])] = { - groupedPlaceholders.toSeq - .map { - case (outerKey, _) => - val placeholderValues = placeholders.getOrElse( - outerKey, - throw new RuntimeException(s"Failed to find values of $outerKey in placeholders")) - outerKey -> placeholderValues - } - } - - def crossProduct[T](seqOfSeqOfItems: Seq[Seq[T]]): Seq[List[T]] = { - if (seqOfSeqOfItems.isEmpty) { - List(Nil) - } else { - for { - // for every item in the head list - item <- seqOfSeqOfItems.head - // for every result (List) based on the cross-product of tail - resultList <- crossProduct(seqOfSeqOfItems.tail) - } yield { - item :: resultList - } - } - } - - def generateTemplateKey(matched: MatchedPlaceholder): String = { - matched.innerKey match { - case None => wrap(matched.outerKey) - case Some(innerKeyString) => wrap(matched.outerKey + "\\[" + innerKeyString + "\\]") - } - } - - // Given an instance and a field name, use reflection to get its value. - def getFieldValue[T: ClassTag]( - mirror: Mirror, - cls: T, - accessors: Map[String, MethodSymbol], - fieldName: String - ): String = { - val instance: InstanceMirror = mirror.reflect(cls) - val accessor = accessors.getOrElse( - fieldName, - throw new RuntimeException(s"Failed to find value of $fieldName for $cls")) - instance.reflectField(accessor).get.toString // .get is safe due to check above - } - - // Given an instance, use reflection to get a mapping for field name -> symbol - def caseAccessors[T: ClassTag](mirror: Mirror, cls: T): Map[String, MethodSymbol] = { - val classSymbol = mirror.classSymbol(cls.getClass) - classSymbol.toType.members.collect { - case m: MethodSymbol if m.isCaseAccessor => (m.name.toString -> m) - }.toMap - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/PlaceholderConfig.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/PlaceholderConfig.docx new file mode 100644 index 000000000..cdea9c429 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/PlaceholderConfig.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/PlaceholderConfig.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/PlaceholderConfig.scala deleted file mode 100644 index 5a81073cc..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/experiments/metrics/PlaceholderConfig.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.product_mixer.component_library.experiments.metrics - -// Base trait for all placeholder values -sealed trait Named { - def name: String -} - -case class Const(override val name: String) extends Named - -// contains only client event patterns -case class CEPattern( - override val name: String, - client: String = "", - page: String = "", - section: String = "", - component: String = "", - element: String = "", - action: String = "", - strainer: String = "") - extends Named { - - override def toString: String = { - "\"" + client + ":" + page + ":" + section + ":" + component + ":" + element + ":" + action + "\"" - } - -} - -case class Topic( - override val name: String, - topicId: String = "") - extends Named - -object PlaceholderConfig { - type PlaceholderKey = String - type Placeholder = Seq[Named] - type PlaceholdersMap = Map[PlaceholderKey, Placeholder] -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/BUILD deleted file mode 100644 index 7da3b156c..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator/featurestorev1", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator/featurestorev1", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/BUILD.docx new file mode 100644 index 000000000..9716225ac Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1QueryUserIdFeature.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1QueryUserIdFeature.docx new file mode 100644 index 000000000..ba95641ea Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1QueryUserIdFeature.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1QueryUserIdFeature.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1QueryUserIdFeature.scala deleted file mode 100644 index 299d9515f..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1QueryUserIdFeature.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.product_mixer.component_library.feature.featurestorev1 - -import com.twitter.ml.api.transform.FeatureRenameTransform -import com.twitter.ml.featurestore.catalog.entities -import com.twitter.ml.featurestore.lib.EntityId -import com.twitter.ml.featurestore.lib.UserId -import com.twitter.ml.featurestore.lib.entity.Entity -import com.twitter.ml.featurestore.lib.entity.EntityWithId -import com.twitter.ml.featurestore.lib.feature.TimelinesAggregationFrameworkFeatureGroup -import com.twitter.ml.featurestore.lib.feature.{Feature => FSv1Feature} -import com.twitter.product_mixer.core.feature.featurestorev1._ -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.FSParam -import scala.reflect.ClassTag -object FeatureStoreV1QueryUserIdFeature { - def apply[Query <: PipelineQuery, Value]( - feature: FSv1Feature[UserId, Value], - legacyName: Option[String] = None, - defaultValue: Option[Value] = None, - enabledParam: Option[FSParam[Boolean]] = None - ): FeatureStoreV1Feature[Query, Query, _ <: EntityId, Value] - with FeatureStoreV1QueryFeature[Query, _ <: EntityId, Value] = - FeatureStoreV1QueryFeature(feature, QueryUserIdEntity, legacyName, defaultValue, enabledParam) -} - -object FeatureStoreV1QueryUserIdAggregateFeature { - def apply[Query <: PipelineQuery]( - featureGroup: TimelinesAggregationFrameworkFeatureGroup[UserId], - enabledParam: Option[FSParam[Boolean]] = None, - keepLegacyNames: Boolean = false, - featureNameTransform: Option[FeatureRenameTransform] = None - ): FeatureStoreV1QueryFeatureGroup[Query, _ <: EntityId] = - FeatureStoreV1QueryFeatureGroup( - featureGroup, - QueryUserIdEntity, - enabledParam, - keepLegacyNames, - featureNameTransform)((implicitly[ClassTag[UserId]])) -} - -object QueryUserIdEntity extends FeatureStoreV1QueryEntity[PipelineQuery, UserId] { - override val entity: Entity[UserId] = entities.core.User - - override def entityWithId(query: PipelineQuery): EntityWithId[UserId] = - entity.withId(UserId(query.getUserIdLoggedOutSupport)) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1QueryUserIdTweetCandidateAuthorIdFeature.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1QueryUserIdTweetCandidateAuthorIdFeature.docx new file mode 100644 index 000000000..c65db2dd5 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1QueryUserIdTweetCandidateAuthorIdFeature.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1QueryUserIdTweetCandidateAuthorIdFeature.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1QueryUserIdTweetCandidateAuthorIdFeature.scala deleted file mode 100644 index bd026c60d..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1QueryUserIdTweetCandidateAuthorIdFeature.scala +++ /dev/null @@ -1,68 +0,0 @@ -package com.twitter.product_mixer.component_library.feature.featurestorev1 - -import com.twitter.ml.api.transform.FeatureRenameTransform -import com.twitter.ml.featurestore.catalog.entities -import com.twitter.ml.featurestore.lib.EdgeEntityId -import com.twitter.ml.featurestore.lib.EntityId -import com.twitter.ml.featurestore.lib.UserId -import com.twitter.ml.featurestore.lib.entity.Entity -import com.twitter.ml.featurestore.lib.entity.EntityWithId -import com.twitter.ml.featurestore.lib.feature.TimelinesAggregationFrameworkFeatureGroup -import com.twitter.ml.featurestore.lib.feature.{Feature => FSv1Feature} -import com.twitter.product_mixer.component_library.model.candidate.TweetAuthorIdFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featurestorev1._ -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.FSParam -import scala.reflect.ClassTag - -object FeatureStoreV1QueryUserIdTweetCandidateAuthorIdFeature { - def apply[Query <: PipelineQuery, Value]( - feature: FSv1Feature[EdgeEntityId[UserId, UserId], Value], - legacyName: Option[String] = None, - defaultValue: Option[Value] = None, - enabledParam: Option[FSParam[Boolean]] = None - ): FeatureStoreV1CandidateFeature[Query, TweetCandidate, _ <: EntityId, Value] = - FeatureStoreV1CandidateFeature( - feature, - QueryUserIdTweetCandidateAuthorIdEntity, - legacyName, - defaultValue, - enabledParam) -} - -object FeatureStoreV1QueryUserIdTweetCandidateAuthorIdAggregateFeature { - def apply[Query <: PipelineQuery]( - featureGroup: TimelinesAggregationFrameworkFeatureGroup[EdgeEntityId[UserId, UserId]], - enabledParam: Option[FSParam[Boolean]] = None, - keepLegacyNames: Boolean = false, - featureNameTransform: Option[FeatureRenameTransform] = None - ): FeatureStoreV1CandidateFeatureGroup[Query, TweetCandidate, _ <: EntityId] = - FeatureStoreV1CandidateFeatureGroup( - featureGroup, - QueryUserIdTweetCandidateAuthorIdEntity, - enabledParam, - keepLegacyNames, - featureNameTransform - )(implicitly[ClassTag[EdgeEntityId[UserId, UserId]]]) -} - -object QueryUserIdTweetCandidateAuthorIdEntity - extends FeatureStoreV1CandidateEntity[ - PipelineQuery, - TweetCandidate, - EdgeEntityId[UserId, UserId] - ] { - override val entity: Entity[EdgeEntityId[UserId, UserId]] = entities.core.UserAuthor - - override def entityWithId( - query: PipelineQuery, - tweet: TweetCandidate, - existingFeatures: FeatureMap - ): EntityWithId[EdgeEntityId[UserId, UserId]] = - entity.withId( - EdgeEntityId( - UserId(query.getUserIdLoggedOutSupport), - UserId(existingFeatures.get(TweetAuthorIdFeature)))) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1QueryUserIdTweetCandidateTweetIdFeature.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1QueryUserIdTweetCandidateTweetIdFeature.docx new file mode 100644 index 000000000..3b6315197 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1QueryUserIdTweetCandidateTweetIdFeature.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1QueryUserIdTweetCandidateTweetIdFeature.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1QueryUserIdTweetCandidateTweetIdFeature.scala deleted file mode 100644 index 9031e88be..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1QueryUserIdTweetCandidateTweetIdFeature.scala +++ /dev/null @@ -1,66 +0,0 @@ -package com.twitter.product_mixer.component_library.feature.featurestorev1 - -import com.twitter.ml.api.transform.FeatureRenameTransform -import com.twitter.ml.featurestore.catalog.entities -import com.twitter.ml.featurestore.lib.EdgeEntityId -import com.twitter.ml.featurestore.lib.EntityId -import com.twitter.ml.featurestore.lib.TweetId -import com.twitter.ml.featurestore.lib.UserId -import com.twitter.ml.featurestore.lib.entity.Entity -import com.twitter.ml.featurestore.lib.entity.EntityWithId -import com.twitter.ml.featurestore.lib.feature.TimelinesAggregationFrameworkFeatureGroup -import com.twitter.ml.featurestore.lib.feature.{Feature => FSv1Feature} -import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featurestorev1._ -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.FSParam -import scala.reflect.ClassTag - -object FeatureStoreV1QueryUserIdTweetCandidateTweetIdFeature { - def apply[Query <: PipelineQuery, Candidate <: BaseTweetCandidate, Value]( - feature: FSv1Feature[EdgeEntityId[UserId, TweetId], Value], - legacyName: Option[String] = None, - defaultValue: Option[Value] = None, - enabledParam: Option[FSParam[Boolean]] = None - ): FeatureStoreV1CandidateFeature[Query, Candidate, _ <: EntityId, Value] = - FeatureStoreV1CandidateFeature( - feature, - QueryUserIdTweetCandidateTweetIdEntity, - legacyName, - defaultValue, - enabledParam) -} - -object FeatureStoreV1QueryUserIdTweetCandidateTweetIdAggregateFeature { - def apply[Query <: PipelineQuery, Candidate <: BaseTweetCandidate]( - featureGroup: TimelinesAggregationFrameworkFeatureGroup[EdgeEntityId[UserId, TweetId]], - enabledParam: Option[FSParam[Boolean]] = None, - keepLegacyNames: Boolean = false, - featureNameTransform: Option[FeatureRenameTransform] = None - ): FeatureStoreV1CandidateFeatureGroup[Query, TweetCandidate, _ <: EntityId] = - FeatureStoreV1CandidateFeatureGroup( - featureGroup, - QueryUserIdTweetCandidateTweetIdEntity, - enabledParam, - keepLegacyNames, - featureNameTransform - )(implicitly[ClassTag[EdgeEntityId[UserId, TweetId]]]) -} - -object QueryUserIdTweetCandidateTweetIdEntity - extends FeatureStoreV1CandidateEntity[ - PipelineQuery, - BaseTweetCandidate, - EdgeEntityId[UserId, TweetId] - ] { - override val entity: Entity[EdgeEntityId[UserId, TweetId]] = entities.core.UserTweet - - override def entityWithId( - query: PipelineQuery, - tweet: BaseTweetCandidate, - existingFeatures: FeatureMap - ): EntityWithId[EdgeEntityId[UserId, TweetId]] = - entity.withId(EdgeEntityId(UserId(query.getUserIdLoggedOutSupport), TweetId(tweet.id))) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1TweetCandidateAuthorIdFeature.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1TweetCandidateAuthorIdFeature.docx new file mode 100644 index 000000000..02e36dd9f Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1TweetCandidateAuthorIdFeature.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1TweetCandidateAuthorIdFeature.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1TweetCandidateAuthorIdFeature.scala deleted file mode 100644 index b78c9569a..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1TweetCandidateAuthorIdFeature.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.product_mixer.component_library.feature.featurestorev1 - -import com.twitter.ml.api.transform.FeatureRenameTransform -import com.twitter.ml.featurestore.catalog.entities -import com.twitter.ml.featurestore.lib.EntityId -import com.twitter.ml.featurestore.lib.UserId -import com.twitter.ml.featurestore.lib.entity.Entity -import com.twitter.ml.featurestore.lib.entity.EntityWithId -import com.twitter.ml.featurestore.lib.feature.TimelinesAggregationFrameworkFeatureGroup -import com.twitter.ml.featurestore.lib.feature.{Feature => FSv1Feature} -import com.twitter.product_mixer.component_library.model.candidate.TweetAuthorIdFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featurestorev1._ -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.FSParam -import scala.reflect.ClassTag - -object FeatureStoreV1TweetCandidateAuthorIdFeature { - def apply[Query <: PipelineQuery, Value]( - feature: FSv1Feature[UserId, Value], - legacyName: Option[String] = None, - defaultValue: Option[Value] = None, - enabledParam: Option[FSParam[Boolean]] = None - ): FeatureStoreV1CandidateFeature[Query, TweetCandidate, _ <: EntityId, Value] = - FeatureStoreV1CandidateFeature( - feature, - TweetCandidateAuthorIdEntity, - legacyName, - defaultValue, - enabledParam) -} - -object FeatureStoreV1TweetCandidateAuthorIdAggregateFeature { - def apply[Query <: PipelineQuery]( - featureGroup: TimelinesAggregationFrameworkFeatureGroup[UserId], - enabledParam: Option[FSParam[Boolean]] = None, - keepLegacyNames: Boolean = false, - featureNameTransform: Option[FeatureRenameTransform] = None - ): FeatureStoreV1CandidateFeatureGroup[Query, TweetCandidate, _ <: EntityId] = - FeatureStoreV1CandidateFeatureGroup( - featureGroup, - TweetCandidateAuthorIdEntity, - enabledParam, - keepLegacyNames, - featureNameTransform - )(implicitly[ClassTag[UserId]]) -} - -object TweetCandidateAuthorIdEntity - extends FeatureStoreV1CandidateEntity[PipelineQuery, TweetCandidate, UserId] { - override val entity: Entity[UserId] = entities.core.Author - - override def entityWithId( - query: PipelineQuery, - tweet: TweetCandidate, - existingFeatures: FeatureMap - ): EntityWithId[UserId] = - entity.withId(UserId(existingFeatures.get(TweetAuthorIdFeature))) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1TweetCandidateTweetIdFeature.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1TweetCandidateTweetIdFeature.docx new file mode 100644 index 000000000..a06ca1fd3 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1TweetCandidateTweetIdFeature.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1TweetCandidateTweetIdFeature.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1TweetCandidateTweetIdFeature.scala deleted file mode 100644 index c75bf21ac..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1TweetCandidateTweetIdFeature.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.product_mixer.component_library.feature.featurestorev1 - -import com.twitter.ml.api.transform.FeatureRenameTransform -import com.twitter.ml.featurestore.catalog.entities -import com.twitter.ml.featurestore.lib.EntityId -import com.twitter.ml.featurestore.lib.TweetId -import com.twitter.ml.featurestore.lib.entity.Entity -import com.twitter.ml.featurestore.lib.entity.EntityWithId -import com.twitter.ml.featurestore.lib.feature.TimelinesAggregationFrameworkFeatureGroup -import com.twitter.ml.featurestore.lib.feature.{Feature => FSv1Feature} -import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featurestorev1._ -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.FSParam - -object FeatureStoreV1TweetCandidateTweetIdFeature { - def apply[Query <: PipelineQuery, Candidate <: BaseTweetCandidate, Value]( - feature: FSv1Feature[TweetId, Value], - legacyName: Option[String] = None, - defaultValue: Option[Value] = None, - enabledParam: Option[FSParam[Boolean]] = None - ): FeatureStoreV1CandidateFeature[Query, Candidate, _ <: EntityId, Value] = - FeatureStoreV1CandidateFeature( - feature, - TweetCandidateTweetIdEntity, - legacyName, - defaultValue, - enabledParam) -} - -object FeatureStoreV1TweetCandidateTweetIdAggregateFeature { - def apply[Query <: PipelineQuery, Candidate <: BaseTweetCandidate]( - featureGroup: TimelinesAggregationFrameworkFeatureGroup[TweetId], - enabledParam: Option[FSParam[Boolean]] = None, - keepLegacyNames: Boolean = false, - featureNameTransform: Option[FeatureRenameTransform] = None - ): FeatureStoreV1CandidateFeatureGroup[Query, Candidate, _ <: EntityId] = - FeatureStoreV1CandidateFeatureGroup( - featureGroup, - TweetCandidateTweetIdEntity, - enabledParam, - keepLegacyNames, - featureNameTransform - ) -} - -object TweetCandidateTweetIdEntity - extends FeatureStoreV1CandidateEntity[PipelineQuery, BaseTweetCandidate, TweetId] { - override val entity: Entity[TweetId] = entities.core.Tweet - - override def entityWithId( - query: PipelineQuery, - tweet: BaseTweetCandidate, - existingFeatures: FeatureMap - ): EntityWithId[TweetId] = - entity.withId(TweetId(tweet.id)) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1UserCandidateUserIdFeature.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1UserCandidateUserIdFeature.docx new file mode 100644 index 000000000..f517907fb Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1UserCandidateUserIdFeature.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1UserCandidateUserIdFeature.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1UserCandidateUserIdFeature.scala deleted file mode 100644 index a0947b995..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/featurestorev1/FeatureStoreV1UserCandidateUserIdFeature.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.product_mixer.component_library.feature.featurestorev1 - -import com.twitter.ml.featurestore.catalog.entities -import com.twitter.ml.featurestore.lib.EntityId -import com.twitter.ml.featurestore.lib.UserId -import com.twitter.ml.featurestore.lib.entity.Entity -import com.twitter.ml.featurestore.lib.entity.EntityWithId -import com.twitter.ml.featurestore.lib.feature.{Feature => FSv1Feature} -import com.twitter.product_mixer.component_library.model.candidate.BaseUserCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featurestorev1._ -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.FSParam - -object FeatureStoreV1UserCandidateUserIdFeature { - def apply[Query <: PipelineQuery, Candidate <: BaseUserCandidate, Value]( - feature: FSv1Feature[UserId, Value], - legacyName: Option[String] = None, - defaultValue: Option[Value] = None, - enabledParam: Option[FSParam[Boolean]] = None - ): FeatureStoreV1CandidateFeature[Query, Candidate, _ <: EntityId, Value] = - FeatureStoreV1CandidateFeature( - feature, - UserCandidateUserIdEntity, - legacyName, - defaultValue, - enabledParam) -} - -object UserCandidateUserIdEntity - extends FeatureStoreV1CandidateEntity[PipelineQuery, BaseUserCandidate, UserId] { - override val entity: Entity[UserId] = entities.core.User - - override def entityWithId( - query: PipelineQuery, - user: BaseUserCandidate, - existingFeatures: FeatureMap - ): EntityWithId[UserId] = - entity.withId(UserId(user.id)) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/BUILD deleted file mode 100644 index 78840577d..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/BUILD +++ /dev/null @@ -1,35 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/src/jvm/com/twitter/storehaus:core", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/module", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "src/thrift/com/twitter/spam/rtf:safety-result-scala", - "src/thrift/com/twitter/timelinescorer:thrift-scala", - "src/thrift/com/twitter/timelinescorer/server/internal:thrift-scala", - "src/thrift/com/twitter/tweetypie:service-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - "stitch/stitch-core", - "stitch/stitch-tweetypie", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/src/jvm/com/twitter/storehaus:core", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/module", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "src/thrift/com/twitter/timelinescorer:thrift-scala", - "src/thrift/com/twitter/timelinescorer/server/internal:thrift-scala", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/BUILD.docx new file mode 100644 index 000000000..93c6f48ed Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/ads/AdvertiserBrandSafetySettingsFeatureHydrator.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/ads/AdvertiserBrandSafetySettingsFeatureHydrator.docx new file mode 100644 index 000000000..99ea736f0 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/ads/AdvertiserBrandSafetySettingsFeatureHydrator.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/ads/AdvertiserBrandSafetySettingsFeatureHydrator.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/ads/AdvertiserBrandSafetySettingsFeatureHydrator.scala deleted file mode 100644 index 107f04fff..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/ads/AdvertiserBrandSafetySettingsFeatureHydrator.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.product_mixer.component_library.feature_hydrator.candidate.ads - -import com.twitter.adserver.{thriftscala => ad} -import com.twitter.product_mixer.component_library.model.candidate.ads.AdsCandidate -import com.twitter.product_mixer.component_library.model.query.ads.AdsQuery -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -import javax.inject.Inject -import javax.inject.Singleton - -object AdvertiserBrandSafetySettingsFeature - extends FeatureWithDefaultOnFailure[AdsCandidate, Option[ad.AdvertiserBrandSafetySettings]] { - override val defaultValue = None -} - -@Singleton -case class AdvertiserBrandSafetySettingsFeatureHydrator[ - Query <: PipelineQuery with AdsQuery, - Candidate <: AdsCandidate] @Inject() ( - advertiserBrandSafetySettingsStore: ReadableStore[Long, ad.AdvertiserBrandSafetySettings]) - extends CandidateFeatureHydrator[Query, Candidate] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( - "AdvertiserBrandSafetySettings") - - override val features: Set[Feature[_, _]] = Set(AdvertiserBrandSafetySettingsFeature) - - override def apply( - query: Query, - candidate: Candidate, - existingFeatures: FeatureMap - ): Stitch[FeatureMap] = { - - val featureMapFuture: Future[FeatureMap] = advertiserBrandSafetySettingsStore - .get(candidate.adImpression.advertiserId) - .map { advertiserBrandSafetySettingsOpt => - FeatureMapBuilder() - .add(AdvertiserBrandSafetySettingsFeature, advertiserBrandSafetySettingsOpt).build() - } - - Stitch.callFuture(featureMapFuture) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/ads/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/ads/BUILD deleted file mode 100644 index 4c66c4346..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/ads/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/twitter/storehaus:core", - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", - "src/thrift/com/twitter/ads/adserver:ads_shared_types-scala", - ], - exports = [ - "3rdparty/jvm/com/twitter/storehaus:core", - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", - "src/thrift/com/twitter/ads/adserver:ads_shared_types-scala", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/ads/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/ads/BUILD.docx new file mode 100644 index 000000000..1ca35ed83 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/ads/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/decay/BUILD.bazel b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/decay/BUILD.bazel deleted file mode 100644 index 3759377bd..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/decay/BUILD.bazel +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/scorer", - "snowflake/src/main/scala/com/twitter/snowflake/id", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/scorer", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/decay/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/decay/BUILD.docx new file mode 100644 index 000000000..11c5fef39 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/decay/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/decay/DecayCandidateFeatureHydrator.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/decay/DecayCandidateFeatureHydrator.docx new file mode 100644 index 000000000..d5feb08c5 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/decay/DecayCandidateFeatureHydrator.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/decay/DecayCandidateFeatureHydrator.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/decay/DecayCandidateFeatureHydrator.scala deleted file mode 100644 index 4f0d99e82..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/decay/DecayCandidateFeatureHydrator.scala +++ /dev/null @@ -1,65 +0,0 @@ -package com.twitter.product_mixer.component_library.feature_hydrator.candidate.decay - -import com.twitter.conversions.DurationOps._ -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.configapi.StaticParam -import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration - -object DecayScore extends Feature[UniversalNoun[Long], Double] - -/** - * Hydrates snowflake ID candidates with a decay score: - * - * It is using exponential decay formula to calculate the score - * exp(k * age) - * where k = ln(0.5) / half-life - * - * Here is an example for half-life = 1 day - * For the brand new tweet it will be exp((ln(0.5)/1)*0) = 1 - * For the tweet which was created 1 day ago it will be exp((ln(0.5)/1)*1) = 0.5 - * For the tweet which was created 10 day ago it will be exp((ln(0.5)/1)*10) = 0.00097 - * - * Reference: https://www.cuemath.com/exponential-decay-formula/ - * - * @note This penalizes but does not filter out the candidate, so "stale" candidates can still appear. - */ -case class DecayCandidateFeatureHydrator[Candidate <: UniversalNoun[Long]]( - halfLife: Param[Duration] = StaticParam[Duration](2.days), - resultFeature: Feature[UniversalNoun[Long], Double] = DecayScore) - extends CandidateFeatureHydrator[PipelineQuery, Candidate] { - - override val features: Set[Feature[_, _]] = Set(resultFeature) - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("Decay") - - override def apply( - query: PipelineQuery, - candidate: Candidate, - existingFeatures: FeatureMap - ): Stitch[FeatureMap] = { - val halfLifeInMillis = query.params(halfLife).inMillis - - val creationTime = SnowflakeId.timeFromId(candidate.id) - val ageInMillis = creationTime.untilNow.inMilliseconds - - // it is using a exponential decay formula: e^(k * tweetAge) - // where k = ln(0.5) / half-life - val k = math.log(0.5D) / halfLifeInMillis - val decayScore = math.exp(k * ageInMillis) - - Stitch.value( - FeatureMapBuilder() - .add(resultFeature, decayScore) - .build()) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/BUILD deleted file mode 100644 index a9e2c7cc4..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/BUILD +++ /dev/null @@ -1,25 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "stitch/stitch-core", - ], - exports = [ - "configapi/configapi-core", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/BUILD.docx new file mode 100644 index 000000000..a6e8b5ed3 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/ParamGatedBulkCandidateFeatureHydrator.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/ParamGatedBulkCandidateFeatureHydrator.docx new file mode 100644 index 000000000..3cc84129c Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/ParamGatedBulkCandidateFeatureHydrator.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/ParamGatedBulkCandidateFeatureHydrator.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/ParamGatedBulkCandidateFeatureHydrator.scala deleted file mode 100644 index 621eed2e9..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/ParamGatedBulkCandidateFeatureHydrator.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated - -import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedBulkCandidateFeatureHydrator.IdentifierPrefix -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.Conditionally -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.Param - -/** - * A [[BulkCandidateFeatureHydrator]] with [[Conditionally]] based on a [[Param]] - * - * @param enabledParam the param to turn this [[BulkCandidateFeatureHydrator]] on and off - * @param bulkCandidateFeatureHydrator the underlying [[BulkCandidateFeatureHydrator]] to run when `enabledParam` is true - * @tparam Query The domain model for the query or request - * @tparam Result The type of the candidates - */ -case class ParamGatedBulkCandidateFeatureHydrator[ - -Query <: PipelineQuery, - Result <: UniversalNoun[Any] -]( - enabledParam: Param[Boolean], - bulkCandidateFeatureHydrator: BulkCandidateFeatureHydrator[Query, Result]) - extends BulkCandidateFeatureHydrator[Query, Result] - with Conditionally[Query] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( - IdentifierPrefix + bulkCandidateFeatureHydrator.identifier.name) - - override val alerts: Seq[Alert] = bulkCandidateFeatureHydrator.alerts - - override val features: Set[Feature[_, _]] = bulkCandidateFeatureHydrator.features - - override def onlyIf(query: Query): Boolean = - Conditionally.and(query, bulkCandidateFeatureHydrator, query.params(enabledParam)) - - override def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[Result]] - ): Stitch[Seq[FeatureMap]] = bulkCandidateFeatureHydrator(query, candidates) -} - -object ParamGatedBulkCandidateFeatureHydrator { - val IdentifierPrefix = "ParamGated" -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/ParamGatedCandidateFeatureHydrator.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/ParamGatedCandidateFeatureHydrator.docx new file mode 100644 index 000000000..4fd0b75d5 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/ParamGatedCandidateFeatureHydrator.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/ParamGatedCandidateFeatureHydrator.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/ParamGatedCandidateFeatureHydrator.scala deleted file mode 100644 index 6409145e5..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/ParamGatedCandidateFeatureHydrator.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated - -import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedCandidateFeatureHydrator.IdentifierPrefix -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.Conditionally -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.Param - -/** - * A [[CandidateFeatureHydrator]] with [[Conditionally]] based on a [[Param]] - * - * @param enabledParam the param to turn this [[CandidateFeatureHydrator]] on and off - * @param candidateFeatureHydrator the underlying [[CandidateFeatureHydrator]] to run when `enabledParam` is true - * @tparam Query The domain model for the query or request - * @tparam Result The type of the candidates - */ -case class ParamGatedCandidateFeatureHydrator[ - -Query <: PipelineQuery, - -Result <: UniversalNoun[Any] -]( - enabledParam: Param[Boolean], - candidateFeatureHydrator: CandidateFeatureHydrator[Query, Result]) - extends CandidateFeatureHydrator[Query, Result] - with Conditionally[Query] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( - IdentifierPrefix + candidateFeatureHydrator.identifier.name) - - override val alerts: Seq[Alert] = candidateFeatureHydrator.alerts - - override val features: Set[Feature[_, _]] = candidateFeatureHydrator.features - - override def onlyIf(query: Query): Boolean = - Conditionally.and(query, candidateFeatureHydrator, query.params(enabledParam)) - - override def apply( - query: Query, - candidate: Result, - existingFeatures: FeatureMap - ): Stitch[FeatureMap] = candidateFeatureHydrator.apply(query, candidate, existingFeatures) -} - -object ParamGatedCandidateFeatureHydrator { - val IdentifierPrefix = "ParamGated" -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/featurestorev1/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/featurestorev1/BUILD deleted file mode 100644 index 7b18d7b06..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/featurestorev1/BUILD +++ /dev/null @@ -1,27 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator/featurestorev1", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "stitch/stitch-core", - ], - exports = [ - "configapi/configapi-core", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator/featurestorev1", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/featurestorev1/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/featurestorev1/BUILD.docx new file mode 100644 index 000000000..410f7b75e Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/featurestorev1/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/featurestorev1/ParamGatedFeatureStoreV1CandidateFeatureHydrator.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/featurestorev1/ParamGatedFeatureStoreV1CandidateFeatureHydrator.docx new file mode 100644 index 000000000..9dc7a6d97 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/featurestorev1/ParamGatedFeatureStoreV1CandidateFeatureHydrator.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/featurestorev1/ParamGatedFeatureStoreV1CandidateFeatureHydrator.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/featurestorev1/ParamGatedFeatureStoreV1CandidateFeatureHydrator.scala deleted file mode 100644 index d151c4374..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated/featurestorev1/ParamGatedFeatureStoreV1CandidateFeatureHydrator.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.featurestorev1 - -import com.twitter.ml.featurestore.lib.EntityId -import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.featurestorev1.ParamGatedFeatureStoreV1CandidateFeatureHydrator.IdentifierPrefix -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featurestorev1.BaseFeatureStoreV1CandidateFeature -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.feature_hydrator.featurestorev1.FeatureStoreV1CandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.featurestorev1.FeatureStoreV1DynamicClientBuilder -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.Conditionally -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.Param - -/** - * A [[FeatureStoreV1CandidateFeatureHydrator]] with [[Conditionally]] based on a [[Param]] - * - * @param enabledParam the param to turn this [[FeatureStoreV1CandidateFeatureHydrator]] on and off - * @param candidateFeatureHydrator the underlying [[FeatureStoreV1CandidateFeatureHydrator]] to run when `enabledParam` is true - * @tparam Query The domain model for the query or request - * @tparam Candidate The type of the candidates - */ -case class ParamGatedFeatureStoreV1CandidateFeatureHydrator[ - Query <: PipelineQuery, - Candidate <: UniversalNoun[Any] -]( - enabledParam: Param[Boolean], - candidateFeatureHydrator: FeatureStoreV1CandidateFeatureHydrator[Query, Candidate]) - extends FeatureStoreV1CandidateFeatureHydrator[Query, Candidate] - with Conditionally[Query] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( - IdentifierPrefix + candidateFeatureHydrator.identifier.name) - - override val alerts: Seq[Alert] = candidateFeatureHydrator.alerts - - override val features: Set[ - BaseFeatureStoreV1CandidateFeature[Query, Candidate, _ <: EntityId, _] - ] = candidateFeatureHydrator.features - - override val clientBuilder: FeatureStoreV1DynamicClientBuilder = - candidateFeatureHydrator.clientBuilder - - override def onlyIf(query: Query): Boolean = - Conditionally.and(query, candidateFeatureHydrator, query.params(enabledParam)) - - override def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[Candidate]] - ): Stitch[Seq[FeatureMap]] = candidateFeatureHydrator(query, candidates) -} - -object ParamGatedFeatureStoreV1CandidateFeatureHydrator { - val IdentifierPrefix = "ParamGated" -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/qualityfactor_gated/BUILD.bazel b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/qualityfactor_gated/BUILD.bazel deleted file mode 100644 index 579f420ee..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/qualityfactor_gated/BUILD.bazel +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/qualityfactor_gated/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/qualityfactor_gated/BUILD.docx new file mode 100644 index 000000000..31ff66d6c Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/qualityfactor_gated/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/qualityfactor_gated/QualityFactorGatedCandidateFeatureHydrator.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/qualityfactor_gated/QualityFactorGatedCandidateFeatureHydrator.docx new file mode 100644 index 000000000..a681c8aa1 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/qualityfactor_gated/QualityFactorGatedCandidateFeatureHydrator.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/qualityfactor_gated/QualityFactorGatedCandidateFeatureHydrator.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/qualityfactor_gated/QualityFactorGatedCandidateFeatureHydrator.scala deleted file mode 100644 index e9d74826c..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/qualityfactor_gated/QualityFactorGatedCandidateFeatureHydrator.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.twitter.product_mixer.component_library.feature_hydrator.candidate.qualityfactor_gated - -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.Conditionally -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.Param - -object QualityFactorGatedCandidateFeatureHydrator { - val IdentifierPrefix = "QfGated" -} - -/** - * A [[CandidateFeatureHydrator]] with [[Conditionally]] based on a qualityFactor threshold. - * @param pipelineIdentifier identifier of the pipeline that associated with observed quality factor - * @param qualityFactorInclusiveThreshold the inclusive threshold of quality factor that value below it results in - * the underlying hydrator being turned off - * @param candidateFeatureHydrator the underlying [[CandidateFeatureHydrator]] to run when quality factor value - * is above the given inclusive threshold - * @tparam Query The domain model for the query or request - * @tparam Result The type of the candidates - */ -case class QualityFactorGatedCandidateFeatureHydrator[ - -Query <: PipelineQuery with HasQualityFactorStatus, - Result <: UniversalNoun[Any] -]( - pipelineIdentifier: ComponentIdentifier, - qualityFactorInclusiveThreshold: Param[Double], - candidateFeatureHydrator: CandidateFeatureHydrator[Query, Result]) - extends CandidateFeatureHydrator[Query, Result] - with Conditionally[Query] { - import QualityFactorGatedCandidateFeatureHydrator._ - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( - IdentifierPrefix + candidateFeatureHydrator.identifier.name) - - override val alerts: Seq[Alert] = candidateFeatureHydrator.alerts - - override val features: Set[Feature[_, _]] = candidateFeatureHydrator.features - - override def onlyIf(query: Query): Boolean = Conditionally.and( - query, - candidateFeatureHydrator, - query.getQualityFactorCurrentValue(pipelineIdentifier) >= query.params( - qualityFactorInclusiveThreshold)) - - override def apply( - query: Query, - candidate: Result, - existingFeatures: FeatureMap - ): Stitch[FeatureMap] = candidateFeatureHydrator.apply(query, candidate, existingFeatures) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw/BUILD deleted file mode 100644 index 3a2b817bf..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw/BUILD +++ /dev/null @@ -1,37 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - scalac_plugins = ["no-roomba"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/src/jvm/com/twitter/storehaus:core", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/module", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "src/thrift/com/twitter/spam/rtf:safety-result-scala", - "src/thrift/com/twitter/timelinescorer:thrift-scala", - "src/thrift/com/twitter/timelinescorer/server/internal:thrift-scala", - "src/thrift/com/twitter/tweetypie:service-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - "stitch/stitch-core", - "stitch/stitch-tweetypie", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/src/jvm/com/twitter/storehaus:core", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/module", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "src/thrift/com/twitter/timelinescorer:thrift-scala", - "src/thrift/com/twitter/timelinescorer/server/internal:thrift-scala", - "stitch/stitch-core", - "stitch/stitch-tweetypie", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw/BUILD.docx new file mode 100644 index 000000000..6598d905b Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw/TweetIsNsfwCandidateFeatureHydrator.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw/TweetIsNsfwCandidateFeatureHydrator.docx new file mode 100644 index 000000000..93a5dd14c Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw/TweetIsNsfwCandidateFeatureHydrator.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw/TweetIsNsfwCandidateFeatureHydrator.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw/TweetIsNsfwCandidateFeatureHydrator.scala deleted file mode 100644 index 651f0f4b0..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw/TweetIsNsfwCandidateFeatureHydrator.scala +++ /dev/null @@ -1,109 +0,0 @@ -package com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_is_nsfw - -import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient} -import com.twitter.tweetypie.{thriftscala => t} -import com.twitter.util.Return -import com.twitter.util.Throw -import com.twitter.util.Try -import com.twitter.util.logging.Logging - -// The VF NsfwHighPrecisionLabel that powers the NSFW determination here has been deprecated and is no longer written to. -@deprecated("Prefer VisibilityReason") -object IsNsfw extends FeatureWithDefaultOnFailure[TweetCandidate, Option[Boolean]] { - - /** - * Generic Logic to evaluate whether a tweet is nsfw - * @param hasNsfwHighPrecisionLabel flag for tweetypieTweet nsfwHighPrecision label - * @param isNsfwUser flag for tweetypieTweet coreData nsfwUser flag - * @param isNsfwAdmin flag for tweetypieTweet coreData nsfwAdmin flag - * @return isNsfw to true if any of the three flags is true - */ - def apply( - hasNsfwHighPrecisionLabel: Option[Boolean], - isNsfwUser: Option[Boolean], - isNsfwAdmin: Option[Boolean] - ): Boolean = { - hasNsfwHighPrecisionLabel - .getOrElse(false) || (isNsfwUser.getOrElse(false) || isNsfwAdmin.getOrElse(false)) - } - - override val defaultValue = None -} - -// The VF NsfwHighPrecisionLabel that powers the NSFW determination here has been deprecated and is no longer written to. -// TODO: Remove after all dependencies have migrated to using TweetCandidateVisibilityReasonFeatureHydrator. -@deprecated("Prefer TweetCandidateVisibilityReasonFeatureHydrator") -case class TweetIsNsfwCandidateFeatureHydrator( - tweetypieStitchClient: TweetypieStitchClient, - tweetVisibilityPolicy: t.TweetVisibilityPolicy) - extends BulkCandidateFeatureHydrator[PipelineQuery, BaseTweetCandidate] - with Logging { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetIsNsfw") - - override def features: Set[Feature[_, _]] = Set(IsNsfw) - - private val NsfwLabelFields: Set[t.TweetInclude] = Set[t.TweetInclude]( - // Tweet fields containing NSFW related attributes, in addition to what exists in coreData. - t.TweetInclude.TweetFieldId(t.Tweet.NsfwHighPrecisionLabelField.id), - t.TweetInclude.TweetFieldId(t.Tweet.CoreDataField.id) - ) - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[BaseTweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - Stitch - .traverse(candidates.map(_.candidate.id)) { tweetId => - tweetypieStitchClient - .getTweetFields( - tweetId = tweetId, - options = t.GetTweetFieldsOptions( - forUserId = query.getOptionalUserId, - tweetIncludes = NsfwLabelFields, - doNotCache = true, - visibilityPolicy = tweetVisibilityPolicy, - safetyLevel = None, - ) - ).liftToTry - }.map { getTweetFieldsResults: Seq[Try[t.GetTweetFieldsResult]] => - val tweets: Seq[Try[t.Tweet]] = getTweetFieldsResults.map { - case Return(t.GetTweetFieldsResult(_, t.TweetFieldsResultState.Found(found), _, _)) => - Return(found.tweet) - case Return(t.GetTweetFieldsResult(_, resultState, _, _)) => - Throw(IsNsfwFeatureHydrationFailure(s"Unexpected tweet result state: ${resultState}")) - case Throw(e) => - Throw(e) - } - - candidates.zip(tweets).map { - case (candidateWithFeatures, tweetTry) => - val isNsfwFeature = tweetTry.map { tweet => - IsNsfw( - hasNsfwHighPrecisionLabel = Some(tweet.nsfwHighPrecisionLabel.isDefined), - isNsfwUser = tweet.coreData.map(_.nsfwUser), - isNsfwAdmin = tweet.coreData.map(_.nsfwAdmin) - ) - } - - FeatureMapBuilder() - .add(IsNsfw, isNsfwFeature.map(Some(_))) - .build() - } - } - } -} - -case class IsNsfwFeatureHydrationFailure(message: String) - extends Exception(s"IsNsfwFeatureHydrationFailure(${message})") diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_tlx/BUILD.bazel b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_tlx/BUILD.bazel deleted file mode 100644 index 9d9d9a421..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_tlx/BUILD.bazel +++ /dev/null @@ -1,42 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/module", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tweet_tlx", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "src/thrift/com/twitter/ml/featurestore/timelines:ml-features-timelines-scala", - "src/thrift/com/twitter/ml/featurestore/timelines:ml-features-timelines-strato", - "src/thrift/com/twitter/timelinescorer:thrift-scala", - "src/thrift/com/twitter/timelinescorer/server/internal:thrift-scala", - "src/thrift/com/twitter/tweetypie:service-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - "stitch/stitch-core", - "strato/config/columns/ml/featureStore:featureStore-strato-client", - "strato/src/main/scala/com/twitter/strato/client", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/module", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tweet_tlx", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "src/thrift/com/twitter/ml/featurestore/timelines:ml-features-timelines-scala", - "src/thrift/com/twitter/ml/featurestore/timelines:ml-features-timelines-strato", - "src/thrift/com/twitter/timelinescorer:thrift-scala", - "src/thrift/com/twitter/timelinescorer/server/internal:thrift-scala", - "stitch/stitch-core", - "strato/config/columns/ml/featureStore:featureStore-strato-client", - "strato/src/main/scala/com/twitter/strato/client", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_tlx/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_tlx/BUILD.docx new file mode 100644 index 000000000..6185c7164 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_tlx/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_tlx/TweetTLXScoreCandidateFeatureHydrator.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_tlx/TweetTLXScoreCandidateFeatureHydrator.docx new file mode 100644 index 000000000..3ffc8605e Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_tlx/TweetTLXScoreCandidateFeatureHydrator.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_tlx/TweetTLXScoreCandidateFeatureHydrator.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_tlx/TweetTLXScoreCandidateFeatureHydrator.scala deleted file mode 100644 index ee985d9c4..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_tlx/TweetTLXScoreCandidateFeatureHydrator.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_tlx - -import com.twitter.ml.featurestore.timelines.thriftscala.TimelineScorerScoreView -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.component_library.scorer.tweet_tlx.TLXScore -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.strato.generated.client.ml.featureStore.TimelineScorerTweetScoresV1ClientColumn -import com.twitter.timelinescorer.thriftscala.v1 -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Hydrate Tweet Scores via Timeline Scorer (TLX) - * - * Note that this is the [[CandidateFeatureHydrator]] version of - * [[com.twitter.product_mixer.component_library.scorer.tweet_tlx.TweetTLXStratoScorer]] - */ -@Singleton -class TweetTLXScoreCandidateFeatureHydrator @Inject() ( - column: TimelineScorerTweetScoresV1ClientColumn) - extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("TweetTLXScore") - - override val features: Set[Feature[_, _]] = Set(TLXScore) - - private val NoScoreMap = FeatureMapBuilder() - .add(TLXScore, None) - .build() - - override def apply( - query: PipelineQuery, - candidate: TweetCandidate, - existingFeatures: FeatureMap - ): Stitch[FeatureMap] = { - query.getOptionalUserId match { - case Some(userId) => - column.fetcher - .fetch(candidate.id, TimelineScorerScoreView(Some(userId))) - .map(scoredTweet => - scoredTweet.v match { - case Some(v1.ScoredTweet(Some(_), score, _, _)) => - FeatureMapBuilder() - .add(TLXScore, score) - .build() - case _ => throw new Exception(s"Invalid response from TLX: ${scoredTweet.v}") - }) - case _ => - Stitch.value(NoScoreMap) - } - } -}