diff --git a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/observer/ResultsObserver.docx b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/observer/ResultsObserver.docx new file mode 100644 index 000000000..1f33d1072 Binary files /dev/null and b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/observer/ResultsObserver.docx differ diff --git a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/observer/ResultsObserver.scala b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/observer/ResultsObserver.scala deleted file mode 100644 index e77676509..000000000 --- a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/observer/ResultsObserver.scala +++ /dev/null @@ -1,281 +0,0 @@ -package com.twitter.product_mixer.shared_library.observer - -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.shared_library.observer.Observer.ArrowObserver -import com.twitter.product_mixer.shared_library.observer.Observer.FunctionObserver -import com.twitter.product_mixer.shared_library.observer.Observer.FutureObserver -import com.twitter.product_mixer.shared_library.observer.Observer.Observer -import com.twitter.product_mixer.shared_library.observer.Observer.StitchObserver -import com.twitter.stitch.Arrow -import com.twitter.stitch.Stitch -import com.twitter.util.Future -import com.twitter.util.Try - -/** - * Helper functions to observe requests, successes, failures, cancellations, exceptions, latency, - * and result counts. Supports native functions and asynchronous operations. - */ -object ResultsObserver { - val Total = "total" - val Found = "found" - val NotFound = "not_found" - - /** - * Helper function to observe a stitch and result counts - * - * @see [[StitchResultsObserver]] - */ - def stitchResults[T]( - size: T => Int, - statsReceiver: StatsReceiver, - scopes: String* - ): StitchResultsObserver[T] = { - new StitchResultsObserver[T](size, statsReceiver, scopes) - } - - /** - * Helper function to observe a stitch and traversable (e.g. Seq, Set) result counts - * - * @see [[StitchResultsObserver]] - */ - def stitchResults[T <: TraversableOnce[_]]( - statsReceiver: StatsReceiver, - scopes: String* - ): StitchResultsObserver[T] = { - new StitchResultsObserver[T](_.size, statsReceiver, scopes) - } - - /** - * Helper function to observe an arrow and result counts - * - * @see [[ArrowResultsObserver]] - */ - def arrowResults[In, Out]( - size: Out => Int, - statsReceiver: StatsReceiver, - scopes: String* - ): ArrowResultsObserver[In, Out] = { - new ArrowResultsObserver[In, Out](size, statsReceiver, scopes) - } - - /** - * Helper function to observe an arrow and traversable (e.g. Seq, Set) result counts - * - * @see [[ArrowResultsObserver]] - */ - def arrowResults[In, Out <: TraversableOnce[_]]( - statsReceiver: StatsReceiver, - scopes: String* - ): ArrowResultsObserver[In, Out] = { - new ArrowResultsObserver[In, Out](_.size, statsReceiver, scopes) - } - - /** - * Helper function to observe an arrow and result counts - * - * @see [[TransformingArrowResultsObserver]] - */ - def transformingArrowResults[In, Out, Transformed]( - transformer: Out => Try[Transformed], - size: Transformed => Int, - statsReceiver: StatsReceiver, - scopes: String* - ): TransformingArrowResultsObserver[In, Out, Transformed] = { - new TransformingArrowResultsObserver[In, Out, Transformed]( - transformer, - size, - statsReceiver, - scopes) - } - - /** - * Helper function to observe an arrow and traversable (e.g. Seq, Set) result counts - * - * @see [[TransformingArrowResultsObserver]] - */ - def transformingArrowResults[In, Out, Transformed <: TraversableOnce[_]]( - transformer: Out => Try[Transformed], - statsReceiver: StatsReceiver, - scopes: String* - ): TransformingArrowResultsObserver[In, Out, Transformed] = { - new TransformingArrowResultsObserver[In, Out, Transformed]( - transformer, - _.size, - statsReceiver, - scopes) - } - - /** - * Helper function to observe a future and result counts - * - * @see [[FutureResultsObserver]] - */ - def futureResults[T]( - size: T => Int, - statsReceiver: StatsReceiver, - scopes: String* - ): FutureResultsObserver[T] = { - new FutureResultsObserver[T](size, statsReceiver, scopes) - } - - /** - * Helper function to observe a future and traversable (e.g. Seq, Set) result counts - * - * @see [[FutureResultsObserver]] - */ - def futureResults[T <: TraversableOnce[_]]( - statsReceiver: StatsReceiver, - scopes: String* - ): FutureResultsObserver[T] = { - new FutureResultsObserver[T](_.size, statsReceiver, scopes) - } - - /** - * Helper function to observe a function and result counts - * - * @see [[FunctionResultsObserver]] - */ - def functionResults[T]( - size: T => Int, - statsReceiver: StatsReceiver, - scopes: String* - ): FunctionResultsObserver[T] = { - new FunctionResultsObserver[T](size, statsReceiver, scopes) - } - - /** - * Helper function to observe a function and traversable (e.g. Seq, Set) result counts - * - * @see [[FunctionResultsObserver]] - */ - def functionResults[T <: TraversableOnce[_]]( - statsReceiver: StatsReceiver, - scopes: String* - ): FunctionResultsObserver[T] = { - new FunctionResultsObserver[T](_.size, statsReceiver, scopes) - } - - /** [[StitchObserver]] that also records result size */ - class StitchResultsObserver[T]( - override val size: T => Int, - override val statsReceiver: StatsReceiver, - override val scopes: Seq[String]) - extends StitchObserver[T](statsReceiver, scopes) - with ResultsObserver[T] { - - override def apply(stitch: => Stitch[T]): Stitch[T] = - super - .apply(stitch) - .onSuccess(observeResults) - } - - /** [[ArrowObserver]] that also records result size */ - class ArrowResultsObserver[In, Out]( - override val size: Out => Int, - override val statsReceiver: StatsReceiver, - override val scopes: Seq[String]) - extends ArrowObserver[In, Out](statsReceiver, scopes) - with ResultsObserver[Out] { - - override def apply(arrow: Arrow[In, Out]): Arrow[In, Out] = - super - .apply(arrow) - .onSuccess(observeResults) - } - - /** - * [[TransformingArrowResultsObserver]] functions like an [[ArrowObserver]] except - * that it transforms the result using [[transformer]] before recording stats. - * - * The original non-transformed result is then returned. - */ - class TransformingArrowResultsObserver[In, Out, Transformed]( - val transformer: Out => Try[Transformed], - override val size: Transformed => Int, - override val statsReceiver: StatsReceiver, - override val scopes: Seq[String]) - extends Observer[Transformed] - with ResultsObserver[Transformed] { - - /** - * Returns a new Arrow that records stats on the result after applying [[transformer]] when it's run. - * The original, non-transformed, result of the Arrow is passed through. - * - * @note the provided Arrow must contain the parts that need to be timed. - * Using this on just the result of the computation the latency stat - * will be incorrect. - */ - def apply(arrow: Arrow[In, Out]): Arrow[In, Out] = { - Arrow - .time(arrow) - .map { - case (response, stitchRunDuration) => - observe(response.flatMap(transformer), stitchRunDuration) - .onSuccess(observeResults) - response - }.lowerFromTry - } - } - - /** [[FutureObserver]] that also records result size */ - class FutureResultsObserver[T]( - override val size: T => Int, - override val statsReceiver: StatsReceiver, - override val scopes: Seq[String]) - extends FutureObserver[T](statsReceiver, scopes) - with ResultsObserver[T] { - - override def apply(future: => Future[T]): Future[T] = - super - .apply(future) - .onSuccess(observeResults) - } - - /** [[FunctionObserver]] that also records result size */ - class FunctionResultsObserver[T]( - override val size: T => Int, - override val statsReceiver: StatsReceiver, - override val scopes: Seq[String]) - extends FunctionObserver[T](statsReceiver, scopes) - with ResultsObserver[T] { - - override def apply(f: => T): T = observeResults(super.apply(f)) - } - - /** [[ResultsObserver]] provides methods for recording stats for the result size */ - trait ResultsObserver[T] { - protected val statsReceiver: StatsReceiver - - /** Scopes that prefix all stats */ - protected val scopes: Seq[String] - - protected val totalCounter: Counter = statsReceiver.counter(scopes :+ Total: _*) - protected val foundCounter: Counter = statsReceiver.counter(scopes :+ Found: _*) - protected val notFoundCounter: Counter = statsReceiver.counter(scopes :+ NotFound: _*) - - /** given a [[T]] returns it's size. */ - protected val size: T => Int - - /** Records the size of the `results` using [[size]] and return the original value. */ - protected def observeResults(results: T): T = { - val resultsSize = size(results) - observeResultsWithSize(results, resultsSize) - } - - /** - * Records the `resultsSize` and returns the `results` - * - * This is useful if the size is already available and is expensive to calculate. - */ - protected def observeResultsWithSize(results: T, resultsSize: Int): T = { - if (resultsSize > 0) { - totalCounter.incr(resultsSize) - foundCounter.incr() - } else { - notFoundCounter.incr() - } - results - } - } -} diff --git a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/observer/ResultsStatsObserver.docx b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/observer/ResultsStatsObserver.docx new file mode 100644 index 000000000..12b092bbc Binary files /dev/null and b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/observer/ResultsStatsObserver.docx differ diff --git a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/observer/ResultsStatsObserver.scala b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/observer/ResultsStatsObserver.scala deleted file mode 100644 index a637e23d8..000000000 --- a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/observer/ResultsStatsObserver.scala +++ /dev/null @@ -1,243 +0,0 @@ -package com.twitter.product_mixer.shared_library.observer - -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.shared_library.observer.Observer.ArrowObserver -import com.twitter.product_mixer.shared_library.observer.Observer.FunctionObserver -import com.twitter.product_mixer.shared_library.observer.Observer.FutureObserver -import com.twitter.product_mixer.shared_library.observer.Observer.Observer -import com.twitter.product_mixer.shared_library.observer.Observer.StitchObserver -import com.twitter.product_mixer.shared_library.observer.ResultsObserver.ResultsObserver -import com.twitter.stitch.Arrow -import com.twitter.stitch.Stitch -import com.twitter.util.Future -import com.twitter.util.Try - -/** - * Helper functions to observe requests, successes, failures, cancellations, exceptions, latency, - * and result counts and time-series stats. Supports native functions and asynchronous operations. - * - * Note that since time-series stats are expensive to compute (relative to counters), prefer - * [[ResultsObserver]] unless a time-series stat is needed. - */ -object ResultsStatsObserver { - val Size = "size" - - /** - * Helper function to observe a stitch and result counts and time-series stats - */ - def stitchResultsStats[T]( - size: T => Int, - statsReceiver: StatsReceiver, - scopes: String* - ): StitchResultsStatsObserver[T] = { - new StitchResultsStatsObserver[T](size, statsReceiver, scopes) - } - - /** - * Helper function to observe a stitch and traversable (e.g. Seq, Set) result counts and - * time-series stats - */ - def stitchResultsStats[T <: TraversableOnce[_]]( - statsReceiver: StatsReceiver, - scopes: String* - ): StitchResultsStatsObserver[T] = { - new StitchResultsStatsObserver[T](_.size, statsReceiver, scopes) - } - - /** - * Helper function to observe an arrow and result counts and time-series stats - */ - def arrowResultsStats[T, U]( - size: U => Int, - statsReceiver: StatsReceiver, - scopes: String* - ): ArrowResultsStatsObserver[T, U] = { - new ArrowResultsStatsObserver[T, U](size, statsReceiver, scopes) - } - - /** - * Helper function to observe an arrow and traversable (e.g. Seq, Set) result counts and - * * time-series stats - */ - def arrowResultsStats[T, U <: TraversableOnce[_]]( - statsReceiver: StatsReceiver, - scopes: String* - ): ArrowResultsStatsObserver[T, U] = { - new ArrowResultsStatsObserver[T, U](_.size, statsReceiver, scopes) - } - - /** - * Helper function to observe an arrow and result counts - * - * @see [[TransformingArrowResultsStatsObserver]] - */ - def transformingArrowResultsStats[In, Out, Transformed]( - transformer: Out => Try[Transformed], - size: Transformed => Int, - statsReceiver: StatsReceiver, - scopes: String* - ): TransformingArrowResultsStatsObserver[In, Out, Transformed] = { - new TransformingArrowResultsStatsObserver[In, Out, Transformed]( - transformer, - size, - statsReceiver, - scopes) - } - - /** - * Helper function to observe an arrow and traversable (e.g. Seq, Set) result counts - * - * @see [[TransformingArrowResultsStatsObserver]] - */ - def transformingArrowResultsStats[In, Out, Transformed <: TraversableOnce[_]]( - transformer: Out => Try[Transformed], - statsReceiver: StatsReceiver, - scopes: String* - ): TransformingArrowResultsStatsObserver[In, Out, Transformed] = { - new TransformingArrowResultsStatsObserver[In, Out, Transformed]( - transformer, - _.size, - statsReceiver, - scopes) - } - - /** - * Helper function to observe a future and result counts and time-series stats - */ - def futureResultsStats[T]( - size: T => Int, - statsReceiver: StatsReceiver, - scopes: String* - ): FutureResultsStatsObserver[T] = { - new FutureResultsStatsObserver[T](size, statsReceiver, scopes) - } - - /** - * Helper function to observe a future and traversable (e.g. Seq, Set) result counts and - * time-series stats - */ - def futureResultsStats[T <: TraversableOnce[_]]( - statsReceiver: StatsReceiver, - scopes: String* - ): FutureResultsStatsObserver[T] = { - new FutureResultsStatsObserver[T](_.size, statsReceiver, scopes) - } - - /** - * Helper function observe a function and result counts and time-series stats - */ - def functionResultsStats[T]( - size: T => Int, - statsReceiver: StatsReceiver, - scopes: String* - ): FunctionResultsStatsObserver[T] = { - new FunctionResultsStatsObserver[T](size, statsReceiver, scopes) - } - - /** - * Helper function observe a function and traversable (e.g. Seq, Set) result counts and - * time-series stats - */ - def functionResultsStats[T <: TraversableOnce[_]]( - statsReceiver: StatsReceiver, - scopes: String* - ): FunctionResultsStatsObserver[T] = { - new FunctionResultsStatsObserver[T](_.size, statsReceiver, scopes) - } - - class StitchResultsStatsObserver[T]( - override val size: T => Int, - override val statsReceiver: StatsReceiver, - override val scopes: Seq[String]) - extends StitchObserver[T](statsReceiver, scopes) - with ResultsStatsObserver[T] { - - override def apply(stitch: => Stitch[T]): Stitch[T] = - super - .apply(stitch) - .onSuccess(observeResults) - } - - class ArrowResultsStatsObserver[T, U]( - override val size: U => Int, - override val statsReceiver: StatsReceiver, - override val scopes: Seq[String]) - extends ArrowObserver[T, U](statsReceiver, scopes) - with ResultsStatsObserver[U] { - - override def apply(arrow: Arrow[T, U]): Arrow[T, U] = - super - .apply(arrow) - .onSuccess(observeResults) - } - - /** - * [[TransformingArrowResultsStatsObserver]] functions like an [[ArrowObserver]] except - * that it transforms the result using [[transformer]] before recording stats. - * - * The original non-transformed result is then returned. - */ - class TransformingArrowResultsStatsObserver[In, Out, Transformed]( - val transformer: Out => Try[Transformed], - override val size: Transformed => Int, - override val statsReceiver: StatsReceiver, - override val scopes: Seq[String]) - extends Observer[Transformed] - with ResultsStatsObserver[Transformed] { - - /** - * Returns a new Arrow that records stats on the result after applying [[transformer]] when it's run. - * The original, non-transformed, result of the Arrow is passed through. - * - * @note the provided Arrow must contain the parts that need to be timed. - * Using this on just the result of the computation the latency stat - * will be incorrect. - */ - def apply(arrow: Arrow[In, Out]): Arrow[In, Out] = { - Arrow - .time(arrow) - .map { - case (response, stitchRunDuration) => - observe(response.flatMap(transformer), stitchRunDuration) - .onSuccess(observeResults) - response - }.lowerFromTry - } - } - - class FutureResultsStatsObserver[T]( - override val size: T => Int, - override val statsReceiver: StatsReceiver, - override val scopes: Seq[String]) - extends FutureObserver[T](statsReceiver, scopes) - with ResultsStatsObserver[T] { - - override def apply(future: => Future[T]): Future[T] = - super - .apply(future) - .onSuccess(observeResults) - } - - class FunctionResultsStatsObserver[T]( - override val size: T => Int, - override val statsReceiver: StatsReceiver, - override val scopes: Seq[String]) - extends FunctionObserver[T](statsReceiver, scopes) - with ResultsStatsObserver[T] { - - override def apply(f: => T): T = { - observeResults(super.apply(f)) - } - } - - trait ResultsStatsObserver[T] extends ResultsObserver[T] { - private val sizeStat: Stat = statsReceiver.stat(scopes :+ Size: _*) - - protected override def observeResults(results: T): T = { - val resultsSize = size(results) - sizeStat.add(resultsSize) - observeResultsWithSize(results, resultsSize) - } - } -} diff --git a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/thrift_client/BUILD b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/thrift_client/BUILD deleted file mode 100644 index 8569f2d61..000000000 --- a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/thrift_client/BUILD +++ /dev/null @@ -1,20 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "finagle/finagle-core/src/main", - "finagle/finagle-thriftmux/src/main/scala", - "finatra-internal/mtls-http/src/main/scala", - "finatra-internal/mtls-thriftmux/src/main/scala", - "util/util-core", - ], - exports = [ - "finagle/finagle-core/src/main", - "finagle/finagle-thriftmux/src/main/scala", - "finatra-internal/mtls-http/src/main/scala", - "finatra-internal/mtls-thriftmux/src/main/scala", - "util/util-core", - ], -) diff --git a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/thrift_client/BUILD.docx b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/thrift_client/BUILD.docx new file mode 100644 index 000000000..9ea04b1d7 Binary files /dev/null and b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/thrift_client/BUILD.docx differ diff --git a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/thrift_client/FinagleThriftClientBuilder.docx b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/thrift_client/FinagleThriftClientBuilder.docx new file mode 100644 index 000000000..17bdf702c Binary files /dev/null and b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/thrift_client/FinagleThriftClientBuilder.docx differ diff --git a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/thrift_client/FinagleThriftClientBuilder.scala b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/thrift_client/FinagleThriftClientBuilder.scala deleted file mode 100644 index b8c25b5a1..000000000 --- a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/thrift_client/FinagleThriftClientBuilder.scala +++ /dev/null @@ -1,198 +0,0 @@ -package com.twitter.product_mixer.shared_library.thrift_client - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.ThriftMux -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.mtls.client.MtlsStackClient._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.finagle.thrift.service.Filterable -import com.twitter.finagle.thrift.service.MethodPerEndpointBuilder -import com.twitter.finagle.thrift.service.ServicePerEndpointBuilder -import com.twitter.finagle.thriftmux.MethodBuilder -import com.twitter.util.Duration -import org.apache.thrift.protocol.TProtocolFactory - -sealed trait Idempotency -case object NonIdempotent extends Idempotency -case class Idempotent(maxExtraLoadPercent: Double) extends Idempotency - -object FinagleThriftClientBuilder { - - /** - * Library to build a Finagle Thrift method per endpoint client is a less error-prone way when - * compared to the builders in Finagle. This is achieved by requiring values for fields that should - * always be set in practice. For example, request timeouts in Finagle are unbounded when not - * explicitly set, and this method requires that timeout durations are passed into the method and - * set on the Finagle builder. - * - * Usage of - * [[com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule]] is almost always preferred, - * and the Product Mixer component library [[com.twitter.product_mixer.component_library.module]] - * package contains numerous examples. However, if multiple versions of a client are needed e.g. - * for different timeout settings, this method is useful to easily provide multiple variants. - * - * @example - * {{{ - * final val SampleServiceClientName = "SampleServiceClient" - * @Provides - * @Singleton - * @Named(SampleServiceClientName) - * def provideSampleServiceClient( - * serviceIdentifier: ServiceIdentifier, - * clientId: ClientId, - * statsReceiver: StatsReceiver, - * ): SampleService.MethodPerEndpoint = - * buildFinagleMethodPerEndpoint[SampleService.ServicePerEndpoint, SampleService.MethodPerEndpoint]( - * serviceIdentifier = serviceIdentifier, - * clientId = clientId, - * dest = "/s/sample/sample", - * label = "sample", - * statsReceiver = statsReceiver, - * idempotency = Idempotent(5.percent), - * timeoutPerRequest = 200.milliseconds, - * timeoutTotal = 400.milliseconds - * ) - * }}} - * @param serviceIdentifier Service ID used to S2S Auth - * @param clientId Client ID - * @param dest Destination as a Wily path e.g. "/s/sample/sample" - * @param label Label of the client - * @param statsReceiver Stats - * @param idempotency Idempotency semantics of the client - * @param timeoutPerRequest Thrift client timeout per request. The Finagle default is - * unbounded which is almost never optimal. - * @param timeoutTotal Thrift client total timeout. The Finagle default is - * unbounded which is almost never optimal. - * If the client is set as idempotent, which adds a - * [[com.twitter.finagle.client.BackupRequestFilter]], - * be sure to leave enough room for the backup request. A - * reasonable (albeit usually too large) starting point is to - * make the total timeout 2x relative to the per request timeout. - * If the client is set as non-idempotent, the total timeout and - * the per request timeout should be the same, as there will be - * no backup requests. - * @param connectTimeout Thrift client transport connect timeout. The Finagle default - * of one second is reasonable but we lower this to match - * acquisitionTimeout for consistency. - * @param acquisitionTimeout Thrift client session acquisition timeout. The Finagle default - * is unbounded which is almost never optimal. - * @param protocolFactoryOverride Override the default protocol factory - * e.g. [[org.apache.thrift.protocol.TCompactProtocol.Factory]] - * @param servicePerEndpointBuilder implicit service per endpoint builder - * @param methodPerEndpointBuilder implicit method per endpoint builder - * - * @see [[https://twitter.github.io/finagle/guide/MethodBuilder.html user guide]] - * @see [[https://twitter.github.io/finagle/guide/MethodBuilder.html#idempotency user guide]] - * @return method per endpoint Finagle Thrift Client - */ - def buildFinagleMethodPerEndpoint[ - ServicePerEndpoint <: Filterable[ServicePerEndpoint], - MethodPerEndpoint - ]( - serviceIdentifier: ServiceIdentifier, - clientId: ClientId, - dest: String, - label: String, - statsReceiver: StatsReceiver, - idempotency: Idempotency, - timeoutPerRequest: Duration, - timeoutTotal: Duration, - connectTimeout: Duration = 500.milliseconds, - acquisitionTimeout: Duration = 500.milliseconds, - protocolFactoryOverride: Option[TProtocolFactory] = None, - )( - implicit servicePerEndpointBuilder: ServicePerEndpointBuilder[ServicePerEndpoint], - methodPerEndpointBuilder: MethodPerEndpointBuilder[ServicePerEndpoint, MethodPerEndpoint] - ): MethodPerEndpoint = { - val service: ServicePerEndpoint = buildFinagleServicePerEndpoint( - serviceIdentifier = serviceIdentifier, - clientId = clientId, - dest = dest, - label = label, - statsReceiver = statsReceiver, - idempotency = idempotency, - timeoutPerRequest = timeoutPerRequest, - timeoutTotal = timeoutTotal, - connectTimeout = connectTimeout, - acquisitionTimeout = acquisitionTimeout, - protocolFactoryOverride = protocolFactoryOverride - ) - - ThriftMux.Client.methodPerEndpoint(service) - } - - /** - * Build a Finagle Thrift service per endpoint client. - * - * @note [[buildFinagleMethodPerEndpoint]] should be preferred over the service per endpoint variant - * - * @param serviceIdentifier Service ID used to S2S Auth - * @param clientId Client ID - * @param dest Destination as a Wily path e.g. "/s/sample/sample" - * @param label Label of the client - * @param statsReceiver Stats - * @param idempotency Idempotency semantics of the client - * @param timeoutPerRequest Thrift client timeout per request. The Finagle default is - * unbounded which is almost never optimal. - * @param timeoutTotal Thrift client total timeout. The Finagle default is - * unbounded which is almost never optimal. - * If the client is set as idempotent, which adds a - * [[com.twitter.finagle.client.BackupRequestFilter]], - * be sure to leave enough room for the backup request. A - * reasonable (albeit usually too large) starting point is to - * make the total timeout 2x relative to the per request timeout. - * If the client is set as non-idempotent, the total timeout and - * the per request timeout should be the same, as there will be - * no backup requests. - * @param connectTimeout Thrift client transport connect timeout. The Finagle default - * of one second is reasonable but we lower this to match - * acquisitionTimeout for consistency. - * @param acquisitionTimeout Thrift client session acquisition timeout. The Finagle default - * is unbounded which is almost never optimal. - * @param protocolFactoryOverride Override the default protocol factory - * e.g. [[org.apache.thrift.protocol.TCompactProtocol.Factory]] - * - * @return service per endpoint Finagle Thrift Client - */ - def buildFinagleServicePerEndpoint[ServicePerEndpoint <: Filterable[ServicePerEndpoint]]( - serviceIdentifier: ServiceIdentifier, - clientId: ClientId, - dest: String, - label: String, - statsReceiver: StatsReceiver, - idempotency: Idempotency, - timeoutPerRequest: Duration, - timeoutTotal: Duration, - connectTimeout: Duration = 500.milliseconds, - acquisitionTimeout: Duration = 500.milliseconds, - protocolFactoryOverride: Option[TProtocolFactory] = None, - )( - implicit servicePerEndpointBuilder: ServicePerEndpointBuilder[ServicePerEndpoint] - ): ServicePerEndpoint = { - val thriftMux: ThriftMux.Client = ThriftMux.client - .withMutualTls(serviceIdentifier) - .withClientId(clientId) - .withLabel(label) - .withStatsReceiver(statsReceiver) - .withTransport.connectTimeout(connectTimeout) - .withSession.acquisitionTimeout(acquisitionTimeout) - - val protocolThriftMux: ThriftMux.Client = protocolFactoryOverride - .map { protocolFactory => - thriftMux.withProtocolFactory(protocolFactory) - }.getOrElse(thriftMux) - - val methodBuilder: MethodBuilder = protocolThriftMux - .methodBuilder(dest) - .withTimeoutPerRequest(timeoutPerRequest) - .withTimeoutTotal(timeoutTotal) - - val idempotencyMethodBuilder: MethodBuilder = idempotency match { - case NonIdempotent => methodBuilder.nonIdempotent - case Idempotent(maxExtraLoad) => methodBuilder.idempotent(maxExtraLoad = maxExtraLoad) - } - - idempotencyMethodBuilder.servicePerEndpoint[ServicePerEndpoint] - } -} diff --git a/pushservice/BUILD.bazel b/pushservice/BUILD.bazel deleted file mode 100644 index 12efdb2e6..000000000 --- a/pushservice/BUILD.bazel +++ /dev/null @@ -1,48 +0,0 @@ -alias( - name = "frigate-pushservice", - target = ":frigate-pushservice_lib", -) - -target( - name = "frigate-pushservice_lib", - dependencies = [ - "frigate/frigate-pushservice-opensource/src/main/scala/com/twitter/frigate/pushservice", - ], -) - -jvm_binary( - name = "bin", - basename = "frigate-pushservice", - main = "com.twitter.frigate.pushservice.PushServiceMain", - runtime_platform = "java11", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/ch/qos/logback:logback-classic", - "finatra/inject/inject-logback/src/main/scala", - "frigate/frigate-pushservice-opensource/src/main/scala/com/twitter/frigate/pushservice", - "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", - "twitter-server/logback-classic/src/main/scala", - ], - excludes = [ - exclude("com.twitter.translations", "translations-twitter"), - exclude("org.apache.hadoop", "hadoop-aws"), - exclude("org.tensorflow"), - scala_exclude("com.twitter", "ckoia-scala"), - ], -) - -jvm_app( - name = "bundle", - basename = "frigate-pushservice-package-dist", - archive = "zip", - binary = ":bin", - tags = ["bazel-compatible"], -) - -python3_library( - name = "mr_model_constants", - sources = [ - "config/deepbird/constants.py", - ], - tags = ["bazel-compatible"], -) diff --git a/pushservice/BUILD.docx b/pushservice/BUILD.docx new file mode 100644 index 000000000..f573ffe10 Binary files /dev/null and b/pushservice/BUILD.docx differ diff --git a/pushservice/README.docx b/pushservice/README.docx new file mode 100644 index 000000000..e65fa2f44 Binary files /dev/null and b/pushservice/README.docx differ diff --git a/pushservice/README.md b/pushservice/README.md deleted file mode 100644 index b1bad0a57..000000000 --- a/pushservice/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# Pushservice - -Pushservice is the main push recommendation service at Twitter used to generate recommendation-based notifications for users. It currently powers two functionalities: - -- RefreshForPushHandler: This handler determines whether to send a recommendation push to a user based on their ID. It generates the best push recommendation item and coordinates with downstream services to deliver it -- SendHandler: This handler determines and manage whether send the push to users based on the given target user details and the provided push recommendation item - -## Overview - -### RefreshForPushHandler - -RefreshForPushHandler follows these steps: - -- Building Target and checking eligibility - - Builds a target user object based on the given user ID - - Performs target-level filterings to determine if the target is eligible for a recommendation push -- Fetch Candidates - - Retrieves a list of potential candidates for the push by querying various candidate sources using the target -- Candidate Hydration - - Hydrates the candidate details with batch calls to different downstream services -- Pre-rank Filtering, also called Light Filtering - - Filters the hydrated candidates with lightweight RPC calls -- Rank - - Perform feature hydration for candidates and target user - - Performs light ranking on candidates - - Performs heavy ranking on candidates -- Take Step, also called Heavy Filtering - - Takes the top-ranked candidates one by one and applies heavy filtering until one candidate passes all filter steps -- Send - - Calls the appropriate downstream service to deliver the eligible candidate as a push and in-app notification to the target user - -### SendHandler - -SendHandler follows these steps: - -- Building Target - - Builds a target user object based on the given user ID -- Candidate Hydration - - Hydrates the candidate details with batch calls to different downstream services -- Feature Hydration - - Perform feature hydration for candidates and target user -- Take Step, also called Heavy Filtering - - Perform filterings and validation checking for the given candidate -- Send - - Calls the appropriate downstream service to deliver the given candidate as a push and/or in-app notification to the target user \ No newline at end of file diff --git a/pushservice/src/main/python/models/heavy_ranking/BUILD b/pushservice/src/main/python/models/heavy_ranking/BUILD deleted file mode 100644 index 2c25693a9..000000000 --- a/pushservice/src/main/python/models/heavy_ranking/BUILD +++ /dev/null @@ -1,169 +0,0 @@ -python37_binary( - name = "update_warm_start_checkpoint", - source = "update_warm_start_checkpoint.py", - tags = ["no-mypy"], - dependencies = [ - ":deep_norm_lib", - "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/heavy_ranking:update_warm_start_checkpoint", - ], -) - -python3_library( - name = "params_lib", - sources = ["params.py"], - tags = ["no-mypy"], - dependencies = [ - "3rdparty/python/pydantic:default", - "src/python/twitter/deepbird/projects/magic_recs/v11/lib:params_lib", - ], -) - -python3_library( - name = "features_lib", - sources = ["features.py"], - tags = ["no-mypy"], - dependencies = [ - ":params_lib", - "src/python/twitter/deepbird/projects/magic_recs/libs", - "twml:twml-nodeps", - ], -) - -python3_library( - name = "model_pools_lib", - sources = ["model_pools.py"], - tags = ["no-mypy"], - dependencies = [ - ":features_lib", - ":params_lib", - "src/python/twitter/deepbird/projects/magic_recs/v11/lib:model_lib", - ], -) - -python3_library( - name = "graph_lib", - sources = ["graph.py"], - tags = ["no-mypy"], - dependencies = [ - ":params_lib", - "src/python/twitter/deepbird/projects/magic_recs/libs", - ], -) - -python3_library( - name = "run_args_lib", - sources = ["run_args.py"], - tags = ["no-mypy"], - dependencies = [ - ":features_lib", - ":params_lib", - "twml:twml-nodeps", - ], -) - -python3_library( - name = "deep_norm_lib", - sources = ["deep_norm.py"], - tags = ["no-mypy"], - dependencies = [ - ":features_lib", - ":graph_lib", - ":model_pools_lib", - ":params_lib", - ":run_args_lib", - "src/python/twitter/deepbird/projects/magic_recs/libs", - "src/python/twitter/deepbird/util/data", - "twml:twml-nodeps", - ], -) - -python3_library( - name = "eval_lib", - sources = ["eval.py"], - tags = ["no-mypy"], - dependencies = [ - ":features_lib", - ":graph_lib", - ":model_pools_lib", - ":params_lib", - ":run_args_lib", - "src/python/twitter/deepbird/projects/magic_recs/libs", - "twml:twml-nodeps", - ], -) - -python37_binary( - name = "deep_norm", - source = "deep_norm.py", - dependencies = [ - ":deep_norm_lib", - "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/heavy_ranking:deep_norm", - "twml", - ], -) - -python37_binary( - name = "eval", - source = "eval.py", - dependencies = [ - ":eval_lib", - "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/heavy_ranking:eval", - "twml", - ], -) - -python3_library( - name = "mlwf_libs", - tags = ["no-mypy"], - dependencies = [ - ":deep_norm_lib", - "twml", - ], -) - -python37_binary( - name = "train_model", - source = "deep_norm.py", - dependencies = [ - ":deep_norm_lib", - "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/heavy_ranking:train_model", - ], -) - -python37_binary( - name = "train_model_local", - source = "deep_norm.py", - dependencies = [ - ":deep_norm_lib", - "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/heavy_ranking:train_model_local", - "twml", - ], -) - -python37_binary( - name = "eval_model_local", - source = "eval.py", - dependencies = [ - ":eval_lib", - "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/heavy_ranking:eval_model_local", - "twml", - ], -) - -python37_binary( - name = "eval_model", - source = "eval.py", - dependencies = [ - ":eval_lib", - "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/heavy_ranking:eval_model", - ], -) - -python37_binary( - name = "mlwf_model", - source = "deep_norm.py", - dependencies = [ - ":mlwf_libs", - "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/heavy_ranking:mlwf_model", - ], -) diff --git a/pushservice/src/main/python/models/heavy_ranking/BUILD.docx b/pushservice/src/main/python/models/heavy_ranking/BUILD.docx new file mode 100644 index 000000000..7f392fe05 Binary files /dev/null and b/pushservice/src/main/python/models/heavy_ranking/BUILD.docx differ diff --git a/pushservice/src/main/python/models/heavy_ranking/README.docx b/pushservice/src/main/python/models/heavy_ranking/README.docx new file mode 100644 index 000000000..b68f99a32 Binary files /dev/null and b/pushservice/src/main/python/models/heavy_ranking/README.docx differ diff --git a/pushservice/src/main/python/models/heavy_ranking/README.md b/pushservice/src/main/python/models/heavy_ranking/README.md deleted file mode 100644 index 75336a09c..000000000 --- a/pushservice/src/main/python/models/heavy_ranking/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# Notification Heavy Ranker Model - -## Model Context -There are 4 major components of Twitter notifications recommendation system: 1) candidate generation 2) light ranking 3) heavy ranking & 4) quality control. This notification heavy ranker model is the core ranking model for the personalised notifications recommendation. It's a multi-task learning model to predict the probabilities that the target users will open and engage with the sent notifications. - - -## Directory Structure -- BUILD: this file defines python library dependencies -- deep_norm.py: this file contains how to set up continuous training, model evaluation and model exporting for the notification heavy ranker model -- eval.py: the main python entry file to set up the overall model evaluation pipeline -- features.py: this file contains importing feature list and support functions for feature engineering -- graph.py: this file defines how to build the tensorflow graph with specified model architecture, loss function and training configuration -- model_pools.py: this file defines the available model types for the heavy ranker -- params.py: this file defines hyper-parameters used in the notification heavy ranker -- run_args.py: this file defines command line parameters to run model training & evaluation -- update_warm_start_checkpoint.py: this file contains the support to modify checkpoints of the given saved heavy ranker model -- lib/BUILD: this file defines python library dependencies for tensorflow model architecture -- lib/layers.py: this file defines different type of convolution layers to be used in the heavy ranker model -- lib/model.py: this file defines the module containing ClemNet, the heavy ranker model type -- lib/params.py: this file defines parameters used in the heavy ranker model diff --git a/pushservice/src/main/python/models/heavy_ranking/__init__.docx b/pushservice/src/main/python/models/heavy_ranking/__init__.docx new file mode 100644 index 000000000..539f73b98 Binary files /dev/null and b/pushservice/src/main/python/models/heavy_ranking/__init__.docx differ diff --git a/pushservice/src/main/python/models/heavy_ranking/__init__.py b/pushservice/src/main/python/models/heavy_ranking/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/pushservice/src/main/python/models/heavy_ranking/deep_norm.docx b/pushservice/src/main/python/models/heavy_ranking/deep_norm.docx new file mode 100644 index 000000000..e219fcc85 Binary files /dev/null and b/pushservice/src/main/python/models/heavy_ranking/deep_norm.docx differ diff --git a/pushservice/src/main/python/models/heavy_ranking/deep_norm.py b/pushservice/src/main/python/models/heavy_ranking/deep_norm.py deleted file mode 100644 index 7db281b4a..000000000 --- a/pushservice/src/main/python/models/heavy_ranking/deep_norm.py +++ /dev/null @@ -1,136 +0,0 @@ -""" -Training job for the heavy ranker of the push notification service. -""" -from datetime import datetime -import json -import os - -import twml - -from ..libs.metric_fn_utils import flip_disliked_labels, get_metric_fn -from ..libs.model_utils import read_config -from ..libs.warm_start_utils import get_feature_list_for_heavy_ranking, warm_start_checkpoint -from .features import get_feature_config -from .model_pools import ALL_MODELS -from .params import load_graph_params -from .run_args import get_training_arg_parser - -import tensorflow.compat.v1 as tf -from tensorflow.compat.v1 import logging - - -def main() -> None: - args, _ = get_training_arg_parser().parse_known_args() - logging.info(f"Parsed args: {args}") - - params = load_graph_params(args) - logging.info(f"Loaded graph params: {params}") - - param_file = os.path.join(args.save_dir, "params.json") - logging.info(f"Saving graph params to: {param_file}") - with tf.io.gfile.GFile(param_file, mode="w") as file: - json.dump(params.json(), file, ensure_ascii=False, indent=4) - - logging.info(f"Get Feature Config: {args.feature_list}") - feature_list = read_config(args.feature_list).items() - feature_config = get_feature_config( - data_spec_path=args.data_spec, - params=params, - feature_list_provided=feature_list, - ) - feature_list_path = args.feature_list - - warm_start_from = args.warm_start_from - if args.warm_start_base_dir: - logging.info(f"Get warm started model from: {args.warm_start_base_dir}.") - - continuous_binary_feat_list_save_path = os.path.join( - args.warm_start_base_dir, "continuous_binary_feat_list.json" - ) - warm_start_folder = os.path.join(args.warm_start_base_dir, "best_checkpoint") - job_name = os.path.basename(args.save_dir) - ws_output_ckpt_folder = os.path.join(args.warm_start_base_dir, f"warm_start_for_{job_name}") - if tf.io.gfile.exists(ws_output_ckpt_folder): - tf.io.gfile.rmtree(ws_output_ckpt_folder) - - tf.io.gfile.mkdir(ws_output_ckpt_folder) - - warm_start_from = warm_start_checkpoint( - warm_start_folder, - continuous_binary_feat_list_save_path, - feature_list_path, - args.data_spec, - ws_output_ckpt_folder, - ) - logging.info(f"Created warm_start_from_ckpt {warm_start_from}.") - - logging.info("Build Trainer.") - metric_fn = get_metric_fn("OONC_Engagement" if len(params.tasks) == 2 else "OONC", False) - - trainer = twml.trainers.DataRecordTrainer( - name="magic_recs", - params=args, - build_graph_fn=lambda *args: ALL_MODELS[params.model.name](params=params)(*args), - save_dir=args.save_dir, - run_config=None, - feature_config=feature_config, - metric_fn=flip_disliked_labels(metric_fn), - warm_start_from=warm_start_from, - ) - - logging.info("Build train and eval input functions.") - train_input_fn = trainer.get_train_input_fn(shuffle=True) - eval_input_fn = trainer.get_eval_input_fn(repeat=False, shuffle=False) - - learn = trainer.learn - if args.distributed or args.num_workers is not None: - learn = trainer.train_and_evaluate - - if not args.directly_export_best: - logging.info("Starting training") - start = datetime.now() - learn( - early_stop_minimize=False, - early_stop_metric="pr_auc_unweighted_OONC", - early_stop_patience=args.early_stop_patience, - early_stop_tolerance=args.early_stop_tolerance, - eval_input_fn=eval_input_fn, - train_input_fn=train_input_fn, - ) - logging.info(f"Total training time: {datetime.now() - start}") - else: - logging.info("Directly exporting the model") - - if not args.export_dir: - args.export_dir = os.path.join(args.save_dir, "exported_models") - - logging.info(f"Exporting the model to {args.export_dir}.") - start = datetime.now() - twml.contrib.export.export_fn.export_all_models( - trainer=trainer, - export_dir=args.export_dir, - parse_fn=feature_config.get_parse_fn(), - serving_input_receiver_fn=feature_config.get_serving_input_receiver_fn(), - export_output_fn=twml.export_output_fns.batch_prediction_continuous_output_fn, - ) - - logging.info(f"Total model export time: {datetime.now() - start}") - logging.info(f"The MLP directory is: {args.save_dir}") - - continuous_binary_feat_list_save_path = os.path.join( - args.save_dir, "continuous_binary_feat_list.json" - ) - logging.info( - f"Saving the list of continuous and binary features to {continuous_binary_feat_list_save_path}." - ) - continuous_binary_feat_list = get_feature_list_for_heavy_ranking( - feature_list_path, args.data_spec - ) - twml.util.write_file( - continuous_binary_feat_list_save_path, continuous_binary_feat_list, encode="json" - ) - - -if __name__ == "__main__": - main() - logging.info("Done.") diff --git a/pushservice/src/main/python/models/heavy_ranking/eval.docx b/pushservice/src/main/python/models/heavy_ranking/eval.docx new file mode 100644 index 000000000..8b383c6f3 Binary files /dev/null and b/pushservice/src/main/python/models/heavy_ranking/eval.docx differ diff --git a/pushservice/src/main/python/models/heavy_ranking/eval.py b/pushservice/src/main/python/models/heavy_ranking/eval.py deleted file mode 100644 index 7f74472fb..000000000 --- a/pushservice/src/main/python/models/heavy_ranking/eval.py +++ /dev/null @@ -1,59 +0,0 @@ -""" -Evaluation job for the heavy ranker of the push notification service. -""" - -from datetime import datetime - -import twml - -from ..libs.metric_fn_utils import get_metric_fn -from ..libs.model_utils import read_config -from .features import get_feature_config -from .model_pools import ALL_MODELS -from .params import load_graph_params -from .run_args import get_eval_arg_parser - -from tensorflow.compat.v1 import logging - - -def main(): - args, _ = get_eval_arg_parser().parse_known_args() - logging.info(f"Parsed args: {args}") - - params = load_graph_params(args) - logging.info(f"Loaded graph params: {params}") - - logging.info(f"Get Feature Config: {args.feature_list}") - feature_list = read_config(args.feature_list).items() - feature_config = get_feature_config( - data_spec_path=args.data_spec, - params=params, - feature_list_provided=feature_list, - ) - - logging.info("Build DataRecordTrainer.") - metric_fn = get_metric_fn("OONC_Engagement" if len(params.tasks) == 2 else "OONC", False) - - trainer = twml.trainers.DataRecordTrainer( - name="magic_recs", - params=args, - build_graph_fn=lambda *args: ALL_MODELS[params.model.name](params=params)(*args), - save_dir=args.save_dir, - run_config=None, - feature_config=feature_config, - metric_fn=metric_fn, - ) - - logging.info("Run the evaluation.") - start = datetime.now() - trainer._estimator.evaluate( - input_fn=trainer.get_eval_input_fn(repeat=False, shuffle=False), - steps=None if (args.eval_steps is not None and args.eval_steps < 0) else args.eval_steps, - checkpoint_path=args.eval_checkpoint, - ) - logging.info(f"Evaluating time: {datetime.now() - start}.") - - -if __name__ == "__main__": - main() - logging.info("Job done.") diff --git a/pushservice/src/main/python/models/heavy_ranking/features.docx b/pushservice/src/main/python/models/heavy_ranking/features.docx new file mode 100644 index 000000000..211b698e1 Binary files /dev/null and b/pushservice/src/main/python/models/heavy_ranking/features.docx differ diff --git a/pushservice/src/main/python/models/heavy_ranking/features.py b/pushservice/src/main/python/models/heavy_ranking/features.py deleted file mode 100644 index ce6a2686a..000000000 --- a/pushservice/src/main/python/models/heavy_ranking/features.py +++ /dev/null @@ -1,138 +0,0 @@ -import os -from typing import Dict - -from twitter.deepbird.projects.magic_recs.libs.model_utils import filter_nans_and_infs -import twml -from twml.layers import full_sparse, sparse_max_norm - -from .params import FeaturesParams, GraphParams, SparseFeaturesParams - -import tensorflow as tf -from tensorflow import Tensor -import tensorflow.compat.v1 as tf1 - - -FEAT_CONFIG_DEFAULT_VAL = 0 -DEFAULT_FEATURE_LIST_PATH = "./feature_list_default.yaml" -FEATURE_LIST_DEFAULT_PATH = os.path.join( - os.path.dirname(os.path.realpath(__file__)), DEFAULT_FEATURE_LIST_PATH -) - - -def get_feature_config(data_spec_path=None, feature_list_provided=[], params: GraphParams = None): - - a_string_feat_list = [feat for feat, feat_type in feature_list_provided if feat_type != "S"] - - builder = twml.contrib.feature_config.FeatureConfigBuilder( - data_spec_path=data_spec_path, debug=False - ) - - builder = builder.extract_feature_group( - feature_regexes=a_string_feat_list, - group_name="continuous_features", - default_value=FEAT_CONFIG_DEFAULT_VAL, - type_filter=["CONTINUOUS"], - ) - - builder = builder.extract_feature_group( - feature_regexes=a_string_feat_list, - group_name="binary_features", - type_filter=["BINARY"], - ) - - if params.model.features.sparse_features: - builder = builder.extract_features_as_hashed_sparse( - feature_regexes=a_string_feat_list, - hash_space_size_bits=params.model.features.sparse_features.bits, - type_filter=["DISCRETE", "STRING", "SPARSE_BINARY"], - output_tensor_name="sparse_not_continuous", - ) - - builder = builder.extract_features_as_hashed_sparse( - feature_regexes=[feat for feat, feat_type in feature_list_provided if feat_type == "S"], - hash_space_size_bits=params.model.features.sparse_features.bits, - type_filter=["SPARSE_CONTINUOUS"], - output_tensor_name="sparse_continuous", - ) - - builder = builder.add_labels([task.label for task in params.tasks] + ["label.ntabDislike"]) - - if params.weight: - builder = builder.define_weight(params.weight) - - return builder.build() - - -def dense_features(features: Dict[str, Tensor], training: bool) -> Tensor: - """ - Performs feature transformations on the raw dense features (continuous and binary). - """ - with tf.name_scope("dense_features"): - x = filter_nans_and_infs(features["continuous_features"]) - - x = tf.sign(x) * tf.math.log(tf.abs(x) + 1) - x = tf1.layers.batch_normalization( - x, momentum=0.9999, training=training, renorm=training, axis=1 - ) - x = tf.clip_by_value(x, -5, 5) - - transformed_continous_features = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x) - - binary_features = filter_nans_and_infs(features["binary_features"]) - binary_features = tf.dtypes.cast(binary_features, tf.float32) - - output = tf.concat([transformed_continous_features, binary_features], axis=1) - - return output - - -def sparse_features( - features: Dict[str, Tensor], training: bool, params: SparseFeaturesParams -) -> Tensor: - """ - Performs feature transformations on the raw sparse features. - """ - - with tf.name_scope("sparse_features"): - with tf.name_scope("sparse_not_continuous"): - sparse_not_continuous = full_sparse( - inputs=features["sparse_not_continuous"], - output_size=params.embedding_size, - use_sparse_grads=training, - use_binary_values=False, - ) - - with tf.name_scope("sparse_continuous"): - shape_enforced_input = twml.util.limit_sparse_tensor_size( - sparse_tf=features["sparse_continuous"], input_size_bits=params.bits, mask_indices=False - ) - - normalized_continuous_sparse = sparse_max_norm( - inputs=shape_enforced_input, is_training=training - ) - - sparse_continuous = full_sparse( - inputs=normalized_continuous_sparse, - output_size=params.embedding_size, - use_sparse_grads=training, - use_binary_values=False, - ) - - output = tf.concat([sparse_not_continuous, sparse_continuous], axis=1) - - return output - - -def get_features(features: Dict[str, Tensor], training: bool, params: FeaturesParams) -> Tensor: - """ - Performs feature transformations on the dense and sparse features and combine the resulting - tensors into a single one. - """ - with tf.name_scope("features"): - x = dense_features(features, training) - tf1.logging.info(f"Dense features: {x.shape}") - - if params.sparse_features: - x = tf.concat([x, sparse_features(features, training, params.sparse_features)], axis=1) - - return x diff --git a/pushservice/src/main/python/models/heavy_ranking/graph.docx b/pushservice/src/main/python/models/heavy_ranking/graph.docx new file mode 100644 index 000000000..6005f888b Binary files /dev/null and b/pushservice/src/main/python/models/heavy_ranking/graph.docx differ diff --git a/pushservice/src/main/python/models/heavy_ranking/graph.py b/pushservice/src/main/python/models/heavy_ranking/graph.py deleted file mode 100644 index 4188736ac..000000000 --- a/pushservice/src/main/python/models/heavy_ranking/graph.py +++ /dev/null @@ -1,129 +0,0 @@ -""" -Graph class defining methods to obtain key quantities such as: - * the logits - * the probabilities - * the final score - * the loss function - * the training operator -""" -from __future__ import annotations - -from abc import ABC, abstractmethod -from typing import Any, Dict - -from twitter.deepbird.hparam import HParams -import twml - -from ..libs.model_utils import generate_disliked_mask -from .params import GraphParams - -import tensorflow as tf -import tensorflow.compat.v1 as tf1 - - -class Graph(ABC): - def __init__(self, params: GraphParams): - self.params = params - - @abstractmethod - def get_logits(self, features: Dict[str, tf.Tensor], mode: tf.estimator.ModeKeys) -> tf.Tensor: - pass - - def get_probabilities(self, logits: tf.Tensor) -> tf.Tensor: - return tf.math.cumprod(tf.nn.sigmoid(logits), axis=1, name="probabilities") - - def get_task_weights(self, labels: tf.Tensor) -> tf.Tensor: - oonc_label = tf.reshape(labels[:, 0], shape=(-1, 1)) - task_weights = tf.concat([tf.ones_like(oonc_label), oonc_label], axis=1) - - n_labels = len(self.params.tasks) - task_weights = tf.reshape(task_weights[:, 0:n_labels], shape=(-1, n_labels)) - - return task_weights - - def get_loss(self, labels: tf.Tensor, logits: tf.Tensor, **kwargs: Any) -> tf.Tensor: - with tf.name_scope("weights"): - disliked_mask = generate_disliked_mask(labels) - - labels = tf.reshape(labels[:, 0:2], shape=[-1, 2]) - - labels = labels * tf.cast(tf.logical_not(disliked_mask), dtype=labels.dtype) - - with tf.name_scope("task_weight"): - task_weights = self.get_task_weights(labels) - - with tf.name_scope("batch_size"): - batch_size = tf.cast(tf.shape(labels)[0], dtype=tf.float32, name="batch_size") - - weights = task_weights / batch_size - - with tf.name_scope("loss"): - loss = tf.reduce_sum( - tf.nn.sigmoid_cross_entropy_with_logits(labels=labels, logits=logits) * weights, - ) - - return loss - - def get_score(self, probabilities: tf.Tensor) -> tf.Tensor: - with tf.name_scope("score_weight"): - score_weights = tf.constant([task.score_weight for task in self.params.tasks]) - score_weights = score_weights / tf.reduce_sum(score_weights, axis=0) - - with tf.name_scope("score"): - score = tf.reshape(tf.reduce_sum(probabilities * score_weights, axis=1), shape=[-1, 1]) - - return score - - def get_train_op(self, loss: tf.Tensor, twml_params) -> Any: - with tf.name_scope("optimizer"): - learning_rate = twml_params.learning_rate - optimizer = tf1.train.GradientDescentOptimizer(learning_rate=learning_rate) - - update_ops = set(tf1.get_collection(tf1.GraphKeys.UPDATE_OPS)) - with tf.control_dependencies(update_ops): - train_op = twml.optimizers.optimize_loss( - loss=loss, - variables=tf1.trainable_variables(), - global_step=tf1.train.get_global_step(), - optimizer=optimizer, - learning_rate=None, - ) - - return train_op - - def __call__( - self, - features: Dict[str, tf.Tensor], - labels: tf.Tensor, - mode: tf.estimator.ModeKeys, - params: HParams, - config=None, - ) -> Dict[str, tf.Tensor]: - training = mode == tf.estimator.ModeKeys.TRAIN - logits = self.get_logits(features=features, training=training) - probabilities = self.get_probabilities(logits=logits) - score = None - loss = None - train_op = None - - if mode == tf.estimator.ModeKeys.PREDICT: - score = self.get_score(probabilities=probabilities) - output = {"loss": loss, "train_op": train_op, "prediction": score} - - elif mode in (tf.estimator.ModeKeys.TRAIN, tf.estimator.ModeKeys.EVAL): - loss = self.get_loss(labels=labels, logits=logits) - - if mode == tf.estimator.ModeKeys.TRAIN: - train_op = self.get_train_op(loss=loss, twml_params=params) - - output = {"loss": loss, "train_op": train_op, "output": probabilities} - - else: - raise ValueError( - f""" - Invalid mode. Possible values are: {tf.estimator.ModeKeys.PREDICT}, {tf.estimator.ModeKeys.TRAIN}, and {tf.estimator.ModeKeys.EVAL} - . Passed: {mode} - """ - ) - - return output diff --git a/pushservice/src/main/python/models/heavy_ranking/lib/BUILD b/pushservice/src/main/python/models/heavy_ranking/lib/BUILD deleted file mode 100644 index a0ed713c4..000000000 --- a/pushservice/src/main/python/models/heavy_ranking/lib/BUILD +++ /dev/null @@ -1,42 +0,0 @@ -python3_library( - name = "params_lib", - sources = [ - "params.py", - ], - tags = [ - "bazel-compatible", - "no-mypy", - ], - dependencies = [ - "3rdparty/python/pydantic:default", - ], -) - -python3_library( - name = "layers_lib", - sources = [ - "layers.py", - ], - tags = [ - "bazel-compatible", - "no-mypy", - ], - dependencies = [ - ], -) - -python3_library( - name = "model_lib", - sources = [ - "model.py", - ], - tags = [ - "bazel-compatible", - "no-mypy", - ], - dependencies = [ - ":layers_lib", - ":params_lib", - "3rdparty/python/absl-py:default", - ], -) diff --git a/pushservice/src/main/python/models/heavy_ranking/lib/BUILD.docx b/pushservice/src/main/python/models/heavy_ranking/lib/BUILD.docx new file mode 100644 index 000000000..65af4a104 Binary files /dev/null and b/pushservice/src/main/python/models/heavy_ranking/lib/BUILD.docx differ diff --git a/pushservice/src/main/python/models/heavy_ranking/lib/layers.docx b/pushservice/src/main/python/models/heavy_ranking/lib/layers.docx new file mode 100644 index 000000000..9af169236 Binary files /dev/null and b/pushservice/src/main/python/models/heavy_ranking/lib/layers.docx differ diff --git a/pushservice/src/main/python/models/heavy_ranking/lib/layers.py b/pushservice/src/main/python/models/heavy_ranking/lib/layers.py deleted file mode 100644 index 33dd6f012..000000000 --- a/pushservice/src/main/python/models/heavy_ranking/lib/layers.py +++ /dev/null @@ -1,128 +0,0 @@ -""" -Different type of convolution layers to be used in the ClemNet. -""" -from typing import Any - -import tensorflow as tf - - -class KerasConv1D(tf.keras.layers.Layer): - """ - Basic Conv1D layer in a wrapper to be compatible with ClemNet. - """ - - def __init__( - self, - kernel_size: int, - filters: int, - strides: int, - padding: str, - use_bias: bool = True, - kernel_initializer: str = "glorot_uniform", - bias_initializer: str = "zeros", - **kwargs: Any, - ): - super(KerasConv1D, self).__init__(**kwargs) - self.kernel_size = kernel_size - self.filters = filters - self.use_bias = use_bias - self.kernel_initializer = kernel_initializer - self.bias_initializer = bias_initializer - self.strides = strides - self.padding = padding - - def build(self, input_shape: tf.TensorShape) -> None: - assert ( - len(input_shape) == 3 - ), f"Tensor shape must be of length 3. Passed tensor of shape {input_shape}." - - self.features = input_shape[1] - - self.w = tf.keras.layers.Conv1D( - kernel_size=self.kernel_size, - filters=self.filters, - strides=self.strides, - padding=self.padding, - use_bias=self.use_bias, - kernel_initializer=self.kernel_initializer, - bias_initializer=self.bias_initializer, - name=self.name, - ) - - def call(self, inputs: tf.Tensor, **kwargs: Any) -> tf.Tensor: - return self.w(inputs) - - -class ChannelWiseDense(tf.keras.layers.Layer): - """ - Dense layer is applied to each channel separately. This is more memory and computationally - efficient than flattening the channels and performing single dense layers over it which is the - default behavior in tf1. - """ - - def __init__( - self, - output_size: int, - use_bias: bool, - kernel_initializer: str = "uniform_glorot", - bias_initializer: str = "zeros", - **kwargs: Any, - ): - super(ChannelWiseDense, self).__init__(**kwargs) - self.output_size = output_size - self.use_bias = use_bias - self.kernel_initializer = kernel_initializer - self.bias_initializer = bias_initializer - - def build(self, input_shape: tf.TensorShape) -> None: - assert ( - len(input_shape) == 3 - ), f"Tensor shape must be of length 3. Passed tensor of shape {input_shape}." - - input_size = input_shape[1] - channels = input_shape[2] - - self.kernel = self.add_weight( - name="kernel", - shape=(channels, input_size, self.output_size), - initializer=self.kernel_initializer, - trainable=True, - ) - - self.bias = self.add_weight( - name="bias", - shape=(channels, self.output_size), - initializer=self.bias_initializer, - trainable=self.use_bias, - ) - - def call(self, inputs: tf.Tensor, **kwargs: Any) -> tf.Tensor: - x = inputs - - transposed_x = tf.transpose(x, perm=[2, 0, 1]) - transposed_residual = ( - tf.transpose(tf.matmul(transposed_x, self.kernel), perm=[1, 0, 2]) + self.bias - ) - output = tf.transpose(transposed_residual, perm=[0, 2, 1]) - - return output - - -class ResidualLayer(tf.keras.layers.Layer): - """ - Layer implementing a 3D-residual connection. - """ - - def build(self, input_shape: tf.TensorShape) -> None: - assert ( - len(input_shape) == 3 - ), f"Tensor shape must be of length 3. Passed tensor of shape {input_shape}." - - def call(self, inputs: tf.Tensor, residual: tf.Tensor, **kwargs: Any) -> tf.Tensor: - shortcut = tf.keras.layers.Conv1D( - filters=int(residual.shape[2]), strides=1, kernel_size=1, padding="SAME", use_bias=False - )(inputs) - - output = tf.add(shortcut, residual) - - return output diff --git a/pushservice/src/main/python/models/heavy_ranking/lib/model.docx b/pushservice/src/main/python/models/heavy_ranking/lib/model.docx new file mode 100644 index 000000000..ba1f867cd Binary files /dev/null and b/pushservice/src/main/python/models/heavy_ranking/lib/model.docx differ diff --git a/pushservice/src/main/python/models/heavy_ranking/lib/model.py b/pushservice/src/main/python/models/heavy_ranking/lib/model.py deleted file mode 100644 index c6c8b1c6b..000000000 --- a/pushservice/src/main/python/models/heavy_ranking/lib/model.py +++ /dev/null @@ -1,76 +0,0 @@ -""" -Module containing ClemNet. -""" -from typing import Any - -from .layers import ChannelWiseDense, KerasConv1D, ResidualLayer -from .params import BlockParams, ClemNetParams - -import tensorflow as tf -import tensorflow.compat.v1 as tf1 - - -class Block2(tf.keras.layers.Layer): - """ - Possible ClemNet block. Architecture is as follow: - Optional(DenseLayer + BN + Act) - Optional(ConvLayer + BN + Act) - Optional(Residual Layer) - - """ - - def __init__(self, params: BlockParams, **kwargs: Any): - super(Block2, self).__init__(**kwargs) - self.params = params - - def build(self, input_shape: tf.TensorShape) -> None: - assert ( - len(input_shape) == 3 - ), f"Tensor shape must be of length 3. Passed tensor of shape {input_shape}." - - def call(self, inputs: tf.Tensor, training: bool) -> tf.Tensor: - x = inputs - if self.params.dense: - x = ChannelWiseDense(**self.params.dense.dict())(inputs=x, training=training) - x = tf1.layers.batch_normalization(x, momentum=0.9999, training=training, axis=1) - x = tf.keras.layers.Activation(self.params.activation)(x) - - if self.params.conv: - x = KerasConv1D(**self.params.conv.dict())(inputs=x, training=training) - x = tf1.layers.batch_normalization(x, momentum=0.9999, training=training, axis=1) - x = tf.keras.layers.Activation(self.params.activation)(x) - - if self.params.residual: - x = ResidualLayer()(inputs=inputs, residual=x) - - return x - - -class ClemNet(tf.keras.layers.Layer): - """ - A residual network stacking residual blocks composed of dense layers and convolutions. - """ - - def __init__(self, params: ClemNetParams, **kwargs: Any): - super(ClemNet, self).__init__(**kwargs) - self.params = params - - def build(self, input_shape: tf.TensorShape) -> None: - assert len(input_shape) in ( - 2, - 3, - ), f"Tensor shape must be of length 3. Passed tensor of shape {input_shape}." - - def call(self, inputs: tf.Tensor, training: bool) -> tf.Tensor: - if len(inputs.shape) < 3: - inputs = tf.expand_dims(inputs, axis=-1) - - x = inputs - for block_params in self.params.blocks: - x = Block2(block_params)(inputs=x, training=training) - - x = tf.keras.layers.Flatten(name="flattened")(x) - if self.params.top: - x = tf.keras.layers.Dense(units=self.params.top.n_labels, name="logits")(x) - - return x diff --git a/pushservice/src/main/python/models/heavy_ranking/lib/params.docx b/pushservice/src/main/python/models/heavy_ranking/lib/params.docx new file mode 100644 index 000000000..abf1e8544 Binary files /dev/null and b/pushservice/src/main/python/models/heavy_ranking/lib/params.docx differ diff --git a/pushservice/src/main/python/models/heavy_ranking/lib/params.py b/pushservice/src/main/python/models/heavy_ranking/lib/params.py deleted file mode 100644 index 721d6ed95..000000000 --- a/pushservice/src/main/python/models/heavy_ranking/lib/params.py +++ /dev/null @@ -1,49 +0,0 @@ -""" -Parameters used in ClemNet. -""" -from typing import List, Optional - -from pydantic import BaseModel, Extra, Field, PositiveInt - - -# checkstyle: noqa - - -class ExtendedBaseModel(BaseModel): - class Config: - extra = Extra.forbid - - -class DenseParams(ExtendedBaseModel): - name: Optional[str] - bias_initializer: str = "zeros" - kernel_initializer: str = "glorot_uniform" - output_size: PositiveInt - use_bias: bool = Field(True) - - -class ConvParams(ExtendedBaseModel): - name: Optional[str] - bias_initializer: str = "zeros" - filters: PositiveInt - kernel_initializer: str = "glorot_uniform" - kernel_size: PositiveInt - padding: str = "SAME" - strides: PositiveInt = 1 - use_bias: bool = Field(True) - - -class BlockParams(ExtendedBaseModel): - activation: Optional[str] - conv: Optional[ConvParams] - dense: Optional[DenseParams] - residual: Optional[bool] - - -class TopLayerParams(ExtendedBaseModel): - n_labels: PositiveInt - - -class ClemNetParams(ExtendedBaseModel): - blocks: List[BlockParams] = [] - top: Optional[TopLayerParams] diff --git a/pushservice/src/main/python/models/heavy_ranking/model_pools.docx b/pushservice/src/main/python/models/heavy_ranking/model_pools.docx new file mode 100644 index 000000000..02b5c1cd2 Binary files /dev/null and b/pushservice/src/main/python/models/heavy_ranking/model_pools.docx differ diff --git a/pushservice/src/main/python/models/heavy_ranking/model_pools.py b/pushservice/src/main/python/models/heavy_ranking/model_pools.py deleted file mode 100644 index de59ee1a6..000000000 --- a/pushservice/src/main/python/models/heavy_ranking/model_pools.py +++ /dev/null @@ -1,34 +0,0 @@ -""" -Candidate architectures for each task's. -""" - -from __future__ import annotations - -from typing import Dict - -from .features import get_features -from .graph import Graph -from .lib.model import ClemNet -from .params import ModelTypeEnum - -import tensorflow as tf - - -class MagicRecsClemNet(Graph): - def get_logits(self, features: Dict[str, tf.Tensor], training: bool) -> tf.Tensor: - - with tf.name_scope("logits"): - inputs = get_features(features=features, training=training, params=self.params.model.features) - - with tf.name_scope("OONC_logits"): - model = ClemNet(params=self.params.model.architecture) - oonc_logit = model(inputs=inputs, training=training) - - with tf.name_scope("EngagementGivenOONC_logits"): - model = ClemNet(params=self.params.model.architecture) - eng_logits = model(inputs=inputs, training=training) - - return tf.concat([oonc_logit, eng_logits], axis=1) - - -ALL_MODELS = {ModelTypeEnum.clemnet: MagicRecsClemNet} diff --git a/pushservice/src/main/python/models/heavy_ranking/params.docx b/pushservice/src/main/python/models/heavy_ranking/params.docx new file mode 100644 index 000000000..38a9b367f Binary files /dev/null and b/pushservice/src/main/python/models/heavy_ranking/params.docx differ diff --git a/pushservice/src/main/python/models/heavy_ranking/params.py b/pushservice/src/main/python/models/heavy_ranking/params.py deleted file mode 100644 index 64a7de2b1..000000000 --- a/pushservice/src/main/python/models/heavy_ranking/params.py +++ /dev/null @@ -1,89 +0,0 @@ -import enum -import json -from typing import List, Optional - -from .lib.params import BlockParams, ClemNetParams, ConvParams, DenseParams, TopLayerParams - -from pydantic import BaseModel, Extra, NonNegativeFloat -import tensorflow.compat.v1 as tf - - -# checkstyle: noqa - - -class ExtendedBaseModel(BaseModel): - class Config: - extra = Extra.forbid - - -class SparseFeaturesParams(ExtendedBaseModel): - bits: int - embedding_size: int - - -class FeaturesParams(ExtendedBaseModel): - sparse_features: Optional[SparseFeaturesParams] - - -class ModelTypeEnum(str, enum.Enum): - clemnet: str = "clemnet" - - -class ModelParams(ExtendedBaseModel): - name: ModelTypeEnum - features: FeaturesParams - architecture: ClemNetParams - - -class TaskNameEnum(str, enum.Enum): - oonc: str = "OONC" - engagement: str = "Engagement" - - -class Task(ExtendedBaseModel): - name: TaskNameEnum - label: str - score_weight: NonNegativeFloat - - -DEFAULT_TASKS = [ - Task(name=TaskNameEnum.oonc, label="label", score_weight=0.9), - Task(name=TaskNameEnum.engagement, label="label.engagement", score_weight=0.1), -] - - -class GraphParams(ExtendedBaseModel): - tasks: List[Task] = DEFAULT_TASKS - model: ModelParams - weight: Optional[str] - - -DEFAULT_ARCHITECTURE_PARAMS = ClemNetParams( - blocks=[ - BlockParams( - activation="relu", - conv=ConvParams(kernel_size=3, filters=5), - dense=DenseParams(output_size=output_size), - residual=False, - ) - for output_size in [1024, 512, 256, 128] - ], - top=TopLayerParams(n_labels=1), -) - -DEFAULT_GRAPH_PARAMS = GraphParams( - model=ModelParams( - name=ModelTypeEnum.clemnet, - architecture=DEFAULT_ARCHITECTURE_PARAMS, - features=FeaturesParams(sparse_features=SparseFeaturesParams(bits=18, embedding_size=50)), - ), -) - - -def load_graph_params(args) -> GraphParams: - params = DEFAULT_GRAPH_PARAMS - if args.param_file: - with tf.io.gfile.GFile(args.param_file, mode="r+") as file: - params = GraphParams.parse_obj(json.load(file)) - - return params diff --git a/pushservice/src/main/python/models/heavy_ranking/run_args.docx b/pushservice/src/main/python/models/heavy_ranking/run_args.docx new file mode 100644 index 000000000..9ea8cb923 Binary files /dev/null and b/pushservice/src/main/python/models/heavy_ranking/run_args.docx differ diff --git a/pushservice/src/main/python/models/heavy_ranking/run_args.py b/pushservice/src/main/python/models/heavy_ranking/run_args.py deleted file mode 100644 index 1cc33a8e0..000000000 --- a/pushservice/src/main/python/models/heavy_ranking/run_args.py +++ /dev/null @@ -1,59 +0,0 @@ -from twml.trainers import DataRecordTrainer - -from .features import FEATURE_LIST_DEFAULT_PATH - - -def get_training_arg_parser(): - parser = DataRecordTrainer.add_parser_arguments() - - parser.add_argument( - "--feature_list", - default=FEATURE_LIST_DEFAULT_PATH, - type=str, - help="Which features to use for training", - ) - - parser.add_argument( - "--param_file", - default=None, - type=str, - help="Path to JSON file containing the graph parameters. If None, model will load default parameters.", - ) - - parser.add_argument( - "--directly_export_best", - default=False, - action="store_true", - help="whether to directly_export best_checkpoint", - ) - - parser.add_argument( - "--warm_start_from", default=None, type=str, help="model dir to warm start from" - ) - - parser.add_argument( - "--warm_start_base_dir", - default=None, - type=str, - help="latest ckpt in this folder will be used to ", - ) - - parser.add_argument( - "--model_type", - default=None, - type=str, - help="Which type of model to train.", - ) - return parser - - -def get_eval_arg_parser(): - parser = get_training_arg_parser() - parser.add_argument( - "--eval_checkpoint", - default=None, - type=str, - help="Which checkpoint to use for evaluation", - ) - - return parser diff --git a/pushservice/src/main/python/models/heavy_ranking/update_warm_start_checkpoint.docx b/pushservice/src/main/python/models/heavy_ranking/update_warm_start_checkpoint.docx new file mode 100644 index 000000000..5f07f01c3 Binary files /dev/null and b/pushservice/src/main/python/models/heavy_ranking/update_warm_start_checkpoint.docx differ diff --git a/pushservice/src/main/python/models/heavy_ranking/update_warm_start_checkpoint.py b/pushservice/src/main/python/models/heavy_ranking/update_warm_start_checkpoint.py deleted file mode 100644 index 04887b9cf..000000000 --- a/pushservice/src/main/python/models/heavy_ranking/update_warm_start_checkpoint.py +++ /dev/null @@ -1,146 +0,0 @@ -""" -Model for modifying the checkpoints of the magic recs cnn Model with addition, deletion, and reordering -of continuous and binary features. -""" - -import os - -from twitter.deepbird.projects.magic_recs.libs.get_feat_config import FEATURE_LIST_DEFAULT_PATH -from twitter.deepbird.projects.magic_recs.libs.warm_start_utils_v11 import ( - get_feature_list_for_heavy_ranking, - mkdirp, - rename_dir, - rmdir, - warm_start_checkpoint, -) -import twml -from twml.trainers import DataRecordTrainer - -import tensorflow.compat.v1 as tf -from tensorflow.compat.v1 import logging - - -def get_arg_parser(): - parser = DataRecordTrainer.add_parser_arguments() - parser.add_argument( - "--model_type", - default="deepnorm_gbdt_inputdrop2_rescale", - type=str, - help="specify the model type to use.", - ) - - parser.add_argument( - "--model_trainer_name", - default="None", - type=str, - help="deprecated, added here just for api compatibility.", - ) - - parser.add_argument( - "--warm_start_base_dir", - default="none", - type=str, - help="latest ckpt in this folder will be used.", - ) - - parser.add_argument( - "--output_checkpoint_dir", - default="none", - type=str, - help="Output folder for warm started ckpt. If none, it will move warm_start_base_dir to backup, and overwrite it", - ) - - parser.add_argument( - "--feature_list", - default="none", - type=str, - help="Which features to use for training", - ) - - parser.add_argument( - "--old_feature_list", - default="none", - type=str, - help="Which features to use for training", - ) - - return parser - - -def get_params(args=None): - parser = get_arg_parser() - if args is None: - return parser.parse_args() - else: - return parser.parse_args(args) - - -def _main(): - opt = get_params() - logging.info("parse is: ") - logging.info(opt) - - if opt.feature_list == "none": - feature_list_path = FEATURE_LIST_DEFAULT_PATH - else: - feature_list_path = opt.feature_list - - if opt.warm_start_base_dir != "none" and tf.io.gfile.exists(opt.warm_start_base_dir): - if opt.output_checkpoint_dir == "none" or opt.output_checkpoint_dir == opt.warm_start_base_dir: - _warm_start_base_dir = os.path.normpath(opt.warm_start_base_dir) + "_backup_warm_start" - _output_folder_dir = opt.warm_start_base_dir - - rename_dir(opt.warm_start_base_dir, _warm_start_base_dir) - tf.logging.info(f"moved {opt.warm_start_base_dir} to {_warm_start_base_dir}") - else: - _warm_start_base_dir = opt.warm_start_base_dir - _output_folder_dir = opt.output_checkpoint_dir - - continuous_binary_feat_list_save_path = os.path.join( - _warm_start_base_dir, "continuous_binary_feat_list.json" - ) - - if opt.old_feature_list != "none": - tf.logging.info("getting old continuous_binary_feat_list") - continuous_binary_feat_list = get_feature_list_for_heavy_ranking( - opt.old_feature_list, opt.data_spec - ) - rmdir(continuous_binary_feat_list_save_path) - twml.util.write_file( - continuous_binary_feat_list_save_path, continuous_binary_feat_list, encode="json" - ) - tf.logging.info(f"Finish writting files to {continuous_binary_feat_list_save_path}") - - warm_start_folder = os.path.join(_warm_start_base_dir, "best_checkpoint") - if not tf.io.gfile.exists(warm_start_folder): - warm_start_folder = _warm_start_base_dir - - rmdir(_output_folder_dir) - mkdirp(_output_folder_dir) - - new_ckpt = warm_start_checkpoint( - warm_start_folder, - continuous_binary_feat_list_save_path, - feature_list_path, - opt.data_spec, - _output_folder_dir, - opt.model_type, - ) - logging.info(f"Created new ckpt {new_ckpt} from {warm_start_folder}") - - tf.logging.info("getting new continuous_binary_feat_list") - new_continuous_binary_feat_list_save_path = os.path.join( - _output_folder_dir, "continuous_binary_feat_list.json" - ) - continuous_binary_feat_list = get_feature_list_for_heavy_ranking( - feature_list_path, opt.data_spec - ) - rmdir(new_continuous_binary_feat_list_save_path) - twml.util.write_file( - new_continuous_binary_feat_list_save_path, continuous_binary_feat_list, encode="json" - ) - tf.logging.info(f"Finish writting files to {new_continuous_binary_feat_list_save_path}") - - -if __name__ == "__main__": - _main() diff --git a/pushservice/src/main/python/models/libs/BUILD b/pushservice/src/main/python/models/libs/BUILD deleted file mode 100644 index 82a014ba5..000000000 --- a/pushservice/src/main/python/models/libs/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -python3_library( - name = "libs", - sources = ["*.py"], - tags = [ - "bazel-compatible", - "no-mypy", - ], - dependencies = [ - "cortex/recsys/src/python/twitter/cortex/recsys/utils", - "magicpony/common/file_access/src/python/twitter/magicpony/common/file_access", - "src/python/twitter/cortex/ml/embeddings/deepbird", - "src/python/twitter/cortex/ml/embeddings/deepbird/grouped_metrics", - "src/python/twitter/deepbird/util/data", - "twml:twml-nodeps", - ], -) diff --git a/pushservice/src/main/python/models/libs/BUILD.docx b/pushservice/src/main/python/models/libs/BUILD.docx new file mode 100644 index 000000000..5ade34483 Binary files /dev/null and b/pushservice/src/main/python/models/libs/BUILD.docx differ diff --git a/pushservice/src/main/python/models/libs/__init__.docx b/pushservice/src/main/python/models/libs/__init__.docx new file mode 100644 index 000000000..539f73b98 Binary files /dev/null and b/pushservice/src/main/python/models/libs/__init__.docx differ diff --git a/pushservice/src/main/python/models/libs/__init__.py b/pushservice/src/main/python/models/libs/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/pushservice/src/main/python/models/libs/customized_full_sparse.docx b/pushservice/src/main/python/models/libs/customized_full_sparse.docx new file mode 100644 index 000000000..14488a41d Binary files /dev/null and b/pushservice/src/main/python/models/libs/customized_full_sparse.docx differ diff --git a/pushservice/src/main/python/models/libs/customized_full_sparse.py b/pushservice/src/main/python/models/libs/customized_full_sparse.py deleted file mode 100644 index b41f7d694..000000000 --- a/pushservice/src/main/python/models/libs/customized_full_sparse.py +++ /dev/null @@ -1,56 +0,0 @@ -# pylint: disable=no-member, arguments-differ, attribute-defined-outside-init, unused-argument -""" -Implementing Full Sparse Layer, allow specify use_binary_value in call() to -overide default action. -""" - -from twml.layers import FullSparse as defaultFullSparse -from twml.layers.full_sparse import sparse_dense_matmul - -import tensorflow.compat.v1 as tf - - -class FullSparse(defaultFullSparse): - def call(self, inputs, use_binary_values=None, **kwargs): # pylint: disable=unused-argument - """The logic of the layer lives here. - - Arguments: - inputs: - A SparseTensor or a list of SparseTensors. - If `inputs` is a list, all tensors must have same `dense_shape`. - - Returns: - - If `inputs` is `SparseTensor`, then returns `bias + inputs * dense_b`. - - If `inputs` is a `list[SparseTensor`, then returns - `bias + add_n([sp_a * dense_b for sp_a in inputs])`. - """ - - if use_binary_values is not None: - default_use_binary_values = use_binary_values - else: - default_use_binary_values = self.use_binary_values - - if isinstance(default_use_binary_values, (list, tuple)): - raise ValueError( - "use_binary_values can not be %s when inputs is %s" - % (type(default_use_binary_values), type(inputs)) - ) - - outputs = sparse_dense_matmul( - inputs, - self.weight, - self.use_sparse_grads, - default_use_binary_values, - name="sparse_mm", - partition_axis=self.partition_axis, - num_partitions=self.num_partitions, - compress_ids=self._use_compression, - cast_indices_dtype=self._cast_indices_dtype, - ) - - if self.bias is not None: - outputs = tf.nn.bias_add(outputs, self.bias) - - if self.activation is not None: - return self.activation(outputs) # pylint: disable=not-callable - return outputs diff --git a/pushservice/src/main/python/models/libs/get_feat_config.docx b/pushservice/src/main/python/models/libs/get_feat_config.docx new file mode 100644 index 000000000..58aa52cf0 Binary files /dev/null and b/pushservice/src/main/python/models/libs/get_feat_config.docx differ diff --git a/pushservice/src/main/python/models/libs/get_feat_config.py b/pushservice/src/main/python/models/libs/get_feat_config.py deleted file mode 100644 index 4d8b3e93c..000000000 --- a/pushservice/src/main/python/models/libs/get_feat_config.py +++ /dev/null @@ -1,176 +0,0 @@ -import os - -from twitter.deepbird.projects.magic_recs.libs.metric_fn_utils import USER_AGE_FEATURE_NAME -from twitter.deepbird.projects.magic_recs.libs.model_utils import read_config -from twml.contrib import feature_config as contrib_feature_config - - -# checkstyle: noqa - -FEAT_CONFIG_DEFAULT_VAL = -1.23456789 - -DEFAULT_INPUT_SIZE_BITS = 18 - -DEFAULT_FEATURE_LIST_PATH = "./feature_list_default.yaml" -FEATURE_LIST_DEFAULT_PATH = os.path.join( - os.path.dirname(os.path.realpath(__file__)), DEFAULT_FEATURE_LIST_PATH -) - -DEFAULT_FEATURE_LIST_LIGHT_RANKING_PATH = "./feature_list_light_ranking.yaml" -FEATURE_LIST_DEFAULT_LIGHT_RANKING_PATH = os.path.join( - os.path.dirname(os.path.realpath(__file__)), DEFAULT_FEATURE_LIST_LIGHT_RANKING_PATH -) - -FEATURE_LIST_DEFAULT = read_config(FEATURE_LIST_DEFAULT_PATH).items() -FEATURE_LIST_LIGHT_RANKING_DEFAULT = read_config(FEATURE_LIST_DEFAULT_LIGHT_RANKING_PATH).items() - - -LABELS = ["label"] -LABELS_MTL = {"OONC": ["label"], "OONC_Engagement": ["label", "label.engagement"]} -LABELS_LR = { - "Sent": ["label.sent"], - "HeavyRankPosition": ["meta.ranking.is_top3"], - "HeavyRankProbability": ["meta.ranking.weighted_oonc_model_score"], -} - - -def _get_new_feature_config_base( - data_spec_path, - labels, - add_sparse_continous=True, - add_gbdt=True, - add_user_id=False, - add_timestamp=False, - add_user_age=False, - feature_list_provided=[], - opt=None, - run_light_ranking_group_metrics_in_bq=False, -): - """ - Getter of the feature config based on specification. - - Args: - data_spec_path: A string indicating the path of the data_spec.json file, which could be - either a local path or a hdfs path. - labels: A list of strings indicating the name of the label in the data spec. - add_sparse_continous: A bool indicating if sparse_continuous feature needs to be included. - add_gbdt: A bool indicating if gbdt feature needs to be included. - add_user_id: A bool indicating if user_id feature needs to be included. - add_timestamp: A bool indicating if timestamp feature needs to be included. This will be useful - for sequential models and meta learning models. - add_user_age: A bool indicating if the user age feature needs to be included. - feature_list_provided: A list of features thats need to be included. If not specified, will use - FEATURE_LIST_DEFAULT by default. - opt: A namespace of arguments indicating the hyparameters. - run_light_ranking_group_metrics_in_bq: A bool indicating if heavy ranker score info needs to be included to compute group metrics in BigQuery. - - Returns: - A twml feature config object. - """ - - input_size_bits = DEFAULT_INPUT_SIZE_BITS if opt is None else opt.input_size_bits - - feature_list = feature_list_provided if feature_list_provided != [] else FEATURE_LIST_DEFAULT - a_string_feat_list = [f[0] for f in feature_list if f[1] != "S"] - - builder = contrib_feature_config.FeatureConfigBuilder(data_spec_path=data_spec_path) - - builder = builder.extract_feature_group( - feature_regexes=a_string_feat_list, - group_name="continuous", - default_value=FEAT_CONFIG_DEFAULT_VAL, - type_filter=["CONTINUOUS"], - ) - - builder = builder.extract_features_as_hashed_sparse( - feature_regexes=a_string_feat_list, - output_tensor_name="sparse_no_continuous", - hash_space_size_bits=input_size_bits, - type_filter=["BINARY", "DISCRETE", "STRING", "SPARSE_BINARY"], - ) - - if add_gbdt: - builder = builder.extract_features_as_hashed_sparse( - feature_regexes=["ads\..*"], - output_tensor_name="gbdt_sparse", - hash_space_size_bits=input_size_bits, - ) - - if add_sparse_continous: - s_string_feat_list = [f[0] for f in feature_list if f[1] == "S"] - - builder = builder.extract_features_as_hashed_sparse( - feature_regexes=s_string_feat_list, - output_tensor_name="sparse_continuous", - hash_space_size_bits=input_size_bits, - type_filter=["SPARSE_CONTINUOUS"], - ) - - if add_user_id: - builder = builder.extract_feature("meta.user_id") - if add_timestamp: - builder = builder.extract_feature("meta.timestamp") - if add_user_age: - builder = builder.extract_feature(USER_AGE_FEATURE_NAME) - - if run_light_ranking_group_metrics_in_bq: - builder = builder.extract_feature("meta.trace_id") - builder = builder.extract_feature("meta.ranking.weighted_oonc_model_score") - - builder = builder.add_labels(labels).define_weight("meta.weight") - - return builder.build() - - -def get_feature_config_with_sparse_continuous( - data_spec_path, - feature_list_provided=[], - opt=None, - add_user_id=False, - add_timestamp=False, - add_user_age=False, -): - task_name = opt.task_name if getattr(opt, "task_name", None) is not None else "OONC" - if task_name not in LABELS_MTL: - raise ValueError("Invalid Task Name !") - - return _get_new_feature_config_base( - data_spec_path=data_spec_path, - labels=LABELS_MTL[task_name], - add_sparse_continous=True, - add_user_id=add_user_id, - add_timestamp=add_timestamp, - add_user_age=add_user_age, - feature_list_provided=feature_list_provided, - opt=opt, - ) - - -def get_feature_config_light_ranking( - data_spec_path, - feature_list_provided=[], - opt=None, - add_user_id=True, - add_timestamp=False, - add_user_age=False, - add_gbdt=False, - run_light_ranking_group_metrics_in_bq=False, -): - task_name = opt.task_name if getattr(opt, "task_name", None) is not None else "HeavyRankPosition" - if task_name not in LABELS_LR: - raise ValueError("Invalid Task Name !") - if not feature_list_provided: - feature_list_provided = FEATURE_LIST_LIGHT_RANKING_DEFAULT - - return _get_new_feature_config_base( - data_spec_path=data_spec_path, - labels=LABELS_LR[task_name], - add_sparse_continous=False, - add_gbdt=add_gbdt, - add_user_id=add_user_id, - add_timestamp=add_timestamp, - add_user_age=add_user_age, - feature_list_provided=feature_list_provided, - opt=opt, - run_light_ranking_group_metrics_in_bq=run_light_ranking_group_metrics_in_bq, - ) diff --git a/pushservice/src/main/python/models/libs/graph_utils.docx b/pushservice/src/main/python/models/libs/graph_utils.docx new file mode 100644 index 000000000..4b92cbca0 Binary files /dev/null and b/pushservice/src/main/python/models/libs/graph_utils.docx differ diff --git a/pushservice/src/main/python/models/libs/graph_utils.py b/pushservice/src/main/python/models/libs/graph_utils.py deleted file mode 100644 index 4a4626a59..000000000 --- a/pushservice/src/main/python/models/libs/graph_utils.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -Utilties that aid in building the magic recs graph. -""" - -import re - -import tensorflow.compat.v1 as tf - - -def get_trainable_variables(all_trainable_variables, trainable_regexes): - """Returns a subset of trainable variables for training. - - Given a collection of trainable variables, this will return all those that match the given regexes. - Will also log those variables. - - Args: - all_trainable_variables (a collection of trainable tf.Variable): The variables to search through. - trainable_regexes (a collection of regexes): Variables that match any regex will be included. - - Returns a list of tf.Variable - """ - if trainable_regexes is None or len(trainable_regexes) == 0: - tf.logging.info("No trainable regexes found. Not using get_trainable_variables behavior.") - return None - - assert any( - tf.is_tensor(var) for var in all_trainable_variables - ), f"Non TF variable found: {all_trainable_variables}" - trainable_variables = list( - filter( - lambda var: any(re.match(regex, var.name, re.IGNORECASE) for regex in trainable_regexes), - all_trainable_variables, - ) - ) - tf.logging.info(f"Using filtered trainable variables: {trainable_variables}") - - assert ( - trainable_variables - ), "Did not find trainable variables after filtering after filtering from {} number of vars originaly. All vars: {} and train regexes: {}".format( - len(all_trainable_variables), all_trainable_variables, trainable_regexes - ) - return trainable_variables diff --git a/pushservice/src/main/python/models/libs/group_metrics.docx b/pushservice/src/main/python/models/libs/group_metrics.docx new file mode 100644 index 000000000..8a531f0d7 Binary files /dev/null and b/pushservice/src/main/python/models/libs/group_metrics.docx differ diff --git a/pushservice/src/main/python/models/libs/group_metrics.py b/pushservice/src/main/python/models/libs/group_metrics.py deleted file mode 100644 index eeef3c501..000000000 --- a/pushservice/src/main/python/models/libs/group_metrics.py +++ /dev/null @@ -1,114 +0,0 @@ -import os -import time - -from twitter.cortex.ml.embeddings.deepbird.grouped_metrics.computation import ( - write_grouped_metrics_to_mldash, -) -from twitter.cortex.ml.embeddings.deepbird.grouped_metrics.configuration import ( - ClassificationGroupedMetricsConfiguration, - NDCGGroupedMetricsConfiguration, -) -import twml - -from .light_ranking_metrics import ( - CGRGroupedMetricsConfiguration, - ExpectedLossGroupedMetricsConfiguration, - RecallGroupedMetricsConfiguration, -) - -import numpy as np -import tensorflow.compat.v1 as tf -from tensorflow.compat.v1 import logging - - -# checkstyle: noqa - - -def run_group_metrics(trainer, data_dir, model_path, parse_fn, group_feature_name="meta.user_id"): - - start_time = time.time() - logging.info("Evaluating with group metrics.") - - metrics = write_grouped_metrics_to_mldash( - trainer=trainer, - data_dir=data_dir, - model_path=model_path, - group_fn=lambda datarecord: str( - datarecord.discreteFeatures[twml.feature_id(group_feature_name)[0]] - ), - parse_fn=parse_fn, - metric_configurations=[ - ClassificationGroupedMetricsConfiguration(), - NDCGGroupedMetricsConfiguration(k=[5, 10, 20]), - ], - total_records_to_read=1000000000, - shuffle=False, - mldash_metrics_name="grouped_metrics", - ) - - end_time = time.time() - logging.info(f"Evaluated Group Metics: {metrics}.") - logging.info(f"Group metrics evaluation time {end_time - start_time}.") - - -def run_group_metrics_light_ranking( - trainer, data_dir, model_path, parse_fn, group_feature_name="meta.trace_id" -): - - start_time = time.time() - logging.info("Evaluating with group metrics.") - - metrics = write_grouped_metrics_to_mldash( - trainer=trainer, - data_dir=data_dir, - model_path=model_path, - group_fn=lambda datarecord: str( - datarecord.discreteFeatures[twml.feature_id(group_feature_name)[0]] - ), - parse_fn=parse_fn, - metric_configurations=[ - CGRGroupedMetricsConfiguration(lightNs=[50, 100, 200], heavyKs=[1, 3, 10, 20, 50]), - RecallGroupedMetricsConfiguration(n=[50, 100, 200], k=[1, 3, 10, 20, 50]), - ExpectedLossGroupedMetricsConfiguration(lightNs=[50, 100, 200]), - ], - total_records_to_read=10000000, - num_batches_to_load=50, - batch_size=1024, - shuffle=False, - mldash_metrics_name="grouped_metrics_for_light_ranking", - ) - - end_time = time.time() - logging.info(f"Evaluated Group Metics for Light Ranking: {metrics}.") - logging.info(f"Group metrics evaluation time {end_time - start_time}.") - - -def run_group_metrics_light_ranking_in_bq(trainer, params, checkpoint_path): - logging.info("getting Test Predictions for Light Ranking Group Metrics in BigQuery !!!") - eval_input_fn = trainer.get_eval_input_fn(repeat=False, shuffle=False) - info_pool = [] - - for result in trainer.estimator.predict( - eval_input_fn, checkpoint_path=checkpoint_path, yield_single_examples=False - ): - traceID = result["trace_id"] - pred = result["prediction"] - label = result["target"] - info = np.concatenate([traceID, pred, label], axis=1) - info_pool.append(info) - - info_pool = np.concatenate(info_pool) - - locname = "/tmp/000/" - if not os.path.exists(locname): - os.makedirs(locname) - - locfile = locname + params.pred_file_name - columns = ["trace_id", "model_prediction", "meta__ranking__weighted_oonc_model_score"] - np.savetxt(locfile, info_pool, delimiter=",", header=",".join(columns)) - tf.io.gfile.copy(locfile, params.pred_file_path + params.pred_file_name, overwrite=True) - - if os.path.isfile(locfile): - os.remove(locfile) - - logging.info("Done Prediction for Light Ranking Group Metrics in BigQuery.") diff --git a/pushservice/src/main/python/models/libs/initializer.docx b/pushservice/src/main/python/models/libs/initializer.docx new file mode 100644 index 000000000..f2bf18437 Binary files /dev/null and b/pushservice/src/main/python/models/libs/initializer.docx differ diff --git a/pushservice/src/main/python/models/libs/initializer.py b/pushservice/src/main/python/models/libs/initializer.py deleted file mode 100644 index 8bba00216..000000000 --- a/pushservice/src/main/python/models/libs/initializer.py +++ /dev/null @@ -1,118 +0,0 @@ -import numpy as np -from tensorflow.keras import backend as K - - -class VarianceScaling(object): - """Initializer capable of adapting its scale to the shape of weights. - With `distribution="normal"`, samples are drawn from a truncated normal - distribution centered on zero, with `stddev = sqrt(scale / n)` where n is: - - number of input units in the weight tensor, if mode = "fan_in" - - number of output units, if mode = "fan_out" - - average of the numbers of input and output units, if mode = "fan_avg" - With `distribution="uniform"`, - samples are drawn from a uniform distribution - within [-limit, limit], with `limit = sqrt(3 * scale / n)`. - # Arguments - scale: Scaling factor (positive float). - mode: One of "fan_in", "fan_out", "fan_avg". - distribution: Random distribution to use. One of "normal", "uniform". - seed: A Python integer. Used to seed the random generator. - # Raises - ValueError: In case of an invalid value for the "scale", mode" or - "distribution" arguments.""" - - def __init__( - self, - scale=1.0, - mode="fan_in", - distribution="normal", - seed=None, - fan_in=None, - fan_out=None, - ): - self.fan_in = fan_in - self.fan_out = fan_out - if scale <= 0.0: - raise ValueError("`scale` must be a positive float. Got:", scale) - mode = mode.lower() - if mode not in {"fan_in", "fan_out", "fan_avg"}: - raise ValueError( - "Invalid `mode` argument: " 'expected on of {"fan_in", "fan_out", "fan_avg"} ' "but got", - mode, - ) - distribution = distribution.lower() - if distribution not in {"normal", "uniform"}: - raise ValueError( - "Invalid `distribution` argument: " 'expected one of {"normal", "uniform"} ' "but got", - distribution, - ) - self.scale = scale - self.mode = mode - self.distribution = distribution - self.seed = seed - - def __call__(self, shape, dtype=None, partition_info=None): - fan_in = shape[-2] if self.fan_in is None else self.fan_in - fan_out = shape[-1] if self.fan_out is None else self.fan_out - - scale = self.scale - if self.mode == "fan_in": - scale /= max(1.0, fan_in) - elif self.mode == "fan_out": - scale /= max(1.0, fan_out) - else: - scale /= max(1.0, float(fan_in + fan_out) / 2) - if self.distribution == "normal": - stddev = np.sqrt(scale) / 0.87962566103423978 - return K.truncated_normal(shape, 0.0, stddev, dtype=dtype, seed=self.seed) - else: - limit = np.sqrt(3.0 * scale) - return K.random_uniform(shape, -limit, limit, dtype=dtype, seed=self.seed) - - def get_config(self): - return { - "scale": self.scale, - "mode": self.mode, - "distribution": self.distribution, - "seed": self.seed, - } - - -def customized_glorot_uniform(seed=None, fan_in=None, fan_out=None): - """Glorot uniform initializer, also called Xavier uniform initializer. - It draws samples from a uniform distribution within [-limit, limit] - where `limit` is `sqrt(6 / (fan_in + fan_out))` - where `fan_in` is the number of input units in the weight tensor - and `fan_out` is the number of output units in the weight tensor. - # Arguments - seed: A Python integer. Used to seed the random generator. - # Returns - An initializer.""" - return VarianceScaling( - scale=1.0, - mode="fan_avg", - distribution="uniform", - seed=seed, - fan_in=fan_in, - fan_out=fan_out, - ) - - -def customized_glorot_norm(seed=None, fan_in=None, fan_out=None): - """Glorot norm initializer, also called Xavier uniform initializer. - It draws samples from a uniform distribution within [-limit, limit] - where `limit` is `sqrt(6 / (fan_in + fan_out))` - where `fan_in` is the number of input units in the weight tensor - and `fan_out` is the number of output units in the weight tensor. - # Arguments - seed: A Python integer. Used to seed the random generator. - # Returns - An initializer.""" - return VarianceScaling( - scale=1.0, - mode="fan_avg", - distribution="normal", - seed=seed, - fan_in=fan_in, - fan_out=fan_out, - ) diff --git a/pushservice/src/main/python/models/libs/light_ranking_metrics.docx b/pushservice/src/main/python/models/libs/light_ranking_metrics.docx new file mode 100644 index 000000000..1c98ad5f0 Binary files /dev/null and b/pushservice/src/main/python/models/libs/light_ranking_metrics.docx differ diff --git a/pushservice/src/main/python/models/libs/light_ranking_metrics.py b/pushservice/src/main/python/models/libs/light_ranking_metrics.py deleted file mode 100644 index b83fcf3ae..000000000 --- a/pushservice/src/main/python/models/libs/light_ranking_metrics.py +++ /dev/null @@ -1,255 +0,0 @@ -from functools import partial - -from twitter.cortex.ml.embeddings.deepbird.grouped_metrics.configuration import ( - GroupedMetricsConfiguration, -) -from twitter.cortex.ml.embeddings.deepbird.grouped_metrics.helpers import ( - extract_prediction_from_prediction_record, -) - - -# checkstyle: noqa - - -def score_loss_at_n(labels, predictions, lightN): - """ - Compute the absolute ScoreLoss ranking metric - Args: - labels (list) : A list of label values (HeavyRanking Reference) - predictions (list): A list of prediction values (LightRanking Predictions) - lightN (int): size of the list at which of Initial candidates to compute ScoreLoss. (LightRanking) - """ - assert len(labels) == len(predictions) - - if lightN <= 0: - return None - - labels_with_predictions = zip(labels, predictions) - labels_with_sorted_predictions = sorted( - labels_with_predictions, key=lambda x: x[1], reverse=True - )[:lightN] - labels_top1_light = max([label for label, _ in labels_with_sorted_predictions]) - labels_top1_heavy = max(labels) - - return labels_top1_heavy - labels_top1_light - - -def cgr_at_nk(labels, predictions, lightN, heavyK): - """ - Compute Cumulative Gain Ratio (CGR) ranking metric - Args: - labels (list) : A list of label values (HeavyRanking Reference) - predictions (list): A list of prediction values (LightRanking Predictions) - lightN (int): size of the list at which of Initial candidates to compute CGR. (LightRanking) - heavyK (int): size of the list at which of Refined candidates to compute CGR. (HeavyRanking) - """ - assert len(labels) == len(predictions) - - if (not lightN) or (not heavyK): - out = None - elif lightN <= 0 or heavyK <= 0: - out = None - else: - - labels_with_predictions = zip(labels, predictions) - labels_with_sorted_predictions = sorted( - labels_with_predictions, key=lambda x: x[1], reverse=True - )[:lightN] - labels_topN_light = [label for label, _ in labels_with_sorted_predictions] - - if lightN <= heavyK: - cg_light = sum(labels_topN_light) - else: - labels_topK_heavy_from_light = sorted(labels_topN_light, reverse=True)[:heavyK] - cg_light = sum(labels_topK_heavy_from_light) - - ideal_ordering = sorted(labels, reverse=True) - cg_heavy = sum(ideal_ordering[: min(lightN, heavyK)]) - - out = 0.0 - if cg_heavy != 0: - out = max(cg_light / cg_heavy, 0) - - return out - - -def _get_weight(w, atK): - if not w: - return 1.0 - elif len(w) <= atK: - return 0.0 - else: - return w[atK] - - -def recall_at_nk(labels, predictions, n=None, k=None, w=None): - """ - Recall at N-K ranking metric - Args: - labels (list): A list of label values - predictions (list): A list of prediction values - n (int): size of the list at which of predictions to compute recall. (Light Ranking Predictions) - The default is None in which case the length of the provided predictions is used as L - k (int): size of the list at which of labels to compute recall. (Heavy Ranking Predictions) - The default is None in which case the length of the provided labels is used as L - w (list): weight vector sorted by labels - """ - assert len(labels) == len(predictions) - - if not any(labels): - out = None - else: - - safe_n = len(predictions) if not n else min(len(predictions), n) - safe_k = len(labels) if not k else min(len(labels), k) - - labels_with_predictions = zip(labels, predictions) - sorted_labels_with_predictions = sorted( - labels_with_predictions, key=lambda x: x[0], reverse=True - ) - - order_sorted_labels_predictions = zip(range(len(labels)), *zip(*sorted_labels_with_predictions)) - - order_with_predictions = [ - (order, pred) for order, label, pred in order_sorted_labels_predictions - ] - order_with_sorted_predictions = sorted(order_with_predictions, key=lambda x: x[1], reverse=True) - - pred_sorted_order_at_n = [order for order, _ in order_with_sorted_predictions][:safe_n] - - intersection_weight = [ - _get_weight(w, order) if order < safe_k else 0 for order in pred_sorted_order_at_n - ] - - intersection_score = sum(intersection_weight) - full_score = sum(w) if w else float(safe_k) - - out = 0.0 - if full_score != 0: - out = intersection_score / full_score - - return out - - -class ExpectedLossGroupedMetricsConfiguration(GroupedMetricsConfiguration): - """ - This is the Expected Loss Grouped metric computation configuration. - """ - - def __init__(self, lightNs=[]): - """ - Args: - lightNs (list): size of the list at which of Initial candidates to compute Expected Loss. (LightRanking) - """ - self.lightNs = lightNs - - @property - def name(self): - return "ExpectedLoss" - - @property - def metrics_dict(self): - metrics_to_compute = {} - for lightN in self.lightNs: - metric_name = "ExpectedLoss_atLight_" + str(lightN) - metrics_to_compute[metric_name] = partial(score_loss_at_n, lightN=lightN) - return metrics_to_compute - - def extract_label(self, prec, drec, drec_label): - return drec_label - - def extract_prediction(self, prec, drec, drec_label): - return extract_prediction_from_prediction_record(prec) - - -class CGRGroupedMetricsConfiguration(GroupedMetricsConfiguration): - """ - This is the Cumulative Gain Ratio (CGR) Grouped metric computation configuration. - CGR at the max length of each session is the default. - CGR at additional positions can be computed by specifying a list of 'n's and 'k's - """ - - def __init__(self, lightNs=[], heavyKs=[]): - """ - Args: - lightNs (list): size of the list at which of Initial candidates to compute CGR. (LightRanking) - heavyK (int): size of the list at which of Refined candidates to compute CGR. (HeavyRanking) - """ - self.lightNs = lightNs - self.heavyKs = heavyKs - - @property - def name(self): - return "cgr" - - @property - def metrics_dict(self): - metrics_to_compute = {} - for lightN in self.lightNs: - for heavyK in self.heavyKs: - metric_name = "cgr_atLight_" + str(lightN) + "_atHeavy_" + str(heavyK) - metrics_to_compute[metric_name] = partial(cgr_at_nk, lightN=lightN, heavyK=heavyK) - return metrics_to_compute - - def extract_label(self, prec, drec, drec_label): - return drec_label - - def extract_prediction(self, prec, drec, drec_label): - return extract_prediction_from_prediction_record(prec) - - -class RecallGroupedMetricsConfiguration(GroupedMetricsConfiguration): - """ - This is the Recall Grouped metric computation configuration. - Recall at the max length of each session is the default. - Recall at additional positions can be computed by specifying a list of 'n's and 'k's - """ - - def __init__(self, n=[], k=[], w=[]): - """ - Args: - n (list): A list of ints. List of prediction rank thresholds (for light) - k (list): A list of ints. List of label rank thresholds (for heavy) - """ - self.predN = n - self.labelK = k - self.weight = w - - @property - def name(self): - return "group_recall" - - @property - def metrics_dict(self): - metrics_to_compute = {"group_recall_unweighted": recall_at_nk} - if not self.weight: - metrics_to_compute["group_recall_weighted"] = partial(recall_at_nk, w=self.weight) - - if self.predN and self.labelK: - for n in self.predN: - for k in self.labelK: - if n >= k: - metrics_to_compute[ - "group_recall_unweighted_at_L" + str(n) + "_at_H" + str(k) - ] = partial(recall_at_nk, n=n, k=k) - if self.weight: - metrics_to_compute[ - "group_recall_weighted_at_L" + str(n) + "_at_H" + str(k) - ] = partial(recall_at_nk, n=n, k=k, w=self.weight) - - if self.labelK and not self.predN: - for k in self.labelK: - metrics_to_compute["group_recall_unweighted_at_full_at_H" + str(k)] = partial( - recall_at_nk, k=k - ) - if self.weight: - metrics_to_compute["group_recall_weighted_at_full_at_H" + str(k)] = partial( - recall_at_nk, k=k, w=self.weight - ) - return metrics_to_compute - - def extract_label(self, prec, drec, drec_label): - return drec_label - - def extract_prediction(self, prec, drec, drec_label): - return extract_prediction_from_prediction_record(prec) diff --git a/pushservice/src/main/python/models/libs/metric_fn_utils.docx b/pushservice/src/main/python/models/libs/metric_fn_utils.docx new file mode 100644 index 000000000..cfb7853d3 Binary files /dev/null and b/pushservice/src/main/python/models/libs/metric_fn_utils.docx differ diff --git a/pushservice/src/main/python/models/libs/metric_fn_utils.py b/pushservice/src/main/python/models/libs/metric_fn_utils.py deleted file mode 100644 index fc26a1305..000000000 --- a/pushservice/src/main/python/models/libs/metric_fn_utils.py +++ /dev/null @@ -1,294 +0,0 @@ -""" -Utilties for constructing a metric_fn for magic recs. -""" - -from twml.contrib.metrics.metrics import ( - get_dual_binary_tasks_metric_fn, - get_numeric_metric_fn, - get_partial_multi_binary_class_metric_fn, - get_single_binary_task_metric_fn, -) - -from .model_utils import generate_disliked_mask - -import tensorflow.compat.v1 as tf - - -METRIC_BOOK = { - "OONC": ["OONC"], - "OONC_Engagement": ["OONC", "Engagement"], - "Sent": ["Sent"], - "HeavyRankPosition": ["HeavyRankPosition"], - "HeavyRankProbability": ["HeavyRankProbability"], -} - -USER_AGE_FEATURE_NAME = "accountAge" -NEW_USER_AGE_CUTOFF = 0 - - -def remove_padding_and_flatten(tensor, valid_batch_size): - """Remove the padding of the input padded tensor given the valid batch size tensor, - then flatten the output with respect to the first dimension. - Args: - tensor: A tensor of size [META_BATCH_SIZE, BATCH_SIZE, FEATURE_DIM]. - valid_batch_size: A tensor of size [META_BATCH_SIZE], with each element indicating - the effective batch size of the BATCH_SIZE dimension. - - Returns: - A tesnor of size [tf.reduce_sum(valid_batch_size), FEATURE_DIM]. - """ - unpadded_ragged_tensor = tf.RaggedTensor.from_tensor(tensor=tensor, lengths=valid_batch_size) - - return unpadded_ragged_tensor.flat_values - - -def safe_mask(values, mask): - """Mask values if possible. - - Boolean mask inputed values if and only if values is a tensor of the same dimension as mask (or can be broadcasted to that dimension). - - Args: - values (Any or Tensor): Input tensor to mask. Dim 0 should be size N. - mask (boolean tensor): A boolean tensor of size N. - - Returns Values or Values masked. - """ - if values is None: - return values - if not tf.is_tensor(values): - return values - values_shape = values.get_shape() - if not values_shape or len(values_shape) == 0: - return values - if not mask.get_shape().is_compatible_with(values_shape[0]): - return values - return tf.boolean_mask(values, mask) - - -def add_new_user_metrics(metric_fn): - """Will stratify the metric_fn by adding new user metrics. - - Given an input metric_fn, double every metric: One will be the orignal and the other will only include those for new users. - - Args: - metric_fn (python function): Base twml metric_fn. - - Returns a metric_fn with new user metrics included. - """ - - def metric_fn_with_new_users(graph_output, labels, weights): - if USER_AGE_FEATURE_NAME not in graph_output: - raise ValueError( - "In order to get metrics stratified by user age, {name} feature should be added to model graph output. However, only the following output keys were found: {keys}.".format( - name=USER_AGE_FEATURE_NAME, keys=graph_output.keys() - ) - ) - - metric_ops = metric_fn(graph_output, labels, weights) - - is_new = tf.reshape( - tf.math.less_equal( - tf.cast(graph_output[USER_AGE_FEATURE_NAME], tf.int64), - tf.cast(NEW_USER_AGE_CUTOFF, tf.int64), - ), - [-1], - ) - - labels = safe_mask(labels, is_new) - weights = safe_mask(weights, is_new) - graph_output = {key: safe_mask(values, is_new) for key, values in graph_output.items()} - - new_user_metric_ops = metric_fn(graph_output, labels, weights) - new_user_metric_ops = {name + "_new_users": ops for name, ops in new_user_metric_ops.items()} - metric_ops.update(new_user_metric_ops) - return metric_ops - - return metric_fn_with_new_users - - -def get_meta_learn_single_binary_task_metric_fn( - metrics, classnames, top_k=(5, 5, 5), use_top_k=False -): - """Wrapper function to use the metric_fn with meta learning evaluation scheme. - - Args: - metrics: A list of string representing metric names. - classnames: A list of string repsenting class names, In case of multiple binary class models, - the names for each class or label. - top_k: A tuple of int to specify top K metrics. - use_top_k: A boolean value indicating of top K of metrics is used. - - Returns: - A customized metric_fn function. - """ - - def get_eval_metric_ops(graph_output, labels, weights): - """The op func of the eval_metrics. Comparing with normal version, - the difference is we flatten the output, label, and weights. - - Args: - graph_output: A dict of tensors. - labels: A tensor of int32 be the value of either 0 or 1. - weights: A tensor of float32 to indicate the per record weight. - - Returns: - A dict of metric names and values. - """ - metric_op_weighted = get_partial_multi_binary_class_metric_fn( - metrics, predcols=0, classes=classnames - ) - classnames_unweighted = ["unweighted_" + classname for classname in classnames] - metric_op_unweighted = get_partial_multi_binary_class_metric_fn( - metrics, predcols=0, classes=classnames_unweighted - ) - - valid_batch_size = graph_output["valid_batch_size"] - graph_output["output"] = remove_padding_and_flatten(graph_output["output"], valid_batch_size) - labels = remove_padding_and_flatten(labels, valid_batch_size) - weights = remove_padding_and_flatten(weights, valid_batch_size) - - tf.ensure_shape(graph_output["output"], [None, 1]) - tf.ensure_shape(labels, [None, 1]) - tf.ensure_shape(weights, [None, 1]) - - metrics_weighted = metric_op_weighted(graph_output, labels, weights) - metrics_unweighted = metric_op_unweighted(graph_output, labels, None) - metrics_weighted.update(metrics_unweighted) - - if use_top_k: - metric_op_numeric = get_numeric_metric_fn(metrics=None, topK=top_k, predcol=0, labelcol=1) - metrics_numeric = metric_op_numeric(graph_output, labels, weights) - metrics_weighted.update(metrics_numeric) - return metrics_weighted - - return get_eval_metric_ops - - -def get_meta_learn_dual_binary_tasks_metric_fn( - metrics, classnames, top_k=(5, 5, 5), use_top_k=False -): - """Wrapper function to use the metric_fn with meta learning evaluation scheme. - - Args: - metrics: A list of string representing metric names. - classnames: A list of string repsenting class names, In case of multiple binary class models, - the names for each class or label. - top_k: A tuple of int to specify top K metrics. - use_top_k: A boolean value indicating of top K of metrics is used. - - Returns: - A customized metric_fn function. - """ - - def get_eval_metric_ops(graph_output, labels, weights): - """The op func of the eval_metrics. Comparing with normal version, - the difference is we flatten the output, label, and weights. - - Args: - graph_output: A dict of tensors. - labels: A tensor of int32 be the value of either 0 or 1. - weights: A tensor of float32 to indicate the per record weight. - - Returns: - A dict of metric names and values. - """ - metric_op_weighted = get_partial_multi_binary_class_metric_fn( - metrics, predcols=[0, 1], classes=classnames - ) - classnames_unweighted = ["unweighted_" + classname for classname in classnames] - metric_op_unweighted = get_partial_multi_binary_class_metric_fn( - metrics, predcols=[0, 1], classes=classnames_unweighted - ) - - valid_batch_size = graph_output["valid_batch_size"] - graph_output["output"] = remove_padding_and_flatten(graph_output["output"], valid_batch_size) - labels = remove_padding_and_flatten(labels, valid_batch_size) - weights = remove_padding_and_flatten(weights, valid_batch_size) - - tf.ensure_shape(graph_output["output"], [None, 2]) - tf.ensure_shape(labels, [None, 2]) - tf.ensure_shape(weights, [None, 1]) - - metrics_weighted = metric_op_weighted(graph_output, labels, weights) - metrics_unweighted = metric_op_unweighted(graph_output, labels, None) - metrics_weighted.update(metrics_unweighted) - - if use_top_k: - metric_op_numeric = get_numeric_metric_fn(metrics=None, topK=top_k, predcol=2, labelcol=2) - metrics_numeric = metric_op_numeric(graph_output, labels, weights) - metrics_weighted.update(metrics_numeric) - return metrics_weighted - - return get_eval_metric_ops - - -def get_metric_fn(task_name, use_stratify_metrics, use_meta_batch=False): - """Will retrieve the metric_fn for magic recs. - - Args: - task_name (string): Which task is being used for this model. - use_stratify_metrics (boolean): Should we add stratified metrics (new user metrics). - use_meta_batch (boolean): If the output/label/weights are passed in 3D shape instead of - 2D shape. - - Returns: - A metric_fn function to pass in twml Trainer. - """ - if task_name not in METRIC_BOOK: - raise ValueError( - "Task name of {task_name} not recognized. Unable to retrieve metrics.".format( - task_name=task_name - ) - ) - class_names = METRIC_BOOK[task_name] - if use_meta_batch: - get_n_binary_task_metric_fn = ( - get_meta_learn_single_binary_task_metric_fn - if len(class_names) == 1 - else get_meta_learn_dual_binary_tasks_metric_fn - ) - else: - get_n_binary_task_metric_fn = ( - get_single_binary_task_metric_fn if len(class_names) == 1 else get_dual_binary_tasks_metric_fn - ) - - metric_fn = get_n_binary_task_metric_fn(metrics=None, classnames=METRIC_BOOK[task_name]) - - if use_stratify_metrics: - metric_fn = add_new_user_metrics(metric_fn) - - return metric_fn - - -def flip_disliked_labels(metric_fn): - """This function returns an adapted metric_fn which flips the labels of the OONCed evaluation data to 0 if it is disliked. - Args: - metric_fn: A metric_fn function to pass in twml Trainer. - - Returns: - _adapted_metric_fn: A customized metric_fn function with disliked OONC labels flipped. - """ - - def _adapted_metric_fn(graph_output, labels, weights): - """A customized metric_fn function with disliked OONC labels flipped. - - Args: - graph_output: A dict of tensors. - labels: labels of training samples, which is a 2D tensor of shape batch_size x 3: [OONCs, engagements, dislikes] - weights: A tensor of float32 to indicate the per record weight. - - Returns: - A dict of metric names and values. - """ - # We want to multiply the label of the observation by 0 only when it is disliked - disliked_mask = generate_disliked_mask(labels) - - # Extract OONC and engagement labels only. - labels = tf.reshape(labels[:, 0:2], shape=[-1, 2]) - - # Labels will be set to 0 if it is disliked. - adapted_labels = labels * tf.cast(tf.logical_not(disliked_mask), dtype=labels.dtype) - - return metric_fn(graph_output, adapted_labels, weights) - - return _adapted_metric_fn diff --git a/pushservice/src/main/python/models/libs/model_args.docx b/pushservice/src/main/python/models/libs/model_args.docx new file mode 100644 index 000000000..7caacc506 Binary files /dev/null and b/pushservice/src/main/python/models/libs/model_args.docx differ diff --git a/pushservice/src/main/python/models/libs/model_args.py b/pushservice/src/main/python/models/libs/model_args.py deleted file mode 100644 index ae142d818..000000000 --- a/pushservice/src/main/python/models/libs/model_args.py +++ /dev/null @@ -1,231 +0,0 @@ -from twml.trainers import DataRecordTrainer - - -# checkstyle: noqa - - -def get_arg_parser(): - parser = DataRecordTrainer.add_parser_arguments() - - parser.add_argument( - "--input_size_bits", - type=int, - default=18, - help="number of bits allocated to the input size", - ) - parser.add_argument( - "--model_trainer_name", - default="magic_recs_mlp_calibration_MTL_OONC_Engagement", - type=str, - help="specify the model trainer name.", - ) - - parser.add_argument( - "--model_type", - default="deepnorm_gbdt_inputdrop2_rescale", - type=str, - help="specify the model type to use.", - ) - parser.add_argument( - "--feat_config_type", - default="get_feature_config_with_sparse_continuous", - type=str, - help="specify the feature configure function to use.", - ) - - parser.add_argument( - "--directly_export_best", - default=False, - action="store_true", - help="whether to directly_export best_checkpoint", - ) - - parser.add_argument( - "--warm_start_base_dir", - default="none", - type=str, - help="latest ckpt in this folder will be used to ", - ) - - parser.add_argument( - "--feature_list", - default="none", - type=str, - help="Which features to use for training", - ) - parser.add_argument( - "--warm_start_from", default=None, type=str, help="model dir to warm start from" - ) - - parser.add_argument( - "--momentum", default=0.99999, type=float, help="Momentum term for batch normalization" - ) - parser.add_argument( - "--dropout", - default=0.2, - type=float, - help="input_dropout_rate to rescale output by (1 - input_dropout_rate)", - ) - parser.add_argument( - "--out_layer_1_size", default=256, type=int, help="Size of MLP_branch layer 1" - ) - parser.add_argument( - "--out_layer_2_size", default=128, type=int, help="Size of MLP_branch layer 2" - ) - parser.add_argument("--out_layer_3_size", default=64, type=int, help="Size of MLP_branch layer 3") - parser.add_argument( - "--sparse_embedding_size", default=50, type=int, help="Dimensionality of sparse embedding layer" - ) - parser.add_argument( - "--dense_embedding_size", default=128, type=int, help="Dimensionality of dense embedding layer" - ) - - parser.add_argument( - "--use_uam_label", - default=False, - type=str, - help="Whether to use uam_label or not", - ) - - parser.add_argument( - "--task_name", - default="OONC_Engagement", - type=str, - help="specify the task name to use: OONC or OONC_Engagement.", - ) - parser.add_argument( - "--init_weight", - default=0.9, - type=float, - help="Initial OONC Task Weight MTL: OONC+Engagement.", - ) - parser.add_argument( - "--use_engagement_weight", - default=False, - action="store_true", - help="whether to use engagement weight for base model.", - ) - parser.add_argument( - "--mtl_num_extra_layers", - type=int, - default=1, - help="Number of Hidden Layers for each TaskBranch.", - ) - parser.add_argument( - "--mtl_neuron_scale", type=int, default=4, help="Scaling Factor of Neurons in MTL Extra Layers." - ) - parser.add_argument( - "--use_oonc_score", - default=False, - action="store_true", - help="whether to use oonc score only or combined score.", - ) - parser.add_argument( - "--use_stratified_metrics", - default=False, - action="store_true", - help="Use stratified metrics: Break out new-user metrics.", - ) - parser.add_argument( - "--run_group_metrics", - default=False, - action="store_true", - help="Will run evaluation metrics grouped by user.", - ) - parser.add_argument( - "--use_full_scope", - default=False, - action="store_true", - help="Will add extra scope and naming to graph.", - ) - parser.add_argument( - "--trainable_regexes", - default=None, - nargs="*", - help="The union of variables specified by the list of regexes will be considered trainable.", - ) - parser.add_argument( - "--fine_tuning.ckpt_to_initialize_from", - dest="fine_tuning_ckpt_to_initialize_from", - type=str, - default=None, - help="Checkpoint path from which to warm start. Indicates the pre-trained model.", - ) - parser.add_argument( - "--fine_tuning.warm_start_scope_regex", - dest="fine_tuning_warm_start_scope_regex", - type=str, - default=None, - help="All variables matching this will be restored.", - ) - - return parser - - -def get_params(args=None): - parser = get_arg_parser() - if args is None: - return parser.parse_args() - else: - return parser.parse_args(args) - - -def get_arg_parser_light_ranking(): - parser = get_arg_parser() - - parser.add_argument( - "--use_record_weight", - default=False, - action="store_true", - help="whether to use record weight for base model.", - ) - parser.add_argument( - "--min_record_weight", default=0.0, type=float, help="Minimum record weight to use." - ) - parser.add_argument( - "--smooth_weight", default=0.0, type=float, help="Factor to smooth Rank Position Weight." - ) - - parser.add_argument( - "--num_mlp_layers", type=int, default=3, help="Number of Hidden Layers for MLP model." - ) - parser.add_argument( - "--mlp_neuron_scale", type=int, default=4, help="Scaling Factor of Neurons in MLP Layers." - ) - parser.add_argument( - "--run_light_ranking_group_metrics", - default=False, - action="store_true", - help="Will run evaluation metrics grouped by user for Light Ranking.", - ) - parser.add_argument( - "--use_missing_sub_branch", - default=False, - action="store_true", - help="Whether to use missing value sub-branch for Light Ranking.", - ) - parser.add_argument( - "--use_gbdt_features", - default=False, - action="store_true", - help="Whether to use GBDT features for Light Ranking.", - ) - parser.add_argument( - "--run_light_ranking_group_metrics_in_bq", - default=False, - action="store_true", - help="Whether to get_predictions for Light Ranking to compute group metrics in BigQuery.", - ) - parser.add_argument( - "--pred_file_path", - default=None, - type=str, - help="path", - ) - parser.add_argument( - "--pred_file_name", - default=None, - type=str, - help="path", - ) - return parser diff --git a/pushservice/src/main/python/models/libs/model_utils.docx b/pushservice/src/main/python/models/libs/model_utils.docx new file mode 100644 index 000000000..8fc1007f4 Binary files /dev/null and b/pushservice/src/main/python/models/libs/model_utils.docx differ diff --git a/pushservice/src/main/python/models/libs/model_utils.py b/pushservice/src/main/python/models/libs/model_utils.py deleted file mode 100644 index 1c5306911..000000000 --- a/pushservice/src/main/python/models/libs/model_utils.py +++ /dev/null @@ -1,339 +0,0 @@ -import sys - -import twml - -from .initializer import customized_glorot_uniform - -import tensorflow.compat.v1 as tf -import yaml - - -# checkstyle: noqa - - -def read_config(whitelist_yaml_file): - with tf.gfile.FastGFile(whitelist_yaml_file) as f: - try: - return yaml.safe_load(f) - except yaml.YAMLError as exc: - print(exc) - sys.exit(1) - - -def _sparse_feature_fixup(features, input_size_bits): - """Rebuild a sparse tensor feature so that its dense shape attribute is present. - - Arguments: - features (SparseTensor): Sparse feature tensor of shape ``(B, sparse_feature_dim)``. - input_size_bits (int): Number of columns in ``log2`` scale. Must be positive. - - Returns: - SparseTensor: Rebuilt and non-faulty version of `features`.""" - sparse_feature_dim = tf.constant(2**input_size_bits, dtype=tf.int64) - sparse_shape = tf.stack([features.dense_shape[0], sparse_feature_dim]) - sparse_tf = tf.SparseTensor(features.indices, features.values, sparse_shape) - return sparse_tf - - -def self_atten_dense(input, out_dim, activation=None, use_bias=True, name=None): - def safe_concat(base, suffix): - """Concats variables name components if base is given.""" - if not base: - return base - return f"{base}:{suffix}" - - input_dim = input.shape.as_list()[1] - - sigmoid_out = twml.layers.FullDense( - input_dim, dtype=tf.float32, activation=tf.nn.sigmoid, name=safe_concat(name, "sigmoid_out") - )(input) - atten_input = sigmoid_out * input - mlp_out = twml.layers.FullDense( - out_dim, - dtype=tf.float32, - activation=activation, - use_bias=use_bias, - name=safe_concat(name, "mlp_out"), - )(atten_input) - return mlp_out - - -def get_dense_out(input, out_dim, activation, dense_type): - if dense_type == "full_dense": - out = twml.layers.FullDense(out_dim, dtype=tf.float32, activation=activation)(input) - elif dense_type == "self_atten_dense": - out = self_atten_dense(input, out_dim, activation=activation) - return out - - -def get_input_trans_func(bn_normalized_dense, is_training): - gw_normalized_dense = tf.expand_dims(bn_normalized_dense, -1) - group_num = bn_normalized_dense.shape.as_list()[1] - - gw_normalized_dense = GroupWiseTrans(group_num, 1, 8, name="groupwise_1", activation=tf.tanh)( - gw_normalized_dense - ) - gw_normalized_dense = GroupWiseTrans(group_num, 8, 4, name="groupwise_2", activation=tf.tanh)( - gw_normalized_dense - ) - gw_normalized_dense = GroupWiseTrans(group_num, 4, 1, name="groupwise_3", activation=tf.tanh)( - gw_normalized_dense - ) - - gw_normalized_dense = tf.squeeze(gw_normalized_dense, [-1]) - - bn_gw_normalized_dense = tf.layers.batch_normalization( - gw_normalized_dense, - training=is_training, - renorm_momentum=0.9999, - momentum=0.9999, - renorm=is_training, - trainable=True, - ) - - return bn_gw_normalized_dense - - -def tensor_dropout( - input_tensor, - rate, - is_training, - sparse_tensor=None, -): - """ - Implements dropout layer for both dense and sparse input_tensor - - Arguments: - input_tensor: - B x D dense tensor, or a sparse tensor - rate (float32): - dropout rate - is_training (bool): - training stage or not. - sparse_tensor (bool): - whether the input_tensor is sparse tensor or not. Default to be None, this value has to be passed explicitly. - rescale_sparse_dropout (bool): - Do we need to do rescaling or not. - Returns: - tensor dropped out""" - if sparse_tensor == True: - if is_training: - with tf.variable_scope("sparse_dropout"): - values = input_tensor.values - keep_mask = tf.keras.backend.random_binomial( - tf.shape(values), p=1 - rate, dtype=tf.float32, seed=None - ) - keep_mask.set_shape([None]) - keep_mask = tf.cast(keep_mask, tf.bool) - - keep_indices = tf.boolean_mask(input_tensor.indices, keep_mask, axis=0) - keep_values = tf.boolean_mask(values, keep_mask, axis=0) - - dropped_tensor = tf.SparseTensor(keep_indices, keep_values, input_tensor.dense_shape) - return dropped_tensor - else: - return input_tensor - elif sparse_tensor == False: - return tf.layers.dropout(input_tensor, rate=rate, training=is_training) - - -def adaptive_transformation(bn_normalized_dense, is_training, func_type="default"): - assert func_type in [ - "default", - "tiny", - ], f"fun_type can only be one of default and tiny, but get {func_type}" - - gw_normalized_dense = tf.expand_dims(bn_normalized_dense, -1) - group_num = bn_normalized_dense.shape.as_list()[1] - - if func_type == "default": - gw_normalized_dense = FastGroupWiseTrans( - group_num, 1, 8, name="groupwise_1", activation=tf.tanh, init_multiplier=8 - )(gw_normalized_dense) - - gw_normalized_dense = FastGroupWiseTrans( - group_num, 8, 4, name="groupwise_2", activation=tf.tanh, init_multiplier=8 - )(gw_normalized_dense) - - gw_normalized_dense = FastGroupWiseTrans( - group_num, 4, 1, name="groupwise_3", activation=tf.tanh, init_multiplier=8 - )(gw_normalized_dense) - elif func_type == "tiny": - gw_normalized_dense = FastGroupWiseTrans( - group_num, 1, 2, name="groupwise_1", activation=tf.tanh, init_multiplier=8 - )(gw_normalized_dense) - - gw_normalized_dense = FastGroupWiseTrans( - group_num, 2, 1, name="groupwise_2", activation=tf.tanh, init_multiplier=8 - )(gw_normalized_dense) - - gw_normalized_dense = FastGroupWiseTrans( - group_num, 1, 1, name="groupwise_3", activation=tf.tanh, init_multiplier=8 - )(gw_normalized_dense) - - gw_normalized_dense = tf.squeeze(gw_normalized_dense, [-1]) - bn_gw_normalized_dense = tf.layers.batch_normalization( - gw_normalized_dense, - training=is_training, - renorm_momentum=0.9999, - momentum=0.9999, - renorm=is_training, - trainable=True, - ) - - return bn_gw_normalized_dense - - -class FastGroupWiseTrans(object): - """ - used to apply group-wise fully connected layers to the input. - it applies a tiny, unique MLP to each individual feature.""" - - def __init__(self, group_num, input_dim, out_dim, name, activation=None, init_multiplier=1): - self.group_num = group_num - self.input_dim = input_dim - self.out_dim = out_dim - self.activation = activation - self.init_multiplier = init_multiplier - - self.w = tf.get_variable( - name + "_group_weight", - [1, group_num, input_dim, out_dim], - initializer=customized_glorot_uniform( - fan_in=input_dim * init_multiplier, fan_out=out_dim * init_multiplier - ), - trainable=True, - ) - self.b = tf.get_variable( - name + "_group_bias", - [1, group_num, out_dim], - initializer=tf.constant_initializer(0.0), - trainable=True, - ) - - def __call__(self, input_tensor): - """ - input_tensor: batch_size x group_num x input_dim - output_tensor: batch_size x group_num x out_dim""" - input_tensor_expand = tf.expand_dims(input_tensor, axis=-1) - - output_tensor = tf.add( - tf.reduce_sum(tf.multiply(input_tensor_expand, self.w), axis=-2, keepdims=False), - self.b, - ) - - if self.activation is not None: - output_tensor = self.activation(output_tensor) - return output_tensor - - -class GroupWiseTrans(object): - """ - Used to apply group fully connected layers to the input. - """ - - def __init__(self, group_num, input_dim, out_dim, name, activation=None): - self.group_num = group_num - self.input_dim = input_dim - self.out_dim = out_dim - self.activation = activation - - w_list, b_list = [], [] - for idx in range(out_dim): - this_w = tf.get_variable( - name + f"_group_weight_{idx}", - [1, group_num, input_dim], - initializer=tf.keras.initializers.glorot_uniform(), - trainable=True, - ) - this_b = tf.get_variable( - name + f"_group_bias_{idx}", - [1, group_num, 1], - initializer=tf.constant_initializer(0.0), - trainable=True, - ) - w_list.append(this_w) - b_list.append(this_b) - self.w_list = w_list - self.b_list = b_list - - def __call__(self, input_tensor): - """ - input_tensor: batch_size x group_num x input_dim - output_tensor: batch_size x group_num x out_dim - """ - out_tensor_list = [] - for idx in range(self.out_dim): - this_res = ( - tf.reduce_sum(input_tensor * self.w_list[idx], axis=-1, keepdims=True) + self.b_list[idx] - ) - out_tensor_list.append(this_res) - output_tensor = tf.concat(out_tensor_list, axis=-1) - - if self.activation is not None: - output_tensor = self.activation(output_tensor) - return output_tensor - - -def add_scalar_summary(var, name, name_scope="hist_dense_feature/"): - with tf.name_scope("summaries/"): - with tf.name_scope(name_scope): - tf.summary.scalar(name, var) - - -def add_histogram_summary(var, name, name_scope="hist_dense_feature/"): - with tf.name_scope("summaries/"): - with tf.name_scope(name_scope): - tf.summary.histogram(name, tf.reshape(var, [-1])) - - -def sparse_clip_by_value(sparse_tf, min_val, max_val): - new_vals = tf.clip_by_value(sparse_tf.values, min_val, max_val) - return tf.SparseTensor(sparse_tf.indices, new_vals, sparse_tf.dense_shape) - - -def check_numerics_with_msg(tensor, message="", sparse_tensor=False): - if sparse_tensor: - values = tf.debugging.check_numerics(tensor.values, message=message) - return tf.SparseTensor(tensor.indices, values, tensor.dense_shape) - else: - return tf.debugging.check_numerics(tensor, message=message) - - -def pad_empty_sparse_tensor(tensor): - dummy_tensor = tf.SparseTensor( - indices=[[0, 0]], - values=[0.00001], - dense_shape=tensor.dense_shape, - ) - result = tf.cond( - tf.equal(tf.size(tensor.values), 0), - lambda: dummy_tensor, - lambda: tensor, - ) - return result - - -def filter_nans_and_infs(tensor, sparse_tensor=False): - if sparse_tensor: - sparse_values = tensor.values - filtered_val = tf.where( - tf.logical_or(tf.is_nan(sparse_values), tf.is_inf(sparse_values)), - tf.zeros_like(sparse_values), - sparse_values, - ) - return tf.SparseTensor(tensor.indices, filtered_val, tensor.dense_shape) - else: - return tf.where( - tf.logical_or(tf.is_nan(tensor), tf.is_inf(tensor)), tf.zeros_like(tensor), tensor - ) - - -def generate_disliked_mask(labels): - """Generate a disliked mask where only samples with dislike labels are set to 1 otherwise set to 0. - Args: - labels: labels of training samples, which is a 2D tensor of shape batch_size x 3: [OONCs, engagements, dislikes] - Returns: - 1D tensor of shape batch_size x 1: [dislikes (booleans)] - """ - return tf.equal(tf.reshape(labels[:, 2], shape=[-1, 1]), 1) diff --git a/pushservice/src/main/python/models/libs/warm_start_utils.docx b/pushservice/src/main/python/models/libs/warm_start_utils.docx new file mode 100644 index 000000000..07ea51cda Binary files /dev/null and b/pushservice/src/main/python/models/libs/warm_start_utils.docx differ diff --git a/pushservice/src/main/python/models/libs/warm_start_utils.py b/pushservice/src/main/python/models/libs/warm_start_utils.py deleted file mode 100644 index ca83df585..000000000 --- a/pushservice/src/main/python/models/libs/warm_start_utils.py +++ /dev/null @@ -1,309 +0,0 @@ -from collections import OrderedDict -import json -import os -from os.path import join - -from twitter.magicpony.common import file_access -import twml - -from .model_utils import read_config - -import numpy as np -from scipy import stats -import tensorflow.compat.v1 as tf - - -# checkstyle: noqa - - -def get_model_type_to_tensors_to_change_axis(): - model_type_to_tensors_to_change_axis = { - "magic_recs/model/batch_normalization/beta": ([0], "continuous"), - "magic_recs/model/batch_normalization/gamma": ([0], "continuous"), - "magic_recs/model/batch_normalization/moving_mean": ([0], "continuous"), - "magic_recs/model/batch_normalization/moving_stddev": ([0], "continuous"), - "magic_recs/model/batch_normalization/moving_variance": ([0], "continuous"), - "magic_recs/model/batch_normalization/renorm_mean": ([0], "continuous"), - "magic_recs/model/batch_normalization/renorm_stddev": ([0], "continuous"), - "magic_recs/model/logits/EngagementGivenOONC_logits/clem_net_1/block2_4/channel_wise_dense_4/kernel": ( - [1], - "all", - ), - "magic_recs/model/logits/OONC_logits/clem_net/block2/channel_wise_dense/kernel": ([1], "all"), - } - - return model_type_to_tensors_to_change_axis - - -def mkdirp(dirname): - if not tf.io.gfile.exists(dirname): - tf.io.gfile.makedirs(dirname) - - -def rename_dir(dirname, dst): - file_access.hdfs.mv(dirname, dst) - - -def rmdir(dirname): - if tf.io.gfile.exists(dirname): - if tf.io.gfile.isdir(dirname): - tf.io.gfile.rmtree(dirname) - else: - tf.io.gfile.remove(dirname) - - -def get_var_dict(checkpoint_path): - checkpoint = tf.train.get_checkpoint_state(checkpoint_path) - var_dict = OrderedDict() - with tf.Session() as sess: - all_var_list = tf.train.list_variables(checkpoint_path) - for var_name, _ in all_var_list: - # Load the variable - var = tf.train.load_variable(checkpoint_path, var_name) - var_dict[var_name] = var - return var_dict - - -def get_continunous_mapping_from_feat_list(old_feature_list, new_feature_list): - """ - get var_ind for old_feature and corresponding var_ind for new_feature - """ - new_var_ind, old_var_ind = [], [] - for this_new_id, this_new_name in enumerate(new_feature_list): - if this_new_name in old_feature_list: - this_old_id = old_feature_list.index(this_new_name) - new_var_ind.append(this_new_id) - old_var_ind.append(this_old_id) - return np.asarray(old_var_ind), np.asarray(new_var_ind) - - -def get_continuous_mapping_from_feat_dict(old_feature_dict, new_feature_dict): - """ - get var_ind for old_feature and corresponding var_ind for new_feature - """ - old_cont = old_feature_dict["continuous"] - old_bin = old_feature_dict["binary"] - - new_cont = new_feature_dict["continuous"] - new_bin = new_feature_dict["binary"] - - _dummy_sparse_feat = [f"sparse_feature_{_idx}" for _idx in range(100)] - - cont_old_var_ind, cont_new_var_ind = get_continunous_mapping_from_feat_list(old_cont, new_cont) - - all_old_var_ind, all_new_var_ind = get_continunous_mapping_from_feat_list( - old_cont + old_bin + _dummy_sparse_feat, new_cont + new_bin + _dummy_sparse_feat - ) - - _res = { - "continuous": (cont_old_var_ind, cont_new_var_ind), - "all": (all_old_var_ind, all_new_var_ind), - } - - return _res - - -def warm_start_from_var_dict( - old_ckpt_path, - var_ind_dict, - output_dir, - new_len_var, - var_to_change_dict_fn=get_model_type_to_tensors_to_change_axis, -): - """ - Parameters: - old_ckpt_path (str): path to the old checkpoint path - new_var_ind (array of int): index to overlapping features in new var between old and new feature list. - old_var_ind (array of int): index to overlapping features in old var between old and new feature list. - - output_dir (str): dir that used to write modified checkpoint - new_len_var ({str:int}): number of feature in the new feature list. - var_to_change_dict_fn (dict): A function to get the dictionary of format {var_name: dim_to_change} - """ - old_var_dict = get_var_dict(old_ckpt_path) - - ckpt_file_name = os.path.basename(old_ckpt_path) - mkdirp(output_dir) - output_path = join(output_dir, ckpt_file_name) - - tensors_to_change = var_to_change_dict_fn() - tf.compat.v1.reset_default_graph() - - with tf.Session() as sess: - var_name_shape_list = tf.train.list_variables(old_ckpt_path) - count = 0 - - for var_name, var_shape in var_name_shape_list: - old_var = old_var_dict[var_name] - if var_name in tensors_to_change.keys(): - _info_tuple = tensors_to_change[var_name] - dims_to_remove_from, var_type = _info_tuple - - new_var_ind, old_var_ind = var_ind_dict[var_type] - - this_shape = list(old_var.shape) - for this_dim in dims_to_remove_from: - this_shape[this_dim] = new_len_var[var_type] - - stddev = np.std(old_var) - truncated_norm_generator = stats.truncnorm(-0.5, 0.5, loc=0, scale=stddev) - size = np.prod(this_shape) - new_var = truncated_norm_generator.rvs(size).reshape(this_shape) - new_var = new_var.astype(old_var.dtype) - - new_var = copy_feat_based_on_mapping( - new_var, old_var, dims_to_remove_from, new_var_ind, old_var_ind - ) - count = count + 1 - else: - new_var = old_var - var = tf.Variable(new_var, name=var_name) - assert count == len(tensors_to_change.keys()), "not all variables are exchanged.\n" - saver = tf.train.Saver() - sess.run(tf.global_variables_initializer()) - saver.save(sess, output_path) - return output_path - - -def copy_feat_based_on_mapping(new_array, old_array, dims_to_remove_from, new_var_ind, old_var_ind): - if dims_to_remove_from == [0, 1]: - for this_new_ind, this_old_ind in zip(new_var_ind, old_var_ind): - new_array[this_new_ind, new_var_ind] = old_array[this_old_ind, old_var_ind] - elif dims_to_remove_from == [0]: - new_array[new_var_ind] = old_array[old_var_ind] - elif dims_to_remove_from == [1]: - new_array[:, new_var_ind] = old_array[:, old_var_ind] - else: - raise RuntimeError(f"undefined dims_to_remove_from pattern: ({dims_to_remove_from})") - return new_array - - -def read_file(filename, decode=False): - """ - Reads contents from a file and optionally decodes it. - - Arguments: - filename: - path to file where the contents will be loaded from. - Accepts HDFS and local paths. - decode: - False or 'json'. When decode='json', contents is decoded - with json.loads. When False, contents is returned as is. - """ - graph = tf.Graph() - with graph.as_default(): - read = tf.read_file(filename) - - with tf.Session(graph=graph) as sess: - contents = sess.run(read) - if not isinstance(contents, str): - contents = contents.decode() - - if decode == "json": - contents = json.loads(contents) - - return contents - - -def read_feat_list_from_disk(file_path): - return read_file(file_path, decode="json") - - -def get_feature_list_for_light_ranking(feature_list_path, data_spec_path): - feature_list = read_config(feature_list_path).items() - string_feat_list = [f[0] for f in feature_list if f[1] != "S"] - - feature_config_builder = twml.contrib.feature_config.FeatureConfigBuilder( - data_spec_path=data_spec_path - ) - feature_config_builder = feature_config_builder.extract_feature_group( - feature_regexes=string_feat_list, - group_name="continuous", - default_value=-1, - type_filter=["CONTINUOUS"], - ) - feature_config = feature_config_builder.build() - feature_list = feature_config_builder._feature_group_extraction_configs[0].feature_map[ - "CONTINUOUS" - ] - return feature_list - - -def get_feature_list_for_heavy_ranking(feature_list_path, data_spec_path): - feature_list = read_config(feature_list_path).items() - string_feat_list = [f[0] for f in feature_list if f[1] != "S"] - - feature_config_builder = twml.contrib.feature_config.FeatureConfigBuilder( - data_spec_path=data_spec_path - ) - feature_config_builder = feature_config_builder.extract_feature_group( - feature_regexes=string_feat_list, - group_name="continuous", - default_value=-1, - type_filter=["CONTINUOUS"], - ) - - feature_config_builder = feature_config_builder.extract_feature_group( - feature_regexes=string_feat_list, - group_name="binary", - default_value=False, - type_filter=["BINARY"], - ) - - feature_config_builder = feature_config_builder.build() - - continuous_feature_list = feature_config_builder._feature_group_extraction_configs[0].feature_map[ - "CONTINUOUS" - ] - - binary_feature_list = feature_config_builder._feature_group_extraction_configs[1].feature_map[ - "BINARY" - ] - return {"continuous": continuous_feature_list, "binary": binary_feature_list} - - -def warm_start_checkpoint( - old_best_ckpt_folder, - old_feature_list_path, - feature_allow_list_path, - data_spec_path, - output_ckpt_folder, - *args, -): - """ - Reads old checkpoint and the old feature list, and create a new ckpt warm started from old ckpt using new features . - - Arguments: - old_best_ckpt_folder: - path to the best_checkpoint_folder for old model - old_feature_list_path: - path to the json file that stores the list of continuous features used in old models. - feature_allow_list_path: - yaml file that contain the feature allow list. - data_spec_path: - path to the data_spec file - output_ckpt_folder: - folder that contains the modified ckpt. - - Returns: - path to the modified ckpt.""" - old_ckpt_path = tf.train.latest_checkpoint(old_best_ckpt_folder, latest_filename=None) - - new_feature_dict = get_feature_list(feature_allow_list_path, data_spec_path) - old_feature_dict = read_feat_list_from_disk(old_feature_list_path) - - var_ind_dict = get_continuous_mapping_from_feat_dict(new_feature_dict, old_feature_dict) - - new_len_var = { - "continuous": len(new_feature_dict["continuous"]), - "all": len(new_feature_dict["continuous"] + new_feature_dict["binary"]) + 100, - } - - warm_started_ckpt_path = warm_start_from_var_dict( - old_ckpt_path, - var_ind_dict, - output_dir=output_ckpt_folder, - new_len_var=new_len_var, - ) - - return warm_started_ckpt_path diff --git a/pushservice/src/main/python/models/light_ranking/BUILD b/pushservice/src/main/python/models/light_ranking/BUILD deleted file mode 100644 index e88d7de7c..000000000 --- a/pushservice/src/main/python/models/light_ranking/BUILD +++ /dev/null @@ -1,69 +0,0 @@ -#":mlwf_libs", - -python37_binary( - name = "eval_model", - source = "eval_model.py", - dependencies = [ - ":libs", - "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/light_ranking:eval_model", - ], -) - -python37_binary( - name = "train_model", - source = "deep_norm.py", - dependencies = [ - ":libs", - "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/light_ranking:train_model", - ], -) - -python37_binary( - name = "train_model_local", - source = "deep_norm.py", - dependencies = [ - ":libs", - "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/light_ranking:train_model_local", - "twml", - ], -) - -python37_binary( - name = "eval_model_local", - source = "eval_model.py", - dependencies = [ - ":libs", - "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/light_ranking:eval_model_local", - "twml", - ], -) - -python37_binary( - name = "mlwf_model", - source = "deep_norm.py", - dependencies = [ - ":mlwf_libs", - "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/light_ranking:mlwf_model", - ], -) - -python3_library( - name = "libs", - sources = ["**/*.py"], - tags = ["no-mypy"], - dependencies = [ - "src/python/twitter/deepbird/projects/magic_recs/libs", - "src/python/twitter/deepbird/util/data", - "twml:twml-nodeps", - ], -) - -python3_library( - name = "mlwf_libs", - sources = ["**/*.py"], - tags = ["no-mypy"], - dependencies = [ - "src/python/twitter/deepbird/projects/magic_recs/libs", - "twml", - ], -) diff --git a/pushservice/src/main/python/models/light_ranking/BUILD.docx b/pushservice/src/main/python/models/light_ranking/BUILD.docx new file mode 100644 index 000000000..827c80851 Binary files /dev/null and b/pushservice/src/main/python/models/light_ranking/BUILD.docx differ diff --git a/pushservice/src/main/python/models/light_ranking/README.docx b/pushservice/src/main/python/models/light_ranking/README.docx new file mode 100644 index 000000000..f0de667ec Binary files /dev/null and b/pushservice/src/main/python/models/light_ranking/README.docx differ diff --git a/pushservice/src/main/python/models/light_ranking/README.md b/pushservice/src/main/python/models/light_ranking/README.md deleted file mode 100644 index 9d7bd2682..000000000 --- a/pushservice/src/main/python/models/light_ranking/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# Notification Light Ranker Model - -## Model Context -There are 4 major components of Twitter notifications recommendation system: 1) candidate generation 2) light ranking 3) heavy ranking & 4) quality control. This notification light ranker model bridges candidate generation and heavy ranking by pre-selecting highly-relevant candidates from the initial huge candidate pool. It’s a light-weight model to reduce system cost during heavy ranking without hurting user experience. - -## Directory Structure -- BUILD: this file defines python library dependencies -- model_pools_mlp.py: this file defines tensorflow model architecture for the notification light ranker model -- deep_norm.py: this file contains 1) how to build the tensorflow graph with specified model architecture, loss function and training configuration. 2) how to set up the overall model training & evaluation pipeline -- eval_model.py: the main python entry file to set up the overall model evaluation pipeline - - - - diff --git a/pushservice/src/main/python/models/light_ranking/__init__.docx b/pushservice/src/main/python/models/light_ranking/__init__.docx new file mode 100644 index 000000000..3e4a34cdb Binary files /dev/null and b/pushservice/src/main/python/models/light_ranking/__init__.docx differ diff --git a/pushservice/src/main/python/models/light_ranking/__init__.py b/pushservice/src/main/python/models/light_ranking/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/pushservice/src/main/python/models/light_ranking/deep_norm.docx b/pushservice/src/main/python/models/light_ranking/deep_norm.docx new file mode 100644 index 000000000..1c07907ab Binary files /dev/null and b/pushservice/src/main/python/models/light_ranking/deep_norm.docx differ diff --git a/pushservice/src/main/python/models/light_ranking/deep_norm.py b/pushservice/src/main/python/models/light_ranking/deep_norm.py deleted file mode 100644 index bc90deba4..000000000 --- a/pushservice/src/main/python/models/light_ranking/deep_norm.py +++ /dev/null @@ -1,226 +0,0 @@ -from datetime import datetime -from functools import partial -import os - -from twitter.cortex.ml.embeddings.common.helpers import decode_str_or_unicode -import twml -from twml.trainers import DataRecordTrainer - -from ..libs.get_feat_config import get_feature_config_light_ranking, LABELS_LR -from ..libs.graph_utils import get_trainable_variables -from ..libs.group_metrics import ( - run_group_metrics_light_ranking, - run_group_metrics_light_ranking_in_bq, -) -from ..libs.metric_fn_utils import get_metric_fn -from ..libs.model_args import get_arg_parser_light_ranking -from ..libs.model_utils import read_config -from ..libs.warm_start_utils import get_feature_list_for_light_ranking -from .model_pools_mlp import light_ranking_mlp_ngbdt - -import tensorflow.compat.v1 as tf -from tensorflow.compat.v1 import logging - - -# checkstyle: noqa - - -def build_graph( - features, label, mode, params, config=None, run_light_ranking_group_metrics_in_bq=False -): - is_training = mode == tf.estimator.ModeKeys.TRAIN - this_model_func = light_ranking_mlp_ngbdt - model_output = this_model_func(features, is_training, params, label) - - logits = model_output["output"] - graph_output = {} - # -------------------------------------------------------- - # define graph output dict - # -------------------------------------------------------- - if mode == tf.estimator.ModeKeys.PREDICT: - loss = None - output_label = "prediction" - if params.task_name in LABELS_LR: - output = tf.nn.sigmoid(logits) - output = tf.clip_by_value(output, 0, 1) - - if run_light_ranking_group_metrics_in_bq: - graph_output["trace_id"] = features["meta.trace_id"] - graph_output["target"] = features["meta.ranking.weighted_oonc_model_score"] - - else: - raise ValueError("Invalid Task Name !") - - else: - output_label = "output" - weights = tf.cast(features["weights"], dtype=tf.float32, name="RecordWeights") - - if params.task_name in LABELS_LR: - if params.use_record_weight: - weights = tf.clip_by_value( - 1.0 / (1.0 + weights + params.smooth_weight), params.min_record_weight, 1.0 - ) - - loss = tf.reduce_sum( - tf.nn.sigmoid_cross_entropy_with_logits(labels=label, logits=logits) * weights - ) / (tf.reduce_sum(weights)) - else: - loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=label, logits=logits)) - output = tf.nn.sigmoid(logits) - - else: - raise ValueError("Invalid Task Name !") - - train_op = None - if mode == tf.estimator.ModeKeys.TRAIN: - # -------------------------------------------------------- - # get train_op - # -------------------------------------------------------- - optimizer = tf.train.GradientDescentOptimizer(learning_rate=params.learning_rate) - update_ops = set(tf.get_collection(tf.GraphKeys.UPDATE_OPS)) - variables = get_trainable_variables( - all_trainable_variables=tf.trainable_variables(), trainable_regexes=params.trainable_regexes - ) - with tf.control_dependencies(update_ops): - train_op = twml.optimizers.optimize_loss( - loss=loss, - variables=variables, - global_step=tf.train.get_global_step(), - optimizer=optimizer, - learning_rate=params.learning_rate, - learning_rate_decay_fn=twml.learning_rate_decay.get_learning_rate_decay_fn(params), - ) - - graph_output[output_label] = output - graph_output["loss"] = loss - graph_output["train_op"] = train_op - return graph_output - - -def get_params(args=None): - parser = get_arg_parser_light_ranking() - if args is None: - return parser.parse_args() - else: - return parser.parse_args(args) - - -def _main(): - opt = get_params() - logging.info("parse is: ") - logging.info(opt) - - feature_list = read_config(opt.feature_list).items() - feature_config = get_feature_config_light_ranking( - data_spec_path=opt.data_spec, - feature_list_provided=feature_list, - opt=opt, - add_gbdt=opt.use_gbdt_features, - run_light_ranking_group_metrics_in_bq=opt.run_light_ranking_group_metrics_in_bq, - ) - feature_list_path = opt.feature_list - - # -------------------------------------------------------- - # Create Trainer - # -------------------------------------------------------- - trainer = DataRecordTrainer( - name=opt.model_trainer_name, - params=opt, - build_graph_fn=build_graph, - save_dir=opt.save_dir, - run_config=None, - feature_config=feature_config, - metric_fn=get_metric_fn(opt.task_name, use_stratify_metrics=False), - ) - if opt.directly_export_best: - logging.info("Directly exporting the model without training") - else: - # ---------------------------------------------------- - # Model Training & Evaluation - # ---------------------------------------------------- - eval_input_fn = trainer.get_eval_input_fn(repeat=False, shuffle=False) - train_input_fn = trainer.get_train_input_fn(shuffle=True) - - if opt.distributed or opt.num_workers is not None: - learn = trainer.train_and_evaluate - else: - learn = trainer.learn - logging.info("Training...") - start = datetime.now() - - early_stop_metric = "rce_unweighted_" + opt.task_name - learn( - early_stop_minimize=False, - early_stop_metric=early_stop_metric, - early_stop_patience=opt.early_stop_patience, - early_stop_tolerance=opt.early_stop_tolerance, - eval_input_fn=eval_input_fn, - train_input_fn=train_input_fn, - ) - - end = datetime.now() - logging.info("Training time: " + str(end - start)) - - logging.info("Exporting the models...") - - # -------------------------------------------------------- - # Do the model exporting - # -------------------------------------------------------- - start = datetime.now() - if not opt.export_dir: - opt.export_dir = os.path.join(opt.save_dir, "exported_models") - - raw_model_path = twml.contrib.export.export_fn.export_all_models( - trainer=trainer, - export_dir=opt.export_dir, - parse_fn=feature_config.get_parse_fn(), - serving_input_receiver_fn=feature_config.get_serving_input_receiver_fn(), - export_output_fn=twml.export_output_fns.batch_prediction_continuous_output_fn, - ) - export_model_dir = decode_str_or_unicode(raw_model_path) - - logging.info("Model export time: " + str(datetime.now() - start)) - logging.info("The saved model directory is: " + opt.save_dir) - - tf.logging.info("getting default continuous_feature_list") - continuous_feature_list = get_feature_list_for_light_ranking(feature_list_path, opt.data_spec) - continous_feature_list_save_path = os.path.join(opt.save_dir, "continuous_feature_list.json") - twml.util.write_file(continous_feature_list_save_path, continuous_feature_list, encode="json") - tf.logging.info(f"Finish writting files to {continous_feature_list_save_path}") - - if opt.run_light_ranking_group_metrics: - # -------------------------------------------- - # Run Light Ranking Group Metrics - # -------------------------------------------- - run_group_metrics_light_ranking( - trainer=trainer, - data_dir=os.path.join(opt.eval_data_dir, opt.eval_start_datetime), - model_path=export_model_dir, - parse_fn=feature_config.get_parse_fn(), - ) - - if opt.run_light_ranking_group_metrics_in_bq: - # ---------------------------------------------------------------------------------------- - # Get Light/Heavy Ranker Predictions for Light Ranking Group Metrics in BigQuery - # ---------------------------------------------------------------------------------------- - trainer_pred = DataRecordTrainer( - name=opt.model_trainer_name, - params=opt, - build_graph_fn=partial(build_graph, run_light_ranking_group_metrics_in_bq=True), - save_dir=opt.save_dir + "/tmp/", - run_config=None, - feature_config=feature_config, - metric_fn=get_metric_fn(opt.task_name, use_stratify_metrics=False), - ) - checkpoint_folder = os.path.join(opt.save_dir, "best_checkpoint") - checkpoint = tf.train.latest_checkpoint(checkpoint_folder, latest_filename=None) - tf.logging.info("\n\nPrediction from Checkpoint: {:}.\n\n".format(checkpoint)) - run_group_metrics_light_ranking_in_bq( - trainer=trainer_pred, params=opt, checkpoint_path=checkpoint - ) - - tf.logging.info("Done Training & Prediction.") - - -if __name__ == "__main__": - _main() diff --git a/pushservice/src/main/python/models/light_ranking/eval_model.docx b/pushservice/src/main/python/models/light_ranking/eval_model.docx new file mode 100644 index 000000000..596353c91 Binary files /dev/null and b/pushservice/src/main/python/models/light_ranking/eval_model.docx differ diff --git a/pushservice/src/main/python/models/light_ranking/eval_model.py b/pushservice/src/main/python/models/light_ranking/eval_model.py deleted file mode 100644 index 1726685cf..000000000 --- a/pushservice/src/main/python/models/light_ranking/eval_model.py +++ /dev/null @@ -1,89 +0,0 @@ -from datetime import datetime -from functools import partial -import os - -from ..libs.group_metrics import ( - run_group_metrics_light_ranking, - run_group_metrics_light_ranking_in_bq, -) -from ..libs.metric_fn_utils import get_metric_fn -from ..libs.model_args import get_arg_parser_light_ranking -from ..libs.model_utils import read_config -from .deep_norm import build_graph, DataRecordTrainer, get_config_func, logging - - -# checkstyle: noqa - -if __name__ == "__main__": - parser = get_arg_parser_light_ranking() - parser.add_argument( - "--eval_checkpoint", - default=None, - type=str, - help="Which checkpoint to use for evaluation", - ) - parser.add_argument( - "--saved_model_path", - default=None, - type=str, - help="Path to saved model for evaluation", - ) - parser.add_argument( - "--run_binary_metrics", - default=False, - action="store_true", - help="Whether to compute the basic binary metrics for Light Ranking.", - ) - - opt = parser.parse_args() - logging.info("parse is: ") - logging.info(opt) - - feature_list = read_config(opt.feature_list).items() - feature_config = get_config_func(opt.feat_config_type)( - data_spec_path=opt.data_spec, - feature_list_provided=feature_list, - opt=opt, - add_gbdt=opt.use_gbdt_features, - run_light_ranking_group_metrics_in_bq=opt.run_light_ranking_group_metrics_in_bq, - ) - - # ----------------------------------------------- - # Create Trainer - # ----------------------------------------------- - trainer = DataRecordTrainer( - name=opt.model_trainer_name, - params=opt, - build_graph_fn=partial(build_graph, run_light_ranking_group_metrics_in_bq=True), - save_dir=opt.save_dir, - run_config=None, - feature_config=feature_config, - metric_fn=get_metric_fn(opt.task_name, use_stratify_metrics=False), - ) - - # ----------------------------------------------- - # Model Evaluation - # ----------------------------------------------- - logging.info("Evaluating...") - start = datetime.now() - - if opt.run_binary_metrics: - eval_input_fn = trainer.get_eval_input_fn(repeat=False, shuffle=False) - eval_steps = None if (opt.eval_steps is not None and opt.eval_steps < 0) else opt.eval_steps - trainer.estimator.evaluate(eval_input_fn, steps=eval_steps, checkpoint_path=opt.eval_checkpoint) - - if opt.run_light_ranking_group_metrics_in_bq: - run_group_metrics_light_ranking_in_bq( - trainer=trainer, params=opt, checkpoint_path=opt.eval_checkpoint - ) - - if opt.run_light_ranking_group_metrics: - run_group_metrics_light_ranking( - trainer=trainer, - data_dir=os.path.join(opt.eval_data_dir, opt.eval_start_datetime), - model_path=opt.saved_model_path, - parse_fn=feature_config.get_parse_fn(), - ) - - end = datetime.now() - logging.info("Evaluating time: " + str(end - start)) diff --git a/pushservice/src/main/python/models/light_ranking/model_pools_mlp.docx b/pushservice/src/main/python/models/light_ranking/model_pools_mlp.docx new file mode 100644 index 000000000..cf09b7134 Binary files /dev/null and b/pushservice/src/main/python/models/light_ranking/model_pools_mlp.docx differ diff --git a/pushservice/src/main/python/models/light_ranking/model_pools_mlp.py b/pushservice/src/main/python/models/light_ranking/model_pools_mlp.py deleted file mode 100644 index b45c85e47..000000000 --- a/pushservice/src/main/python/models/light_ranking/model_pools_mlp.py +++ /dev/null @@ -1,187 +0,0 @@ -import warnings - -from twml.contrib.layers import ZscoreNormalization - -from ...libs.customized_full_sparse import FullSparse -from ...libs.get_feat_config import FEAT_CONFIG_DEFAULT_VAL as MISSING_VALUE_MARKER -from ...libs.model_utils import ( - _sparse_feature_fixup, - adaptive_transformation, - filter_nans_and_infs, - get_dense_out, - tensor_dropout, -) - -import tensorflow.compat.v1 as tf -# checkstyle: noqa - -def light_ranking_mlp_ngbdt(features, is_training, params, label=None): - return deepnorm_light_ranking( - features, - is_training, - params, - label=label, - decay=params.momentum, - dense_emb_size=params.dense_embedding_size, - base_activation=tf.keras.layers.LeakyReLU(), - input_dropout_rate=params.dropout, - use_gbdt=False, - ) - - -def deepnorm_light_ranking( - features, - is_training, - params, - label=None, - decay=0.99999, - dense_emb_size=128, - base_activation=None, - input_dropout_rate=None, - input_dense_type="self_atten_dense", - emb_dense_type="self_atten_dense", - mlp_dense_type="self_atten_dense", - use_gbdt=False, -): - # -------------------------------------------------------- - # Initial Parameter Checking - # -------------------------------------------------------- - if base_activation is None: - base_activation = tf.keras.layers.LeakyReLU() - - if label is not None: - warnings.warn( - "Label is unused in deepnorm_gbdt. Stop using this argument.", - DeprecationWarning, - ) - - with tf.variable_scope("helper_layers"): - full_sparse_layer = FullSparse( - output_size=params.sparse_embedding_size, - activation=base_activation, - use_sparse_grads=is_training, - use_binary_values=False, - dtype=tf.float32, - ) - input_normalizing_layer = ZscoreNormalization(decay=decay, name="input_normalizing_layer") - - # -------------------------------------------------------- - # Feature Selection & Embedding - # -------------------------------------------------------- - if use_gbdt: - sparse_gbdt_features = _sparse_feature_fixup(features["gbdt_sparse"], params.input_size_bits) - if input_dropout_rate is not None: - sparse_gbdt_features = tensor_dropout( - sparse_gbdt_features, input_dropout_rate, is_training, sparse_tensor=True - ) - - total_embed = full_sparse_layer(sparse_gbdt_features, use_binary_values=True) - - if (input_dropout_rate is not None) and is_training: - total_embed = total_embed / (1 - input_dropout_rate) - - else: - with tf.variable_scope("dense_branch"): - dense_continuous_features = filter_nans_and_infs(features["continuous"]) - - if params.use_missing_sub_branch: - is_missing = tf.equal(dense_continuous_features, MISSING_VALUE_MARKER) - continuous_features_filled = tf.where( - is_missing, - tf.zeros_like(dense_continuous_features), - dense_continuous_features, - ) - normalized_features = input_normalizing_layer( - continuous_features_filled, is_training, tf.math.logical_not(is_missing) - ) - - with tf.variable_scope("missing_sub_branch"): - missing_feature_embed = get_dense_out( - tf.cast(is_missing, tf.float32), - dense_emb_size, - activation=base_activation, - dense_type=input_dense_type, - ) - - else: - continuous_features_filled = dense_continuous_features - normalized_features = input_normalizing_layer(continuous_features_filled, is_training) - - with tf.variable_scope("continuous_sub_branch"): - normalized_features = adaptive_transformation( - normalized_features, is_training, func_type="tiny" - ) - - if input_dropout_rate is not None: - normalized_features = tensor_dropout( - normalized_features, - input_dropout_rate, - is_training, - sparse_tensor=False, - ) - filled_feature_embed = get_dense_out( - normalized_features, - dense_emb_size, - activation=base_activation, - dense_type=input_dense_type, - ) - - if params.use_missing_sub_branch: - dense_embed = tf.concat( - [filled_feature_embed, missing_feature_embed], axis=1, name="merge_dense_emb" - ) - else: - dense_embed = filled_feature_embed - - with tf.variable_scope("sparse_branch"): - sparse_discrete_features = _sparse_feature_fixup( - features["sparse_no_continuous"], params.input_size_bits - ) - if input_dropout_rate is not None: - sparse_discrete_features = tensor_dropout( - sparse_discrete_features, input_dropout_rate, is_training, sparse_tensor=True - ) - - discrete_features_embed = full_sparse_layer(sparse_discrete_features, use_binary_values=True) - - if (input_dropout_rate is not None) and is_training: - discrete_features_embed = discrete_features_embed / (1 - input_dropout_rate) - - total_embed = tf.concat( - [dense_embed, discrete_features_embed], - axis=1, - name="total_embed", - ) - - total_embed = tf.layers.batch_normalization( - total_embed, - training=is_training, - renorm_momentum=decay, - momentum=decay, - renorm=is_training, - trainable=True, - ) - - # -------------------------------------------------------- - # MLP Layers - # -------------------------------------------------------- - with tf.variable_scope("MLP_branch"): - - assert params.num_mlp_layers >= 0 - embed_list = [total_embed] + [None for _ in range(params.num_mlp_layers)] - dense_types = [emb_dense_type] + [mlp_dense_type for _ in range(params.num_mlp_layers - 1)] - - for xl in range(1, params.num_mlp_layers + 1): - neurons = params.mlp_neuron_scale ** (params.num_mlp_layers + 1 - xl) - embed_list[xl] = get_dense_out( - embed_list[xl - 1], neurons, activation=base_activation, dense_type=dense_types[xl - 1] - ) - - if params.task_name in ["Sent", "HeavyRankPosition", "HeavyRankProbability"]: - logits = get_dense_out(embed_list[-1], 1, activation=None, dense_type=mlp_dense_type) - - else: - raise ValueError("Invalid Task Name !") - - output_dict = {"output": logits} - return output_dict diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/BUILD.bazel b/pushservice/src/main/scala/com/twitter/frigate/pushservice/BUILD.bazel deleted file mode 100644 index d53d4e251..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/BUILD.bazel +++ /dev/null @@ -1,337 +0,0 @@ -scala_library( - sources = ["**/*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = [ - "bazel-compatible", - ], - dependencies = [ - "3rdparty/jvm/com/twitter/bijection:scrooge", - "3rdparty/jvm/com/twitter/storehaus:core", - "abdecider", - "abuse/detection/src/main/thrift/com/twitter/abuse/detection/scoring:thrift-scala", - "ann/src/main/scala/com/twitter/ann/common", - "ann/src/main/thrift/com/twitter/ann/common:ann-common-scala", - "audience-rewards/thrift/src/main/thrift:thrift-scala", - "communities/thrift/src/main/thrift/com/twitter/communities:thrift-scala", - "configapi/configapi-core", - "configapi/configapi-decider", - "content-mixer/thrift/src/main/thrift:thrift-scala", - "content-recommender/thrift/src/main/thrift:thrift-scala", - "copyselectionservice/server/src/main/scala/com/twitter/copyselectionservice/algorithms", - "copyselectionservice/thrift/src/main/thrift:copyselectionservice-scala", - "cortex-deepbird/thrift/src/main/thrift:thrift-java", - "cr-mixer/thrift/src/main/thrift:thrift-scala", - "cuad/projects/hashspace/thrift:thrift-scala", - "cuad/projects/tagspace/thrift/src/main/thrift:thrift-scala", - "detopic/thrift/src/main/thrift:thrift-scala", - "discovery-common/src/main/scala/com/twitter/discovery/common/configapi", - "discovery-common/src/main/scala/com/twitter/discovery/common/ddg", - "discovery-common/src/main/scala/com/twitter/discovery/common/environment", - "discovery-common/src/main/scala/com/twitter/discovery/common/fatigue", - "discovery-common/src/main/scala/com/twitter/discovery/common/nackwarmupfilter", - "discovery-common/src/main/scala/com/twitter/discovery/common/server", - "discovery-ds/src/main/thrift/com/twitter/dds/scio/searcher_aggregate_history_srp:searcher_aggregate_history_srp-scala", - "escherbird/src/scala/com/twitter/escherbird/util/metadatastitch", - "escherbird/src/scala/com/twitter/escherbird/util/uttclient", - "escherbird/src/thrift/com/twitter/escherbird/utt:strato-columns-scala", - "eventbus/client", - "eventdetection/event_context/src/main/scala/com/twitter/eventdetection/event_context/util", - "events-recos/events-recos-service/src/main/thrift:events-recos-thrift-scala", - "explore/explore-ranker/thrift/src/main/thrift:thrift-scala", - "featureswitches/featureswitches-core/src/main/scala", - "featureswitches/featureswitches-core/src/main/scala:dynmap", - "featureswitches/featureswitches-core/src/main/scala:recipient", - "featureswitches/featureswitches-core/src/main/scala:useragent", - "featureswitches/featureswitches-core/src/main/scala/com/twitter/featureswitches/v2/builder", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/server", - "finagle-internal/ostrich-stats", - "finagle/finagle-core/src/main", - "finagle/finagle-http/src/main/scala", - "finagle/finagle-memcached/src/main/scala", - "finagle/finagle-stats", - "finagle/finagle-thriftmux", - "finagle/finagle-tunable/src/main/scala", - "finagle/finagle-zipkin-scribe", - "finatra-internal/abdecider", - "finatra-internal/decider", - "finatra-internal/mtls-http/src/main/scala", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/http-client/src/main/scala", - "finatra/http-core/src/main/java/com/twitter/finatra/http", - "finatra/http-core/src/main/scala/com/twitter/finatra/http/response", - "finatra/http-server/src/main/scala/com/twitter/finatra/http", - "finatra/http-server/src/main/scala/com/twitter/finatra/http/filters", - "finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations", - "finatra/inject/inject-app/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-server/src/main/scala", - "finatra/inject/inject-slf4j/src/main/scala/com/twitter/inject", - "finatra/inject/inject-thrift-client/src/main/scala", - "finatra/inject/inject-utils/src/main/scala", - "finatra/utils/src/main/java/com/twitter/finatra/annotations", - "fleets/fleets-proxy/thrift/src/main/thrift:fleet-scala", - "fleets/fleets-proxy/thrift/src/main/thrift/service:baseservice-scala", - "flock-client/src/main/scala", - "flock-client/src/main/thrift:thrift-scala", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "frigate/frigate-common:base", - "frigate/frigate-common:config", - "frigate/frigate-common:debug", - "frigate/frigate-common:entity_graph_client", - "frigate/frigate-common:history", - "frigate/frigate-common:logger", - "frigate/frigate-common:ml-base", - "frigate/frigate-common:ml-feature", - "frigate/frigate-common:ml-prediction", - "frigate/frigate-common:ntab", - "frigate/frigate-common:predicate", - "frigate/frigate-common:rec_types", - "frigate/frigate-common:score_summary", - "frigate/frigate-common:util", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/candidate", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/experiments", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/filter", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/modules/store:semantic_core_stores", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/deviceinfo", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/interests", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/strato", - "frigate/push-mixer/thrift/src/main/thrift:thrift-scala", - "geo/geo-prediction/src/main/thrift:local-viral-tweets-thrift-scala", - "geoduck/service/src/main/scala/com/twitter/geoduck/service/common/clientmodules", - "geoduck/util/country", - "gizmoduck/client/src/main/scala/com/twitter/gizmoduck/testusers/client", - "hermit/hermit-core:model-user_state", - "hermit/hermit-core:predicate", - "hermit/hermit-core:predicate-gizmoduck", - "hermit/hermit-core:predicate-scarecrow", - "hermit/hermit-core:predicate-socialgraph", - "hermit/hermit-core:predicate-tweetypie", - "hermit/hermit-core:store-labeled_push_recs", - "hermit/hermit-core:store-metastore", - "hermit/hermit-core:store-timezone", - "hermit/hermit-core:store-tweetypie", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/constants", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/model", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/gizmoduck", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/scarecrow", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/semantic_core", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/user_htl_session_store", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/user_interest", - "hmli/hss/src/main/thrift/com/twitter/hss:thrift-scala", - "ibis2/service/src/main/scala/com/twitter/ibis2/lib", - "ibis2/service/src/main/thrift/com/twitter/ibis2/service:ibis2-service-scala", - "interests-service/thrift/src/main/thrift:thrift-scala", - "interests_discovery/thrift/src/main/thrift:batch-thrift-scala", - "interests_discovery/thrift/src/main/thrift:service-thrift-scala", - "kujaku/thrift/src/main/thrift:domain-scala", - "live-video-timeline/client/src/main/scala/com/twitter/livevideo/timeline/client/v2", - "live-video-timeline/domain/src/main/scala/com/twitter/livevideo/timeline/domain", - "live-video-timeline/domain/src/main/scala/com/twitter/livevideo/timeline/domain/v2", - "live-video-timeline/thrift/src/main/thrift/com/twitter/livevideo/timeline:thrift-scala", - "live-video/common/src/main/scala/com/twitter/livevideo/common/domain/v2", - "live-video/common/src/main/scala/com/twitter/livevideo/common/ids", - "notifications-platform/inbound-notifications/src/main/thrift/com/twitter/inbound_notifications:exception-scala", - "notifications-platform/inbound-notifications/src/main/thrift/com/twitter/inbound_notifications:thrift-scala", - "notifications-platform/platform-lib/src/main/thrift/com/twitter/notifications/platform:custom-notification-actions-scala", - "notifications-platform/platform-lib/src/main/thrift/com/twitter/notifications/platform:thrift-scala", - "notifications-relevance/src/scala/com/twitter/nrel/heavyranker", - "notifications-relevance/src/scala/com/twitter/nrel/hydration/base", - "notifications-relevance/src/scala/com/twitter/nrel/hydration/frigate", - "notifications-relevance/src/scala/com/twitter/nrel/hydration/push", - "notifications-relevance/src/scala/com/twitter/nrel/lightranker", - "notificationservice/common/src/main/scala/com/twitter/notificationservice/genericfeedbackstore", - "notificationservice/common/src/main/scala/com/twitter/notificationservice/model:alias", - "notificationservice/common/src/main/scala/com/twitter/notificationservice/model/service", - "notificationservice/common/src/test/scala/com/twitter/notificationservice/mocks", - "notificationservice/scribe/src/main/scala/com/twitter/notificationservice/scribe/manhattan:mh_wrapper", - "notificationservice/thrift/src/main/thrift/com/twitter/notificationservice/api:thrift-scala", - "notificationservice/thrift/src/main/thrift/com/twitter/notificationservice/badgecount-api:thrift-scala", - "notificationservice/thrift/src/main/thrift/com/twitter/notificationservice/generic_notifications:thrift-scala", - "notifinfra/ni-lib/src/main/scala/com/twitter/ni/lib/logged_out_transform", - "observability/observability-manhattan-client/src/main/scala", - "onboarding/service/src/main/scala/com/twitter/onboarding/task/service/models/external", - "onboarding/service/thrift/src/main/thrift:thrift-scala", - "people-discovery/api/thrift/src/main/thrift:thrift-scala", - "periscope/api-proxy-thrift/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter", - "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", - "qig-ranker/thrift/src/main/thrift:thrift-scala", - "rux-ds/src/main/thrift/com/twitter/ruxds/jobs/user_past_aggregate:user_past_aggregate-scala", - "rux/common/src/main/scala/com/twitter/rux/common/encode", - "rux/common/thrift/src/main/thrift/rux-context:rux-context-scala", - "rux/common/thrift/src/main/thrift/strato:strato-scala", - "scribelib/marshallers/src/main/scala/com/twitter/scribelib/marshallers", - "scrooge/scrooge-core", - "scrooge/scrooge-serializer/src/main/scala", - "sensitive-ds/src/main/thrift/com/twitter/scio/nsfw_user_segmentation:nsfw_user_segmentation-scala", - "servo/decider/src/main/scala", - "servo/request/src/main/scala", - "servo/util/src/main/scala", - "src/java/com/twitter/ml/api:api-base", - "src/java/com/twitter/ml/prediction/core", - "src/scala/com/twitter/frigate/data_pipeline/common", - "src/scala/com/twitter/frigate/data_pipeline/embedding_cg:embedding_cg-test-user-ids", - "src/scala/com/twitter/frigate/data_pipeline/features_common", - "src/scala/com/twitter/frigate/news_article_recs/news_articles_metadata:thrift-scala", - "src/scala/com/twitter/frontpage/stream/util", - "src/scala/com/twitter/language/normalization", - "src/scala/com/twitter/ml/api/embedding", - "src/scala/com/twitter/ml/api/util:datarecord", - "src/scala/com/twitter/ml/featurestore/catalog/entities/core", - "src/scala/com/twitter/ml/featurestore/catalog/entities/magicrecs", - "src/scala/com/twitter/ml/featurestore/catalog/features/core:aggregate", - "src/scala/com/twitter/ml/featurestore/catalog/features/cuad:aggregate", - "src/scala/com/twitter/ml/featurestore/catalog/features/embeddings", - "src/scala/com/twitter/ml/featurestore/catalog/features/magicrecs:aggregate", - "src/scala/com/twitter/ml/featurestore/catalog/features/topic_signals:aggregate", - "src/scala/com/twitter/ml/featurestore/lib", - "src/scala/com/twitter/ml/featurestore/lib/data", - "src/scala/com/twitter/ml/featurestore/lib/dynamic", - "src/scala/com/twitter/ml/featurestore/lib/entity", - "src/scala/com/twitter/ml/featurestore/lib/online", - "src/scala/com/twitter/recommendation/interests/discovery/core/config", - "src/scala/com/twitter/recommendation/interests/discovery/core/deploy", - "src/scala/com/twitter/recommendation/interests/discovery/core/model", - "src/scala/com/twitter/recommendation/interests/discovery/popgeo/deploy", - "src/scala/com/twitter/simclusters_v2/common", - "src/scala/com/twitter/storehaus_internal/manhattan", - "src/scala/com/twitter/storehaus_internal/manhattan/config", - "src/scala/com/twitter/storehaus_internal/memcache", - "src/scala/com/twitter/storehaus_internal/memcache/config", - "src/scala/com/twitter/storehaus_internal/util", - "src/scala/com/twitter/taxi/common", - "src/scala/com/twitter/taxi/config", - "src/scala/com/twitter/taxi/deploy", - "src/scala/com/twitter/taxi/trending/common", - "src/thrift/com/twitter/ads/adserver:adserver_rpc-scala", - "src/thrift/com/twitter/clientapp/gen:clientapp-scala", - "src/thrift/com/twitter/core_workflows/user_model:user_model-scala", - "src/thrift/com/twitter/escherbird/common:constants-scala", - "src/thrift/com/twitter/escherbird/metadata:megadata-scala", - "src/thrift/com/twitter/escherbird/metadata:metadata-service-scala", - "src/thrift/com/twitter/escherbird/search:search-service-scala", - "src/thrift/com/twitter/expandodo:only-scala", - "src/thrift/com/twitter/frigate:frigate-common-thrift-scala", - "src/thrift/com/twitter/frigate:frigate-ml-thrift-scala", - "src/thrift/com/twitter/frigate:frigate-notification-thrift-scala", - "src/thrift/com/twitter/frigate:frigate-secondary-accounts-thrift-scala", - "src/thrift/com/twitter/frigate:frigate-thrift-scala", - "src/thrift/com/twitter/frigate:frigate-user-media-representation-thrift-scala", - "src/thrift/com/twitter/frigate/data_pipeline:frigate-user-history-thrift-scala", - "src/thrift/com/twitter/frigate/dau_model:frigate-dau-thrift-scala", - "src/thrift/com/twitter/frigate/magic_events:frigate-magic-events-thrift-scala", - "src/thrift/com/twitter/frigate/magic_events/scribe:thrift-scala", - "src/thrift/com/twitter/frigate/pushcap:frigate-pushcap-thrift-scala", - "src/thrift/com/twitter/frigate/pushservice:frigate-pushservice-thrift-scala", - "src/thrift/com/twitter/frigate/scribe:frigate-scribe-thrift-scala", - "src/thrift/com/twitter/frigate/subscribed_search:frigate-subscribed-search-thrift-scala", - "src/thrift/com/twitter/frigate/user_states:frigate-userstates-thrift-scala", - "src/thrift/com/twitter/geoduck:geoduck-scala", - "src/thrift/com/twitter/gizmoduck:thrift-scala", - "src/thrift/com/twitter/gizmoduck:user-thrift-scala", - "src/thrift/com/twitter/hermit:hermit-scala", - "src/thrift/com/twitter/hermit/pop_geo:hermit-pop-geo-scala", - "src/thrift/com/twitter/hermit/stp:hermit-stp-scala", - "src/thrift/com/twitter/ibis:service-scala", - "src/thrift/com/twitter/manhattan:v1-scala", - "src/thrift/com/twitter/manhattan:v2-scala", - "src/thrift/com/twitter/ml/api:data-java", - "src/thrift/com/twitter/ml/api:data-scala", - "src/thrift/com/twitter/ml/featurestore/timelines:ml-features-timelines-scala", - "src/thrift/com/twitter/ml/featurestore/timelines:ml-features-timelines-strato", - "src/thrift/com/twitter/ml/prediction_service:prediction_service-java", - "src/thrift/com/twitter/permissions_storage:thrift-scala", - "src/thrift/com/twitter/pink-floyd/thrift:thrift-scala", - "src/thrift/com/twitter/recos:recos-common-scala", - "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", - "src/thrift/com/twitter/recos/user_user_graph:user_user_graph-scala", - "src/thrift/com/twitter/relevance/feature_store:feature_store-scala", - "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/search/common:features-scala", - "src/thrift/com/twitter/search/query_interaction_graph:query_interaction_graph-scala", - "src/thrift/com/twitter/search/query_interaction_graph/service:qig-service-scala", - "src/thrift/com/twitter/service/metastore/gen:thrift-scala", - "src/thrift/com/twitter/service/scarecrow/gen:scarecrow-scala", - "src/thrift/com/twitter/service/scarecrow/gen:tiered-actions-scala", - "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "src/thrift/com/twitter/spam/rtf:safety-level-scala", - "src/thrift/com/twitter/timelinemixer:thrift-scala", - "src/thrift/com/twitter/timelinemixer/server/internal:thrift-scala", - "src/thrift/com/twitter/timelines/author_features/user_health:thrift-scala", - "src/thrift/com/twitter/timelines/real_graph:real_graph-scala", - "src/thrift/com/twitter/timelinescorer:thrift-scala", - "src/thrift/com/twitter/timelinescorer/server/internal:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala", - "src/thrift/com/twitter/trends/common:common-scala", - "src/thrift/com/twitter/trends/trip_v1:trip-tweets-thrift-scala", - "src/thrift/com/twitter/tweetypie:service-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - "src/thrift/com/twitter/user_session_store:thrift-scala", - "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", - "src/thrift/com/twitter/wtf/interest:interest-thrift-scala", - "src/thrift/com/twitter/wtf/scalding/common:thrift-scala", - "stitch/stitch-core", - "stitch/stitch-gizmoduck", - "stitch/stitch-socialgraph/src/main/scala", - "stitch/stitch-storehaus/src/main/scala", - "stitch/stitch-tweetypie/src/main/scala", - "storage/clients/manhattan/client/src/main/scala", - "strato/config/columns/clients:clients-strato-client", - "strato/config/columns/geo/user:user-strato-client", - "strato/config/columns/globe/curation:curation-strato-client", - "strato/config/columns/interests:interests-strato-client", - "strato/config/columns/ml/featureStore:featureStore-strato-client", - "strato/config/columns/notifications:notifications-strato-client", - "strato/config/columns/notifinfra:notifinfra-strato-client", - "strato/config/columns/periscope:periscope-strato-client", - "strato/config/columns/rux", - "strato/config/columns/rux:rux-strato-client", - "strato/config/columns/rux/open-app:open-app-strato-client", - "strato/config/columns/socialgraph/graphs:graphs-strato-client", - "strato/config/columns/socialgraph/service/soft_users:soft_users-strato-client", - "strato/config/columns/translation/service:service-strato-client", - "strato/config/columns/translation/service/platform:platform-strato-client", - "strato/config/columns/trends/trip:trip-strato-client", - "strato/config/src/thrift/com/twitter/strato/columns/frigate:logged-out-web-notifications-scala", - "strato/config/src/thrift/com/twitter/strato/columns/notifications:thrift-scala", - "strato/src/main/scala/com/twitter/strato/config", - "strato/src/main/scala/com/twitter/strato/response", - "thrift-web-forms", - "timeline-training-service/service/thrift/src/main/thrift:thrift-scala", - "timelines/src/main/scala/com/twitter/timelines/features/app", - "topic-social-proof/server/src/main/thrift:thrift-scala", - "topiclisting/topiclisting-core/src/main/scala/com/twitter/topiclisting", - "topiclisting/topiclisting-utt/src/main/scala/com/twitter/topiclisting/utt", - "trends/common/src/main/thrift/com/twitter/trends/common:thrift-scala", - "tweetypie/src/scala/com/twitter/tweetypie/tweettext", - "twitter-context/src/main/scala", - "twitter-server-internal", - "twitter-server/server/src/main/scala", - "twitter-text/lib/java/src/main/java/com/twitter/twittertext", - "twml/runtime/src/main/scala/com/twitter/deepbird/runtime/prediction_engine:prediction_engine_mkl", - "ubs/common/src/main/thrift/com/twitter/ubs:broadcast-thrift-scala", - "ubs/common/src/main/thrift/com/twitter/ubs:seller_application-thrift-scala", - "user_session_store/src/main/scala/com/twitter/user_session_store/impl/manhattan/readwrite", - "util-internal/scribe", - "util-internal/tunable/src/main/scala/com/twitter/util/tunable", - "util/util-app", - "util/util-hashing/src/main/scala", - "util/util-slf4j-api/src/main/scala", - "util/util-stats/src/main/scala", - "visibility/lib/src/main/scala/com/twitter/visibility/builder", - "visibility/lib/src/main/scala/com/twitter/visibility/interfaces/push_service", - "visibility/lib/src/main/scala/com/twitter/visibility/interfaces/spaces", - "visibility/lib/src/main/scala/com/twitter/visibility/util", - ], - exports = [ - "strato/config/src/thrift/com/twitter/strato/columns/frigate:logged-out-web-notifications-scala", - ], -) diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/BUILD.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/BUILD.docx new file mode 100644 index 000000000..79274f47b Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/BUILD.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/PushMixerThriftServerWarmupHandler.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/PushMixerThriftServerWarmupHandler.docx new file mode 100644 index 000000000..1a3565131 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/PushMixerThriftServerWarmupHandler.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/PushMixerThriftServerWarmupHandler.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/PushMixerThriftServerWarmupHandler.scala deleted file mode 100644 index b13d3b093..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/PushMixerThriftServerWarmupHandler.scala +++ /dev/null @@ -1,93 +0,0 @@ -package com.twitter.frigate.pushservice - -import com.google.inject.Inject -import com.google.inject.Singleton -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.thrift.ClientId -import com.twitter.finatra.thrift.routing.ThriftWarmup -import com.twitter.util.logging.Logging -import com.twitter.inject.utils.Handler -import com.twitter.frigate.pushservice.{thriftscala => t} -import com.twitter.frigate.thriftscala.NotificationDisplayLocation -import com.twitter.util.Stopwatch -import com.twitter.scrooge.Request -import com.twitter.scrooge.Response -import com.twitter.util.Return -import com.twitter.util.Throw -import com.twitter.util.Try - -/** - * Warms up the refresh request path. - * If service is running as pushservice-send then the warmup does nothing. - * - * When making the warmup refresh requests we - * - Set skipFilters to true to execute as much of the request path as possible - * - Set darkWrite to true to prevent sending a push - */ -@Singleton -class PushMixerThriftServerWarmupHandler @Inject() ( - warmup: ThriftWarmup, - serviceIdentifier: ServiceIdentifier) - extends Handler - with Logging { - - private val clientId = ClientId("thrift-warmup-client") - - def handle(): Unit = { - val refreshServices = Set( - "frigate-pushservice", - "frigate-pushservice-canary", - "frigate-pushservice-canary-control", - "frigate-pushservice-canary-treatment" - ) - val isRefresh = refreshServices.contains(serviceIdentifier.service) - if (isRefresh && !serviceIdentifier.isLocal) refreshWarmup() - } - - def refreshWarmup(): Unit = { - val elapsed = Stopwatch.start() - val testIds = Seq( - 1, - 2, - 3 - ) - try { - clientId.asCurrent { - testIds.foreach { id => - val warmupReq = warmupQuery(id) - info(s"Sending warm-up request to service with query: $warmupReq") - warmup.sendRequest( - method = t.PushService.Refresh, - req = Request(t.PushService.Refresh.Args(warmupReq)))(assertWarmupResponse) - } - } - } catch { - case e: Throwable => - error(e.getMessage, e) - } - info(s"Warm up complete. Time taken: ${elapsed().toString}") - } - - private def warmupQuery(userId: Long): t.RefreshRequest = { - t.RefreshRequest( - userId = userId, - notificationDisplayLocation = NotificationDisplayLocation.PushToMobileDevice, - context = Some( - t.PushContext( - skipFilters = Some(true), - darkWrite = Some(true) - )) - ) - } - - private def assertWarmupResponse( - result: Try[Response[t.PushService.Refresh.SuccessType]] - ): Unit = { - result match { - case Return(_) => // ok - case Throw(exception) => - warn("Error performing warm-up request.") - error(exception.getMessage, exception) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/PushServiceMain.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/PushServiceMain.docx new file mode 100644 index 000000000..61d0bc06b Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/PushServiceMain.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/PushServiceMain.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/PushServiceMain.scala deleted file mode 100644 index c60f6e352..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/PushServiceMain.scala +++ /dev/null @@ -1,193 +0,0 @@ -package com.twitter.frigate.pushservice - -import com.twitter.discovery.common.environment.modules.EnvironmentModule -import com.twitter.finagle.Filter -import com.twitter.finatra.annotations.DarkTrafficFilterType -import com.twitter.finatra.decider.modules.DeciderModule -import com.twitter.finatra.http.HttpServer -import com.twitter.finatra.http.filters.CommonFilters -import com.twitter.finatra.http.routing.HttpRouter -import com.twitter.finatra.mtls.http.{Mtls => HttpMtls} -import com.twitter.finatra.mtls.thriftmux.{Mtls => ThriftMtls} -import com.twitter.finatra.mtls.thriftmux.filters.MtlsServerSessionTrackerFilter -import com.twitter.finatra.thrift.ThriftServer -import com.twitter.finatra.thrift.filters.ExceptionMappingFilter -import com.twitter.finatra.thrift.filters.LoggingMDCFilter -import com.twitter.finatra.thrift.filters.StatsFilter -import com.twitter.finatra.thrift.filters.ThriftMDCFilter -import com.twitter.finatra.thrift.filters.TraceIdMDCFilter -import com.twitter.finatra.thrift.routing.ThriftRouter -import com.twitter.frigate.common.logger.MRLoggerGlobalVariables -import com.twitter.frigate.pushservice.controller.PushServiceController -import com.twitter.frigate.pushservice.module._ -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flags -import com.twitter.inject.thrift.modules.ThriftClientIdModule -import com.twitter.logging.BareFormatter -import com.twitter.logging.Level -import com.twitter.logging.LoggerFactory -import com.twitter.logging.{Logging => JLogging} -import com.twitter.logging.QueueingHandler -import com.twitter.logging.ScribeHandler -import com.twitter.product_mixer.core.module.product_mixer_flags.ProductMixerFlagModule -import com.twitter.product_mixer.core.module.ABDeciderModule -import com.twitter.product_mixer.core.module.FeatureSwitchesModule -import com.twitter.product_mixer.core.module.StratoClientModule - -object PushServiceMain extends PushServiceFinatraServer - -class PushServiceFinatraServer - extends ThriftServer - with ThriftMtls - with HttpServer - with HttpMtls - with JLogging { - - override val name = "PushService" - - override val modules: Seq[TwitterModule] = { - Seq( - ABDeciderModule, - DeciderModule, - FeatureSwitchesModule, - FilterModule, - FlagModule, - EnvironmentModule, - ThriftClientIdModule, - DeployConfigModule, - ProductMixerFlagModule, - StratoClientModule, - PushHandlerModule, - PushTargetUserBuilderModule, - PushServiceDarkTrafficModule, - LoggedOutPushTargetUserBuilderModule, - new ThriftWebFormsModule(this), - ) - } - - override def configureThrift(router: ThriftRouter): Unit = { - router - .filter[ExceptionMappingFilter] - .filter[LoggingMDCFilter] - .filter[TraceIdMDCFilter] - .filter[ThriftMDCFilter] - .filter[MtlsServerSessionTrackerFilter] - .filter[StatsFilter] - .filter[Filter.TypeAgnostic, DarkTrafficFilterType] - .add[PushServiceController] - } - - override def configureHttp(router: HttpRouter): Unit = - router - .filter[CommonFilters] - - override protected def start(): Unit = { - MRLoggerGlobalVariables.setRequiredFlags( - traceLogFlag = injector.instance[Boolean](Flags.named(FlagModule.mrLoggerIsTraceAll.name)), - nthLogFlag = injector.instance[Boolean](Flags.named(FlagModule.mrLoggerNthLog.name)), - nthLogValFlag = injector.instance[Long](Flags.named(FlagModule.mrLoggerNthVal.name)) - ) - } - - override protected def warmup(): Unit = { - handle[PushMixerThriftServerWarmupHandler]() - } - - override protected def configureLoggerFactories(): Unit = { - loggerFactories.foreach { _() } - } - - override def loggerFactories: List[LoggerFactory] = { - val scribeScope = statsReceiver.scope("scribe") - List( - LoggerFactory( - level = Some(levelFlag()), - handlers = handlers - ), - LoggerFactory( - node = "request_scribe", - level = Some(Level.INFO), - useParents = false, - handlers = QueueingHandler( - maxQueueSize = 10000, - handler = ScribeHandler( - category = "frigate_pushservice_log", - formatter = BareFormatter, - statsReceiver = scribeScope.scope("frigate_pushservice_log") - ) - ) :: Nil - ), - LoggerFactory( - node = "notification_scribe", - level = Some(Level.INFO), - useParents = false, - handlers = QueueingHandler( - maxQueueSize = 10000, - handler = ScribeHandler( - category = "frigate_notifier", - formatter = BareFormatter, - statsReceiver = scribeScope.scope("frigate_notifier") - ) - ) :: Nil - ), - LoggerFactory( - node = "push_scribe", - level = Some(Level.INFO), - useParents = false, - handlers = QueueingHandler( - maxQueueSize = 10000, - handler = ScribeHandler( - category = "test_frigate_push", - formatter = BareFormatter, - statsReceiver = scribeScope.scope("test_frigate_push") - ) - ) :: Nil - ), - LoggerFactory( - node = "push_subsample_scribe", - level = Some(Level.INFO), - useParents = false, - handlers = QueueingHandler( - maxQueueSize = 2500, - handler = ScribeHandler( - category = "magicrecs_candidates_subsample_scribe", - maxMessagesPerTransaction = 250, - maxMessagesToBuffer = 2500, - formatter = BareFormatter, - statsReceiver = scribeScope.scope("magicrecs_candidates_subsample_scribe") - ) - ) :: Nil - ), - LoggerFactory( - node = "mr_request_scribe", - level = Some(Level.INFO), - useParents = false, - handlers = QueueingHandler( - maxQueueSize = 2500, - handler = ScribeHandler( - category = "mr_request_scribe", - maxMessagesPerTransaction = 250, - maxMessagesToBuffer = 2500, - formatter = BareFormatter, - statsReceiver = scribeScope.scope("mr_request_scribe") - ) - ) :: Nil - ), - LoggerFactory( - node = "high_quality_candidates_scribe", - level = Some(Level.INFO), - useParents = false, - handlers = QueueingHandler( - maxQueueSize = 2500, - handler = ScribeHandler( - category = "frigate_high_quality_candidates_log", - maxMessagesPerTransaction = 250, - maxMessagesToBuffer = 2500, - formatter = BareFormatter, - statsReceiver = scribeScope.scope("high_quality_candidates_scribe") - ) - ) :: Nil - ), - ) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ContentRecommenderMixerAdaptor.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ContentRecommenderMixerAdaptor.docx new file mode 100644 index 000000000..90a716996 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ContentRecommenderMixerAdaptor.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ContentRecommenderMixerAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ContentRecommenderMixerAdaptor.scala deleted file mode 100644 index 946923fb9..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ContentRecommenderMixerAdaptor.scala +++ /dev/null @@ -1,323 +0,0 @@ -package com.twitter.frigate.pushservice.adaptor - -import com.twitter.contentrecommender.thriftscala.MetricTag -import com.twitter.cr_mixer.thriftscala.CrMixerTweetRequest -import com.twitter.cr_mixer.thriftscala.NotificationsContext -import com.twitter.cr_mixer.thriftscala.Product -import com.twitter.cr_mixer.thriftscala.ProductContext -import com.twitter.cr_mixer.thriftscala.{MetricTag => CrMixerMetricTag} -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.AlgorithmScore -import com.twitter.frigate.common.base.CandidateSource -import com.twitter.frigate.common.base.CandidateSourceEligible -import com.twitter.frigate.common.base.CrMixerCandidate -import com.twitter.frigate.common.base.TopicCandidate -import com.twitter.frigate.common.base.TopicProofTweetCandidate -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.common.predicate.CommonOutNetworkTweetCandidatesSourcePredicates.filterOutInNetworkTweets -import com.twitter.frigate.common.predicate.CommonOutNetworkTweetCandidatesSourcePredicates.filterOutReplyTweet -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.store.CrMixerTweetStore -import com.twitter.frigate.pushservice.store.UttEntityHydrationStore -import com.twitter.frigate.pushservice.util.AdaptorUtils -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.frigate.pushservice.util.TopicsUtil -import com.twitter.frigate.pushservice.util.TweetWithTopicProof -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.hermit.predicate.socialgraph.RelationEdge -import com.twitter.product_mixer.core.thriftscala.ClientContext -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.topiclisting.utt.LocalizedEntity -import com.twitter.tsp.thriftscala.TopicSocialProofRequest -import com.twitter.tsp.thriftscala.TopicSocialProofResponse -import com.twitter.util.Future -import scala.collection.Map - -case class ContentRecommenderMixerAdaptor( - crMixerTweetStore: CrMixerTweetStore, - tweetyPieStore: ReadableStore[Long, TweetyPieResult], - edgeStore: ReadableStore[RelationEdge, Boolean], - topicSocialProofServiceStore: ReadableStore[TopicSocialProofRequest, TopicSocialProofResponse], - uttEntityHydrationStore: UttEntityHydrationStore, - globalStats: StatsReceiver) - extends CandidateSource[Target, RawCandidate] - with CandidateSourceEligible[Target, RawCandidate] { - - override val name: String = this.getClass.getSimpleName - - private[this] val stats = globalStats.scope("ContentRecommenderMixerAdaptor") - private[this] val numOfValidAuthors = stats.stat("num_of_valid_authors") - private[this] val numOutOfMaximumDropped = stats.stat("dropped_due_out_of_maximum") - private[this] val totalInputRecs = stats.counter("input_recs") - private[this] val totalOutputRecs = stats.stat("output_recs") - private[this] val totalRequests = stats.counter("total_requests") - private[this] val nonReplyTweetsCounter = stats.counter("non_reply_tweets") - private[this] val totalOutNetworkRecs = stats.counter("out_network_tweets") - private[this] val totalInNetworkRecs = stats.counter("in_network_tweets") - - /** - * Builds OON raw candidates based on input OON Tweets - */ - def buildOONRawCandidates( - inputTarget: Target, - oonTweets: Seq[TweetyPieResult], - tweetScoreMap: Map[Long, Double], - tweetIdToTagsMap: Map[Long, Seq[CrMixerMetricTag]], - maxNumOfCandidates: Int - ): Option[Seq[RawCandidate]] = { - val cands = oonTweets.flatMap { tweetResult => - val tweetId = tweetResult.tweet.id - generateOONRawCandidate( - inputTarget, - tweetId, - Some(tweetResult), - tweetScoreMap, - tweetIdToTagsMap - ) - } - - val candidates = restrict( - maxNumOfCandidates, - cands, - numOutOfMaximumDropped, - totalOutputRecs - ) - - Some(candidates) - } - - /** - * Builds a single RawCandidate With TopicProofTweetCandidate - */ - def buildTopicTweetRawCandidate( - inputTarget: Target, - tweetWithTopicProof: TweetWithTopicProof, - localizedEntity: LocalizedEntity, - tags: Option[Seq[MetricTag]], - ): RawCandidate with TopicProofTweetCandidate = { - new RawCandidate with TopicProofTweetCandidate { - override def target: Target = inputTarget - override def topicListingSetting: Option[String] = Some( - tweetWithTopicProof.topicListingSetting) - override def tweetId: Long = tweetWithTopicProof.tweetId - override def tweetyPieResult: Option[TweetyPieResult] = Some( - tweetWithTopicProof.tweetyPieResult) - override def semanticCoreEntityId: Option[Long] = Some(tweetWithTopicProof.topicId) - override def localizedUttEntity: Option[LocalizedEntity] = Some(localizedEntity) - override def algorithmCR: Option[String] = tweetWithTopicProof.algorithmCR - override def tagsCR: Option[Seq[MetricTag]] = tags - override def isOutOfNetwork: Boolean = tweetWithTopicProof.isOON - } - } - - /** - * Takes a group of TopicTweets and transforms them into RawCandidates - */ - def buildTopicTweetRawCandidates( - inputTarget: Target, - topicProofCandidates: Seq[TweetWithTopicProof], - tweetIdToTagsMap: Map[Long, Seq[CrMixerMetricTag]], - maxNumberOfCands: Int - ): Future[Option[Seq[RawCandidate]]] = { - val semanticCoreEntityIds = topicProofCandidates - .map(_.topicId) - .toSet - - TopicsUtil - .getLocalizedEntityMap(inputTarget, semanticCoreEntityIds, uttEntityHydrationStore) - .map { localizedEntityMap => - val rawCandidates = topicProofCandidates.collect { - case topicSocialProof: TweetWithTopicProof - if localizedEntityMap.contains(topicSocialProof.topicId) => - // Once we deprecate CR calls, we should replace this code to use the CrMixerMetricTag - val tags = tweetIdToTagsMap.get(topicSocialProof.tweetId).map { - _.flatMap { tag => MetricTag.get(tag.value) } - } - buildTopicTweetRawCandidate( - inputTarget, - topicSocialProof, - localizedEntityMap(topicSocialProof.topicId), - tags - ) - } - - val candResult = restrict( - maxNumberOfCands, - rawCandidates, - numOutOfMaximumDropped, - totalOutputRecs - ) - - Some(candResult) - } - } - - private def generateOONRawCandidate( - inputTarget: Target, - id: Long, - result: Option[TweetyPieResult], - tweetScoreMap: Map[Long, Double], - tweetIdToTagsMap: Map[Long, Seq[CrMixerMetricTag]] - ): Option[RawCandidate with TweetCandidate] = { - val tagsFromCR = tweetIdToTagsMap.get(id).map { _.flatMap { tag => MetricTag.get(tag.value) } } - val candidate = new RawCandidate with CrMixerCandidate with TopicCandidate with AlgorithmScore { - override val tweetId = id - override val target = inputTarget - override val tweetyPieResult = result - override val localizedUttEntity = None - override val semanticCoreEntityId = None - override def commonRecType = - getMediaBasedCRT( - CommonRecommendationType.TwistlyTweet, - CommonRecommendationType.TwistlyPhoto, - CommonRecommendationType.TwistlyVideo) - override def tagsCR = tagsFromCR - override def algorithmScore = tweetScoreMap.get(id) - override def algorithmCR = None - } - Some(candidate) - } - - private def restrict( - maxNumToReturn: Int, - candidates: Seq[RawCandidate], - numOutOfMaximumDropped: Stat, - totalOutputRecs: Stat - ): Seq[RawCandidate] = { - val newCandidates = candidates.take(maxNumToReturn) - val numDropped = candidates.length - newCandidates.length - numOutOfMaximumDropped.add(numDropped) - totalOutputRecs.add(newCandidates.size) - newCandidates - } - - private def buildCrMixerRequest( - target: Target, - countryCode: Option[String], - language: Option[String], - seenTweets: Seq[Long] - ): CrMixerTweetRequest = { - CrMixerTweetRequest( - clientContext = ClientContext( - userId = Some(target.targetId), - countryCode = countryCode, - languageCode = language - ), - product = Product.Notifications, - productContext = Some(ProductContext.NotificationsContext(NotificationsContext())), - excludedTweetIds = Some(seenTweets) - ) - } - - private def selectCandidatesToSendBasedOnSettings( - isRecommendationsEligible: Boolean, - isTopicsEligible: Boolean, - oonRawCandidates: Option[Seq[RawCandidate]], - topicTweetCandidates: Option[Seq[RawCandidate]] - ): Option[Seq[RawCandidate]] = { - if (isRecommendationsEligible && isTopicsEligible) { - Some(topicTweetCandidates.getOrElse(Seq.empty) ++ oonRawCandidates.getOrElse(Seq.empty)) - } else if (isRecommendationsEligible) { - oonRawCandidates - } else if (isTopicsEligible) { - topicTweetCandidates - } else None - } - - override def get(target: Target): Future[Option[Seq[RawCandidate]]] = { - Future - .join( - target.seenTweetIds, - target.countryCode, - target.inferredUserDeviceLanguage, - PushDeviceUtil.isTopicsEligible(target), - PushDeviceUtil.isRecommendationsEligible(target) - ).flatMap { - case (seenTweets, countryCode, language, isTopicsEligible, isRecommendationsEligible) => - val request = buildCrMixerRequest(target, countryCode, language, seenTweets) - crMixerTweetStore.getTweetRecommendations(request).flatMap { - case Some(response) => - totalInputRecs.incr(response.tweets.size) - totalRequests.incr() - AdaptorUtils - .getTweetyPieResults( - response.tweets.map(_.tweetId).toSet, - tweetyPieStore).flatMap { tweetyPieResultMap => - filterOutInNetworkTweets( - target, - filterOutReplyTweet(tweetyPieResultMap.toMap, nonReplyTweetsCounter), - edgeStore, - numOfValidAuthors).flatMap { - outNetworkTweetsWithId: Seq[(Long, TweetyPieResult)] => - totalOutNetworkRecs.incr(outNetworkTweetsWithId.size) - totalInNetworkRecs.incr(response.tweets.size - outNetworkTweetsWithId.size) - val outNetworkTweets: Seq[TweetyPieResult] = outNetworkTweetsWithId.map { - case (_, tweetyPieResult) => tweetyPieResult - } - - val tweetIdToTagsMap = response.tweets.map { tweet => - tweet.tweetId -> tweet.metricTags.getOrElse(Seq.empty) - }.toMap - - val tweetScoreMap = response.tweets.map { tweet => - tweet.tweetId -> tweet.score - }.toMap - - val maxNumOfCandidates = - target.params(PushFeatureSwitchParams.NumberOfMaxCrMixerCandidatesParam) - - val oonRawCandidates = - buildOONRawCandidates( - target, - outNetworkTweets, - tweetScoreMap, - tweetIdToTagsMap, - maxNumOfCandidates) - - TopicsUtil - .getTopicSocialProofs( - target, - outNetworkTweets, - topicSocialProofServiceStore, - edgeStore, - PushFeatureSwitchParams.TopicProofTweetCandidatesTopicScoreThreshold).flatMap { - tweetsWithTopicProof => - buildTopicTweetRawCandidates( - target, - tweetsWithTopicProof, - tweetIdToTagsMap, - maxNumOfCandidates) - }.map { topicTweetCandidates => - selectCandidatesToSendBasedOnSettings( - isRecommendationsEligible, - isTopicsEligible, - oonRawCandidates, - topicTweetCandidates) - } - } - } - case _ => Future.None - } - } - } - - /** - * For a user to be available the following news to happen - */ - override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { - Future - .join( - PushDeviceUtil.isRecommendationsEligible(target), - PushDeviceUtil.isTopicsEligible(target) - ).map { - case (isRecommendationsEligible, isTopicsEligible) => - (isRecommendationsEligible || isTopicsEligible) && - target.params(PushParams.ContentRecommenderMixerAdaptorDecider) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/EarlyBirdFirstDegreeCandidateAdaptor.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/EarlyBirdFirstDegreeCandidateAdaptor.docx new file mode 100644 index 000000000..26751076b Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/EarlyBirdFirstDegreeCandidateAdaptor.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/EarlyBirdFirstDegreeCandidateAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/EarlyBirdFirstDegreeCandidateAdaptor.scala deleted file mode 100644 index ab631841a..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/EarlyBirdFirstDegreeCandidateAdaptor.scala +++ /dev/null @@ -1,293 +0,0 @@ -package com.twitter.frigate.pushservice.adaptor - -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.candidate._ -import com.twitter.frigate.common.predicate.CommonOutNetworkTweetCandidatesSourcePredicates.filterOutReplyTweet -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.hermit.store.tweetypie.UserTweet -import com.twitter.recos.recos_common.thriftscala.SocialProofType -import com.twitter.search.common.features.thriftscala.ThriftSearchResultFeatures -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi.Param -import com.twitter.util.Future -import com.twitter.util.Time -import scala.collection.Map - -case class EarlyBirdFirstDegreeCandidateAdaptor( - earlyBirdFirstDegreeCandidates: CandidateSource[ - EarlybirdCandidateSource.Query, - EarlybirdCandidate - ], - tweetyPieStore: ReadableStore[Long, TweetyPieResult], - tweetyPieStoreNoVF: ReadableStore[Long, TweetyPieResult], - userTweetTweetyPieStore: ReadableStore[UserTweet, TweetyPieResult], - maxResultsParam: Param[Int], - globalStats: StatsReceiver) - extends CandidateSource[Target, RawCandidate] - with CandidateSourceEligible[Target, RawCandidate] { - - type EBCandidate = EarlybirdCandidate with TweetDetails - private val stats = globalStats.scope("EarlyBirdFirstDegreeAdaptor") - private val earlyBirdCandsStat: Stat = stats.stat("early_bird_cands_dist") - private val emptyEarlyBirdCands = stats.counter("empty_early_bird_candidates") - private val seedSetEmpty = stats.counter("empty_seedset") - private val seenTweetsStat = stats.stat("filtered_by_seen_tweets") - private val emptyTweetyPieResult = stats.stat("empty_tweetypie_result") - private val nonReplyTweetsCounter = stats.counter("non_reply_tweets") - private val enableRetweets = stats.counter("enable_retweets") - private val f1withoutSocialContexts = stats.counter("f1_without_social_context") - private val userTweetTweetyPieStoreCounter = stats.counter("user_tweet_tweetypie_store") - - override val name: String = earlyBirdFirstDegreeCandidates.name - - private def getAllSocialContextActions( - socialProofTypes: Seq[(SocialProofType, Seq[Long])] - ): Seq[SocialContextAction] = { - socialProofTypes.flatMap { - case (SocialProofType.Favorite, scIds) => - scIds.map { scId => - SocialContextAction( - scId, - Time.now.inMilliseconds, - socialContextActionType = Some(SocialContextActionType.Favorite) - ) - } - case (SocialProofType.Retweet, scIds) => - scIds.map { scId => - SocialContextAction( - scId, - Time.now.inMilliseconds, - socialContextActionType = Some(SocialContextActionType.Retweet) - ) - } - case (SocialProofType.Reply, scIds) => - scIds.map { scId => - SocialContextAction( - scId, - Time.now.inMilliseconds, - socialContextActionType = Some(SocialContextActionType.Reply) - ) - } - case (SocialProofType.Tweet, scIds) => - scIds.map { scId => - SocialContextAction( - scId, - Time.now.inMilliseconds, - socialContextActionType = Some(SocialContextActionType.Tweet) - ) - } - case _ => Nil - } - } - - private def generateRetweetCandidate( - inputTarget: Target, - candidate: EBCandidate, - scIds: Seq[Long], - socialProofTypes: Seq[(SocialProofType, Seq[Long])] - ): RawCandidate = { - val scActions = scIds.map { scId => SocialContextAction(scId, Time.now.inMilliseconds) } - new RawCandidate with TweetRetweetCandidate with EarlybirdTweetFeatures { - override val socialContextActions = scActions - override val socialContextAllTypeActions = getAllSocialContextActions(socialProofTypes) - override val tweetId = candidate.tweetId - override val target = inputTarget - override val tweetyPieResult = candidate.tweetyPieResult - override val features = candidate.features - } - } - - private def generateF1CandidateWithoutSocialContext( - inputTarget: Target, - candidate: EBCandidate - ): RawCandidate = { - f1withoutSocialContexts.incr() - new RawCandidate with F1FirstDegree with EarlybirdTweetFeatures { - override val tweetId = candidate.tweetId - override val target = inputTarget - override val tweetyPieResult = candidate.tweetyPieResult - override val features = candidate.features - } - } - - private def generateEarlyBirdCandidate( - id: Long, - result: Option[TweetyPieResult], - ebFeatures: Option[ThriftSearchResultFeatures] - ): EBCandidate = { - new EarlybirdCandidate with TweetDetails { - override val tweetyPieResult: Option[TweetyPieResult] = result - override val tweetId: Long = id - override val features: Option[ThriftSearchResultFeatures] = ebFeatures - } - } - - private def filterOutSeenTweets(seenTweetIds: Seq[Long], inputTweetIds: Seq[Long]): Seq[Long] = { - inputTweetIds.filterNot(seenTweetIds.contains) - } - - private def filterInvalidTweets( - tweetIds: Seq[Long], - target: Target - ): Future[Seq[(Long, TweetyPieResult)]] = { - - val resMap = { - if (target.params(PushFeatureSwitchParams.EnableF1FromProtectedTweetAuthors)) { - userTweetTweetyPieStoreCounter.incr() - val keys = tweetIds.map { tweetId => - UserTweet(tweetId, Some(target.targetId)) - } - - userTweetTweetyPieStore - .multiGet(keys.toSet).map { - case (userTweet, resultFut) => - userTweet.tweetId -> resultFut - }.toMap - } else { - (target.params(PushFeatureSwitchParams.EnableVFInTweetypie) match { - case true => tweetyPieStore - case false => tweetyPieStoreNoVF - }).multiGet(tweetIds.toSet) - } - } - Future.collect(resMap).map { tweetyPieResultMap => - val cands = filterOutReplyTweet(tweetyPieResultMap, nonReplyTweetsCounter).collect { - case (id: Long, Some(result)) => - id -> result - } - - emptyTweetyPieResult.add(tweetyPieResultMap.size - cands.size) - cands.toSeq - } - } - - private def getEBRetweetCandidates( - inputTarget: Target, - retweets: Seq[(Long, TweetyPieResult)] - ): Seq[RawCandidate] = { - retweets.flatMap { - case (_, tweetypieResult) => - tweetypieResult.tweet.coreData.flatMap { coreData => - tweetypieResult.sourceTweet.map { sourceTweet => - val tweetId = sourceTweet.id - val scId = coreData.userId - val socialProofTypes = Seq((SocialProofType.Retweet, Seq(scId))) - val candidate = generateEarlyBirdCandidate( - tweetId, - Some(TweetyPieResult(sourceTweet, None, None)), - None - ) - generateRetweetCandidate( - inputTarget, - candidate, - Seq(scId), - socialProofTypes - ) - } - } - } - } - - private def getEBFirstDegreeCands( - tweets: Seq[(Long, TweetyPieResult)], - ebTweetIdMap: Map[Long, Option[ThriftSearchResultFeatures]] - ): Seq[EBCandidate] = { - tweets.map { - case (id, tweetypieResult) => - val features = ebTweetIdMap.getOrElse(id, None) - generateEarlyBirdCandidate(id, Some(tweetypieResult), features) - } - } - - /** - * Returns a combination of raw candidates made of: f1 recs, topic social proof recs, sc recs and retweet candidates - */ - def buildRawCandidates( - inputTarget: Target, - firstDegreeCandidates: Seq[EBCandidate], - retweetCandidates: Seq[RawCandidate] - ): Seq[RawCandidate] = { - val hydratedF1Recs = - firstDegreeCandidates.map(generateF1CandidateWithoutSocialContext(inputTarget, _)) - hydratedF1Recs ++ retweetCandidates - } - - override def get(inputTarget: Target): Future[Option[Seq[RawCandidate]]] = { - inputTarget.seedsWithWeight.flatMap { seedsetOpt => - val seedsetMap = seedsetOpt.getOrElse(Map.empty) - - if (seedsetMap.isEmpty) { - seedSetEmpty.incr() - Future.None - } else { - val maxResultsToReturn = inputTarget.params(maxResultsParam) - val maxTweetAge = inputTarget.params(PushFeatureSwitchParams.F1CandidateMaxTweetAgeParam) - val earlybirdQuery = EarlybirdCandidateSource.Query( - maxNumResultsToReturn = maxResultsToReturn, - seedset = seedsetMap, - maxConsecutiveResultsByTheSameUser = Some(1), - maxTweetAge = maxTweetAge, - disableTimelinesMLModel = false, - searcherId = Some(inputTarget.targetId), - isProtectTweetsEnabled = - inputTarget.params(PushFeatureSwitchParams.EnableF1FromProtectedTweetAuthors), - followedUserIds = Some(seedsetMap.keySet.toSeq) - ) - - Future - .join(inputTarget.seenTweetIds, earlyBirdFirstDegreeCandidates.get(earlybirdQuery)) - .flatMap { - case (seenTweetIds, Some(candidates)) => - earlyBirdCandsStat.add(candidates.size) - - val ebTweetIdMap = candidates.map { cand => cand.tweetId -> cand.features }.toMap - - val ebTweetIds = ebTweetIdMap.keys.toSeq - - val tweetIds = filterOutSeenTweets(seenTweetIds, ebTweetIds) - seenTweetsStat.add(ebTweetIds.size - tweetIds.size) - - filterInvalidTweets(tweetIds, inputTarget) - .map { validTweets => - val (retweets, tweets) = validTweets.partition { - case (_, tweetypieResult) => - tweetypieResult.sourceTweet.isDefined - } - - val firstDegreeCandidates = getEBFirstDegreeCands(tweets, ebTweetIdMap) - - val retweetCandidates = { - if (inputTarget.params(PushParams.EarlyBirdSCBasedCandidatesParam) && - inputTarget.params(PushParams.MRTweetRetweetRecsParam)) { - enableRetweets.incr() - getEBRetweetCandidates(inputTarget, retweets) - } else Nil - } - - Some( - buildRawCandidates( - inputTarget, - firstDegreeCandidates, - retweetCandidates - )) - } - - case _ => - emptyEarlyBirdCands.incr() - Future.None - } - } - } - } - - override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { - PushDeviceUtil.isRecommendationsEligible(target) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ExploreVideoTweetCandidateAdaptor.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ExploreVideoTweetCandidateAdaptor.docx new file mode 100644 index 000000000..20cc932c8 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ExploreVideoTweetCandidateAdaptor.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ExploreVideoTweetCandidateAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ExploreVideoTweetCandidateAdaptor.scala deleted file mode 100644 index 345fdbd3c..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ExploreVideoTweetCandidateAdaptor.scala +++ /dev/null @@ -1,120 +0,0 @@ -package com.twitter.frigate.pushservice.adaptor - -import com.twitter.explore_ranker.thriftscala.ExploreRankerProductResponse -import com.twitter.explore_ranker.thriftscala.ExploreRankerRequest -import com.twitter.explore_ranker.thriftscala.ExploreRankerResponse -import com.twitter.explore_ranker.thriftscala.ExploreRecommendation -import com.twitter.explore_ranker.thriftscala.ImmersiveRecsResponse -import com.twitter.explore_ranker.thriftscala.ImmersiveRecsResult -import com.twitter.explore_ranker.thriftscala.NotificationsVideoRecs -import com.twitter.explore_ranker.thriftscala.Product -import com.twitter.explore_ranker.thriftscala.ProductContext -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateSource -import com.twitter.frigate.common.base.CandidateSourceEligible -import com.twitter.frigate.common.base.OutOfNetworkTweetCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.util.AdaptorUtils -import com.twitter.frigate.pushservice.util.MediaCRT -import com.twitter.frigate.pushservice.util.PushAdaptorUtil -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.product_mixer.core.thriftscala.ClientContext -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -case class ExploreVideoTweetCandidateAdaptor( - exploreRankerStore: ReadableStore[ExploreRankerRequest, ExploreRankerResponse], - tweetyPieStore: ReadableStore[Long, TweetyPieResult], - globalStats: StatsReceiver) - extends CandidateSource[Target, RawCandidate] - with CandidateSourceEligible[Target, RawCandidate] { - - override def name: String = this.getClass.getSimpleName - private[this] val stats = globalStats.scope("ExploreVideoTweetCandidateAdaptor") - private[this] val totalInputRecs = stats.stat("input_recs") - private[this] val totalRequests = stats.counter("total_requests") - private[this] val totalEmptyResponse = stats.counter("total_empty_response") - - private def buildExploreRankerRequest( - target: Target, - countryCode: Option[String], - language: Option[String], - ): ExploreRankerRequest = { - ExploreRankerRequest( - clientContext = ClientContext( - userId = Some(target.targetId), - countryCode = countryCode, - languageCode = language, - ), - product = Product.NotificationsVideoRecs, - productContext = Some(ProductContext.NotificationsVideoRecs(NotificationsVideoRecs())), - maxResults = Some(target.params(PushFeatureSwitchParams.MaxExploreVideoTweets)) - ) - } - - override def get(target: Target): Future[Option[Seq[RawCandidate]]] = { - Future - .join( - target.countryCode, - target.inferredUserDeviceLanguage - ).flatMap { - case (countryCode, language) => - val request = buildExploreRankerRequest(target, countryCode, language) - exploreRankerStore.get(request).flatMap { - case Some(response) => - val exploreResonseTweetIds = response match { - case ExploreRankerResponse(ExploreRankerProductResponse - .ImmersiveRecsResponse(ImmersiveRecsResponse(immersiveRecsResult))) => - immersiveRecsResult.collect { - case ImmersiveRecsResult(ExploreRecommendation - .ExploreTweetRecommendation(exploreTweetRecommendation)) => - exploreTweetRecommendation.tweetId - } - case _ => - Seq.empty - } - - totalInputRecs.add(exploreResonseTweetIds.size) - totalRequests.incr() - AdaptorUtils - .getTweetyPieResults(exploreResonseTweetIds.toSet, tweetyPieStore).map { - tweetyPieResultMap => - val candidates = tweetyPieResultMap.values.flatten - .map(buildVideoRawCandidates(target, _)) - Some(candidates.toSeq) - } - case _ => - totalEmptyResponse.incr() - Future.None - } - case _ => - Future.None - } - } - - override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { - PushDeviceUtil.isRecommendationsEligible(target).map { userRecommendationsEligible => - userRecommendationsEligible && target.params(PushFeatureSwitchParams.EnableExploreVideoTweets) - } - } - private def buildVideoRawCandidates( - target: Target, - tweetyPieResult: TweetyPieResult - ): RawCandidate with OutOfNetworkTweetCandidate = { - PushAdaptorUtil.generateOutOfNetworkTweetCandidates( - inputTarget = target, - id = tweetyPieResult.tweet.id, - mediaCRT = MediaCRT( - CommonRecommendationType.ExploreVideoTweet, - CommonRecommendationType.ExploreVideoTweet, - CommonRecommendationType.ExploreVideoTweet - ), - result = Some(tweetyPieResult), - localizedEntity = None - ) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/FRSTweetCandidateAdaptor.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/FRSTweetCandidateAdaptor.docx new file mode 100644 index 000000000..e1cbd65f1 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/FRSTweetCandidateAdaptor.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/FRSTweetCandidateAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/FRSTweetCandidateAdaptor.scala deleted file mode 100644 index 49610c645..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/FRSTweetCandidateAdaptor.scala +++ /dev/null @@ -1,272 +0,0 @@ -package com.twitter.frigate.pushservice.adaptor - -import com.twitter.cr_mixer.thriftscala.FrsTweetRequest -import com.twitter.cr_mixer.thriftscala.NotificationsContext -import com.twitter.cr_mixer.thriftscala.Product -import com.twitter.cr_mixer.thriftscala.ProductContext -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateSource -import com.twitter.frigate.common.base.CandidateSourceEligible -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.predicate.CommonOutNetworkTweetCandidatesSourcePredicates.filterOutReplyTweet -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.store.CrMixerTweetStore -import com.twitter.frigate.pushservice.store.UttEntityHydrationStore -import com.twitter.frigate.pushservice.util.MediaCRT -import com.twitter.frigate.pushservice.util.PushAdaptorUtil -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.frigate.pushservice.util.TopicsUtil -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.hermit.constants.AlgorithmFeedbackTokens -import com.twitter.hermit.model.Algorithm.Algorithm -import com.twitter.hermit.model.Algorithm.CrowdSearchAccounts -import com.twitter.hermit.model.Algorithm.ForwardEmailBook -import com.twitter.hermit.model.Algorithm.ForwardPhoneBook -import com.twitter.hermit.model.Algorithm.ReverseEmailBookIbis -import com.twitter.hermit.model.Algorithm.ReversePhoneBook -import com.twitter.hermit.store.tweetypie.UserTweet -import com.twitter.product_mixer.core.thriftscala.ClientContext -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.tsp.thriftscala.TopicSocialProofRequest -import com.twitter.tsp.thriftscala.TopicSocialProofResponse -import com.twitter.util.Future - -object FRSAlgorithmFeedbackTokenUtil { - private val crtsByAlgoToken = Map( - getAlgorithmToken(ReverseEmailBookIbis) -> CommonRecommendationType.ReverseAddressbookTweet, - getAlgorithmToken(ReversePhoneBook) -> CommonRecommendationType.ReverseAddressbookTweet, - getAlgorithmToken(ForwardEmailBook) -> CommonRecommendationType.ForwardAddressbookTweet, - getAlgorithmToken(ForwardPhoneBook) -> CommonRecommendationType.ForwardAddressbookTweet, - getAlgorithmToken(CrowdSearchAccounts) -> CommonRecommendationType.CrowdSearchTweet - ) - - def getAlgorithmToken(algorithm: Algorithm): Int = { - AlgorithmFeedbackTokens.AlgorithmToFeedbackTokenMap(algorithm) - } - - def getCRTForAlgoToken(algorithmToken: Int): Option[CommonRecommendationType] = { - crtsByAlgoToken.get(algorithmToken) - } -} - -case class FRSTweetCandidateAdaptor( - crMixerTweetStore: CrMixerTweetStore, - tweetyPieStore: ReadableStore[Long, TweetyPieResult], - tweetyPieStoreNoVF: ReadableStore[Long, TweetyPieResult], - userTweetTweetyPieStore: ReadableStore[UserTweet, TweetyPieResult], - uttEntityHydrationStore: UttEntityHydrationStore, - topicSocialProofServiceStore: ReadableStore[TopicSocialProofRequest, TopicSocialProofResponse], - globalStats: StatsReceiver) - extends CandidateSource[Target, RawCandidate] - with CandidateSourceEligible[Target, RawCandidate] { - - private val stats = globalStats.scope(this.getClass.getSimpleName) - private val crtStats = stats.scope("CandidateDistribution") - private val totalRequests = stats.counter("total_requests") - - // Candidate Distribution stats - private val reverseAddressbookCounter = crtStats.counter("reverse_addressbook") - private val forwardAddressbookCounter = crtStats.counter("forward_addressbook") - private val frsTweetCounter = crtStats.counter("frs_tweet") - private val nonReplyTweetsCounter = stats.counter("non_reply_tweets") - private val crtToCounterMapping: Map[CommonRecommendationType, Counter] = Map( - CommonRecommendationType.ReverseAddressbookTweet -> reverseAddressbookCounter, - CommonRecommendationType.ForwardAddressbookTweet -> forwardAddressbookCounter, - CommonRecommendationType.FrsTweet -> frsTweetCounter - ) - - private val emptyTweetyPieResult = stats.stat("empty_tweetypie_result") - - private[this] val numberReturnedCandidates = stats.stat("returned_candidates_from_earlybird") - private[this] val numberCandidateWithTopic: Counter = stats.counter("num_can_with_topic") - private[this] val numberCandidateWithoutTopic: Counter = stats.counter("num_can_without_topic") - - private val userTweetTweetyPieStoreCounter = stats.counter("user_tweet_tweetypie_store") - - override val name: String = this.getClass.getSimpleName - - private def filterInvalidTweets( - tweetIds: Seq[Long], - target: Target - ): Future[Map[Long, TweetyPieResult]] = { - val resMap = { - if (target.params(PushFeatureSwitchParams.EnableF1FromProtectedTweetAuthors)) { - userTweetTweetyPieStoreCounter.incr() - val keys = tweetIds.map { tweetId => - UserTweet(tweetId, Some(target.targetId)) - } - userTweetTweetyPieStore - .multiGet(keys.toSet).map { - case (userTweet, resultFut) => - userTweet.tweetId -> resultFut - }.toMap - } else { - (if (target.params(PushFeatureSwitchParams.EnableVFInTweetypie)) { - tweetyPieStore - } else { - tweetyPieStoreNoVF - }).multiGet(tweetIds.toSet) - } - } - - Future.collect(resMap).map { tweetyPieResultMap => - // Filter out replies and generate earlybird candidates only for non-empty tweetypie result - val cands = filterOutReplyTweet(tweetyPieResultMap, nonReplyTweetsCounter).collect { - case (id: Long, Some(result)) => - id -> result - } - - emptyTweetyPieResult.add(tweetyPieResultMap.size - cands.size) - cands - } - } - - private def buildRawCandidates( - target: Target, - ebCandidates: Seq[FRSTweetCandidate] - ): Future[Option[Seq[RawCandidate with TweetCandidate]]] = { - - val enableTopic = target.params(PushFeatureSwitchParams.EnableFrsTweetCandidatesTopicAnnotation) - val topicScoreThre = - target.params(PushFeatureSwitchParams.FrsTweetCandidatesTopicScoreThreshold) - - val ebTweets = ebCandidates.map { ebCandidate => - ebCandidate.tweetId -> ebCandidate.tweetyPieResult - }.toMap - - val tweetIdLocalizedEntityMapFut = TopicsUtil.getTweetIdLocalizedEntityMap( - target, - ebTweets, - uttEntityHydrationStore, - topicSocialProofServiceStore, - enableTopic, - topicScoreThre - ) - - Future.join(target.deviceInfo, tweetIdLocalizedEntityMapFut).map { - case (Some(deviceInfo), tweetIdLocalizedEntityMap) => - val candidates = ebCandidates - .map { ebCandidate => - val crt = ebCandidate.commonRecType - crtToCounterMapping.get(crt).foreach(_.incr()) - - val tweetId = ebCandidate.tweetId - val localizedEntityOpt = { - if (tweetIdLocalizedEntityMap - .contains(tweetId) && tweetIdLocalizedEntityMap.contains( - tweetId) && deviceInfo.isTopicsEligible) { - tweetIdLocalizedEntityMap(tweetId) - } else { - None - } - } - - PushAdaptorUtil.generateOutOfNetworkTweetCandidates( - inputTarget = target, - id = ebCandidate.tweetId, - mediaCRT = MediaCRT( - crt, - crt, - crt - ), - result = ebCandidate.tweetyPieResult, - localizedEntity = localizedEntityOpt) - }.filter { candidate => - // If user only has the topic setting enabled, filter out all non-topic cands - deviceInfo.isRecommendationsEligible || (deviceInfo.isTopicsEligible && candidate.semanticCoreEntityId.nonEmpty) - } - - candidates.map { candidate => - if (candidate.semanticCoreEntityId.nonEmpty) { - numberCandidateWithTopic.incr() - } else { - numberCandidateWithoutTopic.incr() - } - } - - numberReturnedCandidates.add(candidates.length) - Some(candidates) - case _ => Some(Seq.empty) - } - } - - def getTweetCandidatesFromCrMixer( - inputTarget: Target, - showAllResultsFromFrs: Boolean, - ): Future[Option[Seq[RawCandidate with TweetCandidate]]] = { - Future - .join( - inputTarget.seenTweetIds, - inputTarget.pushRecItems, - inputTarget.countryCode, - inputTarget.targetLanguage).flatMap { - case (seenTweetIds, pastRecItems, countryCode, language) => - val pastUserRecs = pastRecItems.userIds.toSeq - val request = FrsTweetRequest( - clientContext = ClientContext( - userId = Some(inputTarget.targetId), - countryCode = countryCode, - languageCode = language - ), - product = Product.Notifications, - productContext = Some(ProductContext.NotificationsContext(NotificationsContext())), - excludedUserIds = Some(pastUserRecs), - excludedTweetIds = Some(seenTweetIds) - ) - crMixerTweetStore.getFRSTweetCandidates(request).flatMap { - case Some(response) => - val tweetIds = response.tweets.map(_.tweetId) - val validTweets = filterInvalidTweets(tweetIds, inputTarget) - validTweets.flatMap { tweetypieMap => - val ebCandidates = response.tweets - .map { frsTweet => - val candidateTweetId = frsTweet.tweetId - val resultFromTweetyPie = tweetypieMap.get(candidateTweetId) - new FRSTweetCandidate { - override val tweetId = candidateTweetId - override val features = None - override val tweetyPieResult = resultFromTweetyPie - override val feedbackToken = frsTweet.frsPrimarySource - override val commonRecType: CommonRecommendationType = feedbackToken - .flatMap(token => - FRSAlgorithmFeedbackTokenUtil.getCRTForAlgoToken(token)).getOrElse( - CommonRecommendationType.FrsTweet) - } - }.filter { ebCandidate => - showAllResultsFromFrs || ebCandidate.commonRecType == CommonRecommendationType.ReverseAddressbookTweet - } - - numberReturnedCandidates.add(ebCandidates.length) - buildRawCandidates( - inputTarget, - ebCandidates - ) - } - case _ => Future.None - } - } - } - - override def get(inputTarget: Target): Future[Option[Seq[RawCandidate with TweetCandidate]]] = { - totalRequests.incr() - val enableResultsFromFrs = - inputTarget.params(PushFeatureSwitchParams.EnableResultFromFrsCandidates) - getTweetCandidatesFromCrMixer(inputTarget, enableResultsFromFrs) - } - - override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { - lazy val enableFrsCandidates = target.params(PushFeatureSwitchParams.EnableFrsCandidates) - PushDeviceUtil.isRecommendationsEligible(target).flatMap { isEnabledForRecosSetting => - PushDeviceUtil.isTopicsEligible(target).map { topicSettingEnabled => - val isEnabledForTopics = - topicSettingEnabled && target.params( - PushFeatureSwitchParams.EnableFrsTweetCandidatesTopicSetting) - (isEnabledForRecosSetting || isEnabledForTopics) && enableFrsCandidates - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/GenericCandidateAdaptor.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/GenericCandidateAdaptor.docx new file mode 100644 index 000000000..214b17725 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/GenericCandidateAdaptor.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/GenericCandidateAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/GenericCandidateAdaptor.scala deleted file mode 100644 index 24d0cb64a..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/GenericCandidateAdaptor.scala +++ /dev/null @@ -1,107 +0,0 @@ -package com.twitter.frigate.pushservice.adaptor - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.candidate._ -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -object GenericCandidates { - type Target = - TargetUser - with UserDetails - with TargetDecider - with TargetABDecider - with TweetImpressionHistory - with HTLVisitHistory - with MaxTweetAge - with NewUserDetails - with FrigateHistory - with TargetWithSeedUsers -} - -case class GenericCandidateAdaptor( - genericCandidates: CandidateSource[GenericCandidates.Target, Candidate], - tweetyPieStore: ReadableStore[Long, TweetyPieResult], - tweetyPieStoreNoVF: ReadableStore[Long, TweetyPieResult], - stats: StatsReceiver) - extends CandidateSource[Target, RawCandidate] - with CandidateSourceEligible[Target, RawCandidate] { - - override val name: String = genericCandidates.name - - private def generateTweetFavCandidate( - _target: Target, - _tweetId: Long, - _socialContextActions: Seq[SocialContextAction], - socialContextActionsAllTypes: Seq[SocialContextAction], - _tweetyPieResult: Option[TweetyPieResult] - ): RawCandidate = { - new RawCandidate with TweetFavoriteCandidate { - override val socialContextActions = _socialContextActions - override val socialContextAllTypeActions = - socialContextActionsAllTypes - val tweetId = _tweetId - val target = _target - val tweetyPieResult = _tweetyPieResult - } - } - - private def generateTweetRetweetCandidate( - _target: Target, - _tweetId: Long, - _socialContextActions: Seq[SocialContextAction], - socialContextActionsAllTypes: Seq[SocialContextAction], - _tweetyPieResult: Option[TweetyPieResult] - ): RawCandidate = { - new RawCandidate with TweetRetweetCandidate { - override val socialContextActions = _socialContextActions - override val socialContextAllTypeActions = socialContextActionsAllTypes - val tweetId = _tweetId - val target = _target - val tweetyPieResult = _tweetyPieResult - } - } - - override def get(inputTarget: Target): Future[Option[Seq[RawCandidate]]] = { - genericCandidates.get(inputTarget).map { candidatesOpt => - candidatesOpt - .map { candidates => - val candidatesSeq = - candidates.collect { - case tweetRetweet: TweetRetweetCandidate - if inputTarget.params(PushParams.MRTweetRetweetRecsParam) => - generateTweetRetweetCandidate( - inputTarget, - tweetRetweet.tweetId, - tweetRetweet.socialContextActions, - tweetRetweet.socialContextAllTypeActions, - tweetRetweet.tweetyPieResult) - case tweetFavorite: TweetFavoriteCandidate - if inputTarget.params(PushParams.MRTweetFavRecsParam) => - generateTweetFavCandidate( - inputTarget, - tweetFavorite.tweetId, - tweetFavorite.socialContextActions, - tweetFavorite.socialContextAllTypeActions, - tweetFavorite.tweetyPieResult) - } - candidatesSeq.foreach { candidate => - stats.counter(s"${candidate.commonRecType}_count").incr() - } - candidatesSeq - } - } - } - - override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { - PushDeviceUtil.isRecommendationsEligible(target).map { isAvailable => - isAvailable && target.params(PushParams.GenericCandidateAdaptorDecider) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/HighQualityTweetsAdaptor.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/HighQualityTweetsAdaptor.docx new file mode 100644 index 000000000..4919c1c0b Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/HighQualityTweetsAdaptor.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/HighQualityTweetsAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/HighQualityTweetsAdaptor.scala deleted file mode 100644 index 37d11535f..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/HighQualityTweetsAdaptor.scala +++ /dev/null @@ -1,280 +0,0 @@ -package com.twitter.frigate.pushservice.adaptor - -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateSource -import com.twitter.frigate.common.base.CandidateSourceEligible -import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.HighQualityCandidateGroupEnum -import com.twitter.frigate.pushservice.params.HighQualityCandidateGroupEnum._ -import com.twitter.frigate.pushservice.params.PushConstants.targetUserAgeFeatureName -import com.twitter.frigate.pushservice.params.PushConstants.targetUserPreferredLanguage -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} -import com.twitter.frigate.pushservice.predicate.TargetPredicates -import com.twitter.frigate.pushservice.util.MediaCRT -import com.twitter.frigate.pushservice.util.PushAdaptorUtil -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.frigate.pushservice.util.TopicsUtil -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.interests.thriftscala.InterestId.SemanticCore -import com.twitter.interests.thriftscala.UserInterests -import com.twitter.language.normalization.UserDisplayLanguage -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweet -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweets -import com.twitter.util.Future - -object HighQualityTweetsHelper { - def getFollowedTopics( - target: Target, - interestsWithLookupContextStore: ReadableStore[ - InterestsLookupRequestWithContext, - UserInterests - ], - followedTopicsStats: Stat - ): Future[Seq[Long]] = { - TopicsUtil - .getTopicsFollowedByUser(target, interestsWithLookupContextStore, followedTopicsStats).map { - userInterestsOpt => - val userInterests = userInterestsOpt.getOrElse(Seq.empty) - val extractedTopicIds = userInterests.flatMap { - _.interestId match { - case SemanticCore(semanticCore) => Some(semanticCore.id) - case _ => None - } - } - extractedTopicIds - } - } - - def getTripQueries( - target: Target, - enabledGroups: Set[HighQualityCandidateGroupEnum.Value], - interestsWithLookupContextStore: ReadableStore[ - InterestsLookupRequestWithContext, - UserInterests - ], - sourceIds: Seq[String], - stat: Stat - ): Future[Set[TripDomain]] = { - - val followedTopicIdsSetFut: Future[Set[Long]] = if (enabledGroups.contains(Topic)) { - getFollowedTopics(target, interestsWithLookupContextStore, stat).map(topicIds => - topicIds.toSet) - } else { - Future.value(Set.empty) - } - - Future - .join(target.featureMap, target.inferredUserDeviceLanguage, followedTopicIdsSetFut).map { - case ( - featureMap, - deviceLanguageOpt, - followedTopicIds - ) => - val ageBucketOpt = if (enabledGroups.contains(AgeBucket)) { - featureMap.categoricalFeatures.get(targetUserAgeFeatureName) - } else { - None - } - - val languageOptions: Set[Option[String]] = if (enabledGroups.contains(Language)) { - val userPreferredLanguages = featureMap.sparseBinaryFeatures - .getOrElse(targetUserPreferredLanguage, Set.empty[String]) - if (userPreferredLanguages.nonEmpty) { - userPreferredLanguages.map(lang => Some(UserDisplayLanguage.toTweetLanguage(lang))) - } else { - Set(deviceLanguageOpt.map(UserDisplayLanguage.toTweetLanguage)) - } - } else Set(None) - - val followedTopicOptions: Set[Option[Long]] = if (followedTopicIds.nonEmpty) { - followedTopicIds.map(topic => Some(topic)) - } else Set(None) - - val tripQueries = followedTopicOptions.flatMap { topicOption => - languageOptions.flatMap { languageOption => - sourceIds.map { sourceId => - TripDomain( - sourceId = sourceId, - language = languageOption, - placeId = None, - topicId = topicOption, - gender = None, - ageBucket = ageBucketOpt - ) - } - } - } - - tripQueries - } - } -} - -case class HighQualityTweetsAdaptor( - tripTweetCandidateStore: ReadableStore[TripDomain, TripTweets], - interestsWithLookupContextStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests], - tweetyPieStore: ReadableStore[Long, TweetyPieResult], - tweetyPieStoreNoVF: ReadableStore[Long, TweetyPieResult], - globalStats: StatsReceiver) - extends CandidateSource[Target, RawCandidate] - with CandidateSourceEligible[Target, RawCandidate] { - - override def name: String = this.getClass.getSimpleName - - private val stats = globalStats.scope("HighQualityCandidateAdaptor") - private val followedTopicsStats = stats.stat("followed_topics") - private val missingResponseCounter = stats.counter("missing_respond_counter") - private val crtFatigueCounter = stats.counter("fatigue_by_crt") - private val fallbackRequestsCounter = stats.counter("fallback_requests") - - override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { - PushDeviceUtil.isRecommendationsEligible(target).map { - _ && target.params(FS.HighQualityCandidatesEnableCandidateSource) - } - } - - private val highQualityCandidateFrequencyPredicate = { - TargetPredicates - .pushRecTypeFatiguePredicate( - CommonRecommendationType.TripHqTweet, - FS.HighQualityTweetsPushInterval, - FS.MaxHighQualityTweetsPushGivenInterval, - stats - ) - } - - private def getTripCandidatesStrato( - target: Target - ): Future[Map[Long, Set[TripDomain]]] = { - val tripQueriesF: Future[Set[TripDomain]] = HighQualityTweetsHelper.getTripQueries( - target = target, - enabledGroups = target.params(FS.HighQualityCandidatesEnableGroups).toSet, - interestsWithLookupContextStore = interestsWithLookupContextStore, - sourceIds = target.params(FS.TripTweetCandidateSourceIds), - stat = followedTopicsStats - ) - - lazy val fallbackTripQueriesFut: Future[Set[TripDomain]] = - if (target.params(FS.HighQualityCandidatesEnableFallback)) - HighQualityTweetsHelper.getTripQueries( - target = target, - enabledGroups = target.params(FS.HighQualityCandidatesFallbackEnabledGroups).toSet, - interestsWithLookupContextStore = interestsWithLookupContextStore, - sourceIds = target.params(FS.HighQualityCandidatesFallbackSourceIds), - stat = followedTopicsStats - ) - else Future.value(Set.empty) - - val initialTweetsFut: Future[Map[TripDomain, Seq[TripTweet]]] = tripQueriesF.flatMap { - tripQueries => getTripTweetsByDomains(tripQueries) - } - - val tweetsByDomainFut: Future[Map[TripDomain, Seq[TripTweet]]] = - if (target.params(FS.HighQualityCandidatesEnableFallback)) { - initialTweetsFut.flatMap { candidates => - val minCandidatesForFallback: Int = - target.params(FS.HighQualityCandidatesMinNumOfCandidatesToFallback) - val validCandidates = candidates.filter(_._2.size >= minCandidatesForFallback) - - if (validCandidates.nonEmpty) { - Future.value(validCandidates) - } else { - fallbackTripQueriesFut.flatMap { fallbackTripDomains => - fallbackRequestsCounter.incr(fallbackTripDomains.size) - getTripTweetsByDomains(fallbackTripDomains) - } - } - } - } else { - initialTweetsFut - } - - val numOfCandidates: Int = target.params(FS.HighQualityCandidatesNumberOfCandidates) - tweetsByDomainFut.map(tweetsByDomain => reformatDomainTweetMap(tweetsByDomain, numOfCandidates)) - } - - private def getTripTweetsByDomains( - tripQueries: Set[TripDomain] - ): Future[Map[TripDomain, Seq[TripTweet]]] = { - Future.collect(tripTweetCandidateStore.multiGet(tripQueries)).map { response => - response - .filter(p => p._2.exists(_.tweets.nonEmpty)) - .mapValues(_.map(_.tweets).getOrElse(Seq.empty)) - } - } - - private def reformatDomainTweetMap( - tweetsByDomain: Map[TripDomain, Seq[TripTweet]], - numOfCandidates: Int - ): Map[Long, Set[TripDomain]] = tweetsByDomain - .flatMap { - case (tripDomain, tripTweets) => - tripTweets - .sortBy(_.score)(Ordering[Double].reverse) - .take(numOfCandidates) - .map { tweet => (tweet.tweetId, tripDomain) } - }.groupBy(_._1).mapValues(_.map(_._2).toSet) - - private def buildRawCandidate( - target: Target, - tweetyPieResult: TweetyPieResult, - tripDomain: Option[scala.collection.Set[TripDomain]] - ): RawCandidate = { - PushAdaptorUtil.generateOutOfNetworkTweetCandidates( - inputTarget = target, - id = tweetyPieResult.tweet.id, - mediaCRT = MediaCRT( - CommonRecommendationType.TripHqTweet, - CommonRecommendationType.TripHqTweet, - CommonRecommendationType.TripHqTweet - ), - result = Some(tweetyPieResult), - tripTweetDomain = tripDomain - ) - } - - private def getTweetyPieResults( - target: Target, - tweetToTripDomain: Map[Long, Set[TripDomain]] - ): Future[Map[Long, Option[TweetyPieResult]]] = { - Future.collect((if (target.params(FS.EnableVFInTweetypie)) { - tweetyPieStore - } else { - tweetyPieStoreNoVF - }).multiGet(tweetToTripDomain.keySet)) - } - - override def get(target: Target): Future[Option[Seq[RawCandidate]]] = { - for { - tweetsToTripDomainMap <- getTripCandidatesStrato(target) - tweetyPieResults <- getTweetyPieResults(target, tweetsToTripDomainMap) - } yield { - val candidates = tweetyPieResults.flatMap { - case (tweetId, tweetyPieResultOpt) => - tweetyPieResultOpt.map(buildRawCandidate(target, _, tweetsToTripDomainMap.get(tweetId))) - } - if (candidates.nonEmpty) { - highQualityCandidateFrequencyPredicate(Seq(target)) - .map(_.head) - .map { isTargetFatigueEligible => - if (isTargetFatigueEligible) Some(candidates) - else { - crtFatigueCounter.incr() - None - } - } - - Some(candidates.toSeq) - } else { - missingResponseCounter.incr() - None - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ListsToRecommendCandidateAdaptor.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ListsToRecommendCandidateAdaptor.docx new file mode 100644 index 000000000..0368d20c1 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ListsToRecommendCandidateAdaptor.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ListsToRecommendCandidateAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ListsToRecommendCandidateAdaptor.scala deleted file mode 100644 index 59744b375..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ListsToRecommendCandidateAdaptor.scala +++ /dev/null @@ -1,152 +0,0 @@ -package com.twitter.frigate.pushservice.adaptor - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateSource -import com.twitter.frigate.common.base.CandidateSourceEligible -import com.twitter.frigate.common.base.ListPushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.predicate.TargetPredicates -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.geoduck.service.thriftscala.LocationResponse -import com.twitter.interests_discovery.thriftscala.DisplayLocation -import com.twitter.interests_discovery.thriftscala.NonPersonalizedRecommendedLists -import com.twitter.interests_discovery.thriftscala.RecommendedListsRequest -import com.twitter.interests_discovery.thriftscala.RecommendedListsResponse -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -case class ListsToRecommendCandidateAdaptor( - listRecommendationsStore: ReadableStore[String, NonPersonalizedRecommendedLists], - geoDuckV2Store: ReadableStore[Long, LocationResponse], - idsStore: ReadableStore[RecommendedListsRequest, RecommendedListsResponse], - globalStats: StatsReceiver) - extends CandidateSource[Target, RawCandidate] - with CandidateSourceEligible[Target, RawCandidate] { - - override val name: String = this.getClass.getSimpleName - - private[this] val stats = globalStats.scope(name) - private[this] val noLocationCodeCounter = stats.counter("no_location_code") - private[this] val noCandidatesCounter = stats.counter("no_candidates_for_geo") - private[this] val disablePopGeoListsCounter = stats.counter("disable_pop_geo_lists") - private[this] val disableIDSListsCounter = stats.counter("disable_ids_lists") - - private def getListCandidate( - targetUser: Target, - _listId: Long - ): RawCandidate with ListPushCandidate = { - new RawCandidate with ListPushCandidate { - override val listId: Long = _listId - - override val commonRecType: CommonRecommendationType = CommonRecommendationType.List - - override val target: Target = targetUser - } - } - - private def getListsRecommendedFromHistory( - target: Target - ): Future[Seq[Long]] = { - target.history.map { history => - history.sortedHistory.flatMap { - case (_, notif) if notif.commonRecommendationType == List => - notif.listNotification.map(_.listId) - case _ => None - } - } - } - - private def getIDSListRecs( - target: Target, - historicalListIds: Seq[Long] - ): Future[Seq[Long]] = { - val request = RecommendedListsRequest( - target.targetId, - DisplayLocation.ListDiscoveryPage, - Some(historicalListIds) - ) - if (target.params(PushFeatureSwitchParams.EnableIDSListRecommendations)) { - idsStore.get(request).map { - case Some(response) => - response.channels.map(_.id) - case _ => Nil - } - } else { - disableIDSListsCounter.incr() - Future.Nil - } - } - - private def getPopGeoLists( - target: Target, - historicalListIds: Seq[Long] - ): Future[Seq[Long]] = { - if (target.params(PushFeatureSwitchParams.EnablePopGeoListRecommendations)) { - geoDuckV2Store.get(target.targetId).flatMap { - case Some(locationResponse) if locationResponse.geohash.isDefined => - val geoHashLength = - target.params(PushFeatureSwitchParams.ListRecommendationsGeoHashLength) - val geoHash = locationResponse.geohash.get.take(geoHashLength) - listRecommendationsStore - .get(s"geohash_$geoHash") - .map { - case Some(recommendedLists) => - recommendedLists.recommendedListsByAlgo.flatMap { topLists => - topLists.lists.collect { - case list if !historicalListIds.contains(list.listId) => list.listId - } - } - case _ => Nil - } - case _ => - noLocationCodeCounter.incr() - Future.Nil - } - } else { - disablePopGeoListsCounter.incr() - Future.Nil - } - } - - override def get(target: Target): Future[Option[Seq[RawCandidate]]] = { - getListsRecommendedFromHistory(target).flatMap { historicalListIds => - Future - .join( - getPopGeoLists(target, historicalListIds), - getIDSListRecs(target, historicalListIds) - ) - .map { - case (popGeoListsIds, idsListIds) => - val candidates = (idsListIds ++ popGeoListsIds).map(getListCandidate(target, _)) - Some(candidates) - case _ => - noCandidatesCounter.incr() - None - } - } - } - - private val pushCapFatiguePredicate = TargetPredicates.pushRecTypeFatiguePredicate( - CommonRecommendationType.List, - PushFeatureSwitchParams.ListRecommendationsPushInterval, - PushFeatureSwitchParams.MaxListRecommendationsPushGivenInterval, - stats, - ) - override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { - - val isNotFatigued = pushCapFatiguePredicate.apply(Seq(target)).map(_.head) - - Future - .join( - PushDeviceUtil.isRecommendationsEligible(target), - isNotFatigued - ).map { - case (userRecommendationsEligible, isUnderCAP) => - userRecommendationsEligible && isUnderCAP && target.params( - PushFeatureSwitchParams.EnableListRecommendations) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/LoggedOutPushCandidateSourceGenerator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/LoggedOutPushCandidateSourceGenerator.docx new file mode 100644 index 000000000..e3f513298 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/LoggedOutPushCandidateSourceGenerator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/LoggedOutPushCandidateSourceGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/LoggedOutPushCandidateSourceGenerator.scala deleted file mode 100644 index e5ac0b516..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/LoggedOutPushCandidateSourceGenerator.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.frigate.pushservice.adaptor - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateSource -import com.twitter.frigate.common.base.CandidateSourceEligible -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.geoduck.service.thriftscala.LocationResponse -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweets -import com.twitter.content_mixer.thriftscala.ContentMixerRequest -import com.twitter.content_mixer.thriftscala.ContentMixerResponse -import com.twitter.geoduck.common.thriftscala.Location -import com.twitter.hermit.pop_geo.thriftscala.PopTweetsInPlace -import com.twitter.recommendation.interests.discovery.core.model.InterestDomain - -class LoggedOutPushCandidateSourceGenerator( - tripTweetCandidateStore: ReadableStore[TripDomain, TripTweets], - geoDuckV2Store: ReadableStore[Long, LocationResponse], - safeCachedTweetyPieStoreV2: ReadableStore[Long, TweetyPieResult], - cachedTweetyPieStoreV2NoVF: ReadableStore[Long, TweetyPieResult], - cachedTweetyPieStoreV2: ReadableStore[Long, TweetyPieResult], - contentMixerStore: ReadableStore[ContentMixerRequest, ContentMixerResponse], - softUserLocationStore: ReadableStore[Long, Location], - topTweetsByGeoStore: ReadableStore[InterestDomain[String], Map[String, List[(Long, Double)]]], - topTweetsByGeoV2VersionedStore: ReadableStore[String, PopTweetsInPlace], -)( - implicit val globalStats: StatsReceiver) { - val sources: Seq[CandidateSource[Target, RawCandidate] with CandidateSourceEligible[ - Target, - RawCandidate - ]] = { - Seq( - TripGeoCandidatesAdaptor( - tripTweetCandidateStore, - contentMixerStore, - safeCachedTweetyPieStoreV2, - cachedTweetyPieStoreV2NoVF, - globalStats - ), - TopTweetsByGeoAdaptor( - geoDuckV2Store, - softUserLocationStore, - topTweetsByGeoStore, - topTweetsByGeoV2VersionedStore, - cachedTweetyPieStoreV2, - cachedTweetyPieStoreV2NoVF, - globalStats - ) - ) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/OnboardingPushCandidateAdaptor.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/OnboardingPushCandidateAdaptor.docx new file mode 100644 index 000000000..93ba7e2a6 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/OnboardingPushCandidateAdaptor.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/OnboardingPushCandidateAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/OnboardingPushCandidateAdaptor.scala deleted file mode 100644 index 98568e9dc..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/OnboardingPushCandidateAdaptor.scala +++ /dev/null @@ -1,101 +0,0 @@ -package com.twitter.frigate.pushservice.adaptor - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateSource -import com.twitter.frigate.common.base.CandidateSourceEligible -import com.twitter.frigate.common.base.DiscoverTwitterCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} -import com.twitter.frigate.pushservice.predicate.DiscoverTwitterPredicate -import com.twitter.frigate.pushservice.predicate.TargetPredicates -import com.twitter.frigate.pushservice.util.PushAppPermissionUtil -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.frigate.thriftscala.{CommonRecommendationType => CRT} -import com.twitter.util.Future - -class OnboardingPushCandidateAdaptor( - globalStats: StatsReceiver) - extends CandidateSource[Target, RawCandidate] - with CandidateSourceEligible[Target, RawCandidate] { - - override val name: String = this.getClass.getSimpleName - - private[this] val stats = globalStats.scope(name) - private[this] val requestNum = stats.counter("request_num") - private[this] val addressBookCandNum = stats.counter("address_book_cand_num") - private[this] val completeOnboardingCandNum = stats.counter("complete_onboarding_cand_num") - - private def generateOnboardingPushRawCandidate( - _target: Target, - _commonRecType: CRT - ): RawCandidate = { - new RawCandidate with DiscoverTwitterCandidate { - override val target = _target - override val commonRecType = _commonRecType - } - } - - private def getEligibleCandsForTarget( - target: Target - ): Future[Option[Seq[RawCandidate]]] = { - val addressBookFatigue = - TargetPredicates - .pushRecTypeFatiguePredicate( - CRT.AddressBookUploadPush, - FS.FatigueForOnboardingPushes, - FS.MaxOnboardingPushInInterval, - stats)(Seq(target)).map(_.head) - val completeOnboardingFatigue = - TargetPredicates - .pushRecTypeFatiguePredicate( - CRT.CompleteOnboardingPush, - FS.FatigueForOnboardingPushes, - FS.MaxOnboardingPushInInterval, - stats)(Seq(target)).map(_.head) - - Future - .join( - target.appPermissions, - addressBookFatigue, - completeOnboardingFatigue - ).map { - case (appPermissionOpt, addressBookPredicate, completeOnboardingPredicate) => - val addressBookUploaded = - PushAppPermissionUtil.hasTargetUploadedAddressBook(appPermissionOpt) - val abUploadCandidate = - if (!addressBookUploaded && addressBookPredicate && target.params( - FS.EnableAddressBookPush)) { - addressBookCandNum.incr() - Some(generateOnboardingPushRawCandidate(target, CRT.AddressBookUploadPush)) - } else if (!addressBookUploaded && (completeOnboardingPredicate || - target.params(FS.DisableOnboardingPushFatigue)) && target.params( - FS.EnableCompleteOnboardingPush)) { - completeOnboardingCandNum.incr() - Some(generateOnboardingPushRawCandidate(target, CRT.CompleteOnboardingPush)) - } else None - - val allCandidates = - Seq(abUploadCandidate).filter(_.isDefined).flatten - if (allCandidates.nonEmpty) Some(allCandidates) else None - } - } - - override def get(inputTarget: Target): Future[Option[Seq[RawCandidate]]] = { - requestNum.incr() - val minDurationForMRElapsed = - DiscoverTwitterPredicate - .minDurationElapsedSinceLastMrPushPredicate( - name, - FS.MrMinDurationSincePushForOnboardingPushes, - stats)(Seq(inputTarget)).map(_.head) - minDurationForMRElapsed.flatMap { minDurationElapsed => - if (minDurationElapsed) getEligibleCandsForTarget(inputTarget) else Future.None - } - } - - override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { - PushDeviceUtil - .isRecommendationsEligible(target).map(_ && target.params(FS.EnableOnboardingPushes)) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/PushCandidateSourceGenerator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/PushCandidateSourceGenerator.docx new file mode 100644 index 000000000..65e5102c2 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/PushCandidateSourceGenerator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/PushCandidateSourceGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/PushCandidateSourceGenerator.scala deleted file mode 100644 index ea2dcd008..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/PushCandidateSourceGenerator.scala +++ /dev/null @@ -1,162 +0,0 @@ -package com.twitter.frigate.pushservice.adaptor - -import com.twitter.content_mixer.thriftscala.ContentMixerRequest -import com.twitter.content_mixer.thriftscala.ContentMixerResponse -import com.twitter.explore_ranker.thriftscala.ExploreRankerRequest -import com.twitter.explore_ranker.thriftscala.ExploreRankerResponse -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.candidate._ -import com.twitter.frigate.common.store.RecentTweetsQuery -import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.store._ -import com.twitter.geoduck.common.thriftscala.Location -import com.twitter.geoduck.service.thriftscala.LocationResponse -import com.twitter.hermit.pop_geo.thriftscala.PopTweetsInPlace -import com.twitter.hermit.predicate.socialgraph.RelationEdge -import com.twitter.hermit.store.tweetypie.UserTweet -import com.twitter.interests.thriftscala.UserInterests -import com.twitter.interests_discovery.thriftscala.NonPersonalizedRecommendedLists -import com.twitter.interests_discovery.thriftscala.RecommendedListsRequest -import com.twitter.interests_discovery.thriftscala.RecommendedListsResponse -import com.twitter.recommendation.interests.discovery.core.model.InterestDomain -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweets -import com.twitter.tsp.thriftscala.TopicSocialProofRequest -import com.twitter.tsp.thriftscala.TopicSocialProofResponse - -/** - * PushCandidateSourceGenerator generates candidate source list for a given Target user - */ -class PushCandidateSourceGenerator( - earlybirdCandidates: CandidateSource[EarlybirdCandidateSource.Query, EarlybirdCandidate], - userTweetEntityGraphCandidates: CandidateSource[UserTweetEntityGraphCandidates.Target, Candidate], - cachedTweetyPieStoreV2: ReadableStore[Long, TweetyPieResult], - safeCachedTweetyPieStoreV2: ReadableStore[Long, TweetyPieResult], - userTweetTweetyPieStore: ReadableStore[UserTweet, TweetyPieResult], - safeUserTweetTweetyPieStore: ReadableStore[UserTweet, TweetyPieResult], - cachedTweetyPieStoreV2NoVF: ReadableStore[Long, TweetyPieResult], - edgeStore: ReadableStore[RelationEdge, Boolean], - interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests], - uttEntityHydrationStore: UttEntityHydrationStore, - geoDuckV2Store: ReadableStore[Long, LocationResponse], - topTweetsByGeoStore: ReadableStore[InterestDomain[String], Map[String, List[(Long, Double)]]], - topTweetsByGeoV2VersionedStore: ReadableStore[String, PopTweetsInPlace], - tweetImpressionsStore: TweetImpressionsStore, - recommendedTrendsCandidateSource: RecommendedTrendsCandidateSource, - recentTweetsByAuthorStore: ReadableStore[RecentTweetsQuery, Seq[Seq[Long]]], - topicSocialProofServiceStore: ReadableStore[TopicSocialProofRequest, TopicSocialProofResponse], - crMixerStore: CrMixerTweetStore, - contentMixerStore: ReadableStore[ContentMixerRequest, ContentMixerResponse], - exploreRankerStore: ReadableStore[ExploreRankerRequest, ExploreRankerResponse], - softUserLocationStore: ReadableStore[Long, Location], - tripTweetCandidateStore: ReadableStore[TripDomain, TripTweets], - listRecsStore: ReadableStore[String, NonPersonalizedRecommendedLists], - idsStore: ReadableStore[RecommendedListsRequest, RecommendedListsResponse] -)( - implicit val globalStats: StatsReceiver) { - - private val earlyBirdFirstDegreeCandidateAdaptor = EarlyBirdFirstDegreeCandidateAdaptor( - earlybirdCandidates, - cachedTweetyPieStoreV2, - cachedTweetyPieStoreV2NoVF, - userTweetTweetyPieStore, - PushFeatureSwitchParams.NumberOfMaxEarlybirdInNetworkCandidatesParam, - globalStats - ) - - private val frsTweetCandidateAdaptor = FRSTweetCandidateAdaptor( - crMixerStore, - cachedTweetyPieStoreV2, - cachedTweetyPieStoreV2NoVF, - userTweetTweetyPieStore, - uttEntityHydrationStore, - topicSocialProofServiceStore, - globalStats - ) - - private val contentRecommenderMixerAdaptor = ContentRecommenderMixerAdaptor( - crMixerStore, - safeCachedTweetyPieStoreV2, - edgeStore, - topicSocialProofServiceStore, - uttEntityHydrationStore, - globalStats - ) - - private val tripGeoCandidatesAdaptor = TripGeoCandidatesAdaptor( - tripTweetCandidateStore, - contentMixerStore, - safeCachedTweetyPieStoreV2, - cachedTweetyPieStoreV2NoVF, - globalStats - ) - - val sources: Seq[ - CandidateSource[Target, RawCandidate] with CandidateSourceEligible[ - Target, - RawCandidate - ] - ] = { - Seq( - earlyBirdFirstDegreeCandidateAdaptor, - GenericCandidateAdaptor( - userTweetEntityGraphCandidates, - cachedTweetyPieStoreV2, - cachedTweetyPieStoreV2NoVF, - globalStats.scope("UserTweetEntityGraphCandidates") - ), - new OnboardingPushCandidateAdaptor(globalStats), - TopTweetsByGeoAdaptor( - geoDuckV2Store, - softUserLocationStore, - topTweetsByGeoStore, - topTweetsByGeoV2VersionedStore, - cachedTweetyPieStoreV2, - cachedTweetyPieStoreV2NoVF, - globalStats - ), - frsTweetCandidateAdaptor, - TopTweetImpressionsCandidateAdaptor( - recentTweetsByAuthorStore, - cachedTweetyPieStoreV2, - cachedTweetyPieStoreV2NoVF, - tweetImpressionsStore, - globalStats - ), - TrendsCandidatesAdaptor( - softUserLocationStore, - recommendedTrendsCandidateSource, - safeCachedTweetyPieStoreV2, - cachedTweetyPieStoreV2NoVF, - safeUserTweetTweetyPieStore, - globalStats - ), - contentRecommenderMixerAdaptor, - tripGeoCandidatesAdaptor, - HighQualityTweetsAdaptor( - tripTweetCandidateStore, - interestsLookupStore, - cachedTweetyPieStoreV2, - cachedTweetyPieStoreV2NoVF, - globalStats - ), - ExploreVideoTweetCandidateAdaptor( - exploreRankerStore, - cachedTweetyPieStoreV2, - globalStats - ), - ListsToRecommendCandidateAdaptor( - listRecsStore, - geoDuckV2Store, - idsStore, - globalStats - ) - ) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TopTweetImpressionsCandidateAdaptor.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TopTweetImpressionsCandidateAdaptor.docx new file mode 100644 index 000000000..982c40108 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TopTweetImpressionsCandidateAdaptor.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TopTweetImpressionsCandidateAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TopTweetImpressionsCandidateAdaptor.scala deleted file mode 100644 index 25ab31e85..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TopTweetImpressionsCandidateAdaptor.scala +++ /dev/null @@ -1,326 +0,0 @@ -package com.twitter.frigate.pushservice.adaptor - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateSource -import com.twitter.frigate.common.base.CandidateSourceEligible -import com.twitter.frigate.common.base.TopTweetImpressionsCandidate -import com.twitter.frigate.common.store.RecentTweetsQuery -import com.twitter.frigate.common.util.SnowflakeUtils -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} -import com.twitter.frigate.pushservice.store.TweetImpressionsStore -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.FutureOps -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -case class TweetImpressionsCandidate( - tweetId: Long, - tweetyPieResultOpt: Option[TweetyPieResult], - impressionsCountOpt: Option[Long]) - -case class TopTweetImpressionsCandidateAdaptor( - recentTweetsFromTflockStore: ReadableStore[RecentTweetsQuery, Seq[Seq[Long]]], - tweetyPieStore: ReadableStore[Long, TweetyPieResult], - tweetyPieStoreNoVF: ReadableStore[Long, TweetyPieResult], - tweetImpressionsStore: TweetImpressionsStore, - globalStats: StatsReceiver) - extends CandidateSource[Target, RawCandidate] - with CandidateSourceEligible[Target, RawCandidate] { - - private val stats = globalStats.scope("TopTweetImpressionsAdaptor") - private val tweetImpressionsCandsStat = stats.stat("top_tweet_impressions_cands_dist") - - private val eligibleUsersCounter = stats.counter("eligible_users") - private val noneligibleUsersCounter = stats.counter("noneligible_users") - private val meetsMinTweetsRequiredCounter = stats.counter("meets_min_tweets_required") - private val belowMinTweetsRequiredCounter = stats.counter("below_min_tweets_required") - private val aboveMaxInboundFavoritesCounter = stats.counter("above_max_inbound_favorites") - private val meetsImpressionsRequiredCounter = stats.counter("meets_impressions_required") - private val belowImpressionsRequiredCounter = stats.counter("below_impressions_required") - private val meetsFavoritesThresholdCounter = stats.counter("meets_favorites_threshold") - private val aboveFavoritesThresholdCounter = stats.counter("above_favorites_threshold") - private val emptyImpressionsMapCounter = stats.counter("empty_impressions_map") - - private val tflockResultsStat = stats.stat("tflock", "results") - private val emptyTflockResult = stats.counter("tflock", "empty_result") - private val nonEmptyTflockResult = stats.counter("tflock", "non_empty_result") - - private val originalTweetsStat = stats.stat("tweets", "original_tweets") - private val retweetsStat = stats.stat("tweets", "retweets") - private val allRetweetsOnlyCounter = stats.counter("tweets", "all_retweets_only") - private val allOriginalTweetsOnlyCounter = stats.counter("tweets", "all_original_tweets_only") - - private val emptyTweetypieMap = stats.counter("", "empty_tweetypie_map") - private val emptyTweetyPieResult = stats.stat("", "empty_tweetypie_result") - private val allEmptyTweetypieResults = stats.counter("", "all_empty_tweetypie_results") - - private val eligibleUsersAfterImpressionsFilter = - stats.counter("eligible_users_after_impressions_filter") - private val eligibleUsersAfterFavoritesFilter = - stats.counter("eligible_users_after_favorites_filter") - private val eligibleUsersWithEligibleTweets = - stats.counter("eligible_users_with_eligible_tweets") - - private val eligibleTweetCands = stats.stat("eligible_tweet_cands") - private val getCandsRequestCounter = - stats.counter("top_tweet_impressions_get_request") - - override val name: String = this.getClass.getSimpleName - - override def get(inputTarget: Target): Future[Option[Seq[RawCandidate]]] = { - getCandsRequestCounter.incr() - val eligibleCandidatesFut = getTweetImpressionsCandidates(inputTarget) - eligibleCandidatesFut.map { eligibleCandidates => - if (eligibleCandidates.nonEmpty) { - eligibleUsersWithEligibleTweets.incr() - eligibleTweetCands.add(eligibleCandidates.size) - val candidate = getMostImpressionsTweet(eligibleCandidates) - Some( - Seq( - generateTopTweetImpressionsCandidate( - inputTarget, - candidate.tweetId, - candidate.tweetyPieResultOpt, - candidate.impressionsCountOpt.getOrElse(0L)))) - } else None - } - } - - private def getTweetImpressionsCandidates( - inputTarget: Target - ): Future[Seq[TweetImpressionsCandidate]] = { - val originalTweets = getRecentOriginalTweetsForUser(inputTarget) - originalTweets.flatMap { tweetyPieResultsMap => - val numDaysSearchForOriginalTweets = - inputTarget.params(FS.TopTweetImpressionsOriginalTweetsNumDaysSearch) - val moreRecentTweetIds = - getMoreRecentTweetIds(tweetyPieResultsMap.keySet.toSeq, numDaysSearchForOriginalTweets) - val isEligible = isEligibleUser(inputTarget, tweetyPieResultsMap, moreRecentTweetIds) - if (isEligible) filterByEligibility(inputTarget, tweetyPieResultsMap, moreRecentTweetIds) - else Future.Nil - } - } - - private def getRecentOriginalTweetsForUser( - targetUser: Target - ): Future[Map[Long, TweetyPieResult]] = { - val tweetyPieResultsMapFut = getTflockStoreResults(targetUser).flatMap { recentTweetIds => - FutureOps.mapCollect((targetUser.params(FS.EnableVFInTweetypie) match { - case true => tweetyPieStore - case false => tweetyPieStoreNoVF - }).multiGet(recentTweetIds.toSet)) - } - tweetyPieResultsMapFut.map { tweetyPieResultsMap => - if (tweetyPieResultsMap.isEmpty) { - emptyTweetypieMap.incr() - Map.empty - } else removeRetweets(tweetyPieResultsMap) - } - } - - private def getTflockStoreResults(targetUser: Target): Future[Seq[Long]] = { - val maxResults = targetUser.params(FS.TopTweetImpressionsRecentTweetsByAuthorStoreMaxResults) - val maxAge = targetUser.params(FS.TopTweetImpressionsTotalFavoritesLimitNumDaysSearch) - val recentTweetsQuery = - RecentTweetsQuery( - userIds = Seq(targetUser.targetId), - maxResults = maxResults, - maxAge = maxAge.days - ) - recentTweetsFromTflockStore - .get(recentTweetsQuery).map { - case Some(tweetIdsAll) => - val tweetIds = tweetIdsAll.headOption.getOrElse(Seq.empty) - val numTweets = tweetIds.size - if (numTweets > 0) { - tflockResultsStat.add(numTweets) - nonEmptyTflockResult.incr() - } else emptyTflockResult.incr() - tweetIds - case _ => Nil - } - } - - private def removeRetweets( - tweetyPieResultsMap: Map[Long, Option[TweetyPieResult]] - ): Map[Long, TweetyPieResult] = { - val nonEmptyTweetyPieResults: Map[Long, TweetyPieResult] = tweetyPieResultsMap.collect { - case (key, Some(value)) => (key, value) - } - emptyTweetyPieResult.add(tweetyPieResultsMap.size - nonEmptyTweetyPieResults.size) - - if (nonEmptyTweetyPieResults.nonEmpty) { - val originalTweets = nonEmptyTweetyPieResults.filter { - case (_, tweetyPieResult) => - tweetyPieResult.sourceTweet.isEmpty - } - val numOriginalTweets = originalTweets.size - val numRetweets = nonEmptyTweetyPieResults.size - originalTweets.size - originalTweetsStat.add(numOriginalTweets) - retweetsStat.add(numRetweets) - if (numRetweets == 0) allOriginalTweetsOnlyCounter.incr() - if (numOriginalTweets == 0) allRetweetsOnlyCounter.incr() - originalTweets - } else { - allEmptyTweetypieResults.incr() - Map.empty - } - } - - private def getMoreRecentTweetIds( - tweetIds: Seq[Long], - numDays: Int - ): Seq[Long] = { - tweetIds.filter { tweetId => - SnowflakeUtils.isRecent(tweetId, numDays.days) - } - } - - private def isEligibleUser( - inputTarget: Target, - tweetyPieResults: Map[Long, TweetyPieResult], - recentTweetIds: Seq[Long] - ): Boolean = { - val minNumTweets = inputTarget.params(FS.TopTweetImpressionsMinNumOriginalTweets) - lazy val totalFavoritesLimit = - inputTarget.params(FS.TopTweetImpressionsTotalInboundFavoritesLimit) - if (recentTweetIds.size >= minNumTweets) { - meetsMinTweetsRequiredCounter.incr() - val isUnderLimit = isUnderTotalInboundFavoritesLimit(tweetyPieResults, totalFavoritesLimit) - if (isUnderLimit) eligibleUsersCounter.incr() - else { - aboveMaxInboundFavoritesCounter.incr() - noneligibleUsersCounter.incr() - } - isUnderLimit - } else { - belowMinTweetsRequiredCounter.incr() - noneligibleUsersCounter.incr() - false - } - } - - private def getFavoriteCounts( - tweetyPieResult: TweetyPieResult - ): Long = tweetyPieResult.tweet.counts.flatMap(_.favoriteCount).getOrElse(0L) - - private def isUnderTotalInboundFavoritesLimit( - tweetyPieResults: Map[Long, TweetyPieResult], - totalFavoritesLimit: Long - ): Boolean = { - val favoritesIterator = tweetyPieResults.valuesIterator.map(getFavoriteCounts) - val totalInboundFavorites = favoritesIterator.sum - totalInboundFavorites <= totalFavoritesLimit - } - - def filterByEligibility( - inputTarget: Target, - tweetyPieResults: Map[Long, TweetyPieResult], - tweetIds: Seq[Long] - ): Future[Seq[TweetImpressionsCandidate]] = { - lazy val minNumImpressions: Long = inputTarget.params(FS.TopTweetImpressionsMinRequired) - lazy val maxNumLikes: Long = inputTarget.params(FS.TopTweetImpressionsMaxFavoritesPerTweet) - for { - filteredImpressionsMap <- getFilteredImpressionsMap(tweetIds, minNumImpressions) - tweetIdsFilteredByFavorites <- - getTweetIdsFilteredByFavorites(filteredImpressionsMap.keySet, tweetyPieResults, maxNumLikes) - } yield { - if (filteredImpressionsMap.nonEmpty) eligibleUsersAfterImpressionsFilter.incr() - if (tweetIdsFilteredByFavorites.nonEmpty) eligibleUsersAfterFavoritesFilter.incr() - - val candidates = tweetIdsFilteredByFavorites.map { tweetId => - TweetImpressionsCandidate( - tweetId, - tweetyPieResults.get(tweetId), - filteredImpressionsMap.get(tweetId)) - } - tweetImpressionsCandsStat.add(candidates.length) - candidates - } - } - - private def getFilteredImpressionsMap( - tweetIds: Seq[Long], - minNumImpressions: Long - ): Future[Map[Long, Long]] = { - getImpressionsCounts(tweetIds).map { impressionsMap => - if (impressionsMap.isEmpty) emptyImpressionsMapCounter.incr() - impressionsMap.filter { - case (_, numImpressions) => - val isValid = numImpressions >= minNumImpressions - if (isValid) { - meetsImpressionsRequiredCounter.incr() - } else { - belowImpressionsRequiredCounter.incr() - } - isValid - } - } - } - - private def getTweetIdsFilteredByFavorites( - filteredTweetIds: Set[Long], - tweetyPieResults: Map[Long, TweetyPieResult], - maxNumLikes: Long - ): Future[Seq[Long]] = { - val filteredByFavoritesTweetIds = filteredTweetIds.filter { tweetId => - val tweetyPieResultOpt = tweetyPieResults.get(tweetId) - val isValid = tweetyPieResultOpt.exists { tweetyPieResult => - getFavoriteCounts(tweetyPieResult) <= maxNumLikes - } - if (isValid) meetsFavoritesThresholdCounter.incr() - else aboveFavoritesThresholdCounter.incr() - isValid - } - Future(filteredByFavoritesTweetIds.toSeq) - } - - private def getMostImpressionsTweet( - filteredResults: Seq[TweetImpressionsCandidate] - ): TweetImpressionsCandidate = { - val maxImpressions: Long = filteredResults.map { - _.impressionsCountOpt.getOrElse(0L) - }.max - - val mostImpressionsCandidates: Seq[TweetImpressionsCandidate] = - filteredResults.filter(_.impressionsCountOpt.getOrElse(0L) == maxImpressions) - - mostImpressionsCandidates.maxBy(_.tweetId) - } - - private def getImpressionsCounts( - tweetIds: Seq[Long] - ): Future[Map[Long, Long]] = { - val impressionCountMap = tweetIds.map { tweetId => - tweetId -> tweetImpressionsStore - .getCounts(tweetId).map(_.getOrElse(0L)) - }.toMap - Future.collect(impressionCountMap) - } - - private def generateTopTweetImpressionsCandidate( - inputTarget: Target, - _tweetId: Long, - result: Option[TweetyPieResult], - _impressionsCount: Long - ): RawCandidate = { - new RawCandidate with TopTweetImpressionsCandidate { - override val target: Target = inputTarget - override val tweetId: Long = _tweetId - override val tweetyPieResult: Option[TweetyPieResult] = result - override val impressionsCount: Long = _impressionsCount - } - } - - override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { - val enabledTopTweetImpressionsNotification = - target.params(FS.EnableTopTweetImpressionsNotification) - - PushDeviceUtil - .isRecommendationsEligible(target).map(_ && enabledTopTweetImpressionsNotification) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TopTweetsByGeoAdaptor.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TopTweetsByGeoAdaptor.docx new file mode 100644 index 000000000..0da5c4bda Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TopTweetsByGeoAdaptor.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TopTweetsByGeoAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TopTweetsByGeoAdaptor.scala deleted file mode 100644 index 3228760fd..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TopTweetsByGeoAdaptor.scala +++ /dev/null @@ -1,413 +0,0 @@ -package com.twitter.frigate.pushservice.adaptor - -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateSource -import com.twitter.frigate.common.base.CandidateSourceEligible -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.common.predicate.CommonOutNetworkTweetCandidatesSourcePredicates.filterOutReplyTweet -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.model.PushTypes -import com.twitter.frigate.pushservice.params.PopGeoTweetVersion -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.params.TopTweetsForGeoCombination -import com.twitter.frigate.pushservice.params.TopTweetsForGeoRankingFunction -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} -import com.twitter.frigate.pushservice.predicate.DiscoverTwitterPredicate -import com.twitter.frigate.pushservice.predicate.TargetPredicates -import com.twitter.frigate.pushservice.util.MediaCRT -import com.twitter.frigate.pushservice.util.PushAdaptorUtil -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.geoduck.common.thriftscala.{Location => GeoLocation} -import com.twitter.geoduck.service.thriftscala.LocationResponse -import com.twitter.gizmoduck.thriftscala.UserType -import com.twitter.hermit.pop_geo.thriftscala.PopTweetsInPlace -import com.twitter.recommendation.interests.discovery.core.model.InterestDomain -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.FutureOps -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future -import com.twitter.util.Time -import scala.collection.Map - -case class PlaceTweetScore(place: String, tweetId: Long, score: Double) { - def toTweetScore: (Long, Double) = (tweetId, score) -} -case class TopTweetsByGeoAdaptor( - geoduckStoreV2: ReadableStore[Long, LocationResponse], - softUserGeoLocationStore: ReadableStore[Long, GeoLocation], - topTweetsByGeoStore: ReadableStore[InterestDomain[String], Map[String, List[(Long, Double)]]], - topTweetsByGeoStoreV2: ReadableStore[String, PopTweetsInPlace], - tweetyPieStore: ReadableStore[Long, TweetyPieResult], - tweetyPieStoreNoVF: ReadableStore[Long, TweetyPieResult], - globalStats: StatsReceiver) - extends CandidateSource[Target, RawCandidate] - with CandidateSourceEligible[Target, RawCandidate] { - - override def name: String = this.getClass.getSimpleName - - private[this] val stats = globalStats.scope("TopTweetsByGeoAdaptor") - private[this] val noGeohashUserCounter: Counter = stats.counter("users_with_no_geohash_counter") - private[this] val incomingRequestCounter: Counter = stats.counter("incoming_request_counter") - private[this] val incomingLoggedOutRequestCounter: Counter = - stats.counter("incoming_logged_out_request_counter") - private[this] val loggedOutRawCandidatesCounter = - stats.counter("logged_out_raw_candidates_counter") - private[this] val emptyLoggedOutRawCandidatesCounter = - stats.counter("logged_out_empty_raw_candidates") - private[this] val outputTopTweetsByGeoCounter: Stat = - stats.stat("output_top_tweets_by_geo_counter") - private[this] val loggedOutPopByGeoV2CandidatesCounter: Counter = - stats.counter("logged_out_pop_by_geo_candidates") - private[this] val dormantUsersSince14DaysCounter: Counter = - stats.counter("dormant_user_since_14_days_counter") - private[this] val dormantUsersSince30DaysCounter: Counter = - stats.counter("dormant_user_since_30_days_counter") - private[this] val nonDormantUsersSince14DaysCounter: Counter = - stats.counter("non_dormant_user_since_14_days_counter") - private[this] val topTweetsByGeoTake100Counter: Counter = - stats.counter("top_tweets_by_geo_take_100_counter") - private[this] val combinationRequestsCounter = - stats.scope("combination_method_request_counter") - private[this] val popGeoTweetVersionCounter = - stats.scope("popgeo_tweet_version_counter") - private[this] val nonReplyTweetsCounter = stats.counter("non_reply_tweets") - - val MaxGeoHashSize = 4 - - private def constructKeys( - geohash: Option[String], - accountCountryCode: Option[String], - keyLengths: Seq[Int], - version: PopGeoTweetVersion.Value - ): Set[String] = { - val geohashKeys = geohash match { - case Some(hash) => keyLengths.map { version + "_geohash_" + hash.take(_) } - case _ => Seq.empty - } - - val accountCountryCodeKeys = - accountCountryCode.toSeq.map(version + "_country_" + _.toUpperCase) - (geohashKeys ++ accountCountryCodeKeys).toSet - } - - def convertToPlaceTweetScore( - popTweetsInPlace: Seq[PopTweetsInPlace] - ): Seq[PlaceTweetScore] = { - popTweetsInPlace.flatMap { - case p => - p.popTweets.map { - case popTweet => PlaceTweetScore(p.place, popTweet.tweetId, popTweet.score) - } - } - } - - def sortGeoHashTweets( - placeTweetScores: Seq[PlaceTweetScore], - rankingFunction: TopTweetsForGeoRankingFunction.Value - ): Seq[PlaceTweetScore] = { - rankingFunction match { - case TopTweetsForGeoRankingFunction.Score => - placeTweetScores.sortBy(_.score)(Ordering[Double].reverse) - case TopTweetsForGeoRankingFunction.GeohashLengthAndThenScore => - placeTweetScores - .sortBy(row => (row.place.length, row.score))(Ordering[(Int, Double)].reverse) - } - } - - def getResultsForLambdaStore( - inputTarget: Target, - geohash: Option[String], - store: ReadableStore[String, PopTweetsInPlace], - topk: Int, - version: PopGeoTweetVersion.Value - ): Future[Seq[(Long, Double)]] = { - inputTarget.accountCountryCode.flatMap { countryCode => - val keys = { - if (inputTarget.params(FS.EnableCountryCodeBackoffTopTweetsByGeo)) - constructKeys(geohash, countryCode, inputTarget.params(FS.GeoHashLengthList), version) - else - constructKeys(geohash, None, inputTarget.params(FS.GeoHashLengthList), version) - } - FutureOps - .mapCollect(store.multiGet(keys)).map { - case geohashTweetMap => - val popTweets = - geohashTweetMap.values.flatten.toSeq - val results = sortGeoHashTweets( - convertToPlaceTweetScore(popTweets), - inputTarget.params(FS.RankingFunctionForTopTweetsByGeo)) - .map(_.toTweetScore).take(topk) - results - } - } - } - - def getPopGeoTweetsForLoggedOutUsers( - inputTarget: Target, - store: ReadableStore[String, PopTweetsInPlace] - ): Future[Seq[(Long, Double)]] = { - inputTarget.countryCode.flatMap { countryCode => - val keys = constructKeys(None, countryCode, Seq(4), PopGeoTweetVersion.Prod) - FutureOps.mapCollect(store.multiGet(keys)).map { - case tweetMap => - val tweets = tweetMap.values.flatten.toSeq - loggedOutPopByGeoV2CandidatesCounter.incr(tweets.size) - val popTweets = sortGeoHashTweets( - convertToPlaceTweetScore(tweets), - TopTweetsForGeoRankingFunction.Score).map(_.toTweetScore) - popTweets - } - } - } - - def getRankedTweets( - inputTarget: Target, - geohash: Option[String] - ): Future[Seq[(Long, Double)]] = { - val MaxTopTweetsByGeoCandidatesToTake = - inputTarget.params(FS.MaxTopTweetsByGeoCandidatesToTake) - val scoringFn: String = inputTarget.params(FS.ScoringFuncForTopTweetsByGeo) - val combinationMethod = inputTarget.params(FS.TopTweetsByGeoCombinationParam) - val popGeoTweetVersion = inputTarget.params(FS.PopGeoTweetVersionParam) - - inputTarget.isHeavyUserState.map { isHeavyUser => - stats - .scope(combinationMethod.toString).scope(popGeoTweetVersion.toString).scope( - "IsHeavyUser_" + isHeavyUser.toString).counter().incr() - } - combinationRequestsCounter.scope(combinationMethod.toString).counter().incr() - popGeoTweetVersionCounter.scope(popGeoTweetVersion.toString).counter().incr() - lazy val geoStoreResults = if (geohash.isDefined) { - val hash = geohash.get.take(MaxGeoHashSize) - topTweetsByGeoStore - .get( - InterestDomain[String](hash) - ) - .map { - case Some(scoringFnToTweetsMapOpt) => - val tweetsWithScore = scoringFnToTweetsMapOpt - .getOrElse(scoringFn, List.empty) - val sortedResults = sortGeoHashTweets( - tweetsWithScore.map { - case (tweetId, score) => PlaceTweetScore(hash, tweetId, score) - }, - TopTweetsForGeoRankingFunction.Score - ).map(_.toTweetScore).take( - MaxTopTweetsByGeoCandidatesToTake - ) - sortedResults - case _ => Seq.empty - } - } else Future.value(Seq.empty) - lazy val versionPopGeoTweetResults = - getResultsForLambdaStore( - inputTarget, - geohash, - topTweetsByGeoStoreV2, - MaxTopTweetsByGeoCandidatesToTake, - popGeoTweetVersion - ) - combinationMethod match { - case TopTweetsForGeoCombination.Default => geoStoreResults - case TopTweetsForGeoCombination.AccountsTweetFavAsBackfill => - Future.join(geoStoreResults, versionPopGeoTweetResults).map { - case (geoStoreTweets, versionPopGeoTweets) => - (geoStoreTweets ++ versionPopGeoTweets).take(MaxTopTweetsByGeoCandidatesToTake) - } - case TopTweetsForGeoCombination.AccountsTweetFavIntermixed => - Future.join(geoStoreResults, versionPopGeoTweetResults).map { - case (geoStoreTweets, versionPopGeoTweets) => - CandidateSource.interleaveSeqs(Seq(geoStoreTweets, versionPopGeoTweets)) - } - } - } - - override def get(inputTarget: Target): Future[Option[Seq[RawCandidate]]] = { - if (inputTarget.isLoggedOutUser) { - incomingLoggedOutRequestCounter.incr() - val rankedTweets = getPopGeoTweetsForLoggedOutUsers(inputTarget, topTweetsByGeoStoreV2) - val rawCandidates = { - rankedTweets.map { rt => - FutureOps - .mapCollect( - tweetyPieStore - .multiGet(rt.map { case (tweetId, _) => tweetId }.toSet)) - .map { tweetyPieResultMap => - val results = buildTopTweetsByGeoRawCandidates( - inputTarget, - None, - tweetyPieResultMap - ) - if (results.isEmpty) { - emptyLoggedOutRawCandidatesCounter.incr() - } - loggedOutRawCandidatesCounter.incr(results.size) - Some(results) - } - }.flatten - } - rawCandidates - } else { - incomingRequestCounter.incr() - getGeoHashForUsers(inputTarget).flatMap { geohash => - if (geohash.isEmpty) noGeohashUserCounter.incr() - getRankedTweets(inputTarget, geohash).map { rt => - if (rt.size == 100) { - topTweetsByGeoTake100Counter.incr(1) - } - FutureOps - .mapCollect((inputTarget.params(FS.EnableVFInTweetypie) match { - case true => tweetyPieStore - case false => tweetyPieStoreNoVF - }).multiGet(rt.map { case (tweetId, _) => tweetId }.toSet)) - .map { tweetyPieResultMap => - Some( - buildTopTweetsByGeoRawCandidates( - inputTarget, - None, - filterOutReplyTweet( - tweetyPieResultMap, - nonReplyTweetsCounter - ) - ) - ) - } - }.flatten - } - } - } - - private def getGeoHashForUsers( - inputTarget: Target - ): Future[Option[String]] = { - - inputTarget.targetUser.flatMap { - case Some(user) => - user.userType match { - case UserType.Soft => - softUserGeoLocationStore - .get(inputTarget.targetId) - .map(_.flatMap(_.geohash.flatMap(_.stringGeohash))) - - case _ => - geoduckStoreV2.get(inputTarget.targetId).map(_.flatMap(_.geohash)) - } - - case None => Future.None - } - } - - private def buildTopTweetsByGeoRawCandidates( - target: PushTypes.Target, - locationName: Option[String], - topTweets: Map[Long, Option[TweetyPieResult]] - ): Seq[RawCandidate with TweetCandidate] = { - val candidates = topTweets.map { tweetIdTweetyPieResultMap => - PushAdaptorUtil.generateOutOfNetworkTweetCandidates( - inputTarget = target, - id = tweetIdTweetyPieResultMap._1, - mediaCRT = MediaCRT( - CommonRecommendationType.GeoPopTweet, - CommonRecommendationType.GeoPopTweet, - CommonRecommendationType.GeoPopTweet - ), - result = tweetIdTweetyPieResultMap._2, - localizedEntity = None - ) - }.toSeq - outputTopTweetsByGeoCounter.add(candidates.length) - candidates - } - - private val topTweetsByGeoFrequencyPredicate = { - TargetPredicates - .pushRecTypeFatiguePredicate( - CommonRecommendationType.GeoPopTweet, - FS.TopTweetsByGeoPushInterval, - FS.MaxTopTweetsByGeoPushGivenInterval, - stats - ) - } - - def getAvailabilityForDormantUser(target: Target): Future[Boolean] = { - lazy val isDormantUserNotFatigued = topTweetsByGeoFrequencyPredicate(Seq(target)).map(_.head) - lazy val enableTopTweetsByGeoForDormantUsers = - target.params(FS.EnableTopTweetsByGeoCandidatesForDormantUsers) - - target.lastHTLVisitTimestamp.flatMap { - case Some(lastHTLTimestamp) => - val minTimeSinceLastLogin = - target.params(FS.MinimumTimeSinceLastLoginForGeoPopTweetPush).ago - val timeSinceInactive = target.params(FS.TimeSinceLastLoginForGeoPopTweetPush).ago - val lastActiveTimestamp = Time.fromMilliseconds(lastHTLTimestamp) - if (lastActiveTimestamp > minTimeSinceLastLogin) { - nonDormantUsersSince14DaysCounter.incr() - Future.False - } else { - dormantUsersSince14DaysCounter.incr() - isDormantUserNotFatigued.map { isUserNotFatigued => - lastActiveTimestamp < timeSinceInactive && - enableTopTweetsByGeoForDormantUsers && - isUserNotFatigued - } - } - case _ => - dormantUsersSince30DaysCounter.incr() - isDormantUserNotFatigued.map { isUserNotFatigued => - enableTopTweetsByGeoForDormantUsers && isUserNotFatigued - } - } - } - - def getAvailabilityForPlaybookSetUp(target: Target): Future[Boolean] = { - lazy val enableTopTweetsByGeoForNewUsers = target.params(FS.EnableTopTweetsByGeoCandidates) - val isTargetEligibleForMrFatigueCheck = target.isAccountAtleastNDaysOld( - target.params(FS.MrMinDurationSincePushForTopTweetsByGeoPushes)) - val isMrFatigueCheckEnabled = - target.params(FS.EnableMrMinDurationSinceMrPushFatigue) - val applyPredicateForTopTweetsByGeo = - if (isMrFatigueCheckEnabled) { - if (isTargetEligibleForMrFatigueCheck) { - DiscoverTwitterPredicate - .minDurationElapsedSinceLastMrPushPredicate( - name, - FS.MrMinDurationSincePushForTopTweetsByGeoPushes, - stats - ).andThen( - topTweetsByGeoFrequencyPredicate - )(Seq(target)).map(_.head) - } else { - Future.False - } - } else { - topTweetsByGeoFrequencyPredicate(Seq(target)).map(_.head) - } - applyPredicateForTopTweetsByGeo.map { predicateResult => - predicateResult && enableTopTweetsByGeoForNewUsers - } - } - - override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { - if (target.isLoggedOutUser) { - Future.True - } else { - PushDeviceUtil - .isRecommendationsEligible(target).map( - _ && target.params(PushParams.PopGeoCandidatesDecider)).flatMap { isAvailable => - if (isAvailable) { - Future - .join(getAvailabilityForDormantUser(target), getAvailabilityForPlaybookSetUp(target)) - .map { - case (isAvailableForDormantUser, isAvailableForPlaybook) => - isAvailableForDormantUser || isAvailableForPlaybook - case _ => false - } - } else Future.False - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TrendsCandidatesAdaptor.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TrendsCandidatesAdaptor.docx new file mode 100644 index 000000000..c2e6b9326 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TrendsCandidatesAdaptor.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TrendsCandidatesAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TrendsCandidatesAdaptor.scala deleted file mode 100644 index 4e7ec3314..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TrendsCandidatesAdaptor.scala +++ /dev/null @@ -1,215 +0,0 @@ -package com.twitter.frigate.pushservice.adaptor - -import com.twitter.events.recos.thriftscala.DisplayLocation -import com.twitter.events.recos.thriftscala.TrendsContext -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateSource -import com.twitter.frigate.common.base.CandidateSourceEligible -import com.twitter.frigate.common.base.TrendTweetCandidate -import com.twitter.frigate.common.base.TrendsCandidate -import com.twitter.frigate.common.candidate.RecommendedTrendsCandidateSource -import com.twitter.frigate.common.candidate.RecommendedTrendsCandidateSource.Query -import com.twitter.frigate.common.predicate.CommonOutNetworkTweetCandidatesSourcePredicates.filterOutReplyTweet -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.adaptor.TrendsCandidatesAdaptor._ -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.predicate.TargetPredicates -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.geoduck.common.thriftscala.Location -import com.twitter.gizmoduck.thriftscala.UserType -import com.twitter.hermit.store.tweetypie.UserTweet -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future -import scala.collection.Map - -object TrendsCandidatesAdaptor { - type TweetId = Long - type EventId = Long -} - -case class TrendsCandidatesAdaptor( - softUserGeoLocationStore: ReadableStore[Long, Location], - recommendedTrendsCandidateSource: RecommendedTrendsCandidateSource, - tweetyPieStore: ReadableStore[Long, TweetyPieResult], - tweetyPieStoreNoVF: ReadableStore[Long, TweetyPieResult], - safeUserTweetTweetyPieStore: ReadableStore[UserTweet, TweetyPieResult], - statsReceiver: StatsReceiver) - extends CandidateSource[Target, RawCandidate] - with CandidateSourceEligible[Target, RawCandidate] { - override val name = this.getClass.getSimpleName - - private val trendAdaptorStats = statsReceiver.scope("TrendsCandidatesAdaptor") - private val trendTweetCandidateNumber = trendAdaptorStats.counter("trend_tweet_candidate") - private val nonReplyTweetsCounter = trendAdaptorStats.counter("non_reply_tweets") - - private def getQuery(target: Target): Future[Query] = { - def getUserCountryCode(target: Target): Future[Option[String]] = { - target.targetUser.flatMap { - case Some(user) if user.userType == UserType.Soft => - softUserGeoLocationStore - .get(user.id) - .map(_.flatMap(_.simpleRgcResult.flatMap(_.countryCodeAlpha2))) - - case _ => target.accountCountryCode - } - } - - for { - countryCode <- getUserCountryCode(target) - inferredLanguage <- target.inferredUserDeviceLanguage - } yield { - Query( - userId = target.targetId, - displayLocation = DisplayLocation.MagicRecs, - languageCode = inferredLanguage, - countryCode = countryCode, - maxResults = target.params(PushFeatureSwitchParams.MaxRecommendedTrendsToQuery) - ) - } - } - - /** - * Query candidates only if sent at most [[PushFeatureSwitchParams.MaxTrendTweetNotificationsInDuration]] - * trend tweet notifications in [[PushFeatureSwitchParams.TrendTweetNotificationsFatigueDuration]] - */ - val trendTweetFatiguePredicate = TargetPredicates.pushRecTypeFatiguePredicate( - CommonRecommendationType.TrendTweet, - PushFeatureSwitchParams.TrendTweetNotificationsFatigueDuration, - PushFeatureSwitchParams.MaxTrendTweetNotificationsInDuration, - trendAdaptorStats - ) - - private val recommendedTrendsWithTweetsCandidateSource: CandidateSource[ - Target, - RawCandidate with TrendsCandidate - ] = recommendedTrendsCandidateSource - .convert[Target, TrendsCandidate]( - getQuery, - recommendedTrendsCandidateSource.identityCandidateMapper - ) - .batchMapValues[Target, RawCandidate with TrendsCandidate]( - trendsCandidatesToTweetCandidates(_, _, getTweetyPieResults)) - - private def getTweetyPieResults( - tweetIds: Seq[TweetId], - target: Target - ): Future[Map[TweetId, TweetyPieResult]] = { - if (target.params(PushFeatureSwitchParams.EnableSafeUserTweetTweetypieStore)) { - Future - .collect( - safeUserTweetTweetyPieStore.multiGet( - tweetIds.toSet.map(UserTweet(_, Some(target.targetId))))).map { - _.collect { - case (userTweet, Some(tweetyPieResult)) => userTweet.tweetId -> tweetyPieResult - } - } - } else { - Future - .collect((target.params(PushFeatureSwitchParams.EnableVFInTweetypie) match { - case true => tweetyPieStore - case false => tweetyPieStoreNoVF - }).multiGet(tweetIds.toSet)).map { tweetyPieResultMap => - filterOutReplyTweet(tweetyPieResultMap, nonReplyTweetsCounter).collect { - case (tweetId, Some(tweetyPieResult)) => tweetId -> tweetyPieResult - } - } - } - } - - /** - * - * @param _target: [[Target]] object representing notificaion recipient user - * @param trendsCandidates: Sequence of [[TrendsCandidate]] returned from ERS - * @return: Seq of trends candidates expanded to associated tweets. - */ - private def trendsCandidatesToTweetCandidates( - _target: Target, - trendsCandidates: Seq[TrendsCandidate], - getTweetyPieResults: (Seq[TweetId], Target) => Future[Map[TweetId, TweetyPieResult]] - ): Future[Seq[RawCandidate with TrendsCandidate]] = { - - def generateTrendTweetCandidates( - trendCandidate: TrendsCandidate, - tweetyPieResults: Map[TweetId, TweetyPieResult] - ) = { - val tweetIds = trendCandidate.context.curatedRepresentativeTweets.getOrElse(Seq.empty) ++ - trendCandidate.context.algoRepresentativeTweets.getOrElse(Seq.empty) - - tweetIds.flatMap { tweetId => - tweetyPieResults.get(tweetId).map { _tweetyPieResult => - new RawCandidate with TrendTweetCandidate { - override val trendId: String = trendCandidate.trendId - override val trendName: String = trendCandidate.trendName - override val landingUrl: String = trendCandidate.landingUrl - override val timeBoundedLandingUrl: Option[String] = - trendCandidate.timeBoundedLandingUrl - override val context: TrendsContext = trendCandidate.context - override val tweetyPieResult: Option[TweetyPieResult] = Some(_tweetyPieResult) - override val tweetId: TweetId = _tweetyPieResult.tweet.id - override val target: Target = _target - } - } - } - } - - // collect all tweet ids associated with all trends - val allTweetIds = trendsCandidates.flatMap { trendsCandidate => - val context = trendsCandidate.context - context.curatedRepresentativeTweets.getOrElse(Seq.empty) ++ - context.algoRepresentativeTweets.getOrElse(Seq.empty) - } - - getTweetyPieResults(allTweetIds, _target) - .map { tweetIdToTweetyPieResult => - val trendTweetCandidates = trendsCandidates.flatMap { trendCandidate => - val allTrendTweetCandidates = generateTrendTweetCandidates( - trendCandidate, - tweetIdToTweetyPieResult - ) - - val (tweetCandidatesFromCuratedTrends, tweetCandidatesFromNonCuratedTrends) = - allTrendTweetCandidates.partition(_.isCuratedTrend) - - tweetCandidatesFromCuratedTrends.filter( - _.target.params(PushFeatureSwitchParams.EnableCuratedTrendTweets)) ++ - tweetCandidatesFromNonCuratedTrends.filter( - _.target.params(PushFeatureSwitchParams.EnableNonCuratedTrendTweets)) - } - - trendTweetCandidateNumber.incr(trendTweetCandidates.size) - trendTweetCandidates - } - } - - /** - * - * @param target: [[Target]] user - * @return: true if customer is eligible to receive trend tweet notifications - * - */ - override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { - PushDeviceUtil - .isRecommendationsEligible(target) - .map(target.params(PushParams.TrendsCandidateDecider) && _) - } - - override def get(target: Target): Future[Option[Seq[RawCandidate with TrendsCandidate]]] = { - recommendedTrendsWithTweetsCandidateSource - .get(target) - .flatMap { - case Some(candidates) if candidates.nonEmpty => - trendTweetFatiguePredicate(Seq(target)) - .map(_.head) - .map { isTargetFatigueEligible => - if (isTargetFatigueEligible) Some(candidates) - else None - } - - case _ => Future.None - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TripGeoCandidatesAdaptor.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TripGeoCandidatesAdaptor.docx new file mode 100644 index 000000000..be8584f0b Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TripGeoCandidatesAdaptor.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TripGeoCandidatesAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TripGeoCandidatesAdaptor.scala deleted file mode 100644 index 2bdef162c..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TripGeoCandidatesAdaptor.scala +++ /dev/null @@ -1,188 +0,0 @@ -package com.twitter.frigate.pushservice.adaptor - -import com.twitter.content_mixer.thriftscala.ContentMixerProductResponse -import com.twitter.content_mixer.thriftscala.ContentMixerRequest -import com.twitter.content_mixer.thriftscala.ContentMixerResponse -import com.twitter.content_mixer.thriftscala.NotificationsTripTweetsProductContext -import com.twitter.content_mixer.thriftscala.Product -import com.twitter.content_mixer.thriftscala.ProductContext -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateSource -import com.twitter.frigate.common.base.CandidateSourceEligible -import com.twitter.frigate.common.predicate.CommonOutNetworkTweetCandidatesSourcePredicates.filterOutReplyTweet -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.util.MediaCRT -import com.twitter.frigate.pushservice.util.PushAdaptorUtil -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.geoduck.util.country.CountryInfo -import com.twitter.product_mixer.core.thriftscala.ClientContext -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweets -import com.twitter.util.Future - -case class TripGeoCandidatesAdaptor( - tripTweetCandidateStore: ReadableStore[TripDomain, TripTweets], - contentMixerStore: ReadableStore[ContentMixerRequest, ContentMixerResponse], - tweetyPieStore: ReadableStore[Long, TweetyPieResult], - tweetyPieStoreNoVF: ReadableStore[Long, TweetyPieResult], - statsReceiver: StatsReceiver) - extends CandidateSource[Target, RawCandidate] - with CandidateSourceEligible[Target, RawCandidate] { - - override def name: String = this.getClass.getSimpleName - - private val stats = statsReceiver.scope(name.stripSuffix("$")) - - private val contentMixerRequests = stats.counter("getTripCandidatesContentMixerRequests") - private val loggedOutTripTweetIds = stats.counter("logged_out_trip_tweet_ids_count") - private val loggedOutRawCandidates = stats.counter("logged_out_raw_candidates_count") - private val rawCandidates = stats.counter("raw_candidates_count") - private val loggedOutEmptyplaceId = stats.counter("logged_out_empty_place_id_count") - private val loggedOutPlaceId = stats.counter("logged_out_place_id_count") - private val nonReplyTweetsCounter = stats.counter("non_reply_tweets") - - override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { - if (target.isLoggedOutUser) { - Future.True - } else { - for { - isRecommendationsSettingEnabled <- PushDeviceUtil.isRecommendationsEligible(target) - inferredLanguage <- target.inferredUserDeviceLanguage - } yield { - isRecommendationsSettingEnabled && - inferredLanguage.nonEmpty && - target.params(PushParams.TripGeoTweetCandidatesDecider) - } - } - - } - - private def buildRawCandidate(target: Target, tweetyPieResult: TweetyPieResult): RawCandidate = { - PushAdaptorUtil.generateOutOfNetworkTweetCandidates( - inputTarget = target, - id = tweetyPieResult.tweet.id, - mediaCRT = MediaCRT( - CommonRecommendationType.TripGeoTweet, - CommonRecommendationType.TripGeoTweet, - CommonRecommendationType.TripGeoTweet - ), - result = Some(tweetyPieResult), - localizedEntity = None - ) - } - - override def get(target: Target): Future[Option[Seq[RawCandidate]]] = { - if (target.isLoggedOutUser) { - for { - tripTweetIds <- getTripCandidatesForLoggedOutTarget(target) - tweetyPieResults <- Future.collect(tweetyPieStoreNoVF.multiGet(tripTweetIds)) - } yield { - val candidates = tweetyPieResults.values.flatten.map(buildRawCandidate(target, _)) - if (candidates.nonEmpty) { - loggedOutRawCandidates.incr(candidates.size) - Some(candidates.toSeq) - } else None - } - } else { - for { - tripTweetIds <- getTripCandidatesContentMixer(target) - tweetyPieResults <- - Future.collect((target.params(PushFeatureSwitchParams.EnableVFInTweetypie) match { - case true => tweetyPieStore - case false => tweetyPieStoreNoVF - }).multiGet(tripTweetIds)) - } yield { - val nonReplyTweets = filterOutReplyTweet(tweetyPieResults, nonReplyTweetsCounter) - val candidates = nonReplyTweets.values.flatten.map(buildRawCandidate(target, _)) - if (candidates.nonEmpty && target.params( - PushFeatureSwitchParams.TripTweetCandidateReturnEnable)) { - rawCandidates.incr(candidates.size) - Some(candidates.toSeq) - } else None - } - } - } - - private def getTripCandidatesContentMixer( - target: Target - ): Future[Set[Long]] = { - contentMixerRequests.incr() - Future - .join( - target.inferredUserDeviceLanguage, - target.deviceInfo - ) - .flatMap { - case (languageOpt, deviceInfoOpt) => - contentMixerStore - .get( - ContentMixerRequest( - clientContext = ClientContext( - userId = Some(target.targetId), - languageCode = languageOpt, - userAgent = deviceInfoOpt.flatMap(_.guessedPrimaryDeviceUserAgent.map(_.toString)) - ), - product = Product.NotificationsTripTweets, - productContext = Some( - ProductContext.NotificationsTripTweetsProductContext( - NotificationsTripTweetsProductContext() - )), - cursor = None, - maxResults = - Some(target.params(PushFeatureSwitchParams.TripTweetMaxTotalCandidates)) - ) - ).map { - _.map { rawResponse => - val tripResponse = - rawResponse.contentMixerProductResponse - .asInstanceOf[ - ContentMixerProductResponse.NotificationsTripTweetsProductResponse] - .notificationsTripTweetsProductResponse - - tripResponse.results.map(_.tweetResult.tweetId).toSet - }.getOrElse(Set.empty) - } - } - } - - private def getTripCandidatesForLoggedOutTarget( - target: Target - ): Future[Set[Long]] = { - Future.join(target.targetLanguage, target.countryCode).flatMap { - case (Some(lang), Some(country)) => - val placeId = CountryInfo.lookupByCode(country).map(_.placeIdLong) - if (placeId.nonEmpty) { - loggedOutPlaceId.incr() - } else { - loggedOutEmptyplaceId.incr() - } - val tripSource = "TOP_GEO_V3_LR" - val tripQuery = TripDomain( - sourceId = tripSource, - language = Some(lang), - placeId = placeId, - topicId = None - ) - val response = tripTweetCandidateStore.get(tripQuery) - val tripTweetIds = - response.map { res => - if (res.isDefined) { - res.get.tweets - .sortBy(_.score)(Ordering[Double].reverse).map(_.tweetId).toSet - } else { - Set.empty[Long] - } - } - tripTweetIds.map { ids => loggedOutTripTweetIds.incr(ids.size) } - tripTweetIds - - case (_, _) => Future.value(Set.empty) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/Config.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/Config.docx new file mode 100644 index 000000000..45b206ad2 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/Config.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/Config.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/Config.scala deleted file mode 100644 index 3a0e1dc70..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/Config.scala +++ /dev/null @@ -1,461 +0,0 @@ -package com.twitter.frigate.pushservice.config - -import com.twitter.abdecider.LoggingABDecider -import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringRequest -import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringResponse -import com.twitter.audience_rewards.thriftscala.HasSuperFollowingRelationshipRequest -import com.twitter.channels.common.thriftscala.ApiList -import com.twitter.datatools.entityservice.entities.sports.thriftscala._ -import com.twitter.decider.Decider -import com.twitter.discovery.common.configapi.ConfigParamsBuilder -import com.twitter.escherbird.common.thriftscala.QualifiedId -import com.twitter.escherbird.metadata.thriftscala.EntityMegadata -import com.twitter.eventbus.client.EventBusPublisher -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.candidate._ -import com.twitter.frigate.common.history._ -import com.twitter.frigate.common.ml.base._ -import com.twitter.frigate.common.ml.feature._ -import com.twitter.frigate.common.store._ -import com.twitter.frigate.common.store.deviceinfo.DeviceInfo -import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext -import com.twitter.frigate.common.store.interests.UserId -import com.twitter.frigate.common.util._ -import com.twitter.frigate.data_pipeline.features_common._ -import com.twitter.frigate.data_pipeline.thriftscala.UserHistoryKey -import com.twitter.frigate.data_pipeline.thriftscala.UserHistoryValue -import com.twitter.frigate.dau_model.thriftscala.DauProbability -import com.twitter.frigate.magic_events.thriftscala.FanoutEvent -import com.twitter.frigate.pushcap.thriftscala.PushcapUserHistory -import com.twitter.frigate.pushservice.ml._ -import com.twitter.frigate.pushservice.params.DeciderKey -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushFeatureSwitches -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.send_handler.SendHandlerPushCandidateHydrator -import com.twitter.frigate.pushservice.refresh_handler.PushCandidateHydrator -import com.twitter.frigate.pushservice.store._ -import com.twitter.frigate.pushservice.store.{Ibis2Store => PushIbis2Store} -import com.twitter.frigate.pushservice.take.NotificationServiceRequest -import com.twitter.frigate.pushservice.thriftscala.PushRequestScribe -import com.twitter.frigate.scribe.thriftscala.NotificationScribe -import com.twitter.frigate.thriftscala._ -import com.twitter.frigate.user_states.thriftscala.MRUserHmmState -import com.twitter.geoduck.common.thriftscala.{Location => GeoLocation} -import com.twitter.geoduck.service.thriftscala.LocationResponse -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.hermit.pop_geo.thriftscala.PopTweetsInPlace -import com.twitter.hermit.predicate.socialgraph.RelationEdge -import com.twitter.hermit.predicate.tweetypie.Perspective -import com.twitter.hermit.predicate.tweetypie.UserTweet -import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery -import com.twitter.hermit.store.tweetypie.{UserTweet => TweetyPieUserTweet} -import com.twitter.hermit.stp.thriftscala.STPResult -import com.twitter.hss.api.thriftscala.UserHealthSignalResponse -import com.twitter.interests.thriftscala.InterestId -import com.twitter.interests.thriftscala.{UserInterests => Interests} -import com.twitter.interests_discovery.thriftscala.NonPersonalizedRecommendedLists -import com.twitter.interests_discovery.thriftscala.RecommendedListsRequest -import com.twitter.interests_discovery.thriftscala.RecommendedListsResponse -import com.twitter.livevideo.timeline.domain.v2.{Event => LiveEvent} -import com.twitter.ml.api.thriftscala.{DataRecord => ThriftDataRecord} -import com.twitter.ml.featurestore.lib.dynamic.DynamicFeatureStoreClient -import com.twitter.notificationservice.genericfeedbackstore.FeedbackPromptValue -import com.twitter.notificationservice.genericfeedbackstore.GenericFeedbackStore -import com.twitter.notificationservice.scribe.manhattan.GenericNotificationsFeedbackRequest -import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationResponse -import com.twitter.nrel.heavyranker.CandidateFeatureHydrator -import com.twitter.nrel.heavyranker.{FeatureHydrator => MRFeatureHydrator} -import com.twitter.nrel.heavyranker.{TargetFeatureHydrator => RelevanceTargetFeatureHydrator} -import com.twitter.onboarding.task.service.thriftscala.FatigueFlowEnrollment -import com.twitter.permissions_storage.thriftscala.AppPermission -import com.twitter.recommendation.interests.discovery.core.model.InterestDomain -import com.twitter.recos.user_tweet_entity_graph.thriftscala.RecommendTweetEntityRequest -import com.twitter.recos.user_tweet_entity_graph.thriftscala.RecommendTweetEntityResponse -import com.twitter.recos.user_user_graph.thriftscala.RecommendUserRequest -import com.twitter.recos.user_user_graph.thriftscala.RecommendUserResponse -import com.twitter.rux.common.strato.thriftscala.UserTargetingProperty -import com.twitter.scio.nsfw_user_segmentation.thriftscala.NSFWProducer -import com.twitter.scio.nsfw_user_segmentation.thriftscala.NSFWUserSegmentation -import com.twitter.search.common.features.thriftscala.ThriftSearchResultFeatures -import com.twitter.search.earlybird.thriftscala.EarlybirdRequest -import com.twitter.search.earlybird.thriftscala.ThriftSearchResult -import com.twitter.service.gen.scarecrow.thriftscala.Event -import com.twitter.service.gen.scarecrow.thriftscala.TieredActionResult -import com.twitter.service.metastore.gen.thriftscala.Location -import com.twitter.service.metastore.gen.thriftscala.UserLanguages -import com.twitter.servo.decider.DeciderGateBuilder -import com.twitter.simclusters_v2.thriftscala.SimClustersInferredEntities -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.columns.frigate.logged_out_web_notifications.thriftscala.LOWebNotificationMetadata -import com.twitter.strato.columns.notifications.thriftscala.SourceDestUserRequest -import com.twitter.strato.client.{UserId => StratoUserId} -import com.twitter.timelines.configapi -import com.twitter.timelines.configapi.CompositeConfig -import com.twitter.timelinescorer.thriftscala.v1.ScoredTweet -import com.twitter.topiclisting.TopicListing -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweets -import com.twitter.tsp.thriftscala.TopicSocialProofRequest -import com.twitter.tsp.thriftscala.TopicSocialProofResponse -import com.twitter.ubs.thriftscala.SellerTrack -import com.twitter.ubs.thriftscala.AudioSpace -import com.twitter.ubs.thriftscala.Participants -import com.twitter.ubs.thriftscala.SellerApplicationState -import com.twitter.user_session_store.thriftscala.UserSession -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.wtf.scalding.common.thriftscala.UserFeatures - -trait Config { - self => - - def isServiceLocal: Boolean - - def localConfigRepoPath: String - - def inMemCacheOff: Boolean - - def historyStore: PushServiceHistoryStore - - def emailHistoryStore: PushServiceHistoryStore - - def strongTiesStore: ReadableStore[Long, STPResult] - - def safeUserStore: ReadableStore[Long, User] - - def deviceInfoStore: ReadableStore[Long, DeviceInfo] - - def edgeStore: ReadableStore[RelationEdge, Boolean] - - def socialGraphServiceProcessStore: ReadableStore[RelationEdge, Boolean] - - def userUtcOffsetStore: ReadableStore[Long, Duration] - - def cachedTweetyPieStoreV2: ReadableStore[Long, TweetyPieResult] - - def safeCachedTweetyPieStoreV2: ReadableStore[Long, TweetyPieResult] - - def userTweetTweetyPieStore: ReadableStore[TweetyPieUserTweet, TweetyPieResult] - - def safeUserTweetTweetyPieStore: ReadableStore[TweetyPieUserTweet, TweetyPieResult] - - def cachedTweetyPieStoreV2NoVF: ReadableStore[Long, TweetyPieResult] - - def tweetContentFeatureCacheStore: ReadableStore[Long, ThriftDataRecord] - - def scarecrowCheckEventStore: ReadableStore[Event, TieredActionResult] - - def userTweetPerspectiveStore: ReadableStore[UserTweet, Perspective] - - def userCountryStore: ReadableStore[Long, Location] - - def pushInfoStore: ReadableStore[Long, UserForPushTargeting] - - def loggedOutPushInfoStore: ReadableStore[Long, LOWebNotificationMetadata] - - def tweetImpressionStore: ReadableStore[Long, Seq[Long]] - - def audioSpaceStore: ReadableStore[String, AudioSpace] - - def basketballGameScoreStore: ReadableStore[QualifiedId, BasketballGameLiveUpdate] - - def baseballGameScoreStore: ReadableStore[QualifiedId, BaseballGameLiveUpdate] - - def cricketMatchScoreStore: ReadableStore[QualifiedId, CricketMatchLiveUpdate] - - def soccerMatchScoreStore: ReadableStore[QualifiedId, SoccerMatchLiveUpdate] - - def nflGameScoreStore: ReadableStore[QualifiedId, NflFootballGameLiveUpdate] - - def topicSocialProofServiceStore: ReadableStore[TopicSocialProofRequest, TopicSocialProofResponse] - - def spaceDeviceFollowStore: ReadableStore[SourceDestUserRequest, Boolean] - - def audioSpaceParticipantsStore: ReadableStore[String, Participants] - - def notificationServiceSender: ReadableStore[ - NotificationServiceRequest, - CreateGenericNotificationResponse - ] - - def ocfFatigueStore: ReadableStore[OCFHistoryStoreKey, FatigueFlowEnrollment] - - def dauProbabilityStore: ReadableStore[Long, DauProbability] - - def hydratedLabeledPushRecsStore: ReadableStore[UserHistoryKey, UserHistoryValue] - - def userHTLLastVisitStore: ReadableStore[Long, Seq[Long]] - - def userLanguagesStore: ReadableStore[Long, UserLanguages] - - def topTweetsByGeoStore: ReadableStore[InterestDomain[String], Map[String, List[ - (Long, Double) - ]]] - - def topTweetsByGeoV2VersionedStore: ReadableStore[String, PopTweetsInPlace] - - lazy val pushRecItemStore: ReadableStore[PushRecItemsKey, RecItems] = PushRecItemStore( - hydratedLabeledPushRecsStore - ) - - lazy val labeledPushRecsVerifyingStore: ReadableStore[ - LabeledPushRecsVerifyingStoreKey, - LabeledPushRecsVerifyingStoreResponse - ] = - LabeledPushRecsVerifyingStore( - hydratedLabeledPushRecsStore, - historyStore - ) - - lazy val labeledPushRecsDecideredStore: ReadableStore[LabeledPushRecsStoreKey, UserHistoryValue] = - LabeledPushRecsDecideredStore( - labeledPushRecsVerifyingStore, - useHydratedLabeledSendsForFeaturesDeciderKey, - verifyHydratedLabeledSendsForFeaturesDeciderKey - ) - - def onlineUserHistoryStore: ReadableStore[OnlineUserHistoryKey, UserHistoryValue] - - def nsfwConsumerStore: ReadableStore[Long, NSFWUserSegmentation] - - def nsfwProducerStore: ReadableStore[Long, NSFWProducer] - - def popGeoLists: ReadableStore[String, NonPersonalizedRecommendedLists] - - def listAPIStore: ReadableStore[Long, ApiList] - - def openedPushByHourAggregatedStore: ReadableStore[Long, Map[Int, Int]] - - def userHealthSignalStore: ReadableStore[Long, UserHealthSignalResponse] - - def reactivatedUserInfoStore: ReadableStore[Long, String] - - def weightedOpenOrNtabClickModelScorer: PushMLModelScorer - - def optoutModelScorer: PushMLModelScorer - - def filteringModelScorer: PushMLModelScorer - - def recentFollowsStore: ReadableStore[Long, Seq[Long]] - - def geoDuckV2Store: ReadableStore[UserId, LocationResponse] - - def realGraphScoresTop500InStore: ReadableStore[Long, Map[Long, Double]] - - def tweetEntityGraphStore: ReadableStore[ - RecommendTweetEntityRequest, - RecommendTweetEntityResponse - ] - - def userUserGraphStore: ReadableStore[RecommendUserRequest, RecommendUserResponse] - - def userFeaturesStore: ReadableStore[Long, UserFeatures] - - def userTargetingPropertyStore: ReadableStore[Long, UserTargetingProperty] - - def timelinesUserSessionStore: ReadableStore[Long, UserSession] - - def optOutUserInterestsStore: ReadableStore[UserId, Seq[InterestId]] - - def ntabCaretFeedbackStore: ReadableStore[GenericNotificationsFeedbackRequest, Seq[ - CaretFeedbackDetails - ]] - - def genericFeedbackStore: ReadableStore[FeedbackRequest, Seq[ - FeedbackPromptValue - ]] - - def genericNotificationFeedbackStore: GenericFeedbackStore - - def semanticCoreMegadataStore: ReadableStore[ - SemanticEntityForQuery, - EntityMegadata - ] - - def tweetHealthScoreStore: ReadableStore[TweetScoringRequest, TweetScoringResponse] - - def earlybirdFeatureStore: ReadableStore[Long, ThriftSearchResultFeatures] - - def earlybirdFeatureBuilder: FeatureBuilder[Long] - - // Feature builders - - def tweetAuthorLocationFeatureBuilder: FeatureBuilder[Location] - - def tweetAuthorLocationFeatureBuilderById: FeatureBuilder[Long] - - def socialContextActionsFeatureBuilder: FeatureBuilder[SocialContextActions] - - def tweetContentFeatureBuilder: FeatureBuilder[Long] - - def tweetAuthorRecentRealGraphFeatureBuilder: FeatureBuilder[RealGraphEdge] - - def socialContextRecentRealGraphFeatureBuilder: FeatureBuilder[Set[RealGraphEdge]] - - def tweetSocialProofFeatureBuilder: FeatureBuilder[TweetSocialProofKey] - - def targetUserFullRealGraphFeatureBuilder: FeatureBuilder[TargetFullRealGraphFeatureKey] - - def postProcessingFeatureBuilder: PostProcessingFeatureBuilder - - def mrOfflineUserCandidateSparseAggregatesFeatureBuilder: FeatureBuilder[ - OfflineSparseAggregateKey - ] - - def mrOfflineUserAggregatesFeatureBuilder: FeatureBuilder[Long] - - def mrOfflineUserCandidateAggregatesFeatureBuilder: FeatureBuilder[OfflineAggregateKey] - - def tweetAnnotationsFeatureBuilder: FeatureBuilder[Long] - - def targetUserMediaRepresentationFeatureBuilder: FeatureBuilder[Long] - - def targetLevelFeatureBuilder: FeatureBuilder[MrRequestContextForFeatureStore] - - def candidateLevelFeatureBuilder: FeatureBuilder[EntityRequestContextForFeatureStore] - - def targetFeatureHydrator: RelevanceTargetFeatureHydrator - - def useHydratedLabeledSendsForFeaturesDeciderKey: String = - DeciderKey.useHydratedLabeledSendsForFeaturesDeciderKey.toString - - def verifyHydratedLabeledSendsForFeaturesDeciderKey: String = - DeciderKey.verifyHydratedLabeledSendsForFeaturesDeciderKey.toString - - def lexServiceStore: ReadableStore[EventRequest, LiveEvent] - - def userMediaRepresentationStore: ReadableStore[Long, UserMediaRepresentation] - - def producerMediaRepresentationStore: ReadableStore[Long, UserMediaRepresentation] - - def mrUserStatePredictionStore: ReadableStore[Long, MRUserHmmState] - - def pushcapDynamicPredictionStore: ReadableStore[Long, PushcapUserHistory] - - def earlybirdCandidateSource: EarlybirdCandidateSource - - def earlybirdSearchStore: ReadableStore[EarlybirdRequest, Seq[ThriftSearchResult]] - - def earlybirdSearchDest: String - - def pushserviceThriftClientId: ClientId - - def simClusterToEntityStore: ReadableStore[Int, SimClustersInferredEntities] - - def fanoutMetadataStore: ReadableStore[(Long, Long), FanoutEvent] - - /** - * PostRanking Feature Store Client - */ - def postRankingFeatureStoreClient: DynamicFeatureStoreClient[MrRequestContextForFeatureStore] - - /** - * ReadableStore to fetch [[UserInterests]] from INTS service - */ - def interestsWithLookupContextStore: ReadableStore[InterestsLookupRequestWithContext, Interests] - - /** - * - * @return: [[TopicListing]] object to fetch paused topics and scope from productId - */ - def topicListing: TopicListing - - /** - * - * @return: [[UttEntityHydrationStore]] object - */ - def uttEntityHydrationStore: UttEntityHydrationStore - - def appPermissionStore: ReadableStore[(Long, (String, String)), AppPermission] - - lazy val userTweetEntityGraphCandidates: UserTweetEntityGraphCandidates = - UserTweetEntityGraphCandidates( - cachedTweetyPieStoreV2, - tweetEntityGraphStore, - PushParams.UTEGTweetCandidateSourceParam, - PushFeatureSwitchParams.NumberOfMaxUTEGCandidatesQueriedParam, - PushParams.AllowOneSocialProofForTweetInUTEGParam, - PushParams.OutNetworkTweetsOnlyForUTEGParam, - PushFeatureSwitchParams.MaxTweetAgeParam - )(statsReceiver) - - def pushSendEventBusPublisher: EventBusPublisher[NotificationScribe] - - // miscs. - - def isProd: Boolean - - implicit def statsReceiver: StatsReceiver - - def decider: Decider - - def abDecider: LoggingABDecider - - def casLock: CasLock - - def pushIbisV2Store: PushIbis2Store - - // scribe - def notificationScribe(data: NotificationScribe): Unit - - def requestScribe(data: PushRequestScribe): Unit - - def init(): Future[Unit] = Future.Done - - def configParamsBuilder: ConfigParamsBuilder - - def candidateFeatureHydrator: CandidateFeatureHydrator - - def featureHydrator: MRFeatureHydrator - - def candidateHydrator: PushCandidateHydrator - - def sendHandlerCandidateHydrator: SendHandlerPushCandidateHydrator - - lazy val overridesConfig: configapi.Config = { - val pushFeatureSwitchConfigs: configapi.Config = PushFeatureSwitches( - deciderGateBuilder = new DeciderGateBuilder(decider), - statsReceiver = statsReceiver - ).config - - new CompositeConfig(Seq(pushFeatureSwitchConfigs)) - } - - def realTimeClientEventStore: RealTimeClientEventStore - - def inlineActionHistoryStore: ReadableStore[Long, Seq[(Long, String)]] - - def softUserGeoLocationStore: ReadableStore[Long, GeoLocation] - - def tweetTranslationStore: ReadableStore[TweetTranslationStore.Key, TweetTranslationStore.Value] - - def tripTweetCandidateStore: ReadableStore[TripDomain, TripTweets] - - def softUserFollowingStore: ReadableStore[User, Seq[Long]] - - def superFollowEligibilityUserStore: ReadableStore[Long, Boolean] - - def superFollowCreatorTweetCountStore: ReadableStore[StratoUserId, Int] - - def hasSuperFollowingRelationshipStore: ReadableStore[ - HasSuperFollowingRelationshipRequest, - Boolean - ] - - def superFollowApplicationStatusStore: ReadableStore[(Long, SellerTrack), SellerApplicationState] - - def recentHistoryCacheClient: RecentHistoryCacheClient - - def openAppUserStore: ReadableStore[Long, Boolean] - - def loggedOutHistoryStore: PushServiceHistoryStore - - def idsStore: ReadableStore[RecommendedListsRequest, RecommendedListsResponse] - - def htlScoreStore(userId: Long): ReadableStore[Long, ScoredTweet] -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/DeployConfig.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/DeployConfig.docx new file mode 100644 index 000000000..2a16974c1 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/DeployConfig.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/DeployConfig.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/DeployConfig.scala deleted file mode 100644 index 8d6e95a67..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/DeployConfig.scala +++ /dev/null @@ -1,2150 +0,0 @@ -package com.twitter.frigate.pushservice.config - -import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringRequest -import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringResponse -import com.twitter.audience_rewards.thriftscala.HasSuperFollowingRelationshipRequest -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.bijection.scrooge.CompactScalaCodec -import com.twitter.channels.common.thriftscala.ApiList -import com.twitter.channels.common.thriftscala.ApiListDisplayLocation -import com.twitter.channels.common.thriftscala.ApiListView -import com.twitter.content_mixer.thriftscala.ContentMixer -import com.twitter.conversions.DurationOps._ -import com.twitter.cortex.deepbird.thriftjava.DeepbirdPredictionService -import com.twitter.cr_mixer.thriftscala.CrMixer -import com.twitter.datatools.entityservice.entities.sports.thriftscala.BaseballGameLiveUpdate -import com.twitter.datatools.entityservice.entities.sports.thriftscala.BasketballGameLiveUpdate -import com.twitter.datatools.entityservice.entities.sports.thriftscala.CricketMatchLiveUpdate -import com.twitter.datatools.entityservice.entities.sports.thriftscala.NflFootballGameLiveUpdate -import com.twitter.datatools.entityservice.entities.sports.thriftscala.SoccerMatchLiveUpdate -import com.twitter.discovery.common.configapi.ConfigParamsBuilder -import com.twitter.discovery.common.configapi.FeatureContextBuilder -import com.twitter.discovery.common.environment.{Environment => NotifEnvironment} -import com.twitter.escherbird.common.thriftscala.Domains -import com.twitter.escherbird.common.thriftscala.QualifiedId -import com.twitter.escherbird.metadata.thriftscala.EntityMegadata -import com.twitter.escherbird.metadata.thriftscala.MetadataService -import com.twitter.escherbird.util.metadatastitch.MetadataStitchClient -import com.twitter.escherbird.util.uttclient -import com.twitter.escherbird.util.uttclient.CacheConfigV2 -import com.twitter.escherbird.util.uttclient.CachedUttClientV2 -import com.twitter.escherbird.utt.strato.thriftscala.Environment -import com.twitter.eventbus.client.EventBusPublisherBuilder -import com.twitter.events.recos.thriftscala.EventsRecosService -import com.twitter.explore_ranker.thriftscala.ExploreRanker -import com.twitter.featureswitches.v2.FeatureSwitches -import com.twitter.finagle.Memcached -import com.twitter.finagle.ThriftMux -import com.twitter.finagle.client.BackupRequestFilter -import com.twitter.finagle.client.ClientRegistry -import com.twitter.finagle.loadbalancer.Balancers -import com.twitter.finagle.memcached.Client -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.mtls.client.MtlsStackClient._ -import com.twitter.finagle.mux.transport.OpportunisticTls -import com.twitter.finagle.service.Retries -import com.twitter.finagle.service.RetryPolicy -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.finagle.thrift.RichClientParam -import com.twitter.finagle.util.DefaultTimer -import com.twitter.flockdb.client._ -import com.twitter.flockdb.client.thriftscala.FlockDB -import com.twitter.frigate.common.base.RandomRanker -import com.twitter.frigate.common.candidate._ -import com.twitter.frigate.common.config.RateLimiterGenerator -import com.twitter.frigate.common.entity_graph_client.RecommendedTweetEntitiesStore -import com.twitter.frigate.common.filter.DynamicRequestMeterFilter -import com.twitter.frigate.common.history._ -import com.twitter.frigate.common.ml.feature._ -import com.twitter.frigate.common.store._ -import com.twitter.frigate.common.store.deviceinfo.DeviceInfoStore -import com.twitter.frigate.common.store.deviceinfo.MobileSdkStore -import com.twitter.frigate.common.store.interests._ -import com.twitter.frigate.common.store.strato.StratoFetchableStore -import com.twitter.frigate.common.store.strato.StratoScannableStore -import com.twitter.frigate.common.util.Finagle.readOnlyThriftService -import com.twitter.frigate.common.util._ -import com.twitter.frigate.data_pipeline.features_common.FeatureStoreUtil -import com.twitter.frigate.data_pipeline.features_common._ -import com.twitter.frigate.data_pipeline.thriftscala.UserHistoryKey -import com.twitter.frigate.data_pipeline.thriftscala.UserHistoryValue -import com.twitter.frigate.dau_model.thriftscala.DauProbability -import com.twitter.frigate.magic_events.thriftscala.FanoutEvent -import com.twitter.frigate.pushcap.thriftscala.PushcapUserHistory -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.adaptor.LoggedOutPushCandidateSourceGenerator -import com.twitter.frigate.pushservice.adaptor.PushCandidateSourceGenerator -import com.twitter.frigate.pushservice.config.mlconfig.DeepbirdV2ModelConfig -import com.twitter.frigate.pushservice.ml._ -import com.twitter.frigate.pushservice.params._ -import com.twitter.frigate.pushservice.rank.LoggedOutRanker -import com.twitter.frigate.pushservice.rank.RFPHLightRanker -import com.twitter.frigate.pushservice.rank.RFPHRanker -import com.twitter.frigate.pushservice.rank.SubscriptionCreatorRanker -import com.twitter.frigate.pushservice.refresh_handler._ -import com.twitter.frigate.pushservice.refresh_handler.cross.CandidateCopyExpansion -import com.twitter.frigate.pushservice.send_handler.SendHandlerPushCandidateHydrator -import com.twitter.frigate.pushservice.store._ -import com.twitter.frigate.pushservice.take.CandidateNotifier -import com.twitter.frigate.pushservice.take.NotificationSender -import com.twitter.frigate.pushservice.take.NotificationServiceRequest -import com.twitter.frigate.pushservice.take.NotificationServiceSender -import com.twitter.frigate.pushservice.take.NtabOnlyChannelSelector -import com.twitter.frigate.pushservice.take.history.EventBusWriter -import com.twitter.frigate.pushservice.take.history.HistoryWriter -import com.twitter.frigate.pushservice.take.sender.Ibis2Sender -import com.twitter.frigate.pushservice.take.sender.NtabSender -import com.twitter.frigate.pushservice.take.LoggedOutRefreshForPushNotifier -import com.twitter.frigate.pushservice.util.RFPHTakeStepUtil -import com.twitter.frigate.pushservice.util.SendHandlerPredicateUtil -import com.twitter.frigate.scribe.thriftscala.NotificationScribe -import com.twitter.frigate.thriftscala._ -import com.twitter.frigate.user_states.thriftscala.MRUserHmmState -import com.twitter.geoduck.backend.hydration.thriftscala.Hydration -import com.twitter.geoduck.common.thriftscala.PlaceQueryFields -import com.twitter.geoduck.common.thriftscala.PlaceType -import com.twitter.geoduck.common.thriftscala.{Location => GeoLocation} -import com.twitter.geoduck.service.common.clientmodules.GeoduckUserLocate -import com.twitter.geoduck.service.common.clientmodules.GeoduckUserLocateModule -import com.twitter.geoduck.service.thriftscala.LocationResponse -import com.twitter.geoduck.thriftscala.LocationService -import com.twitter.gizmoduck.context.thriftscala.ReadConfig -import com.twitter.gizmoduck.context.thriftscala.TestUserConfig -import com.twitter.gizmoduck.testusers.client.TestUserClientBuilder -import com.twitter.gizmoduck.thriftscala.LookupContext -import com.twitter.gizmoduck.thriftscala.QueryFields -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.gizmoduck.thriftscala.UserService -import com.twitter.hermit.pop_geo.thriftscala.PopTweetsInPlace -import com.twitter.hermit.predicate.socialgraph.SocialGraphPredicate -import com.twitter.hermit.predicate.tweetypie.PerspectiveReadableStore -import com.twitter.hermit.store._ -import com.twitter.hermit.store.common._ -import com.twitter.hermit.store.gizmoduck.GizmoduckUserStore -import com.twitter.hermit.store.metastore.UserCountryStore -import com.twitter.hermit.store.metastore.UserLanguagesStore -import com.twitter.hermit.store.scarecrow.ScarecrowCheckEventStore -import com.twitter.hermit.store.semantic_core.MetaDataReadableStore -import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery -import com.twitter.hermit.store.timezone.GizmoduckUserUtcOffsetStore -import com.twitter.hermit.store.timezone.UtcOffsetStore -import com.twitter.hermit.store.tweetypie.TweetyPieStore -import com.twitter.hermit.store.tweetypie.UserTweet -import com.twitter.hermit.store.user_htl_session_store.UserHTLLastVisitReadableStore -import com.twitter.hermit.stp.thriftscala.STPResult -import com.twitter.hss.api.thriftscala.UserHealthSignal -import com.twitter.hss.api.thriftscala.UserHealthSignal._ -import com.twitter.hss.api.thriftscala.UserHealthSignalResponse -import com.twitter.interests.thriftscala.InterestId -import com.twitter.interests.thriftscala.InterestsThriftService -import com.twitter.interests.thriftscala.{UserInterests => Interests} -import com.twitter.interests_discovery.thriftscala.InterestsDiscoveryService -import com.twitter.interests_discovery.thriftscala.NonPersonalizedRecommendedLists -import com.twitter.interests_discovery.thriftscala.RecommendedListsRequest -import com.twitter.interests_discovery.thriftscala.RecommendedListsResponse -import com.twitter.kujaku.domain.thriftscala.MachineTranslationResponse -import com.twitter.livevideo.timeline.client.v2.LiveVideoTimelineClient -import com.twitter.livevideo.timeline.domain.v2.{Event => LiveEvent} -import com.twitter.livevideo.timeline.thrift.thriftscala.TimelineService -import com.twitter.logging.Logger -import com.twitter.ml.api.thriftscala.{DataRecord => ThriftDataRecord} -import com.twitter.ml.featurestore.catalog.entities.core.{Author => TweetAuthorEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{User => TargetUserEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{UserAuthor => UserAuthorEntity} -import com.twitter.ml.featurestore.catalog.entities.magicrecs.{SocialContext => SocialContextEntity} -import com.twitter.ml.featurestore.catalog.entities.magicrecs.{UserSocialContext => TargetUserSocialContextEntity} -import com.twitter.ml.featurestore.timelines.thriftscala.TimelineScorerScoreView -import com.twitter.notificationservice.api.thriftscala.DeleteCurrentTimelineForUserRequest -import com.twitter.notificationservice.genericfeedbackstore.FeedbackPromptValue -import com.twitter.notificationservice.genericfeedbackstore.GenericFeedbackStore -import com.twitter.notificationservice.genericfeedbackstore.GenericFeedbackStoreBuilder -import com.twitter.notificationservice.scribe.manhattan.FeedbackSignalManhattanClient -import com.twitter.notificationservice.scribe.manhattan.GenericNotificationsFeedbackRequest -import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationRequest -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationResponse -import com.twitter.notificationservice.thriftscala.DeleteGenericNotificationRequest -import com.twitter.notificationservice.thriftscala.GenericNotificationOverrideKey -import com.twitter.notificationservice.thriftscala.NotificationService$FinagleClient -import com.twitter.nrel.heavyranker.CandidateFeatureHydrator -import com.twitter.nrel.heavyranker.FeatureHydrator -import com.twitter.nrel.heavyranker.{PushPredictionServiceStore => RelevancePushPredictionServiceStore} -import com.twitter.nrel.heavyranker.{TargetFeatureHydrator => RelevanceTargetFeatureHydrator} -import com.twitter.nrel.lightranker.MagicRecsServeDataRecordLightRanker -import com.twitter.nrel.lightranker.{Config => LightRankerConfig} -import com.twitter.onboarding.task.service.thriftscala.FatigueFlowEnrollment -import com.twitter.periscope.api.thriftscala.AudioSpacesLookupContext -import com.twitter.permissions_storage.thriftscala.AppPermission -import com.twitter.recommendation.interests.discovery.core.config.{DeployConfig => InterestDeployConfig} -import com.twitter.recommendation.interests.discovery.popgeo.deploy.PopGeoInterestProvider -import com.twitter.recos.user_tweet_entity_graph.thriftscala.UserTweetEntityGraph -import com.twitter.recos.user_user_graph.thriftscala.UserUserGraph -import com.twitter.rux.common.strato.thriftscala.UserTargetingProperty -import com.twitter.scio.nsfw_user_segmentation.thriftscala.NSFWProducer -import com.twitter.scio.nsfw_user_segmentation.thriftscala.NSFWUserSegmentation -import com.twitter.search.earlybird.thriftscala.EarlybirdService -import com.twitter.service.gen.scarecrow.thriftscala.ScarecrowService -import com.twitter.service.metastore.gen.thriftscala.Location -import com.twitter.simclusters_v2.thriftscala.SimClustersInferredEntities -import com.twitter.socialgraph.thriftscala.SocialGraphService -import com.twitter.spam.rtf.thriftscala.SafetyLevel -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storage.client.manhattan.kv.Guarantee -import com.twitter.storage.client.manhattan.kv.ManhattanKVClient -import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams -import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpoint -import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpointBuilder -import com.twitter.storehaus.ReadableStore -import com.twitter.storehaus_internal.manhattan.Apollo -import com.twitter.storehaus_internal.manhattan.Athena -import com.twitter.storehaus_internal.manhattan.Dataset -import com.twitter.storehaus_internal.manhattan.ManhattanStore -import com.twitter.storehaus_internal.manhattan.Nash -import com.twitter.storehaus_internal.manhattan.Omega -import com.twitter.storehaus_internal.memcache.MemcacheStore -import com.twitter.storehaus_internal.util.ClientName -import com.twitter.storehaus_internal.util.ZkEndPoint -import com.twitter.strato.catalog.Scan.Slice -import com.twitter.strato.client.Strato -import com.twitter.strato.client.UserId -import com.twitter.strato.columns.frigate.logged_out_web_notifications.thriftscala.LOWebNotificationMetadata -import com.twitter.strato.columns.notifications.thriftscala.SourceDestUserRequest -import com.twitter.strato.generated.client.geo.user.FrequentSoftUserLocationClientColumn -import com.twitter.strato.generated.client.ml.featureStore.TimelineScorerTweetScoresV1ClientColumn -import com.twitter.strato.generated.client.notifications.space_device_follow_impl.SpaceDeviceFollowingClientColumn -import com.twitter.strato.generated.client.periscope.CoreOnAudioSpaceClientColumn -import com.twitter.strato.generated.client.periscope.ParticipantsOnAudioSpaceClientColumn -import com.twitter.strato.generated.client.rux.TargetingPropertyOnUserClientColumn -import com.twitter.strato.generated.client.socialgraph.graphs.creatorSubscriptionTimeline.{CountEdgesBySourceClientColumn => CreatorSubscriptionNumTweetsColumn} -import com.twitter.strato.generated.client.translation.service.IsTweetTranslatableClientColumn -import com.twitter.strato.generated.client.translation.service.platform.MachineTranslateTweetClientColumn -import com.twitter.strato.generated.client.trends.trip.TripTweetsAirflowProdClientColumn -import com.twitter.strato.thrift.ScroogeConvImplicits._ -import com.twitter.taxi.common.AppId -import com.twitter.taxi.deploy.Cluster -import com.twitter.taxi.deploy.Env -import com.twitter.topiclisting.TopicListing -import com.twitter.topiclisting.TopicListingBuilder -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweets -import com.twitter.tsp.thriftscala.TopicSocialProofRequest -import com.twitter.tsp.thriftscala.TopicSocialProofResponse -import com.twitter.tweetypie.thriftscala.GetTweetOptions -import com.twitter.tweetypie.thriftscala.Tweet.VisibleTextRangeField -import com.twitter.tweetypie.thriftscala.TweetService -import com.twitter.ubs.thriftscala.AudioSpace -import com.twitter.ubs.thriftscala.Participants -import com.twitter.ubs.thriftscala.SellerApplicationState -import com.twitter.user_session_store.thriftscala.UserSession -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.util.Timer -import com.twitter.util.tunable.TunableMap -import com.twitter.wtf.scalding.common.thriftscala.UserFeatures -import org.apache.thrift.protocol.TCompactProtocol -import com.twitter.timelinescorer.thriftscala.v1.ScoredTweet -import com.twitter.ubs.thriftscala.SellerTrack -import com.twitter.wtf.candidate.thriftscala.CandidateSeq - -trait DeployConfig extends Config { - // Any finagle clients should not be defined as lazy. If defined lazy, - // ClientRegistry.expAllRegisteredClientsResolved() call in init will not ensure that the clients - // are active before thrift endpoint is active. We want the clients to be active, because zookeeper - // resolution triggered by first request(s) might result in the request(s) failing. - - def serviceIdentifier: ServiceIdentifier - - def tunableMap: TunableMap - - def featureSwitches: FeatureSwitches - - override val isProd: Boolean = - serviceIdentifier.environment == PushConstants.ServiceProdEnvironmentName - - def shardParams: ShardParams - - def log: Logger - - implicit def statsReceiver: StatsReceiver - - implicit val timer: Timer = DefaultTimer - - def notifierThriftClientId: ClientId - - def loggedOutNotifierThriftClientId: ClientId - - def pushserviceThriftClientId: ClientId - - def deepbirdv2PredictionServiceDest: String - - def featureStoreUtil: FeatureStoreUtil - - def targetLevelFeaturesConfig: PushFeaturesConfig - - private val manhattanClientMtlsParams = ManhattanKVClientMtlsParams( - serviceIdentifier = serviceIdentifier, - opportunisticTls = OpportunisticTls.Required - ) - - // Commonly used clients - val gizmoduckClient = { - - val client = ThriftMux.client - .withMutualTls(serviceIdentifier) - .withClientId(pushserviceThriftClientId) - .build[UserService.MethodPerEndpoint]( - dest = "/s/gizmoduck/gizmoduck" - ) - - /** - * RequestContext test user config to allow reading test user accounts on pushservice for load - * testing - */ - val GizmoduckTestUserConfig = TestUserConfig( - clientId = Some(pushserviceThriftClientId.name), - readConfig = Some(ReadConfig(includeTestUsers = true)) - ) - - TestUserClientBuilder[UserService.MethodPerEndpoint] - .withClient(client) - .withConfig(GizmoduckTestUserConfig) - .build() - } - - val sgsClient = { - val service = readOnlyThriftService( - "", - "/s/socialgraph/socialgraph", - statsReceiver, - pushserviceThriftClientId, - mTLSServiceIdentifier = Some(serviceIdentifier) - ) - new SocialGraphService.FinagledClient(service) - } - - val tweetyPieClient = { - val service = readOnlyThriftService( - "", - "/s/tweetypie/tweetypie", - statsReceiver, - notifierThriftClientId, - mTLSServiceIdentifier = Some(serviceIdentifier) - ) - new TweetService.FinagledClient(service) - } - - lazy val geoduckHydrationClient: Hydration.MethodPerEndpoint = { - val servicePerEndpoint = ThriftMux.client - .withLabel("geoduck_hydration") - .withClientId(pushserviceThriftClientId) - .withMutualTls(serviceIdentifier) - .methodBuilder("/s/geo/hydration") - .withTimeoutPerRequest(10.seconds) - .withTimeoutTotal(10.seconds) - .idempotent(maxExtraLoad = 0.0) - .servicePerEndpoint[Hydration.ServicePerEndpoint] - Hydration.MethodPerEndpoint(servicePerEndpoint) - } - - lazy val geoduckLocationClient: LocationService.MethodPerEndpoint = { - val servicePerEndpoint = ThriftMux.client - .withLabel("geoduck_location") - .withClientId(pushserviceThriftClientId) - .withMutualTls(serviceIdentifier) - .methodBuilder("/s/geo/geoduck_locationservice") - .withTimeoutPerRequest(10.seconds) - .withTimeoutTotal(10.seconds) - .idempotent(maxExtraLoad = 0.0) - .servicePerEndpoint[LocationService.ServicePerEndpoint] - LocationService.MethodPerEndpoint(servicePerEndpoint) - } - - override val geoDuckV2Store: ReadableStore[Long, LocationResponse] = { - val geoduckLocate: GeoduckUserLocate = GeoduckUserLocateModule.providesGeoduckUserLocate( - locationServiceClient = geoduckLocationClient, - hydrationClient = geoduckHydrationClient, - unscopedStatsReceiver = statsReceiver - ) - - val store: ReadableStore[Long, LocationResponse] = ReadableStore - .convert[GeoduckRequest, Long, LocationResponse, LocationResponse]( - GeoduckStoreV2(geoduckLocate))({ userId: Long => - GeoduckRequest( - userId, - placeTypes = Set( - PlaceType.City, - PlaceType.Metro, - PlaceType.Country, - PlaceType.ZipCode, - PlaceType.Admin0, - PlaceType.Admin1), - placeFields = Set(PlaceQueryFields.PlaceNames), - includeCountryCode = true - ) - })({ locationResponse: LocationResponse => Future.value(locationResponse) }) - - val _cacheName = "geoduckv2_in_memory_cache" - ObservedCachedReadableStore.from( - store, - ttl = 20.seconds, - maxKeys = 1000, - cacheName = _cacheName, - windowSize = 10000L - )(statsReceiver.scope(_cacheName)) - } - - private val deepbirdServiceBase = ThriftMux.client - .withClientId(pushserviceThriftClientId) - .withMutualTls(serviceIdentifier) - .withLoadBalancer(Balancers.p2c()) - .newService(deepbirdv2PredictionServiceDest, "DeepbirdV2PredictionService") - val deepbirdPredictionServiceClient = new DeepbirdPredictionService.ServiceToClient( - Finagle - .retryReadFilter( - tries = 3, - statsReceiver = statsReceiver.scope("DeepbirdV2PredictionService")) - .andThen(Finagle.timeoutFilter(timeout = 10.seconds)) - .andThen(deepbirdServiceBase), - RichClientParam(serviceName = "DeepbirdV2PredictionService", clientStats = statsReceiver) - ) - - val manhattanStarbuckAppId = "frigate_pushservice_starbuck" - val metastoreLocationAppId = "frigate_notifier_metastore_location" - val manhattanMetastoreAppId = "frigate_pushservice_penguin" - - def pushServiceMHCacheDest: String - def pushServiceCoreSvcsCacheDest: String - def poptartImpressionsCacheDest: String = "/srv#/prod/local/cache/poptart_impressions" - def entityGraphCacheDest: String - - val pushServiceCacheClient: Client = MemcacheStore.memcachedClient( - name = ClientName("memcache-pushservice"), - dest = ZkEndPoint(pushServiceMHCacheDest), - statsReceiver = statsReceiver, - timeout = 2.seconds, - serviceIdentifier = serviceIdentifier - ) - - val pushServiceCoreSvcsCacheClient: Client = - MemcacheStore.memcachedClient( - name = ClientName("memcache-pushservice-core-svcs"), - dest = ZkEndPoint(pushServiceCoreSvcsCacheDest), - statsReceiver = statsReceiver, - serviceIdentifier = serviceIdentifier, - timeout = 2.seconds, - ) - - val poptartImpressionsCacheClient: Client = - MemcacheStore.memcachedClient( - name = ClientName("memcache-pushservice-poptart-impressions"), - dest = ZkEndPoint(poptartImpressionsCacheDest), - statsReceiver = statsReceiver, - serviceIdentifier = serviceIdentifier, - timeout = 2.seconds - ) - - val entityGraphCacheClient: Client = MemcacheStore.memcachedClient( - name = ClientName("memcache-pushservice-entity-graph"), - dest = ZkEndPoint(entityGraphCacheDest), - statsReceiver = statsReceiver, - serviceIdentifier = serviceIdentifier, - timeout = 2.seconds - ) - - val stratoClient = { - val pushserviceThriftClient = ThriftMux.client.withClientId(pushserviceThriftClientId) - val baseBuilder = Strato - .Client(pushserviceThriftClient) - .withMutualTls(serviceIdentifier) - val finalBuilder = if (isServiceLocal) { - baseBuilder.withRequestTimeout(Duration.fromSeconds(15)) - } else { - baseBuilder.withRequestTimeout(Duration.fromSeconds(3)) - } - finalBuilder.build() - } - - val interestThriftServiceClient = ThriftMux.client - .withClientId(pushserviceThriftClientId) - .withMutualTls(serviceIdentifier) - .withRequestTimeout(3.seconds) - .configured(Retries.Policy(RetryPolicy.tries(1))) - .configured(BackupRequestFilter.Configured(maxExtraLoad = 0.0, sendInterrupts = false)) - .withStatsReceiver(statsReceiver) - .build[InterestsThriftService.MethodPerEndpoint]( - dest = "/s/interests-thrift-service/interests-thrift-service", - label = "interests-lookup" - ) - - def memcacheCASDest: String - - override val casLock: CasLock = { - val magicrecsCasMemcacheClient = Memcached.client - .withMutualTls(serviceIdentifier) - .withLabel("mr-cas-memcache-client") - .withRequestTimeout(3.seconds) - .withStatsReceiver(statsReceiver) - .configured(Retries.Policy(RetryPolicy.tries(3))) - .newTwemcacheClient(memcacheCASDest) - .withStrings - - MemcacheCasLock(magicrecsCasMemcacheClient) - } - - override val pushInfoStore: ReadableStore[Long, UserForPushTargeting] = { - StratoFetchableStore.withUnitView[Long, UserForPushTargeting]( - stratoClient, - "frigate/magicrecs/pushRecsTargeting.User") - } - - override val loggedOutPushInfoStore: ReadableStore[Long, LOWebNotificationMetadata] = { - StratoFetchableStore.withUnitView[Long, LOWebNotificationMetadata]( - stratoClient, - "frigate/magicrecs/web/loggedOutWebUserStoreMh" - ) - } - - // Setting up model stores - override val dauProbabilityStore: ReadableStore[Long, DauProbability] = { - StratoFetchableStore - .withUnitView[Long, DauProbability](stratoClient, "frigate/magicrecs/dauProbability.User") - } - - override val nsfwConsumerStore = { - StratoFetchableStore.withUnitView[Long, NSFWUserSegmentation]( - stratoClient, - "frigate/nsfw-user-segmentation/nsfwUserSegmentation.User") - } - - override val nsfwProducerStore = { - StratoFetchableStore.withUnitView[Long, NSFWProducer]( - stratoClient, - "frigate/nsfw-user-segmentation/nsfwProducer.User" - ) - } - - override val idsStore: ReadableStore[RecommendedListsRequest, RecommendedListsResponse] = { - val service = Finagle.readOnlyThriftService( - name = "interests-discovery-service", - dest = "/s/interests_discovery/interests_discovery", - statsReceiver, - pushserviceThriftClientId, - requestTimeout = 4.seconds, - tries = 2, - mTLSServiceIdentifier = Some(serviceIdentifier) - ) - val client = new InterestsDiscoveryService.FinagledClient( - service = service, - RichClientParam(serviceName = "interests-discovery-service") - ) - - InterestDiscoveryStore(client) - } - - override val popGeoLists = { - StratoFetchableStore.withUnitView[String, NonPersonalizedRecommendedLists]( - stratoClient, - column = "recommendations/interests_discovery/recommendations_mh/OrganicPopgeoLists" - ) - } - - override val listAPIStore = { - val fetcher = stratoClient - .fetcher[Long, ApiListView, ApiList]("channels/hydration/apiList.List") - StratoFetchableStore.withView[Long, ApiListView, ApiList]( - fetcher, - ApiListView(ApiListDisplayLocation.Recommendations) - ) - } - - override val reactivatedUserInfoStore = { - val stratoFetchableStore = StratoFetchableStore - .withUnitView[Long, String](stratoClient, "ml/featureStore/recentReactivationTime.User") - - ObservedReadableStore( - stratoFetchableStore - )(statsReceiver.scope("RecentReactivationTime")) - } - - override val openedPushByHourAggregatedStore: ReadableStore[Long, Map[Int, Int]] = { - StratoFetchableStore - .withUnitView[Long, Map[Int, Int]]( - stratoClient, - "frigate/magicrecs/opendPushByHourAggregated.User") - } - - private val lexClient: LiveVideoTimelineClient = { - val lexService = - new TimelineService.FinagledClient( - readOnlyThriftService( - name = "lex", - dest = lexServiceDest, - statsReceiver = statsReceiver.scope("lex-service"), - thriftClientId = pushserviceThriftClientId, - requestTimeout = 5.seconds, - mTLSServiceIdentifier = Some(serviceIdentifier) - ), - clientParam = RichClientParam(serviceName = "lex") - ) - new LiveVideoTimelineClient(lexService) - } - - override val lexServiceStore = { - ObservedCachedReadableStore.from[EventRequest, LiveEvent]( - buildStore(LexServiceStore(lexClient), "lexServiceStore"), - ttl = 1.hour, - maxKeys = 1000, - cacheName = "lexServiceStore_cache", - windowSize = 10000L - )(statsReceiver.scope("lexServiceStore_cache")) - } - - val inferredEntitiesFromInterestedInKeyedByClusterColumn = - "recommendations/simclusters_v2/inferred_entities/inferredEntitiesFromInterestedInKeyedByCluster" - override val simClusterToEntityStore: ReadableStore[Int, SimClustersInferredEntities] = { - val store = StratoFetchableStore - .withUnitView[Int, SimClustersInferredEntities]( - stratoClient, - inferredEntitiesFromInterestedInKeyedByClusterColumn) - ObservedCachedReadableStore.from[Int, SimClustersInferredEntities]( - buildStore(store, "simcluster_entity_store_cache"), - ttl = 6.hours, - maxKeys = 1000, - cacheName = "simcluster_entity_store_cache", - windowSize = 10000L - )(statsReceiver.scope("simcluster_entity_store_cache")) - } - - def fanoutMetadataColumn: String - - override val fanoutMetadataStore: ReadableStore[(Long, Long), FanoutEvent] = { - val store = StratoFetchableStore - .withUnitView[(Long, Long), FanoutEvent](stratoClient, fanoutMetadataColumn) - ObservedCachedReadableStore.from[(Long, Long), FanoutEvent]( - buildStore(store, "fanoutMetadataStore"), - ttl = 10.minutes, - maxKeys = 1000, - cacheName = "fanoutMetadataStore_cache", - windowSize = 10000L - )(statsReceiver.scope("fanoutMetadataStore_cache")) - } - - /** - * PostRanking Feature Store Client - */ - override def postRankingFeatureStoreClient = { - val clientStats = statsReceiver.scope("post_ranking_feature_store_client") - val clientConfig = - FeatureStoreClientBuilder.getClientConfig(PostRankingFeaturesConfig(), featureStoreUtil) - - FeatureStoreClientBuilder.getDynamicFeatureStoreClient(clientConfig, clientStats) - } - - /** - * Interests lookup store - */ - override val interestsWithLookupContextStore = { - ObservedCachedReadableStore.from[InterestsLookupRequestWithContext, Interests]( - buildStore( - new InterestsWithLookupContextStore(interestThriftServiceClient, statsReceiver), - "InterestsWithLookupContextStore" - ), - ttl = 1.minute, - maxKeys = 1000, - cacheName = "interestsWithLookupContextStore_cache", - windowSize = 10000L - ) - } - - /** - * OptOutInterestsStore - */ - override lazy val optOutUserInterestsStore: ReadableStore[Long, Seq[InterestId]] = { - buildStore( - InterestsOptOutwithLookUpContextStore(interestThriftServiceClient), - "InterestsOptOutStore" - ) - } - - override val topicListing: TopicListing = - if (isServiceLocal) { - new TopicListingBuilder(statsReceiver.scope("topiclisting"), Some(localConfigRepoPath)).build - } else { - new TopicListingBuilder(statsReceiver.scope("topiclisting"), None).build - } - - val cachedUttClient = { - val DefaultUttCacheConfig = CacheConfigV2(capacity = 100) - val uttClientCacheConfigs = uttclient.UttClientCacheConfigsV2( - DefaultUttCacheConfig, - DefaultUttCacheConfig, - DefaultUttCacheConfig, - DefaultUttCacheConfig - ) - new CachedUttClientV2(stratoClient, Environment.Prod, uttClientCacheConfigs, statsReceiver) - } - - override val uttEntityHydrationStore = - new UttEntityHydrationStore(cachedUttClient, statsReceiver, log) - - private lazy val dbv2PredictionServiceScoreStore: RelevancePushPredictionServiceStore = - DeepbirdV2ModelConfig.buildPredictionServiceScoreStore( - deepbirdPredictionServiceClient, - "deepbirdv2_magicrecs" - ) - - // Customized model to PredictionServiceStoreMap - // It is used to specify the predictionServiceStore for the models not in the default dbv2PredictionServiceScoreStore - private lazy val modelToPredictionServiceStoreMap: Map[ - WeightedOpenOrNtabClickModel.ModelNameType, - RelevancePushPredictionServiceStore - ] = Map() - - override lazy val weightedOpenOrNtabClickModelScorer = new PushMLModelScorer( - PushMLModel.WeightedOpenOrNtabClickProbability, - modelToPredictionServiceStoreMap, - dbv2PredictionServiceScoreStore, - statsReceiver.scope("weighted_oonc_scoring") - ) - - override lazy val optoutModelScorer = new PushMLModelScorer( - PushMLModel.OptoutProbability, - Map.empty, - dbv2PredictionServiceScoreStore, - statsReceiver.scope("optout_scoring") - ) - - override lazy val filteringModelScorer = new PushMLModelScorer( - PushMLModel.FilteringProbability, - Map.empty, - dbv2PredictionServiceScoreStore, - statsReceiver.scope("filtering_scoring") - ) - - private val queryFields: Set[QueryFields] = Set( - QueryFields.Profile, - QueryFields.Account, - QueryFields.Roles, - QueryFields.Discoverability, - QueryFields.Safety, - QueryFields.Takedowns, - QueryFields.Labels, - QueryFields.Counts, - QueryFields.ExtendedProfile - ) - - // Setting up safeUserStore - override val safeUserStore = - // in-memory cache - ObservedCachedReadableStore.from[Long, User]( - ObservedReadableStore( - GizmoduckUserStore.safeStore( - client = gizmoduckClient, - queryFields = queryFields, - safetyLevel = SafetyLevel.FilterNone, - statsReceiver = statsReceiver - ) - )(statsReceiver.scope("SafeUserStore")), - ttl = 1.minute, - maxKeys = 5e4.toInt, - cacheName = "safeUserStore_cache", - windowSize = 10000L - )(statsReceiver.scope("safeUserStore_cache")) - - val mobileSdkStore = MobileSdkStore( - "frigate_mobile_sdk_version_apollo", - "mobile_sdk_versions_scalding", - manhattanClientMtlsParams, - Apollo - ) - - val deviceUserStore = ObservedReadableStore( - GizmoduckUserStore( - client = gizmoduckClient, - queryFields = Set(QueryFields.Devices), - context = LookupContext(includeSoftUsers = true), - statsReceiver = statsReceiver - ) - )(statsReceiver.scope("devicesUserStore")) - - override val deviceInfoStore = DeviceInfoStore( - ObservedMemcachedReadableStore.fromCacheClient( - backingStore = ObservedReadableStore( - mobileSdkStore - )(statsReceiver.scope("uncachedMobileSdkVersionsStore")), - cacheClient = pushServiceCacheClient, - ttl = 12.hours - )( - valueInjection = BinaryScalaCodec(SdkVersionValue), - statsReceiver = statsReceiver.scope("MobileSdkVersionsStore"), - keyToString = { - case SdkVersionKey(Some(userId), Some(clientId)) => - s"DeviceInfoStore/$userId/$clientId" - case SdkVersionKey(Some(userId), None) => s"DeviceInfoStore/$userId/_" - case SdkVersionKey(None, Some(clientId)) => - s"DeviceInfoStore/_/$clientId" - case SdkVersionKey(None, None) => s"DeviceInfoStore/_" - } - ), - deviceUserStore - ) - - // Setting up edgeStore - override val edgeStore = SocialGraphPredicate.buildEdgeStore(sgsClient) - - override val socialGraphServiceProcessStore = SocialGraphServiceProcessStore(edgeStore) - - def userTweetEntityGraphDest: String - def userUserGraphDest: String - def lexServiceDest: String - - // Setting up the history store - def frigateHistoryCacheDest: String - - val notificationHistoryStore: NotificationHistoryStore = { - - val manhattanStackBasedClient = ThriftMux.client - .withClientId(notifierThriftClientId) - .withOpportunisticTls(OpportunisticTls.Required) - .withMutualTls( - serviceIdentifier - ) - - val manhattanHistoryMethodBuilder = manhattanStackBasedClient - .withLabel("manhattan_history_v2") - .withRequestTimeout(10.seconds) - .withStatsReceiver(statsReceiver) - .methodBuilder(Omega.wilyName) - .withMaxRetries(3) - - NotificationHistoryStore.build( - "frigate_notifier", - "frigate_notifications_v2", - manhattanHistoryMethodBuilder, - maxRetryCount = 3 - ) - } - - val emailNotificationHistoryStore: ReadOnlyHistoryStore = { - val client = ManhattanKVClient( - appId = "frigate_email_history", - dest = "/s/manhattan/omega.native-thrift", - mtlsParams = ManhattanKVClientMtlsParams( - serviceIdentifier = serviceIdentifier, - opportunisticTls = OpportunisticTls.Required - ) - ) - val endpoint = ManhattanKVEndpointBuilder(client) - .defaultGuarantee(Guarantee.SoftDcReadMyWrites) - .statsReceiver(statsReceiver) - .build() - - ReadOnlyHistoryStore(ManhattanKVHistoryStore(endpoint, dataset = "frigate_email_history"))( - statsReceiver) - } - - val manhattanKVLoggedOutHistoryStoreEndpoint: ManhattanKVEndpoint = { - val mhClient = ManhattanKVClient( - "frigate_notification_logged_out_history", - Nash.wilyName, - manhattanClientMtlsParams) - ManhattanKVEndpointBuilder(mhClient) - .defaultGuarantee(Guarantee.SoftDcReadMyWrites) - .defaultMaxTimeout(5.seconds) - .maxRetryCount(3) - .statsReceiver(statsReceiver) - .build() - } - - val manhattanKVNtabHistoryStoreEndpoint: ManhattanKVEndpoint = { - val mhClient = ManhattanKVClient("frigate_ntab", Omega.wilyName, manhattanClientMtlsParams) - ManhattanKVEndpointBuilder(mhClient) - .defaultGuarantee(Guarantee.SoftDcReadMyWrites) - .defaultMaxTimeout(5.seconds) - .maxRetryCount(3) - .statsReceiver(statsReceiver) - .build() - } - - val nTabHistoryStore: ReadableWritableStore[(Long, String), GenericNotificationOverrideKey] = { - ObservedReadableWritableStore( - NTabHistoryStore(manhattanKVNtabHistoryStoreEndpoint, "frigate_ntab_generic_notif_history") - )(statsReceiver.scope("NTabHistoryStore")) - } - - override lazy val ocfFatigueStore: ReadableStore[OCFHistoryStoreKey, FatigueFlowEnrollment] = - new OCFPromptHistoryStore( - manhattanAppId = "frigate_pushservice_ocf_fatigue_store", - dataset = "fatigue_v1", - manhattanClientMtlsParams - ) - - def historyStore: PushServiceHistoryStore - - def emailHistoryStore: PushServiceHistoryStore - - def loggedOutHistoryStore: PushServiceHistoryStore - - override val hydratedLabeledPushRecsStore: ReadableStore[UserHistoryKey, UserHistoryValue] = { - val labeledHistoryMemcacheClient = { - MemcacheStore.memcachedClient( - name = ClientName("history-memcache"), - dest = ZkEndPoint(frigateHistoryCacheDest), - statsReceiver = statsReceiver, - timeout = 2.seconds, - serviceIdentifier = serviceIdentifier - ) - } - - implicit val keyCodec = CompactScalaCodec(UserHistoryKey) - implicit val valueCodec = CompactScalaCodec(UserHistoryValue) - val dataset: Dataset[UserHistoryKey, UserHistoryValue] = - Dataset( - "", - "frigate_data_pipeline_pushservice", - "labeled_push_recs_aggregated_hydrated", - Athena - ) - ObservedMemcachedReadableStore.fromCacheClient( - backingStore = ObservedReadableStore(buildManhattanStore(dataset))( - statsReceiver.scope("UncachedHydratedLabeledPushRecsStore") - ), - cacheClient = labeledHistoryMemcacheClient, - ttl = 6.hours - )( - valueInjection = valueCodec, - statsReceiver = statsReceiver.scope("HydratedLabeledPushRecsStore"), - keyToString = { - case UserHistoryKey.UserId(userId) => s"HLPRS/$userId" - case unknownKey => - throw new IllegalArgumentException(s"Unknown userHistoryStore cache key $unknownKey") - } - ) - } - - override val realTimeClientEventStore: RealTimeClientEventStore = { - val client = ManhattanKVClient( - "frigate_eventstream", - "/s/manhattan/omega.native-thrift", - manhattanClientMtlsParams - ) - val endpoint = - ManhattanKVEndpointBuilder(client) - .defaultGuarantee(Guarantee.SoftDcReadMyWrites) - .defaultMaxTimeout(3.seconds) - .statsReceiver(statsReceiver) - .build() - - ManhattanRealTimeClientEventStore(endpoint, "realtime_client_events", statsReceiver, None) - } - - override val onlineUserHistoryStore: ReadableStore[OnlineUserHistoryKey, UserHistoryValue] = { - OnlineUserHistoryStore(realTimeClientEventStore) - } - - override val userMediaRepresentationStore = UserMediaRepresentationStore( - "user_media_representation", - "user_media_representation_dataset", - manhattanClientMtlsParams - ) - - override val producerMediaRepresentationStore = ObservedMemcachedReadableStore.fromCacheClient( - backingStore = UserMediaRepresentationStore( - "user_media_representation", - "producer_media_representation_dataset", - manhattanClientMtlsParams - )(statsReceiver.scope("UncachedProducerMediaRepStore")), - cacheClient = pushServiceCacheClient, - ttl = 4.hours - )( - valueInjection = BinaryScalaCodec(UserMediaRepresentation), - keyToString = { k: Long => s"ProducerMediaRepStore/$k" }, - statsReceiver.scope("ProducerMediaRepStore") - ) - - override val mrUserStatePredictionStore = { - StratoFetchableStore.withUnitView[Long, MRUserHmmState]( - stratoClient, - "frigate/magicrecs/mrUserStatePrediction.User") - } - - override val userHTLLastVisitStore = - UserHTLLastVisitReadableStore( - "pushservice_htl_user_session", - "tls_user_session_store", - statsReceiver.scope("userHTLLastVisitStore"), - manhattanClientMtlsParams - ) - - val crMixerClient: CrMixer.MethodPerEndpoint = new CrMixer.FinagledClient( - readOnlyThriftService( - "cr-mixer", - "/s/cr-mixer/cr-mixer-plus", - statsReceiver, - pushserviceThriftClientId, - requestTimeout = 5.seconds, - mTLSServiceIdentifier = Some(serviceIdentifier) - ), - clientParam = RichClientParam(serviceName = "cr-mixer") - ) - - val crMixerStore = CrMixerTweetStore(crMixerClient)(statsReceiver.scope("CrMixerTweetStore")) - - val contentMixerClient: ContentMixer.MethodPerEndpoint = new ContentMixer.FinagledClient( - readOnlyThriftService( - "content-mixer", - "/s/corgi-shared/content-mixer", - statsReceiver, - pushserviceThriftClientId, - requestTimeout = 5.seconds, - mTLSServiceIdentifier = Some(serviceIdentifier) - ), - clientParam = RichClientParam(serviceName = "content-mixer") - ) - - val exploreRankerClient: ExploreRanker.MethodPerEndpoint = - new ExploreRanker.FinagledClient( - readOnlyThriftService( - "explore-ranker", - "/s/explore-ranker/explore-ranker", - statsReceiver, - pushserviceThriftClientId, - requestTimeout = 5.seconds, - mTLSServiceIdentifier = Some(serviceIdentifier) - ), - clientParam = RichClientParam(serviceName = "explore-ranker") - ) - - val contentMixerStore = { - ObservedReadableStore(ContentMixerStore(contentMixerClient))( - statsReceiver.scope("ContentMixerStore")) - } - - val exploreRankerStore = { - ObservedReadableStore(ExploreRankerStore(exploreRankerClient))( - statsReceiver.scope("ExploreRankerStore") - ) - } - - val gizmoduckUtcOffsetStore = ObservedReadableStore( - GizmoduckUserUtcOffsetStore.fromUserStore(safeUserStore) - )(statsReceiver.scope("GizmoUserUtcOffsetStore")) - - override val userUtcOffsetStore = - UtcOffsetStore - .makeMemcachedUtcOffsetStore( - gizmoduckUtcOffsetStore, - pushServiceCoreSvcsCacheClient, - ReadableStore.empty, - manhattanStarbuckAppId, - manhattanClientMtlsParams - )(statsReceiver) - .mapValues(Duration.fromSeconds) - - override val cachedTweetyPieStoreV2 = { - val getTweetOptions = Some( - GetTweetOptions( - safetyLevel = Some(SafetyLevel.MagicRecsV2), - includeRetweetCount = true, - includeReplyCount = true, - includeFavoriteCount = true, - includeQuotedTweet = true, - additionalFieldIds = Seq(VisibleTextRangeField.id) - ) - ) - buildCachedTweetyPieStore(getTweetOptions, "tp_v2") - } - - override val cachedTweetyPieStoreV2NoVF = { - val getTweetOptions = Some( - GetTweetOptions( - safetyLevel = Some(SafetyLevel.FilterDefault), - includeRetweetCount = true, - includeReplyCount = true, - includeFavoriteCount = true, - includeQuotedTweet = true, - additionalFieldIds = Seq(VisibleTextRangeField.id), - ) - ) - buildCachedTweetyPieStore(getTweetOptions, "tp_v2_noVF") - } - - override val safeCachedTweetyPieStoreV2 = { - val getTweetOptions = Some( - GetTweetOptions( - safetyLevel = Some(SafetyLevel.MagicRecsAggressiveV2), - includeRetweetCount = true, - includeReplyCount = true, - includeFavoriteCount = true, - includeQuotedTweet = true, - additionalFieldIds = Seq(VisibleTextRangeField.id) - ) - ) - buildCachedTweetyPieStore(getTweetOptions, "sftp_v2") - } - - override val userTweetTweetyPieStore: ReadableStore[UserTweet, TweetyPieResult] = { - val getTweetOptions = Some( - GetTweetOptions( - safetyLevel = Some(SafetyLevel.MagicRecsV2), - includeRetweetCount = true, - includeReplyCount = true, - includeFavoriteCount = true, - includeQuotedTweet = true, - additionalFieldIds = Seq(VisibleTextRangeField.id) - ) - ) - TweetyPieStore.buildUserTweetStore( - client = tweetyPieClient, - options = getTweetOptions - ) - } - - override val safeUserTweetTweetyPieStore: ReadableStore[UserTweet, TweetyPieResult] = { - val getTweetOptions = Some( - GetTweetOptions( - safetyLevel = Some(SafetyLevel.MagicRecsAggressiveV2), - includeRetweetCount = true, - includeReplyCount = true, - includeFavoriteCount = true, - includeQuotedTweet = true, - additionalFieldIds = Seq(VisibleTextRangeField.id) - ) - ) - TweetyPieStore.buildUserTweetStore( - client = tweetyPieClient, - options = getTweetOptions - ) - } - - override val tweetContentFeatureCacheStore: ReadableStore[Long, ThriftDataRecord] = { - ObservedMemcachedReadableStore.fromCacheClient( - backingStore = TweetContentFeatureReadableStore(stratoClient), - cacheClient = poptartImpressionsCacheClient, - ttl = 12.hours - )( - valueInjection = BinaryScalaCodec(ThriftDataRecord), - statsReceiver = statsReceiver.scope("TweetContentFeaturesCacheStore"), - keyToString = { k: Long => s"tcf/$k" } - ) - } - - lazy val tweetTranslationStore: ReadableStore[ - TweetTranslationStore.Key, - TweetTranslationStore.Value - ] = { - val isTweetTranslatableStore = - StratoFetchableStore - .withUnitView[IsTweetTranslatableClientColumn.Key, Boolean]( - fetcher = new IsTweetTranslatableClientColumn(stratoClient).fetcher - ) - - val translateTweetStore = - StratoFetchableStore - .withUnitView[MachineTranslateTweetClientColumn.Key, MachineTranslationResponse]( - fetcher = new MachineTranslateTweetClientColumn(stratoClient).fetcher - ) - - ObservedReadableStore( - TweetTranslationStore(translateTweetStore, isTweetTranslatableStore, statsReceiver) - )(statsReceiver.scope("tweetTranslationStore")) - } - - val scarecrowClient = new ScarecrowService.FinagledClient( - readOnlyThriftService( - "", - "/s/abuse/scarecrow", - statsReceiver, - notifierThriftClientId, - requestTimeout = 5.second, - mTLSServiceIdentifier = Some(serviceIdentifier) - ), - clientParam = RichClientParam(serviceName = "") - ) - - // Setting up scarecrow store - override val scarecrowCheckEventStore = { - ScarecrowCheckEventStore(scarecrowClient) - } - - // setting up the perspective store - override val userTweetPerspectiveStore = { - val service = new DynamicRequestMeterFilter( - tunableMap(PushServiceTunableKeys.TweetPerspectiveStoreQpsLimit), - RateLimiterGenerator.asTuple(_, shardParams.numShards, 40), - PushQPSLimitConstants.PerspectiveStoreQPS)(timer) - .andThen( - readOnlyThriftService( - "tweetypie_perspective_service", - "/s/tweetypie/tweetypie", - statsReceiver, - notifierThriftClientId, - mTLSServiceIdentifier = Some(serviceIdentifier) - ) - ) - - val client = new TweetService.FinagledClient( - service, - clientParam = RichClientParam(serviceName = "tweetypie_perspective_client")) - ObservedReadableStore( - PerspectiveReadableStore(client) - )(statsReceiver.scope("TweetPerspectiveStore")) - } - - //user country code store, used in RecsWithheldContentPredicate - wrapped by memcache based cache - override val userCountryStore = - ObservedMemcachedReadableStore.fromCacheClient( - backingStore = ObservedReadableStore( - UserCountryStore(metastoreLocationAppId, manhattanClientMtlsParams) - )(statsReceiver.scope("userCountryStore")), - cacheClient = pushServiceCacheClient, - ttl = 12.hours - )( - valueInjection = BinaryScalaCodec(Location), - statsReceiver = statsReceiver.scope("UserCountryStore"), - keyToString = { k: Long => s"UserCountryStore/$k" } - ) - - override val audioSpaceParticipantsStore: ReadableStore[String, Participants] = { - val store = StratoFetchableStore - .DefaultStratoFetchableStore( - fetcher = new ParticipantsOnAudioSpaceClientColumn(stratoClient).fetcher - ).composeKeyMapping[String](broadcastId => - (broadcastId, AudioSpacesLookupContext(forUserId = None))) - - ObservedCachedReadableStore - .from( - store = buildStore(store, "AudioSpaceParticipantsStore"), - ttl = 20.seconds, - maxKeys = 200, - cacheName = "AudioSpaceParticipantsStore", - windowSize = 200 - ) - - } - - override val topicSocialProofServiceStore: ReadableStore[ - TopicSocialProofRequest, - TopicSocialProofResponse - ] = { - StratoFetchableStore.withUnitView[TopicSocialProofRequest, TopicSocialProofResponse]( - stratoClient, - "topic-signals/tsp/topic-social-proof") - } - - override val spaceDeviceFollowStore: ReadableStore[SourceDestUserRequest, Boolean] = { - StratoFetchableStore.withUnitView( - fetcher = new SpaceDeviceFollowingClientColumn(stratoClient).fetcher - ) - } - - override val audioSpaceStore: ReadableStore[String, AudioSpace] = { - val store = StratoFetchableStore - .DefaultStratoFetchableStore( - fetcher = new CoreOnAudioSpaceClientColumn(stratoClient).fetcher - ).composeKeyMapping[String] { broadcastId => - (broadcastId, AudioSpacesLookupContext(forUserId = None)) - } - - ObservedCachedReadableStore - .from( - store = buildStore(store, "AudioSpaceVisibilityStore"), - ttl = 1.minute, - maxKeys = 5000, - cacheName = "AudioSpaceVisibilityStore", - windowSize = 10000L) - } - - override val userLanguagesStore = UserLanguagesStore( - manhattanMetastoreAppId, - manhattanClientMtlsParams, - statsReceiver.scope("user_languages_store") - ) - - val tflockClient: TFlockClient = new TFlockClient( - new FlockDB.FinagledClient( - readOnlyThriftService( - "tflockClient", - "/s/tflock/tflock", - statsReceiver, - pushserviceThriftClientId, - mTLSServiceIdentifier = Some(serviceIdentifier) - ), - serviceName = "tflock", - stats = statsReceiver - ), - defaultPageSize = 1000 - ) - - val rawFlockClient = ThriftMux.client - .withClientId(pushserviceThriftClientId) - .withMutualTls(serviceIdentifier) - .build[FlockDB.MethodPerEndpoint]("/s/flock/flock") - - val flockClient: FlockClient = new FlockClient( - rawFlockClient, - defaultPageSize = 100 - ) - - override val recentFollowsStore: FlockFollowStore = { - val dStats = statsReceiver.scope("FlockRecentFollowsStore") - FlockFollowStore(flockClient, dStats) - } - - def notificationServiceClient: NotificationService$FinagleClient - - def notificationServiceSend( - target: Target, - request: CreateGenericNotificationRequest - ): Future[CreateGenericNotificationResponse] - - def notificationServiceDelete( - request: DeleteGenericNotificationRequest - ): Future[Unit] - - def notificationServiceDeleteTimeline( - request: DeleteCurrentTimelineForUserRequest - ): Future[Unit] - - override val notificationServiceSender: ReadableStore[ - NotificationServiceRequest, - CreateGenericNotificationResponse - ] = { - new NotificationServiceSender( - notificationServiceSend, - PushParams.EnableWritesToNotificationServiceParam, - PushParams.EnableWritesToNotificationServiceForAllEmployeesParam, - PushParams.EnableWritesToNotificationServiceForEveryoneParam - ) - } - - val eventRecosServiceClient = { - val dest = "/s/events-recos/events-recos-service" - new EventsRecosService.FinagledClient( - readOnlyThriftService( - "EventRecosService", - dest, - statsReceiver, - pushserviceThriftClientId, - mTLSServiceIdentifier = Some(serviceIdentifier) - ), - clientParam = RichClientParam(serviceName = "EventRecosService") - ) - } - - lazy val recommendedTrendsCandidateSource = RecommendedTrendsCandidateSource( - TrendsRecommendationStore(eventRecosServiceClient, statsReceiver)) - - override val softUserGeoLocationStore: ReadableStore[Long, GeoLocation] = - StratoFetchableStore.withUnitView[Long, GeoLocation](fetcher = - new FrequentSoftUserLocationClientColumn(stratoClient).fetcher) - - lazy val candidateSourceGenerator = new PushCandidateSourceGenerator( - earlybirdCandidateSource, - userTweetEntityGraphCandidates, - cachedTweetyPieStoreV2, - safeCachedTweetyPieStoreV2, - userTweetTweetyPieStore, - safeUserTweetTweetyPieStore, - cachedTweetyPieStoreV2NoVF, - edgeStore, - interestsWithLookupContextStore, - uttEntityHydrationStore, - geoDuckV2Store, - topTweetsByGeoStore, - topTweetsByGeoV2VersionedStore, - ruxTweetImpressionsStore, - recommendedTrendsCandidateSource, - recentTweetsByAuthorsStore, - topicSocialProofServiceStore, - crMixerStore, - contentMixerStore, - exploreRankerStore, - softUserGeoLocationStore, - tripTweetCandidateStore, - popGeoLists, - idsStore - ) - - lazy val loCandidateSourceGenerator = new LoggedOutPushCandidateSourceGenerator( - tripTweetCandidateStore, - geoDuckV2Store, - safeCachedTweetyPieStoreV2, - cachedTweetyPieStoreV2NoVF, - cachedTweetyPieStoreV2, - contentMixerStore, - softUserGeoLocationStore, - topTweetsByGeoStore, - topTweetsByGeoV2VersionedStore - ) - - lazy val rfphStatsRecorder = new RFPHStatsRecorder() - - lazy val rfphRestrictStep = new RFPHRestrictStep() - - lazy val rfphTakeStepUtil = new RFPHTakeStepUtil()(statsReceiver) - - lazy val rfphPrerankFilter = new RFPHPrerankFilter()(statsReceiver) - - lazy val rfphLightRanker = new RFPHLightRanker(lightRanker, statsReceiver) - - lazy val sendHandlerPredicateUtil = new SendHandlerPredicateUtil()(statsReceiver) - - lazy val ntabSender = - new NtabSender( - notificationServiceSender, - nTabHistoryStore, - notificationServiceDelete, - notificationServiceDeleteTimeline - ) - - lazy val ibis2Sender = new Ibis2Sender(pushIbisV2Store, tweetTranslationStore, statsReceiver) - - lazy val historyWriter = new HistoryWriter(historyStore, statsReceiver) - - lazy val loggedOutHistoryWriter = new HistoryWriter(loggedOutHistoryStore, statsReceiver) - - lazy val eventBusWriter = new EventBusWriter(pushSendEventBusPublisher, statsReceiver) - - lazy val ntabOnlyChannelSelector = new NtabOnlyChannelSelector - - lazy val notificationSender = - new NotificationSender( - ibis2Sender, - ntabSender, - statsReceiver, - notificationScribe - ) - - lazy val candidateNotifier = - new CandidateNotifier( - notificationSender, - casLock = casLock, - historyWriter = historyWriter, - eventBusWriter = eventBusWriter, - ntabOnlyChannelSelector = ntabOnlyChannelSelector - )(statsReceiver) - - lazy val loggedOutCandidateNotifier = new CandidateNotifier( - notificationSender, - casLock = casLock, - historyWriter = loggedOutHistoryWriter, - eventBusWriter = null, - ntabOnlyChannelSelector = ntabOnlyChannelSelector - )(statsReceiver) - - lazy val rfphNotifier = - new RefreshForPushNotifier(rfphStatsRecorder, candidateNotifier)(statsReceiver) - - lazy val loRfphNotifier = - new LoggedOutRefreshForPushNotifier(rfphStatsRecorder, loggedOutCandidateNotifier)( - statsReceiver) - - lazy val rfphRanker = { - val randomRanker = RandomRanker[Target, PushCandidate]() - val subscriptionCreatorRanker = - new SubscriptionCreatorRanker(superFollowEligibilityUserStore, statsReceiver) - new RFPHRanker( - randomRanker, - weightedOpenOrNtabClickModelScorer, - subscriptionCreatorRanker, - userHealthSignalStore, - producerMediaRepresentationStore, - statsReceiver - ) - } - - lazy val rfphFeatureHydrator = new RFPHFeatureHydrator(featureHydrator) - lazy val loggedOutRFPHRanker = new LoggedOutRanker(cachedTweetyPieStoreV2, statsReceiver) - - override val userFeaturesStore: ReadableStore[Long, UserFeatures] = { - implicit val valueCodec = new BinaryScalaCodec(UserFeatures) - val dataset: Dataset[Long, UserFeatures] = - Dataset( - "", - "user_features_pushservice_apollo", - "recommendations_user_features_apollo", - Apollo) - - ObservedMemcachedReadableStore.fromCacheClient( - backingStore = ObservedReadableStore(buildManhattanStore(dataset))( - statsReceiver.scope("UncachedUserFeaturesStore") - ), - cacheClient = pushServiceCacheClient, - ttl = 24.hours - )( - valueInjection = valueCodec, - statsReceiver = statsReceiver.scope("UserFeaturesStore"), - keyToString = { k: Long => s"ufts/$k" } - ) - } - - override def htlScoreStore(userId: Long): ReadableStore[Long, ScoredTweet] = { - val fetcher = new TimelineScorerTweetScoresV1ClientColumn(stratoClient).fetcher - val htlStore = buildStore( - StratoFetchableStore.withView[Long, TimelineScorerScoreView, ScoredTweet]( - fetcher, - TimelineScorerScoreView(Some(userId)) - ), - "htlScoreStore" - ) - htlStore - } - - override val userTargetingPropertyStore: ReadableStore[Long, UserTargetingProperty] = { - val name = "userTargetingPropertyStore" - val store = StratoFetchableStore - .withUnitView(new TargetingPropertyOnUserClientColumn(stratoClient).fetcher) - buildStore(store, name) - } - - override val timelinesUserSessionStore: ReadableStore[Long, UserSession] = { - implicit val valueCodec = new CompactScalaCodec(UserSession) - val dataset: Dataset[Long, UserSession] = Dataset[Long, UserSession]( - "", - "frigate_realgraph", - "real_graph_user_features", - Apollo - ) - - ObservedMemcachedReadableStore.fromCacheClient( - backingStore = ObservedReadableStore(buildManhattanStore(dataset))( - statsReceiver.scope("UncachedTimelinesUserSessionStore") - ), - cacheClient = pushServiceCacheClient, - ttl = 6.hours - )( - valueInjection = valueCodec, - statsReceiver = statsReceiver.scope("timelinesUserSessionStore"), - keyToString = { k: Long => s"tluss/$k" } - ) - } - - lazy val recentTweetsFromTflockStore: ReadableStore[Long, Seq[Long]] = - ObservedReadableStore( - RecentTweetsByAuthorsStore.usingRecentTweetsConfig( - tflockClient, - RecentTweetsConfig(maxResults = 1, maxAge = 3.days) - ) - )(statsReceiver.scope("RecentTweetsFromTflockStore")) - - lazy val recentTweetsByAuthorsStore: ReadableStore[RecentTweetsQuery, Seq[Seq[Long]]] = - ObservedReadableStore( - RecentTweetsByAuthorsStore(tflockClient) - )(statsReceiver.scope("RecentTweetsByAuthorsStore")) - - val jobConfig = PopGeoInterestProvider - .getPopularTweetsJobConfig( - InterestDeployConfig( - AppId("PopularTweetsByInterestProd"), - Cluster.ATLA, - Env.Prod, - serviceIdentifier, - manhattanClientMtlsParams - )) - .withManhattanAppId("frigate_pop_by_geo_tweets") - - override val topTweetsByGeoStore = TopTweetsStore.withMemCache( - jobConfig, - pushServiceCacheClient, - 10.seconds - )(statsReceiver) - - override val topTweetsByGeoV2VersionedStore: ReadableStore[String, PopTweetsInPlace] = { - StratoFetchableStore.withUnitView[String, PopTweetsInPlace]( - stratoClient, - "recommendations/popgeo/popGeoTweetsVersioned") - } - - override lazy val pushcapDynamicPredictionStore: ReadableStore[Long, PushcapUserHistory] = { - StratoFetchableStore.withUnitView[Long, PushcapUserHistory]( - stratoClient, - "frigate/magicrecs/pushcapDynamicPrediction.User") - } - - override val tweetAuthorLocationFeatureBuilder = - UserLocationFeatureBuilder(Some("TweetAuthor")) - .withStats() - - override val tweetAuthorLocationFeatureBuilderById = - UserLocationFeatureBuilderById( - userCountryStore, - tweetAuthorLocationFeatureBuilder - ).withStats() - - override val socialContextActionsFeatureBuilder = - SocialContextActionsFeatureBuilder().withStats() - - override val tweetContentFeatureBuilder = - TweetContentFeatureBuilder(tweetContentFeatureCacheStore).withStats() - - override val tweetAuthorRecentRealGraphFeatureBuilder = - RecentRealGraphFeatureBuilder( - stratoClient, - UserAuthorEntity, - TargetUserEntity, - TweetAuthorEntity, - TweetAuthorRecentRealGraphFeatures(statsReceiver.scope("TweetAuthorRecentRealGraphFeatures")) - ).withStats() - - override val socialContextRecentRealGraphFeatureBuilder = - SocialContextRecentRealGraphFeatureBuilder( - RecentRealGraphFeatureBuilder( - stratoClient, - TargetUserSocialContextEntity, - TargetUserEntity, - SocialContextEntity, - SocialContextRecentRealGraphFeatures( - statsReceiver.scope("SocialContextRecentRealGraphFeatures")) - )(statsReceiver - .scope("SocialContextRecentRealGraphFeatureBuilder").scope("RecentRealGraphFeatureBuilder")) - ).withStats() - - override val tweetSocialProofFeatureBuilder = - TweetSocialProofFeatureBuilder(Some("TargetUser")).withStats() - - override val targetUserFullRealGraphFeatureBuilder = - TargetFullRealGraphFeatureBuilder(Some("TargetUser")).withStats() - - override val postProcessingFeatureBuilder: PostProcessingFeatureBuilder = - PostProcessingFeatureBuilder() - - override val mrOfflineUserCandidateSparseAggregatesFeatureBuilder = - MrOfflineUserCandidateSparseAggregatesFeatureBuilder(stratoClient, featureStoreUtil).withStats() - - override val mrOfflineUserAggregatesFeatureBuilder = - MrOfflineUserAggregatesFeatureBuilder(stratoClient, featureStoreUtil).withStats() - - override val mrOfflineUserCandidateAggregatesFeatureBuilder = - MrOfflineUserCandidateAggregatesFeatureBuilder(stratoClient, featureStoreUtil).withStats() - - override val tweetAnnotationsFeatureBuilder = - TweetAnnotationsFeatureBuilder(stratoClient).withStats() - - override val targetUserMediaRepresentationFeatureBuilder = - UserMediaRepresentationFeatureBuilder(userMediaRepresentationStore).withStats() - - override val targetLevelFeatureBuilder = - TargetLevelFeatureBuilder(featureStoreUtil, targetLevelFeaturesConfig).withStats() - - override val candidateLevelFeatureBuilder = - CandidateLevelFeatureBuilder(featureStoreUtil).withStats() - - override lazy val targetFeatureHydrator = RelevanceTargetFeatureHydrator( - targetUserFullRealGraphFeatureBuilder, - postProcessingFeatureBuilder, - targetUserMediaRepresentationFeatureBuilder, - targetLevelFeatureBuilder - ) - - override lazy val featureHydrator = - FeatureHydrator(targetFeatureHydrator, candidateFeatureHydrator) - - val pushServiceLightRankerConfig: LightRankerConfig = new LightRankerConfig( - pushserviceThriftClientId, - serviceIdentifier, - statsReceiver.scope("lightRanker"), - deepbirdv2PredictionServiceDest, - "DeepbirdV2PredictionService" - ) - val lightRanker: MagicRecsServeDataRecordLightRanker = - pushServiceLightRankerConfig.lightRanker - - override val tweetImpressionStore: ReadableStore[Long, Seq[Long]] = { - val name = "htl_impression_store" - val store = buildStore( - HtlTweetImpressionStore.createStoreWithTweetIds( - requestTimeout = 6.seconds, - label = "htl_tweet_impressions", - serviceIdentifier = serviceIdentifier, - statsReceiver = statsReceiver - ), - name - ) - val numTweetsReturned = - statsReceiver.scope(name).stat("num_tweets_returned_per_user") - new TransformedReadableStore(store)((userId: Long, tweetIds: Seq[Long]) => { - numTweetsReturned.add(tweetIds.size) - Future.value(Some(tweetIds)) - }) - } - - val ruxTweetImpressionsStore = new TweetImpressionsStore(stratoClient) - - override val strongTiesStore: ReadableStore[Long, STPResult] = { - implicit val valueCodec = new BinaryScalaCodec(STPResult) - val strongTieScoringDataset: Dataset[Long, STPResult] = - Dataset("", "frigate_stp", "stp_result_rerank", Athena) - buildManhattanStore(strongTieScoringDataset) - } - - override lazy val earlybirdFeatureStore = ObservedReadableStore( - EarlybirdFeatureStore( - clientId = pushserviceThriftClientId.name, - earlybirdSearchStore = earlybirdSearchStore - ) - )(statsReceiver.scope("EarlybirdFeatureStore")) - - override lazy val earlybirdFeatureBuilder = EarlybirdFeatureBuilder(earlybirdFeatureStore) - - override lazy val earlybirdSearchStore = { - val earlybirdClientName: String = "earlybird" - val earlybirdSearchStoreName: String = "EarlybirdSearchStore" - - val earlybirdClient = new EarlybirdService.FinagledClient( - readOnlyThriftService( - earlybirdClientName, - earlybirdSearchDest, - statsReceiver, - pushserviceThriftClientId, - tries = 1, - requestTimeout = 3.seconds, - mTLSServiceIdentifier = Some(serviceIdentifier) - ), - clientParam = RichClientParam(protocolFactory = new TCompactProtocol.Factory) - ) - - ObservedReadableStore( - EarlybirdSearchStore(earlybirdClient)(statsReceiver.scope(earlybirdSearchStoreName)) - )(statsReceiver.scope(earlybirdSearchStoreName)) - } - - override lazy val earlybirdCandidateSource: EarlybirdCandidateSource = EarlybirdCandidateSource( - clientId = pushserviceThriftClientId.name, - earlybirdSearchStore = earlybirdSearchStore - ) - - override val realGraphScoresTop500InStore: RealGraphScoresTop500InStore = { - val stratoRealGraphInStore = - StratoFetchableStore - .withUnitView[Long, CandidateSeq]( - stratoClient, - "frigate/magicrecs/fanoutCoi500pRealGraphV2") - - RealGraphScoresTop500InStore( - ObservedMemcachedReadableStore.fromCacheClient( - backingStore = stratoRealGraphInStore, - cacheClient = entityGraphCacheClient, - ttl = 24.hours - )( - valueInjection = BinaryScalaCodec(CandidateSeq), - statsReceiver = statsReceiver.scope("CachedRealGraphScoresTop500InStore"), - keyToString = { k: Long => s"500p_test/$k" } - ) - ) - } - - override val tweetEntityGraphStore = { - val tweetEntityGraphClient = new UserTweetEntityGraph.FinagledClient( - Finagle.readOnlyThriftService( - "user_tweet_entity_graph", - userTweetEntityGraphDest, - statsReceiver, - pushserviceThriftClientId, - requestTimeout = 5.seconds, - mTLSServiceIdentifier = Some(serviceIdentifier) - ) - ) - ObservedReadableStore( - RecommendedTweetEntitiesStore( - tweetEntityGraphClient, - statsReceiver.scope("RecommendedTweetEntitiesStore") - ) - )(statsReceiver.scope("RecommendedTweetEntitiesStore")) - } - - override val userUserGraphStore = { - val userUserGraphClient = new UserUserGraph.FinagledClient( - Finagle.readOnlyThriftService( - "user_user_graph", - userUserGraphDest, - statsReceiver, - pushserviceThriftClientId, - requestTimeout = 5.seconds, - mTLSServiceIdentifier = Some(serviceIdentifier) - ), - clientParam = RichClientParam(serviceName = "user_user_graph") - ) - ObservedReadableStore( - UserUserGraphStore(userUserGraphClient, statsReceiver.scope("UserUserGraphStore")) - )(statsReceiver.scope("UserUserGraphStore")) - } - - override val ntabCaretFeedbackStore: ReadableStore[GenericNotificationsFeedbackRequest, Seq[ - CaretFeedbackDetails - ]] = { - val client = ManhattanKVClient( - "pushservice_ntab_caret_feedback_omega", - Omega.wilyName, - manhattanClientMtlsParams - ) - val endpoint = ManhattanKVEndpointBuilder(client) - .defaultGuarantee(Guarantee.SoftDcReadMyWrites) - .defaultMaxTimeout(3.seconds) - .maxRetryCount(2) - .statsReceiver(statsReceiver) - .build() - - val feedbackSignalManhattanClient = - FeedbackSignalManhattanClient(endpoint, statsReceiver.scope("FeedbackSignalManhattanClient")) - NtabCaretFeedbackStore(feedbackSignalManhattanClient) - } - - override val genericFeedbackStore: ReadableStore[FeedbackRequest, Seq[ - FeedbackPromptValue - ]] = { - FeedbackStore( - GenericFeedbackStoreBuilder.build( - manhattanKVClientAppId = "frigate_pushservice_ntabfeedback_prompt", - environment = NotifEnvironment.apply(serviceIdentifier.environment), - svcIdentifier = serviceIdentifier, - statsReceiver = statsReceiver - )) - } - - override val genericNotificationFeedbackStore: GenericFeedbackStore = { - - GenericFeedbackStoreBuilder.build( - manhattanKVClientAppId = "frigate_pushservice_ntabfeedback_prompt", - environment = NotifEnvironment.apply(serviceIdentifier.environment), - svcIdentifier = serviceIdentifier, - statsReceiver = statsReceiver - ) - } - - override val earlybirdSearchDest = "/s/earlybird-root-superroot/root-superroot" - - // low latency as compared to default `semanticCoreMetadataClient` - private val lowLatencySemanticCoreMetadataClient: MetadataService.MethodPerEndpoint = - new MetadataService.FinagledClient( - Finagle.readOnlyThriftService( - name = "semantic_core_metadata_service", - dest = "/s/escherbird/metadataservice", - statsReceiver = statsReceiver, - thriftClientId = pushserviceThriftClientId, - tries = 2, // total number of tries. number of retries = tries - 1 - requestTimeout = 2.seconds, - mTLSServiceIdentifier = Some(serviceIdentifier) - ) - ) - - private val semanticCoreMetadataStitchClient = new MetadataStitchClient( - lowLatencySemanticCoreMetadataClient - ) - - override val semanticCoreMegadataStore: ReadableStore[SemanticEntityForQuery, EntityMegadata] = { - val name = "semantic_core_megadata_store_cached" - val store = MetaDataReadableStore.getMegadataReadableStore( - metadataStitchClient = semanticCoreMetadataStitchClient, - typedMetadataDomains = Some(Set(Domains.EventsEntityService)) - ) - ObservedCachedReadableStore - .from( - store = ObservedReadableStore(store)( - statsReceiver - .scope("store") - .scope("semantic_core_megadata_store") - ), - ttl = 1.hour, - maxKeys = 1000, - cacheName = "semantic_core_megadata_cache", - windowSize = 10000L - )(statsReceiver.scope("store", name)) - } - - override val basketballGameScoreStore: ReadableStore[QualifiedId, BasketballGameLiveUpdate] = { - StratoFetchableStore.withUnitView[QualifiedId, BasketballGameLiveUpdate]( - stratoClient, - "semanticCore/basketballGameScore.Entity") - } - - override val baseballGameScoreStore: ReadableStore[QualifiedId, BaseballGameLiveUpdate] = { - StratoFetchableStore.withUnitView[QualifiedId, BaseballGameLiveUpdate]( - stratoClient, - "semanticCore/baseballGameScore.Entity") - } - - override val cricketMatchScoreStore: ReadableStore[QualifiedId, CricketMatchLiveUpdate] = { - StratoFetchableStore.withUnitView[QualifiedId, CricketMatchLiveUpdate]( - stratoClient, - "semanticCore/cricketMatchScore.Entity") - } - - override val soccerMatchScoreStore: ReadableStore[QualifiedId, SoccerMatchLiveUpdate] = { - ObservedCachedReadableStore - .from( - store = StratoFetchableStore.withUnitView[QualifiedId, SoccerMatchLiveUpdate]( - stratoClient, - "semanticCore/soccerMatchScore.Entity"), - ttl = 10.seconds, - maxKeys = 100, - cacheName = "SoccerMatchCachedStore", - windowSize = 100L - )(statsReceiver.scope("SoccerMatchCachedStore")) - - } - - override val nflGameScoreStore: ReadableStore[QualifiedId, NflFootballGameLiveUpdate] = { - ObservedCachedReadableStore - .from( - store = StratoFetchableStore.withUnitView[QualifiedId, NflFootballGameLiveUpdate]( - stratoClient, - "semanticCore/nflFootballGameScore.Entity"), - ttl = 10.seconds, - maxKeys = 100, - cacheName = "NFLMatchCachedStore", - windowSize = 100L - )(statsReceiver.scope("NFLMatchCachedStore")) - - } - - override val userHealthSignalStore: ReadableStore[Long, UserHealthSignalResponse] = { - val userHealthSignalFetcher = - stratoClient.fetcher[Long, Seq[UserHealthSignal], UserHealthSignalResponse]( - "hss/user_signals/api/healthSignals.User" - ) - - val store = buildStore( - StratoFetchableStore.withView[Long, Seq[UserHealthSignal], UserHealthSignalResponse]( - userHealthSignalFetcher, - Seq( - AgathaRecentAbuseStrikeDouble, - AgathaCalibratedNsfwDouble, - AgathaCseDouble, - NsfwTextUserScoreDouble, - NsfwConsumerScoreDouble)), - "UserHealthSignalFetcher" - ) - if (!inMemCacheOff) { - ObservedCachedReadableStore - .from( - store = ObservedReadableStore(store)( - statsReceiver.scope("store").scope("user_health_model_score_store")), - ttl = 12.hours, - maxKeys = 16777215, - cacheName = "user_health_model_score_store_cache", - windowSize = 10000L - )(statsReceiver.scope("store", "user_health_model_score_store_cached")) - } else { - store - } - } - - override val tweetHealthScoreStore: ReadableStore[TweetScoringRequest, TweetScoringResponse] = { - val tweetHealthScoreFetcher = - stratoClient.fetcher[TweetScoringRequest, Unit, TweetScoringResponse]( - "abuse/detection/tweetHealthModelScore" - ) - - val store = buildStore( - StratoFetchableStore.withUnitView(tweetHealthScoreFetcher), - "TweetHealthScoreFetcher" - ) - - ObservedCachedReadableStore - .from( - store = ObservedReadableStore(store)( - statsReceiver.scope("store").scope("tweet_health_model_score_store")), - ttl = 30.minutes, - maxKeys = 1000, - cacheName = "tweet_health_model_score_store_cache", - windowSize = 10000L - )(statsReceiver.scope("store", "tweet_health_model_score_store_cached")) - } - - override val appPermissionStore: ReadableStore[(Long, (String, String)), AppPermission] = { - val store = StratoFetchableStore - .withUnitView[(Long, (String, String)), AppPermission]( - stratoClient, - "clients/permissionsState") - ObservedCachedReadableStore.from[(Long, (String, String)), AppPermission]( - buildStore(store, "mr_app_permission_store"), - ttl = 30.minutes, - maxKeys = 1000, - cacheName = "mr_app_permission_store_cache", - windowSize = 10000L - )(statsReceiver.scope("mr_app_permission_store_cached")) - } - - def pushSendEventStreamName: String - - override val pushSendEventBusPublisher = EventBusPublisherBuilder() - .clientId("frigate_pushservice") - .streamName(pushSendEventStreamName) - .thriftStruct(NotificationScribe) - .statsReceiver(statsReceiver.scope("push_send_eventbus")) - .build() - - override lazy val candidateFeatureHydrator: CandidateFeatureHydrator = - CandidateFeatureHydrator( - socialContextActionsFeatureBuilder = Some(socialContextActionsFeatureBuilder), - tweetSocialProofFeatureBuilder = Some(tweetSocialProofFeatureBuilder), - earlybirdFeatureBuilder = Some(earlybirdFeatureBuilder), - tweetContentFeatureBuilder = Some(tweetContentFeatureBuilder), - tweetAuthorRecentRealGraphFeatureBuilder = Some(tweetAuthorRecentRealGraphFeatureBuilder), - socialContextRecentRealGraphFeatureBuilder = Some(socialContextRecentRealGraphFeatureBuilder), - tweetAnnotationsFeatureBuilder = Some(tweetAnnotationsFeatureBuilder), - mrOfflineUserCandidateSparseAggregatesFeatureBuilder = - Some(mrOfflineUserCandidateSparseAggregatesFeatureBuilder), - candidateLevelFeatureBuilder = Some(candidateLevelFeatureBuilder) - )(statsReceiver.scope("push_feature_hydrator")) - - private val candidateCopyCross = - new CandidateCopyExpansion(statsReceiver.scope("refresh_handler/cross")) - - override lazy val candidateHydrator: PushCandidateHydrator = - PushCandidateHydrator( - this.socialGraphServiceProcessStore, - safeUserStore, - listAPIStore, - candidateCopyCross)( - statsReceiver.scope("push_candidate_hydrator"), - weightedOpenOrNtabClickModelScorer) - - override lazy val sendHandlerCandidateHydrator: SendHandlerPushCandidateHydrator = - SendHandlerPushCandidateHydrator( - lexServiceStore, - fanoutMetadataStore, - semanticCoreMegadataStore, - safeUserStore, - simClusterToEntityStore, - audioSpaceStore, - interestsWithLookupContextStore, - uttEntityHydrationStore, - superFollowCreatorTweetCountStore - )( - statsReceiver.scope("push_candidate_hydrator"), - weightedOpenOrNtabClickModelScorer - ) - - def mrRequestScriberNode: String - def loggedOutMrRequestScriberNode: String - - override lazy val configParamsBuilder: ConfigParamsBuilder = ConfigParamsBuilder( - config = overridesConfig, - featureContextBuilder = FeatureContextBuilder(featureSwitches), - statsReceiver = statsReceiver - ) - - def buildStore[K, V](store: ReadableStore[K, V], name: String): ReadableStore[K, V] = { - ObservedReadableStore(store)(statsReceiver.scope("store").scope(name)) - } - - def buildManhattanStore[K, V](dataset: Dataset[K, V]): ReadableStore[K, V] = { - val manhattanKVClientParams = ManhattanKVClientMtlsParams( - serviceIdentifier = serviceIdentifier, - opportunisticTls = OpportunisticTls.Required - ) - ManhattanStore - .fromDatasetWithMtls[K, V]( - dataset, - mtlsParams = manhattanKVClientParams, - statsReceiver = statsReceiver.scope(dataset.datasetName)) - } - - def buildCachedTweetyPieStore( - getTweetOptions: Option[GetTweetOptions], - keyPrefix: String - ): ReadableStore[Long, TweetyPieResult] = { - def discardAdditionalMediaInfo(tweetypieResult: TweetyPieResult) = { - val updatedMedia = tweetypieResult.tweet.media.map { mediaSeq => - mediaSeq.map { media => media.copy(additionalMetadata = None, sizes = Nil.toSet) } - } - val updatedTweet = tweetypieResult.tweet.copy(media = updatedMedia) - tweetypieResult.copy(tweet = updatedTweet) - } - - val tweetypieStoreWithoutAdditionalMediaInfo = TweetyPieStore( - tweetyPieClient, - getTweetOptions, - transformTweetypieResult = discardAdditionalMediaInfo - )(statsReceiver.scope("tweetypie_without_additional_media_info")) - - ObservedMemcachedReadableStore.fromCacheClient( - backingStore = tweetypieStoreWithoutAdditionalMediaInfo, - cacheClient = pushServiceCoreSvcsCacheClient, - ttl = 12.hours - )( - valueInjection = TweetyPieResultInjection, - statsReceiver = statsReceiver.scope("TweetyPieStore"), - keyToString = { k: Long => s"$keyPrefix/$k" } - ) - } - - override def init(): Future[Unit] = - ClientRegistry.expAllRegisteredClientsResolved().map { clients => - log.info("Done resolving clients: " + clients.mkString("[", ", ", "]")) - } - - val InlineActionsMhColumn = - "frigate/magicrecs/inlineActionsMh" - - override val inlineActionHistoryStore: ReadableStore[Long, Seq[(Long, String)]] = - StratoScannableStore - .withUnitView[(Long, Slice[Long]), (Long, Long), String](stratoClient, InlineActionsMhColumn) - .composeKeyMapping[Long] { userId => - (userId, Slice[Long](from = None, to = None, limit = None)) - }.mapValues { response => - response.map { - case (key, value) => (key._2, value) - } - } - - override val tripTweetCandidateStore: ReadableStore[TripDomain, TripTweets] = { - StratoFetchableStore - .withUnitView[TripDomain, TripTweets]( - new TripTweetsAirflowProdClientColumn(stratoClient).fetcher) - } - - override val softUserFollowingStore: ReadableStore[User, Seq[Long]] = new SoftUserFollowingStore( - stratoClient) - - override val superFollowEligibilityUserStore: ReadableStore[Long, Boolean] = { - StratoFetchableStore.withUnitView[Long, Boolean]( - stratoClient, - "audiencerewards/audienceRewardsService/getSuperFollowEligibility.User") - } - - override val superFollowCreatorTweetCountStore: ReadableStore[UserId, Int] = { - ObservedCachedReadableStore - .from( - store = StratoFetchableStore - .withUnitView[UserId, Int](new CreatorSubscriptionNumTweetsColumn(stratoClient).fetcher), - ttl = 5.minutes, - maxKeys = 1000, - cacheName = "SuperFollowCreatorTweetCountStore", - windowSize = 10000L - )(statsReceiver.scope("SuperFollowCreatorTweetCountStore")) - - } - - override val hasSuperFollowingRelationshipStore: ReadableStore[ - HasSuperFollowingRelationshipRequest, - Boolean - ] = { - StratoFetchableStore.withUnitView[HasSuperFollowingRelationshipRequest, Boolean]( - stratoClient, - "audiencerewards/superFollows/hasSuperFollowingRelationshipV2") - } - - override val superFollowApplicationStatusStore: ReadableStore[ - (Long, SellerTrack), - SellerApplicationState - ] = { - StratoFetchableStore.withUnitView[(Long, SellerTrack), SellerApplicationState]( - stratoClient, - "periscope/eligibility/applicationStatus") - } - - def historyStoreMemcacheDest: String - - override lazy val recentHistoryCacheClient = { - RecentHistoryCacheClient.build(historyStoreMemcacheDest, serviceIdentifier, statsReceiver) - } - - override val openAppUserStore: ReadableStore[Long, Boolean] = { - buildStore(OpenAppUserStore(stratoClient), "OpenAppUserStore") - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/ExperimentsWithStats.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/ExperimentsWithStats.docx new file mode 100644 index 000000000..5e3b7ca13 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/ExperimentsWithStats.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/ExperimentsWithStats.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/ExperimentsWithStats.scala deleted file mode 100644 index 923a785ee..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/ExperimentsWithStats.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.frigate.pushservice.config - -import com.twitter.frigate.common.util.Experiments - -object ExperimentsWithStats { - - /** - * Add an experiment here to collect detailed pushservice stats. - * - * ! Important ! - * Keep this set small and remove experiments when you don't need the stats anymore. - */ - final val PushExperiments: Set[String] = Set( - Experiments.MRAndroidInlineActionHoldback.exptName, - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/ProdConfig.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/ProdConfig.docx new file mode 100644 index 000000000..c8902b8cf Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/ProdConfig.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/ProdConfig.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/ProdConfig.scala deleted file mode 100644 index 7edc8d46d..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/ProdConfig.scala +++ /dev/null @@ -1,230 +0,0 @@ -package com.twitter.frigate.pushservice.config - -import com.twitter.abdecider.LoggingABDecider -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.bijection.Base64String -import com.twitter.bijection.Injection -import com.twitter.conversions.DurationOps._ -import com.twitter.decider.Decider -import com.twitter.featureswitches.v2.FeatureSwitches -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.finagle.thrift.RichClientParam -import com.twitter.finagle.util.DefaultTimer -import com.twitter.frigate.common.config.RateLimiterGenerator -import com.twitter.frigate.common.filter.DynamicRequestMeterFilter -import com.twitter.frigate.common.history.ManhattanHistoryStore -import com.twitter.frigate.common.history.InvalidatingAfterWritesPushServiceHistoryStore -import com.twitter.frigate.common.history.ManhattanKVHistoryStore -import com.twitter.frigate.common.history.PushServiceHistoryStore -import com.twitter.frigate.common.history.SimplePushServiceHistoryStore -import com.twitter.frigate.common.util._ -import com.twitter.frigate.data_pipeline.features_common.FeatureStoreUtil -import com.twitter.frigate.data_pipeline.features_common.TargetLevelFeaturesConfig -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.DeciderKey -import com.twitter.frigate.pushservice.params.PushQPSLimitConstants -import com.twitter.frigate.pushservice.params.PushServiceTunableKeys -import com.twitter.frigate.pushservice.params.ShardParams -import com.twitter.frigate.pushservice.store.PushIbis2Store -import com.twitter.frigate.pushservice.thriftscala.PushRequestScribe -import com.twitter.frigate.scribe.thriftscala.NotificationScribe -import com.twitter.ibis2.service.thriftscala.Ibis2Service -import com.twitter.logging.Logger -import com.twitter.notificationservice.api.thriftscala.DeleteCurrentTimelineForUserRequest -import com.twitter.notificationservice.api.thriftscala.NotificationApi -import com.twitter.notificationservice.api.thriftscala.NotificationApi$FinagleClient -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationRequest -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationResponse -import com.twitter.notificationservice.thriftscala.DeleteGenericNotificationRequest -import com.twitter.notificationservice.thriftscala.NotificationService -import com.twitter.notificationservice.thriftscala.NotificationService$FinagleClient -import com.twitter.servo.decider.DeciderGateBuilder -import com.twitter.util.tunable.TunableMap -import com.twitter.util.Future -import com.twitter.util.Timer - -case class ProdConfig( - override val isServiceLocal: Boolean, - override val localConfigRepoPath: String, - override val inMemCacheOff: Boolean, - override val decider: Decider, - override val abDecider: LoggingABDecider, - override val featureSwitches: FeatureSwitches, - override val shardParams: ShardParams, - override val serviceIdentifier: ServiceIdentifier, - override val tunableMap: TunableMap, -)( - implicit val statsReceiver: StatsReceiver) - extends { - // Due to trait initialization logic in Scala, any abstract members declared in Config or - // DeployConfig should be declared in this block. Otherwise the abstract member might initialize to - // null if invoked before object creation finishing. - - val log = Logger("ProdConfig") - - // Deciders - val isPushserviceCanaryDeepbirdv2CanaryClusterEnabled = decider - .feature(DeciderKey.enablePushserviceDeepbirdv2CanaryClusterDeciderKey.toString).isAvailable - - // Client ids - val notifierThriftClientId = ClientId("frigate-notifier.prod") - val loggedOutNotifierThriftClientId = ClientId("frigate-logged-out-notifier.prod") - val pushserviceThriftClientId: ClientId = ClientId("frigate-pushservice.prod") - - // Dests - val frigateHistoryCacheDest = "/s/cache/frigate_history" - val memcacheCASDest = "/s/cache/magic_recs_cas:twemcaches" - val historyStoreMemcacheDest = - "/srv#/prod/local/cache/magic_recs_history:twemcaches" - - val deepbirdv2PredictionServiceDest = - if (serviceIdentifier.service.equals("frigate-pushservice-canary") && - isPushserviceCanaryDeepbirdv2CanaryClusterEnabled) - "/s/frigate/deepbirdv2-magicrecs-canary" - else "/s/frigate/deepbirdv2-magicrecs" - - override val fanoutMetadataColumn = "frigate/magicfanout/prod/mh/fanoutMetadata" - - override val timer: Timer = DefaultTimer - override val featureStoreUtil = FeatureStoreUtil.withParams(Some(serviceIdentifier)) - override val targetLevelFeaturesConfig = TargetLevelFeaturesConfig() - val pushServiceMHCacheDest = "/s/cache/pushservice_mh" - - val pushServiceCoreSvcsCacheDest = "/srv#/prod/local/cache/pushservice_core_svcs" - - val userTweetEntityGraphDest = "/s/cassowary/user_tweet_entity_graph" - val userUserGraphDest = "/s/cassowary/user_user_graph" - val lexServiceDest = "/s/live-video/timeline-thrift" - val entityGraphCacheDest = "/s/cache/pushservice_entity_graph" - - override val pushIbisV2Store = { - val service = Finagle.readOnlyThriftService( - "ibis-v2-service", - "/s/ibis2/ibis2", - statsReceiver, - notifierThriftClientId, - requestTimeout = 3.seconds, - tries = 3, - mTLSServiceIdentifier = Some(serviceIdentifier) - ) - - // according to ibis team, it is safe to retry on timeout, write & channel closed exceptions. - val pushIbisClient = new Ibis2Service.FinagledClient( - new DynamicRequestMeterFilter( - tunableMap(PushServiceTunableKeys.IbisQpsLimitTunableKey), - RateLimiterGenerator.asTuple(_, shardParams.numShards, 20), - PushQPSLimitConstants.IbisOrNTabQPSForRFPH - )(timer).andThen(service), - RichClientParam(serviceName = "ibis-v2-service") - ) - - PushIbis2Store(pushIbisClient) - } - - val notificationServiceClient: NotificationService$FinagleClient = { - val service = Finagle.readWriteThriftService( - "notificationservice", - "/s/notificationservice/notificationservice", - statsReceiver, - pushserviceThriftClientId, - requestTimeout = 10.seconds, - mTLSServiceIdentifier = Some(serviceIdentifier) - ) - - new NotificationService.FinagledClient( - new DynamicRequestMeterFilter( - tunableMap(PushServiceTunableKeys.NtabQpsLimitTunableKey), - RateLimiterGenerator.asTuple(_, shardParams.numShards, 20), - PushQPSLimitConstants.IbisOrNTabQPSForRFPH)(timer).andThen(service), - RichClientParam(serviceName = "notificationservice") - ) - } - - val notificationServiceApiClient: NotificationApi$FinagleClient = { - val service = Finagle.readWriteThriftService( - "notificationservice-api", - "/s/notificationservice/notificationservice-api:thrift", - statsReceiver, - pushserviceThriftClientId, - requestTimeout = 10.seconds, - mTLSServiceIdentifier = Some(serviceIdentifier) - ) - - new NotificationApi.FinagledClient( - new DynamicRequestMeterFilter( - tunableMap(PushServiceTunableKeys.NtabQpsLimitTunableKey), - RateLimiterGenerator.asTuple(_, shardParams.numShards, 20), - PushQPSLimitConstants.IbisOrNTabQPSForRFPH)(timer).andThen(service), - RichClientParam(serviceName = "notificationservice-api") - ) - } - - val mrRequestScriberNode = "mr_request_scribe" - val loggedOutMrRequestScriberNode = "lo_mr_request_scribe" - - override val pushSendEventStreamName = "frigate_pushservice_send_event_prod" -} with DeployConfig { - // Scribe - private val notificationScribeLog = Logger("notification_scribe") - private val notificationScribeInjection: Injection[NotificationScribe, String] = BinaryScalaCodec( - NotificationScribe - ) andThen Injection.connect[Array[Byte], Base64String, String] - - override def notificationScribe(data: NotificationScribe): Unit = { - val logEntry: String = notificationScribeInjection(data) - notificationScribeLog.info(logEntry) - } - - // History Store - Invalidates cached history after writes - override val historyStore = new InvalidatingAfterWritesPushServiceHistoryStore( - ManhattanHistoryStore(notificationHistoryStore, statsReceiver), - recentHistoryCacheClient, - new DeciderGateBuilder(decider) - .idGate(DeciderKey.enableInvalidatingCachedHistoryStoreAfterWrites) - ) - - override val emailHistoryStore: PushServiceHistoryStore = { - statsReceiver.scope("frigate_email_history").counter("request").incr() - new SimplePushServiceHistoryStore(emailNotificationHistoryStore) - } - - override val loggedOutHistoryStore = - new InvalidatingAfterWritesPushServiceHistoryStore( - ManhattanKVHistoryStore( - manhattanKVLoggedOutHistoryStoreEndpoint, - "frigate_notification_logged_out_history"), - recentHistoryCacheClient, - new DeciderGateBuilder(decider) - .idGate(DeciderKey.enableInvalidatingCachedLoggedOutHistoryStoreAfterWrites) - ) - - private val requestScribeLog = Logger("request_scribe") - private val requestScribeInjection: Injection[PushRequestScribe, String] = BinaryScalaCodec( - PushRequestScribe - ) andThen Injection.connect[Array[Byte], Base64String, String] - - override def requestScribe(data: PushRequestScribe): Unit = { - val logEntry: String = requestScribeInjection(data) - requestScribeLog.info(logEntry) - } - - // generic notification server - override def notificationServiceSend( - target: Target, - request: CreateGenericNotificationRequest - ): Future[CreateGenericNotificationResponse] = - notificationServiceClient.createGenericNotification(request) - - // generic notification server - override def notificationServiceDelete( - request: DeleteGenericNotificationRequest - ): Future[Unit] = notificationServiceClient.deleteGenericNotification(request) - - // NTab-api - override def notificationServiceDeleteTimeline( - request: DeleteCurrentTimelineForUserRequest - ): Future[Unit] = notificationServiceApiClient.deleteCurrentTimelineForUser(request) - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/StagingConfig.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/StagingConfig.docx new file mode 100644 index 000000000..927868d5f Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/StagingConfig.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/StagingConfig.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/StagingConfig.scala deleted file mode 100644 index c93ca0ea8..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/StagingConfig.scala +++ /dev/null @@ -1,193 +0,0 @@ -package com.twitter.frigate.pushservice.config - -import com.twitter.abdecider.LoggingABDecider -import com.twitter.conversions.DurationOps._ -import com.twitter.decider.Decider -import com.twitter.featureswitches.v2.FeatureSwitches -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.finagle.thrift.RichClientParam -import com.twitter.finagle.util.DefaultTimer -import com.twitter.frigate.common.config.RateLimiterGenerator -import com.twitter.frigate.common.filter.DynamicRequestMeterFilter -import com.twitter.frigate.common.history.InvalidatingAfterWritesPushServiceHistoryStore -import com.twitter.frigate.common.history.ManhattanHistoryStore -import com.twitter.frigate.common.history.ManhattanKVHistoryStore -import com.twitter.frigate.common.history.ReadOnlyHistoryStore -import com.twitter.frigate.common.history.PushServiceHistoryStore -import com.twitter.frigate.common.history.SimplePushServiceHistoryStore -import com.twitter.frigate.common.util.Finagle -import com.twitter.frigate.data_pipeline.features_common.FeatureStoreUtil -import com.twitter.frigate.data_pipeline.features_common.TargetLevelFeaturesConfig -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.DeciderKey -import com.twitter.frigate.pushservice.params.PushQPSLimitConstants -import com.twitter.frigate.pushservice.params.PushServiceTunableKeys -import com.twitter.frigate.pushservice.params.ShardParams -import com.twitter.frigate.pushservice.store._ -import com.twitter.frigate.pushservice.thriftscala.PushRequestScribe -import com.twitter.frigate.scribe.thriftscala.NotificationScribe -import com.twitter.ibis2.service.thriftscala.Ibis2Service -import com.twitter.logging.Logger -import com.twitter.notificationservice.api.thriftscala.DeleteCurrentTimelineForUserRequest -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationRequest -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationResponse -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationResponseType -import com.twitter.notificationservice.thriftscala.DeleteGenericNotificationRequest -import com.twitter.notificationservice.thriftscala.NotificationService -import com.twitter.notificationservice.thriftscala.NotificationService$FinagleClient -import com.twitter.servo.decider.DeciderGateBuilder -import com.twitter.util.tunable.TunableMap -import com.twitter.util.Future -import com.twitter.util.Timer - -case class StagingConfig( - override val isServiceLocal: Boolean, - override val localConfigRepoPath: String, - override val inMemCacheOff: Boolean, - override val decider: Decider, - override val abDecider: LoggingABDecider, - override val featureSwitches: FeatureSwitches, - override val shardParams: ShardParams, - override val serviceIdentifier: ServiceIdentifier, - override val tunableMap: TunableMap, -)( - implicit val statsReceiver: StatsReceiver) - extends { - // Due to trait initialization logic in Scala, any abstract members declared in Config or - // DeployConfig should be declared in this block. Otherwise the abstract member might initialize to - // null if invoked before object creation finishing. - - val log = Logger("StagingConfig") - - // Client ids - val notifierThriftClientId = ClientId("frigate-notifier.dev") - val loggedOutNotifierThriftClientId = ClientId("frigate-logged-out-notifier.dev") - val pushserviceThriftClientId: ClientId = ClientId("frigate-pushservice.staging") - - override val fanoutMetadataColumn = "frigate/magicfanout/staging/mh/fanoutMetadata" - - // dest - val frigateHistoryCacheDest = "/srv#/test/local/cache/twemcache_frigate_history" - val memcacheCASDest = "/srv#/test/local/cache/twemcache_magic_recs_cas_dev:twemcaches" - val pushServiceMHCacheDest = "/srv#/test/local/cache/twemcache_pushservice_test" - val entityGraphCacheDest = "/srv#/test/local/cache/twemcache_pushservice_test" - val pushServiceCoreSvcsCacheDest = "/srv#/test/local/cache/twemcache_pushservice_core_svcs_test" - val historyStoreMemcacheDest = "/srv#/test/local/cache/twemcache_eventstream_test:twemcaches" - val userTweetEntityGraphDest = "/cluster/local/cassowary/staging/user_tweet_entity_graph" - val userUserGraphDest = "/cluster/local/cassowary/staging/user_user_graph" - val lexServiceDest = "/srv#/staging/local/live-video/timeline-thrift" - val deepbirdv2PredictionServiceDest = "/cluster/local/frigate/staging/deepbirdv2-magicrecs" - - override val featureStoreUtil = FeatureStoreUtil.withParams(Some(serviceIdentifier)) - override val targetLevelFeaturesConfig = TargetLevelFeaturesConfig() - val mrRequestScriberNode = "validation_mr_request_scribe" - val loggedOutMrRequestScriberNode = "lo_mr_request_scribe" - - override val timer: Timer = DefaultTimer - - override val pushSendEventStreamName = "frigate_pushservice_send_event_staging" - - override val pushIbisV2Store = { - val service = Finagle.readWriteThriftService( - "ibis-v2-service", - "/s/ibis2/ibis2", - statsReceiver, - notifierThriftClientId, - requestTimeout = 6.seconds, - mTLSServiceIdentifier = Some(serviceIdentifier) - ) - - val pushIbisClient = new Ibis2Service.FinagledClient( - new DynamicRequestMeterFilter( - tunableMap(PushServiceTunableKeys.IbisQpsLimitTunableKey), - RateLimiterGenerator.asTuple(_, shardParams.numShards, 20), - PushQPSLimitConstants.IbisOrNTabQPSForRFPH - )(timer).andThen(service), - RichClientParam(serviceName = "ibis-v2-service") - ) - - StagingIbis2Store(PushIbis2Store(pushIbisClient)) - } - - val notificationServiceClient: NotificationService$FinagleClient = { - val service = Finagle.readWriteThriftService( - "notificationservice", - "/s/notificationservice/notificationservice", - statsReceiver, - pushserviceThriftClientId, - requestTimeout = 10.seconds, - mTLSServiceIdentifier = Some(serviceIdentifier) - ) - - new NotificationService.FinagledClient( - new DynamicRequestMeterFilter( - tunableMap(PushServiceTunableKeys.NtabQpsLimitTunableKey), - RateLimiterGenerator.asTuple(_, shardParams.numShards, 20), - PushQPSLimitConstants.IbisOrNTabQPSForRFPH)(timer).andThen(service), - RichClientParam(serviceName = "notificationservice") - ) - } -} with DeployConfig { - - // Scribe - private val notificationScribeLog = Logger("StagingNotificationScribe") - - override def notificationScribe(data: NotificationScribe): Unit = { - notificationScribeLog.info(data.toString) - } - private val requestScribeLog = Logger("StagingRequestScribe") - - override def requestScribe(data: PushRequestScribe): Unit = { - requestScribeLog.info(data.toString) - } - - // history store - override val historyStore = new InvalidatingAfterWritesPushServiceHistoryStore( - ReadOnlyHistoryStore( - ManhattanHistoryStore(notificationHistoryStore, statsReceiver) - ), - recentHistoryCacheClient, - new DeciderGateBuilder(decider) - .idGate(DeciderKey.enableInvalidatingCachedHistoryStoreAfterWrites) - ) - - override val emailHistoryStore: PushServiceHistoryStore = new SimplePushServiceHistoryStore( - emailNotificationHistoryStore) - - // history store - override val loggedOutHistoryStore = - new InvalidatingAfterWritesPushServiceHistoryStore( - ReadOnlyHistoryStore( - ManhattanKVHistoryStore( - manhattanKVLoggedOutHistoryStoreEndpoint, - "frigate_notification_logged_out_history")), - recentHistoryCacheClient, - new DeciderGateBuilder(decider) - .idGate(DeciderKey.enableInvalidatingCachedLoggedOutHistoryStoreAfterWrites) - ) - - override def notificationServiceSend( - target: Target, - request: CreateGenericNotificationRequest - ): Future[CreateGenericNotificationResponse] = - target.isTeamMember.flatMap { isTeamMember => - if (isTeamMember) { - notificationServiceClient.createGenericNotification(request) - } else { - log.info(s"Mock creating generic notification $request for user: ${target.targetId}") - Future.value( - CreateGenericNotificationResponse(CreateGenericNotificationResponseType.Success) - ) - } - } - - override def notificationServiceDelete( - request: DeleteGenericNotificationRequest - ): Future[Unit] = Future.Unit - - override def notificationServiceDeleteTimeline( - request: DeleteCurrentTimelineForUserRequest - ): Future[Unit] = Future.Unit -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/mlconfig/DeepbirdV2ModelConfig.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/mlconfig/DeepbirdV2ModelConfig.docx new file mode 100644 index 000000000..0a0d3e52f Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/mlconfig/DeepbirdV2ModelConfig.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/mlconfig/DeepbirdV2ModelConfig.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/mlconfig/DeepbirdV2ModelConfig.scala deleted file mode 100644 index c45ccc72e..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/mlconfig/DeepbirdV2ModelConfig.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.frigate.pushservice.config.mlconfig - -import com.twitter.cortex.deepbird.thriftjava.DeepbirdPredictionService -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.ml.prediction.DeepbirdPredictionEngineServiceStore -import com.twitter.nrel.heavyranker.PushDBv2PredictionServiceStore - -object DeepbirdV2ModelConfig { - def buildPredictionServiceScoreStore( - predictionServiceClient: DeepbirdPredictionService.ServiceToClient, - serviceName: String - )( - implicit statsReceiver: StatsReceiver - ): PushDBv2PredictionServiceStore = { - - val stats = statsReceiver.scope(serviceName) - val serviceStats = statsReceiver.scope("dbv2PredictionServiceStore") - - new PushDBv2PredictionServiceStore( - DeepbirdPredictionEngineServiceStore(predictionServiceClient, batchSize = Some(32))(stats) - )(serviceStats) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/controller/PushServiceController.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/controller/PushServiceController.docx new file mode 100644 index 000000000..43b0514df Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/controller/PushServiceController.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/controller/PushServiceController.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/controller/PushServiceController.scala deleted file mode 100644 index d271e5a57..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/controller/PushServiceController.scala +++ /dev/null @@ -1,114 +0,0 @@ -package com.twitter.frigate.pushservice.controller - -import com.google.inject.Inject -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.finatra.thrift.Controller -import com.twitter.frigate.pushservice.exception.DisplayLocationNotSupportedException -import com.twitter.frigate.pushservice.refresh_handler.RefreshForPushHandler -import com.twitter.frigate.pushservice.send_handler.SendHandler -import com.twitter.frigate.pushservice.refresh_handler.LoggedOutRefreshForPushHandler -import com.twitter.frigate.pushservice.thriftscala.PushService.Loggedout -import com.twitter.frigate.pushservice.thriftscala.PushService.Refresh -import com.twitter.frigate.pushservice.thriftscala.PushService.Send -import com.twitter.frigate.pushservice.{thriftscala => t} -import com.twitter.frigate.thriftscala.NotificationDisplayLocation -import com.twitter.util.logging.Logging -import com.twitter.util.Future - -class PushServiceController @Inject() ( - sendHandler: SendHandler, - refreshForPushHandler: RefreshForPushHandler, - loggedOutRefreshForPushHandler: LoggedOutRefreshForPushHandler, - statsReceiver: StatsReceiver) - extends Controller(t.PushService) - with Logging { - - private val stats: StatsReceiver = statsReceiver.scope(s"${this.getClass.getSimpleName}") - private val failureCount = stats.counter("failures") - private val failureStatsScope = stats.scope("failures") - private val uncaughtErrorCount = failureStatsScope.counter("uncaught") - private val uncaughtErrorScope = failureStatsScope.scope("uncaught") - private val clientIdScope = stats.scope("client_id") - - handle(t.PushService.Send) { request: Send.Args => - send(request) - } - - handle(t.PushService.Refresh) { args: Refresh.Args => - refresh(args) - } - - handle(t.PushService.Loggedout) { request: Loggedout.Args => - loggedOutRefresh(request) - } - - private def loggedOutRefresh( - request: t.PushService.Loggedout.Args - ): Future[t.PushService.Loggedout.SuccessType] = { - val fut = request.request.notificationDisplayLocation match { - case NotificationDisplayLocation.PushToMobileDevice => - loggedOutRefreshForPushHandler.refreshAndSend(request.request) - case _ => - Future.exception( - new DisplayLocationNotSupportedException( - "Specified notification display location is not supported")) - } - fut.onFailure { ex => - logger.error( - s"Failure in push service for logged out refresh request: $request - ${ex.getMessage} - ${ex.getStackTrace - .mkString(", \n\t")}", - ex) - failureCount.incr() - uncaughtErrorCount.incr() - uncaughtErrorScope.counter(ex.getClass.getCanonicalName).incr() - } - } - - private def refresh( - request: t.PushService.Refresh.Args - ): Future[t.PushService.Refresh.SuccessType] = { - - val fut = request.request.notificationDisplayLocation match { - case NotificationDisplayLocation.PushToMobileDevice => - val clientId: String = - ClientId.current - .flatMap { cid => Option(cid.name) } - .getOrElse("none") - clientIdScope.counter(clientId).incr() - refreshForPushHandler.refreshAndSend(request.request) - case _ => - Future.exception( - new DisplayLocationNotSupportedException( - "Specified notification display location is not supported")) - } - fut.onFailure { ex => - logger.error( - s"Failure in push service for refresh request: $request - ${ex.getMessage} - ${ex.getStackTrace - .mkString(", \n\t")}", - ex - ) - - failureCount.incr() - uncaughtErrorCount.incr() - uncaughtErrorScope.counter(ex.getClass.getCanonicalName).incr() - } - - } - - private def send( - request: t.PushService.Send.Args - ): Future[t.PushService.Send.SuccessType] = { - sendHandler(request.request).onFailure { ex => - logger.error( - s"Failure in push service for send request: $request - ${ex.getMessage} - ${ex.getStackTrace - .mkString(", \n\t")}", - ex - ) - - failureCount.incr() - uncaughtErrorCount.incr() - uncaughtErrorScope.counter(ex.getClass.getCanonicalName).incr() - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/DisplayLocationNotSupportedException.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/DisplayLocationNotSupportedException.docx new file mode 100644 index 000000000..614d515ed Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/DisplayLocationNotSupportedException.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/DisplayLocationNotSupportedException.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/DisplayLocationNotSupportedException.scala deleted file mode 100644 index 08399c934..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/DisplayLocationNotSupportedException.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.frigate.pushservice.exception - -import scala.util.control.NoStackTrace - -/** - * Throw exception if DisplayLocation is not supported - * - * @param message Exception message - */ -class DisplayLocationNotSupportedException(private val message: String) - extends Exception(message) - with NoStackTrace diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/InvalidSportDomainException.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/InvalidSportDomainException.docx new file mode 100644 index 000000000..f38e92f50 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/InvalidSportDomainException.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/InvalidSportDomainException.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/InvalidSportDomainException.scala deleted file mode 100644 index 8f0d2b988..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/InvalidSportDomainException.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.frigate.pushservice.exception - -import scala.util.control.NoStackTrace - -/** - * Throw exception if the sport domain is not supported by MagicFanoutSports - * - * @param message Exception message - */ -class InvalidSportDomainException(private val message: String) - extends Exception(message) - with NoStackTrace diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/TweetNTabRequestHydratorException.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/TweetNTabRequestHydratorException.docx new file mode 100644 index 000000000..fad02a98a Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/TweetNTabRequestHydratorException.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/TweetNTabRequestHydratorException.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/TweetNTabRequestHydratorException.scala deleted file mode 100644 index 069e65d79..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/TweetNTabRequestHydratorException.scala +++ /dev/null @@ -1,7 +0,0 @@ -package com.twitter.frigate.pushservice.exception - -import scala.util.control.NoStackTrace - -class TweetNTabRequestHydratorException(private val message: String) - extends Exception(message) - with NoStackTrace diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/UnsupportedCrtException.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/UnsupportedCrtException.docx new file mode 100644 index 000000000..1da18ef9b Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/UnsupportedCrtException.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/UnsupportedCrtException.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/UnsupportedCrtException.scala deleted file mode 100644 index 5ed6c1c28..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/UnsupportedCrtException.scala +++ /dev/null @@ -1,11 +0,0 @@ -package com.twitter.frigate.pushservice.exception - -import scala.util.control.NoStackTrace - -/** - * Exception for CRT not expected in the scope - * @param message Exception message to log the UnsupportedCrt - */ -class UnsupportedCrtException(private val message: String) - extends Exception(message) - with NoStackTrace diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/UttEntityNotFoundException.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/UttEntityNotFoundException.docx new file mode 100644 index 000000000..4930f0cbd Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/UttEntityNotFoundException.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/UttEntityNotFoundException.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/UttEntityNotFoundException.scala deleted file mode 100644 index 3ac069dac..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/UttEntityNotFoundException.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.frigate.pushservice.exception - -import scala.util.control.NoStackTrace - -/** - * Throw exception if UttEntity is not found where it might be a required data field - * - * @param message Exception message - */ -class UttEntityNotFoundException(private val message: String) - extends Exception(message) - with NoStackTrace diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/HealthFeatureGetter.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/HealthFeatureGetter.docx new file mode 100644 index 000000000..fce7780b2 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/HealthFeatureGetter.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/HealthFeatureGetter.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/HealthFeatureGetter.scala deleted file mode 100644 index addf5b438..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/HealthFeatureGetter.scala +++ /dev/null @@ -1,220 +0,0 @@ -package com.twitter.frigate.pushservice.ml - -import com.twitter.abuse.detection.scoring.thriftscala.{Model => TweetHealthModel} -import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringRequest -import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringResponse -import com.twitter.frigate.common.base.FeatureMap -import com.twitter.frigate.common.base.TweetAuthor -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushConstants -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.predicate.HealthPredicates.userHealthSignalValueToDouble -import com.twitter.frigate.pushservice.util.CandidateHydrationUtil -import com.twitter.frigate.pushservice.util.CandidateUtil -import com.twitter.frigate.pushservice.util.MediaAnnotationsUtil -import com.twitter.frigate.thriftscala.UserMediaRepresentation -import com.twitter.hss.api.thriftscala.SignalValue -import com.twitter.hss.api.thriftscala.UserHealthSignal -import com.twitter.hss.api.thriftscala.UserHealthSignal.AgathaCalibratedNsfwDouble -import com.twitter.hss.api.thriftscala.UserHealthSignal.NsfwTextUserScoreDouble -import com.twitter.hss.api.thriftscala.UserHealthSignalResponse -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future -import com.twitter.util.Time - -object HealthFeatureGetter { - - def getFeatures( - pushCandidate: PushCandidate, - producerMediaRepresentationStore: ReadableStore[Long, UserMediaRepresentation], - userHealthScoreStore: ReadableStore[Long, UserHealthSignalResponse], - tweetHealthScoreStoreOpt: Option[ReadableStore[TweetScoringRequest, TweetScoringResponse]] = - None - ): Future[FeatureMap] = { - - pushCandidate match { - case cand: PushCandidate with TweetCandidate with TweetAuthor with TweetAuthorDetails => - val pMediaNsfwRequest = - TweetScoringRequest(cand.tweetId, TweetHealthModel.ExperimentalHealthModelScore4) - val pTweetTextNsfwRequest = - TweetScoringRequest(cand.tweetId, TweetHealthModel.ExperimentalHealthModelScore1) - - cand.authorId match { - case Some(authorId) => - Future - .join( - userHealthScoreStore.get(authorId), - producerMediaRepresentationStore.get(authorId), - tweetHealthScoreStoreOpt.map(_.get(pMediaNsfwRequest)).getOrElse(Future.None), - tweetHealthScoreStoreOpt.map(_.get(pTweetTextNsfwRequest)).getOrElse(Future.None), - cand.tweetAuthor - ).map { - case ( - healthSignalsResponseOpt, - producerMuOpt, - pMediaNsfwOpt, - pTweetTextNsfwOpt, - tweetAuthorOpt) => - val healthSignalScoreMap = healthSignalsResponseOpt - .map(_.signalValues).getOrElse(Map.empty[UserHealthSignal, SignalValue]) - val agathaNSFWScore = userHealthSignalValueToDouble( - healthSignalScoreMap - .getOrElse(AgathaCalibratedNsfwDouble, SignalValue.DoubleValue(0.5))) - val userTextNSFWScore = userHealthSignalValueToDouble( - healthSignalScoreMap - .getOrElse(NsfwTextUserScoreDouble, SignalValue.DoubleValue(0.15))) - val pMediaNsfwScore = pMediaNsfwOpt.map(_.score).getOrElse(0.0) - val pTweetTextNsfwScore = pTweetTextNsfwOpt.map(_.score).getOrElse(0.0) - - val mediaRepresentationMap = - producerMuOpt.map(_.mediaRepresentation).getOrElse(Map.empty[String, Double]) - val sumScore: Double = mediaRepresentationMap.values.sum - val nudityRate = - if (sumScore > 0) - mediaRepresentationMap.getOrElse( - MediaAnnotationsUtil.nudityCategoryId, - 0.0) / sumScore - else 0.0 - val beautyRate = - if (sumScore > 0) - mediaRepresentationMap.getOrElse( - MediaAnnotationsUtil.beautyCategoryId, - 0.0) / sumScore - else 0.0 - val singlePersonRate = - if (sumScore > 0) - mediaRepresentationMap.getOrElse( - MediaAnnotationsUtil.singlePersonCategoryId, - 0.0) / sumScore - else 0.0 - val dislikeCt = cand.numericFeatures.getOrElse( - "tweet.magic_recs_tweet_real_time_aggregates_v2.pair.v2.magicrecs.realtime.is_ntab_disliked.any_feature.Duration.Top.count", - 0.0) - val sentCt = cand.numericFeatures.getOrElse( - "tweet.magic_recs_tweet_real_time_aggregates_v2.pair.v2.magicrecs.realtime.is_sent.any_feature.Duration.Top.count", - 0.0) - val dislikeRate = if (sentCt > 0) dislikeCt / sentCt else 0.0 - - val authorDislikeCt = cand.numericFeatures.getOrElse( - "tweet_author_aggregate.pair.label.ntab.isDisliked.any_feature.28.days.count", - 0.0) - val authorReportCt = cand.numericFeatures.getOrElse( - "tweet_author_aggregate.pair.label.reportTweetDone.any_feature.28.days.count", - 0.0) - val authorSentCt = cand.numericFeatures - .getOrElse( - "tweet_author_aggregate.pair.any_label.any_feature.28.days.count", - 0.0) - val authorDislikeRate = - if (authorSentCt > 0) authorDislikeCt / authorSentCt else 0.0 - val authorReportRate = - if (authorSentCt > 0) authorReportCt / authorSentCt else 0.0 - - val (isNsfwAccount, authorAccountAge) = tweetAuthorOpt match { - case Some(tweetAuthor) => - ( - CandidateHydrationUtil.isNsfwAccount( - tweetAuthor, - cand.target.params(PushFeatureSwitchParams.NsfwTokensParam)), - (Time.now - Time.fromMilliseconds(tweetAuthor.createdAtMsec)).inHours - ) - case _ => (false, 0) - } - - val tweetSemanticCoreIds = cand.sparseBinaryFeatures - .getOrElse(PushConstants.TweetSemanticCoreIdFeature, Set.empty[String]) - - val continuousFeatures = Map[String, Double]( - "agathaNsfwScore" -> agathaNSFWScore, - "textNsfwScore" -> userTextNSFWScore, - "pMediaNsfwScore" -> pMediaNsfwScore, - "pTweetTextNsfwScore" -> pTweetTextNsfwScore, - "nudityRate" -> nudityRate, - "beautyRate" -> beautyRate, - "singlePersonRate" -> singlePersonRate, - "numSources" -> CandidateUtil.getTagsCRCount(cand), - "favCount" -> cand.numericFeatures - .getOrElse("tweet.core.tweet_counts.favorite_count", 0.0), - "activeFollowers" -> cand.numericFeatures - .getOrElse("RecTweetAuthor.User.ActiveFollowers", 0.0), - "favorsRcvd28Days" -> cand.numericFeatures - .getOrElse("RecTweetAuthor.User.FavorsRcvd28Days", 0.0), - "tweets28Days" -> cand.numericFeatures - .getOrElse("RecTweetAuthor.User.Tweets28Days", 0.0), - "dislikeCount" -> dislikeCt, - "dislikeRate" -> dislikeRate, - "sentCount" -> sentCt, - "authorDislikeCount" -> authorDislikeCt, - "authorDislikeRate" -> authorDislikeRate, - "authorReportCount" -> authorReportCt, - "authorReportRate" -> authorReportRate, - "authorSentCount" -> authorSentCt, - "authorAgeInHour" -> authorAccountAge.toDouble - ) - - val booleanFeatures = Map[String, Boolean]( - "isSimclusterBased" -> RecTypes.simclusterBasedTweets - .contains(cand.commonRecType), - "isTopicTweet" -> RecTypes.isTopicTweetType(cand.commonRecType), - "isHashSpace" -> RecTypes.tagspaceTypes.contains(cand.commonRecType), - "isFRS" -> RecTypes.frsTypes.contains(cand.commonRecType), - "isModelingBased" -> RecTypes.mrModelingBasedTypes.contains(cand.commonRecType), - "isGeoPop" -> RecTypes.GeoPopTweetTypes.contains(cand.commonRecType), - "hasPhoto" -> cand.booleanFeatures - .getOrElse("RecTweet.TweetyPieResult.HasPhoto", false), - "hasVideo" -> cand.booleanFeatures - .getOrElse("RecTweet.TweetyPieResult.HasVideo", false), - "hasUrl" -> cand.booleanFeatures - .getOrElse("RecTweet.TweetyPieResult.HasUrl", false), - "isMrTwistly" -> CandidateUtil.isMrTwistlyCandidate(cand), - "abuseStrikeTop2Percent" -> tweetSemanticCoreIds.contains( - PushConstants.AbuseStrike_Top2Percent_Id), - "abuseStrikeTop1Percent" -> tweetSemanticCoreIds.contains( - PushConstants.AbuseStrike_Top1Percent_Id), - "abuseStrikeTop05Percent" -> tweetSemanticCoreIds.contains( - PushConstants.AbuseStrike_Top05Percent_Id), - "abuseStrikeTop025Percent" -> tweetSemanticCoreIds.contains( - PushConstants.AbuseStrike_Top025Percent_Id), - "allSpamReportsPerFavTop1Percent" -> tweetSemanticCoreIds.contains( - PushConstants.AllSpamReportsPerFav_Top1Percent_Id), - "reportsPerFavTop1Percent" -> tweetSemanticCoreIds.contains( - PushConstants.ReportsPerFav_Top1Percent_Id), - "reportsPerFavTop2Percent" -> tweetSemanticCoreIds.contains( - PushConstants.ReportsPerFav_Top2Percent_Id), - "isNudity" -> tweetSemanticCoreIds.contains( - PushConstants.MediaUnderstanding_Nudity_Id), - "beautyStyleFashion" -> tweetSemanticCoreIds.contains( - PushConstants.MediaUnderstanding_Beauty_Id), - "singlePerson" -> tweetSemanticCoreIds.contains( - PushConstants.MediaUnderstanding_SinglePerson_Id), - "pornList" -> tweetSemanticCoreIds.contains(PushConstants.PornList_Id), - "pornographyAndNsfwContent" -> tweetSemanticCoreIds.contains( - PushConstants.PornographyAndNsfwContent_Id), - "sexLife" -> tweetSemanticCoreIds.contains(PushConstants.SexLife_Id), - "sexLifeOrSexualOrientation" -> tweetSemanticCoreIds.contains( - PushConstants.SexLifeOrSexualOrientation_Id), - "profanity" -> tweetSemanticCoreIds.contains(PushConstants.ProfanityFilter_Id), - "isVerified" -> cand.booleanFeatures - .getOrElse("RecTweetAuthor.User.IsVerified", false), - "hasNsfwToken" -> isNsfwAccount - ) - - val stringFeatures = Map[String, String]( - "tweetLanguage" -> cand.categoricalFeatures - .getOrElse("tweet.core.tweet_text.language", "") - ) - - FeatureMap( - booleanFeatures = booleanFeatures, - numericFeatures = continuousFeatures, - categoricalFeatures = stringFeatures) - } - case _ => Future.value(FeatureMap()) - } - case _ => Future.value(FeatureMap()) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/HydrationContextBuilder.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/HydrationContextBuilder.docx new file mode 100644 index 000000000..40d9808cc Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/HydrationContextBuilder.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/HydrationContextBuilder.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/HydrationContextBuilder.scala deleted file mode 100644 index 023adb81e..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/HydrationContextBuilder.scala +++ /dev/null @@ -1,179 +0,0 @@ -package com.twitter.frigate.pushservice.ml - -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.ml.feature.TweetSocialProofKey -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.predicate.quality_model_predicate.PDauCohortUtil -import com.twitter.nrel.hydration.base.FeatureInput -import com.twitter.nrel.hydration.push.HydrationContext -import com.twitter.nrel.hydration.frigate.{FeatureInputs => FI} -import com.twitter.util.Future - -object HydrationContextBuilder { - - private def getRecUserInputs( - pushCandidate: PushCandidate - ): Set[FI.RecUser] = { - pushCandidate match { - case userCandidate: UserCandidate => - Set(FI.RecUser(userCandidate.userId)) - case _ => Set.empty - } - } - - private def getRecTweetInputs( - pushCandidate: PushCandidate - ): Set[FI.RecTweet] = - pushCandidate match { - case tweetCandidateWithAuthor: TweetCandidate with TweetAuthor with TweetAuthorDetails => - val authorIdOpt = tweetCandidateWithAuthor.authorId - Set(FI.RecTweet(tweetCandidateWithAuthor.tweetId, authorIdOpt)) - case _ => Set.empty - } - - private def getMediaInputs( - pushCandidate: PushCandidate - ): Set[FI.Media] = - pushCandidate match { - case tweetCandidateWithMedia: TweetCandidate with TweetDetails => - tweetCandidateWithMedia.mediaKeys - .map { mk => - Set(FI.Media(mk)) - }.getOrElse(Set.empty) - case _ => Set.empty - } - - private def getEventInputs( - pushCandidate: PushCandidate - ): Set[FI.Event] = pushCandidate match { - case mrEventCandidate: EventCandidate => - Set(FI.Event(mrEventCandidate.eventId)) - case mfEventCandidate: MagicFanoutEventCandidate => - Set(FI.Event(mfEventCandidate.eventId)) - case _ => Set.empty - } - - private def getTopicInputs( - pushCandidate: PushCandidate - ): Set[FI.Topic] = - pushCandidate match { - case mrTopicCandidate: TopicCandidate => - mrTopicCandidate.semanticCoreEntityId match { - case Some(topicId) => Set(FI.Topic(topicId)) - case _ => Set.empty - } - case _ => Set.empty - } - - private def getTweetSocialProofKey( - pushCandidate: PushCandidate - ): Future[Set[FI.SocialProofKey]] = { - pushCandidate match { - case candidate: TweetCandidate with SocialContextActions => - val target = pushCandidate.target - target.seedsWithWeight.map { seedsWithWeightOpt => - Set( - FI.SocialProofKey( - TweetSocialProofKey( - seedsWithWeightOpt.getOrElse(Map.empty), - candidate.socialContextAllTypeActions - )) - ) - } - case _ => Future.value(Set.empty) - } - } - - private def getSocialContextInputs( - pushCandidate: PushCandidate - ): Future[Set[FeatureInput]] = - pushCandidate match { - case candidateWithSC: Candidate with SocialContextActions => - val tweetSocialProofKeyFut = getTweetSocialProofKey(pushCandidate) - tweetSocialProofKeyFut.map { tweetSocialProofKeyOpt => - val socialContextUsers = FI.SocialContextUsers(candidateWithSC.socialContextUserIds.toSet) - val socialContextActions = - FI.SocialContextActions(candidateWithSC.socialContextAllTypeActions) - val socialProofKeyOpt = tweetSocialProofKeyOpt - Set(Set(socialContextUsers), Set(socialContextActions), socialProofKeyOpt).flatten - } - case _ => Future.value(Set.empty) - } - - private def getPushStringGroupInputs( - pushCandidate: PushCandidate - ): Set[FI.PushStringGroup] = - Set( - FI.PushStringGroup( - pushCandidate.getPushCopy.flatMap(_.pushStringGroup).map(_.toString).getOrElse("") - )) - - private def getCRTInputs( - pushCandidate: PushCandidate - ): Set[FI.CommonRecommendationType] = - Set(FI.CommonRecommendationType(pushCandidate.commonRecType)) - - private def getFrigateNotification( - pushCandidate: PushCandidate - ): Set[FI.CandidateFrigateNotification] = - Set(FI.CandidateFrigateNotification(pushCandidate.frigateNotification)) - - private def getCopyId( - pushCandidate: PushCandidate - ): Set[FI.CopyId] = - Set(FI.CopyId(pushCandidate.pushCopyId, pushCandidate.ntabCopyId)) - - def build(candidate: PushCandidate): Future[HydrationContext] = { - val socialContextInputsFut = getSocialContextInputs(candidate) - socialContextInputsFut.map { socialContextInputs => - val featureInputs: Set[FeatureInput] = - socialContextInputs ++ - getRecUserInputs(candidate) ++ - getRecTweetInputs(candidate) ++ - getEventInputs(candidate) ++ - getTopicInputs(candidate) ++ - getCRTInputs(candidate) ++ - getPushStringGroupInputs(candidate) ++ - getMediaInputs(candidate) ++ - getFrigateNotification(candidate) ++ - getCopyId(candidate) - - HydrationContext( - candidate.target.targetId, - featureInputs - ) - } - } - - def build(target: Target): Future[HydrationContext] = { - val realGraphFeaturesFut = target.realGraphFeatures - for { - realGraphFeaturesOpt <- realGraphFeaturesFut - dauProb <- PDauCohortUtil.getDauProb(target) - mrUserStateOpt <- target.targetMrUserState - historyInputOpt <- - if (target.params(PushFeatureSwitchParams.EnableHydratingOnlineMRHistoryFeatures)) { - target.onlineLabeledPushRecs.map { mrHistoryValueOpt => - mrHistoryValueOpt.map(FI.MrHistory) - } - } else Future.None - } yield { - val realGraphFeaturesInputOpt = realGraphFeaturesOpt.map { realGraphFeatures => - FI.TargetRealGraphFeatures(realGraphFeatures) - } - val dauProbInput = FI.DauProb(dauProb) - val mrUserStateInput = FI.MrUserState(mrUserStateOpt.map(_.name).getOrElse("unknown")) - HydrationContext( - target.targetId, - Seq( - realGraphFeaturesInputOpt, - historyInputOpt, - Some(dauProbInput), - Some(mrUserStateInput) - ).flatten.toSet - ) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/PushMLModelScorer.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/PushMLModelScorer.docx new file mode 100644 index 000000000..a170e0f0a Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/PushMLModelScorer.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/PushMLModelScorer.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/PushMLModelScorer.scala deleted file mode 100644 index fd702cc3c..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/PushMLModelScorer.scala +++ /dev/null @@ -1,188 +0,0 @@ -package com.twitter.frigate.pushservice.ml - -import com.twitter.cortex.deepbird.thriftjava.ModelSelector -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.common.base.FeatureMap -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushMLModel -import com.twitter.frigate.pushservice.params.PushModelName -import com.twitter.frigate.pushservice.params.WeightedOpenOrNtabClickModel -import com.twitter.nrel.heavyranker.PushCandidateHydrationContextWithModel -import com.twitter.nrel.heavyranker.PushPredictionServiceStore -import com.twitter.nrel.heavyranker.TargetFeatureMapWithModel -import com.twitter.timelines.configapi.FSParam -import com.twitter.util.Future - -/** - * PushMLModelScorer scores the Candidates and populates their ML scores - * - * @param pushMLModel Enum to specify which model to use for scoring the Candidates - * @param modelToPredictionServiceStoreMap Supports all other prediction services. Specifies model ID -> dbv2 ReadableStore - * @param defaultDBv2PredictionServiceStore: Supports models that are not specified in the previous maps (which will be directly configured in the config repo) - * @param scoringStats StatsReceiver for scoping stats - */ -class PushMLModelScorer( - pushMLModel: PushMLModel.Value, - modelToPredictionServiceStoreMap: Map[ - WeightedOpenOrNtabClickModel.ModelNameType, - PushPredictionServiceStore - ], - defaultDBv2PredictionServiceStore: PushPredictionServiceStore, - scoringStats: StatsReceiver) { - - val queriesOutsideTheModelMaps: StatsReceiver = - scoringStats.scope("queries_outside_the_model_maps") - val totalQueriesOutsideTheModelMaps: Counter = - queriesOutsideTheModelMaps.counter("total") - - private def scoreByBatchPredictionForModelFromMultiModelService( - predictionServiceStore: PushPredictionServiceStore, - modelVersion: WeightedOpenOrNtabClickModel.ModelNameType, - candidatesDetails: Seq[CandidateDetails[PushCandidate]], - useCommonFeatures: Boolean, - overridePushMLModel: PushMLModel.Value - ): Seq[CandidateDetails[PushCandidate]] = { - val modelName = - PushModelName(overridePushMLModel, modelVersion).toString - val modelSelector = new ModelSelector() - modelSelector.setId(modelName) - - val candidateHydrationWithFeaturesMap = candidatesDetails.map { candidatesDetail => - ( - candidatesDetail.candidate.candidateHydrationContext, - candidatesDetail.candidate.candidateFeatureMap()) - } - if (candidatesDetails.nonEmpty) { - val candidatesWithScore = predictionServiceStore.getBatchPredictionsForModel( - candidatesDetails.head.candidate.target.targetHydrationContext, - candidatesDetails.head.candidate.target.featureMap, - candidateHydrationWithFeaturesMap, - Some(modelSelector), - useCommonFeatures - ) - candidatesDetails.zip(candidatesWithScore).foreach { - case (candidateDetail, (_, scoreOptFut)) => - candidateDetail.candidate.populateQualityModelScore( - overridePushMLModel, - modelVersion, - scoreOptFut - ) - } - } - - candidatesDetails - } - - private def scoreByBatchPrediction( - modelVersion: WeightedOpenOrNtabClickModel.ModelNameType, - candidatesDetails: Seq[CandidateDetails[PushCandidate]], - useCommonFeaturesForDBv2Service: Boolean, - overridePushMLModel: PushMLModel.Value - ): Seq[CandidateDetails[PushCandidate]] = { - if (modelToPredictionServiceStoreMap.contains(modelVersion)) { - scoreByBatchPredictionForModelFromMultiModelService( - modelToPredictionServiceStoreMap(modelVersion), - modelVersion, - candidatesDetails, - useCommonFeaturesForDBv2Service, - overridePushMLModel - ) - } else { - totalQueriesOutsideTheModelMaps.incr() - queriesOutsideTheModelMaps.counter(modelVersion).incr() - scoreByBatchPredictionForModelFromMultiModelService( - defaultDBv2PredictionServiceStore, - modelVersion, - candidatesDetails, - useCommonFeaturesForDBv2Service, - overridePushMLModel - ) - } - } - - def scoreByBatchPredictionForModelVersion( - target: Target, - candidatesDetails: Seq[CandidateDetails[PushCandidate]], - modelVersionParam: FSParam[WeightedOpenOrNtabClickModel.ModelNameType], - useCommonFeaturesForDBv2Service: Boolean = true, - overridePushMLModelOpt: Option[PushMLModel.Value] = None - ): Seq[CandidateDetails[PushCandidate]] = { - scoreByBatchPrediction( - target.params(modelVersionParam), - candidatesDetails, - useCommonFeaturesForDBv2Service, - overridePushMLModelOpt.getOrElse(pushMLModel) - ) - } - - def singlePredicationForModelVersion( - modelVersion: String, - candidate: PushCandidate, - overridePushMLModelOpt: Option[PushMLModel.Value] = None - ): Future[Option[Double]] = { - val modelSelector = new ModelSelector() - modelSelector.setId( - PushModelName(overridePushMLModelOpt.getOrElse(pushMLModel), modelVersion).toString - ) - if (modelToPredictionServiceStoreMap.contains(modelVersion)) { - modelToPredictionServiceStoreMap(modelVersion).get( - PushCandidateHydrationContextWithModel( - candidate.target.targetHydrationContext, - candidate.target.featureMap, - candidate.candidateHydrationContext, - candidate.candidateFeatureMap(), - Some(modelSelector) - ) - ) - } else { - totalQueriesOutsideTheModelMaps.incr() - queriesOutsideTheModelMaps.counter(modelVersion).incr() - defaultDBv2PredictionServiceStore.get( - PushCandidateHydrationContextWithModel( - candidate.target.targetHydrationContext, - candidate.target.featureMap, - candidate.candidateHydrationContext, - candidate.candidateFeatureMap(), - Some(modelSelector) - ) - ) - } - } - - def singlePredictionForTargetLevel( - modelVersion: String, - targetId: Long, - featureMap: Future[FeatureMap] - ): Future[Option[Double]] = { - val modelSelector = new ModelSelector() - modelSelector.setId( - PushModelName(pushMLModel, modelVersion).toString - ) - defaultDBv2PredictionServiceStore.getForTargetLevel( - TargetFeatureMapWithModel(targetId, featureMap, Some(modelSelector)) - ) - } - - def getScoreHistogramCounters( - stats: StatsReceiver, - scopeName: String, - histogramBinSize: Double - ): IndexedSeq[Counter] = { - val histogramScopedStatsReceiver = stats.scope(scopeName) - val numBins = math.ceil(1.0 / histogramBinSize).toInt - - (0 to numBins) map { k => - if (k == 0) - histogramScopedStatsReceiver.counter("candidates_with_scores_zero") - else { - val counterName = "candidates_with_scores_from_%s_to_%s".format( - "%.2f".format(histogramBinSize * (k - 1)).replace(".", ""), - "%.2f".format(math.min(1.0, histogramBinSize * k)).replace(".", "")) - histogramScopedStatsReceiver.counter(counterName) - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/DiscoverTwitter.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/DiscoverTwitter.docx new file mode 100644 index 000000000..96fd1836f Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/DiscoverTwitter.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/DiscoverTwitter.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/DiscoverTwitter.scala deleted file mode 100644 index dc350a740..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/DiscoverTwitter.scala +++ /dev/null @@ -1,89 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.DiscoverTwitterCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.DiscoverTwitterPushIbis2Hydrator -import com.twitter.frigate.pushservice.model.ntab.DiscoverTwitterNtabRequestHydrator -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate -import com.twitter.frigate.pushservice.take.predicates.BasicRFPHPredicates -import com.twitter.frigate.pushservice.take.predicates.OutOfNetworkTweetPredicates -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.hermit.predicate.NamedPredicate - -class DiscoverTwitterPushCandidate( - candidate: RawCandidate with DiscoverTwitterCandidate, - copyIds: CopyIds, -)( - implicit val statsScoped: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with DiscoverTwitterCandidate - with DiscoverTwitterPushIbis2Hydrator - with DiscoverTwitterNtabRequestHydrator { - - override val pushCopyId: Option[Int] = copyIds.pushCopyId - - override val ntabCopyId: Option[Int] = copyIds.ntabCopyId - - override val copyAggregationId: Option[String] = copyIds.aggregationId - - override val target: Target = candidate.target - - override lazy val commonRecType: CommonRecommendationType = candidate.commonRecType - - override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - - override val statsReceiver: StatsReceiver = - statsScoped.scope("DiscoverTwitterPushCandidate") -} - -case class AddressBookPushCandidatePredicates(config: Config) - extends BasicRFPHPredicates[DiscoverTwitterPushCandidate] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override val predicates: List[ - NamedPredicate[DiscoverTwitterPushCandidate] - ] = - List( - PredicatesForCandidate.paramPredicate( - PushFeatureSwitchParams.EnableAddressBookPush - ) - ) -} - -case class CompleteOnboardingPushCandidatePredicates(config: Config) - extends BasicRFPHPredicates[DiscoverTwitterPushCandidate] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override val predicates: List[ - NamedPredicate[DiscoverTwitterPushCandidate] - ] = - List( - PredicatesForCandidate.paramPredicate( - PushFeatureSwitchParams.EnableCompleteOnboardingPush - ) - ) -} - -case class PopGeoTweetCandidatePredicates(override val config: Config) - extends OutOfNetworkTweetPredicates[OutOfNetworkTweetPushCandidate] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override def postCandidateSpecificPredicates: List[ - NamedPredicate[OutOfNetworkTweetPushCandidate] - ] = List( - PredicatesForCandidate.htlFatiguePredicate( - PushFeatureSwitchParams.NewUserPlaybookAllowedLastLoginHours - ) - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/F1FirstdegreeTweet.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/F1FirstdegreeTweet.docx new file mode 100644 index 000000000..29b0eeb8f Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/F1FirstdegreeTweet.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/F1FirstdegreeTweet.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/F1FirstdegreeTweet.scala deleted file mode 100644 index a4e8bec68..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/F1FirstdegreeTweet.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.F1FirstDegree -import com.twitter.frigate.common.base.SocialContextAction -import com.twitter.frigate.common.base.SocialGraphServiceRelationshipMap -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes._ -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.F1FirstDegreeTweetIbis2HydratorForCandidate -import com.twitter.frigate.pushservice.model.ntab.F1FirstDegreeTweetNTabRequestHydrator -import com.twitter.frigate.pushservice.take.predicates.BasicTweetPredicatesForRFPHWithoutSGSPredicates -import com.twitter.frigate.pushservice.util.CandidateHydrationUtil.TweetWithSocialContextTraits -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.hermit.predicate.socialgraph.RelationEdge -import com.twitter.stitch.tweetypie.TweetyPie -import com.twitter.util.Future - -class F1TweetPushCandidate( - candidate: RawCandidate with TweetWithSocialContextTraits, - author: Future[Option[User]], - socialGraphServiceResultMap: Map[RelationEdge, Boolean], - copyIds: CopyIds -)( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with F1FirstDegree - with TweetAuthorDetails - with SocialGraphServiceRelationshipMap - with F1FirstDegreeTweetNTabRequestHydrator - with F1FirstDegreeTweetIbis2HydratorForCandidate { - override val socialContextActions: Seq[SocialContextAction] = - candidate.socialContextActions - override val socialContextAllTypeActions: Seq[SocialContextAction] = - candidate.socialContextActions - override val statsReceiver: StatsReceiver = stats - override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - override val tweetId: Long = candidate.tweetId - override lazy val tweetyPieResult: Option[TweetyPie.TweetyPieResult] = - candidate.tweetyPieResult - override lazy val tweetAuthor: Future[Option[User]] = author - override val target: PushTypes.Target = candidate.target - override lazy val commonRecType: CommonRecommendationType = - candidate.commonRecType - override val pushCopyId: Option[Int] = copyIds.pushCopyId - override val ntabCopyId: Option[Int] = copyIds.ntabCopyId - override val copyAggregationId: Option[String] = copyIds.aggregationId - - override val relationshipMap: Map[RelationEdge, Boolean] = socialGraphServiceResultMap -} - -case class F1TweetCandidatePredicates(override val config: Config) - extends BasicTweetPredicatesForRFPHWithoutSGSPredicates[F1TweetPushCandidate] { - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ListRecommendationPushCandidate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ListRecommendationPushCandidate.docx new file mode 100644 index 000000000..1ced0ef00 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ListRecommendationPushCandidate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ListRecommendationPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ListRecommendationPushCandidate.scala deleted file mode 100644 index 412dfbf00..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ListRecommendationPushCandidate.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.channels.common.thriftscala.ApiList -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.ListPushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.ListIbis2Hydrator -import com.twitter.frigate.pushservice.model.ntab.ListCandidateNTabRequestHydrator -import com.twitter.frigate.pushservice.predicate.ListPredicates -import com.twitter.frigate.pushservice.take.predicates.BasicRFPHPredicates -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -class ListRecommendationPushCandidate( - val apiListStore: ReadableStore[Long, ApiList], - candidate: RawCandidate with ListPushCandidate, - copyIds: CopyIds -)( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with ListPushCandidate - with ListIbis2Hydrator - with ListCandidateNTabRequestHydrator { - - override val commonRecType: CommonRecommendationType = candidate.commonRecType - - override val pushCopyId: Option[Int] = copyIds.pushCopyId - - override val ntabCopyId: Option[Int] = copyIds.ntabCopyId - - override val copyAggregationId: Option[String] = copyIds.aggregationId - - override val statsReceiver: StatsReceiver = stats - - override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - - override val target: PushTypes.Target = candidate.target - - override val listId: Long = candidate.listId - - lazy val apiList: Future[Option[ApiList]] = apiListStore.get(listId) - - lazy val listName: Future[Option[String]] = apiList.map { apiListOpt => - apiListOpt.map(_.name) - } - - lazy val listOwnerId: Future[Option[Long]] = apiList.map { apiListOpt => - apiListOpt.map(_.ownerId) - } - -} - -case class ListRecommendationPredicates(config: Config) - extends BasicRFPHPredicates[ListRecommendationPushCandidate] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override val predicates: List[NamedPredicate[ListRecommendationPushCandidate]] = List( - ListPredicates.listNameExistsPredicate(), - ListPredicates.listAuthorExistsPredicate(), - ListPredicates.listAuthorAcceptableToTargetUser(config.edgeStore), - ListPredicates.listAcceptablePredicate(), - ListPredicates.listSubscriberCountPredicate() - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutCreatorEventPushCandidate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutCreatorEventPushCandidate.docx new file mode 100644 index 000000000..afab3efed Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutCreatorEventPushCandidate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutCreatorEventPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutCreatorEventPushCandidate.scala deleted file mode 100644 index 65633259c..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutCreatorEventPushCandidate.scala +++ /dev/null @@ -1,136 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.HydratedMagicFanoutCreatorEventCandidate -import com.twitter.frigate.common.base.MagicFanoutCreatorEventCandidate -import com.twitter.frigate.magic_events.thriftscala.CreatorFanoutType -import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.MagicFanoutCreatorEventIbis2Hydrator -import com.twitter.frigate.pushservice.model.ntab.MagicFanoutCreatorEventNtabRequestHydrator -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate -import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutPredicatesForCandidate -import com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue.MagicFanoutNtabCaretFatiguePredicate -import com.twitter.frigate.pushservice.take.predicates.BasicSendHandlerPredicates -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.client.UserId -import com.twitter.util.Future -import scala.util.control.NoStackTrace - -class MagicFanoutCreatorEventPushCandidateHydratorException(private val message: String) - extends Exception(message) - with NoStackTrace - -class MagicFanoutCreatorEventPushCandidate( - candidate: RawCandidate with MagicFanoutCreatorEventCandidate, - creatorUser: Option[User], - copyIds: CopyIds, - creatorTweetCountStore: ReadableStore[UserId, Int] -)( - implicit val statsScoped: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with MagicFanoutCreatorEventIbis2Hydrator - with MagicFanoutCreatorEventNtabRequestHydrator - with MagicFanoutCreatorEventCandidate - with HydratedMagicFanoutCreatorEventCandidate { - override def creatorId: Long = candidate.creatorId - - override def hydratedCreator: Option[User] = creatorUser - - override lazy val numberOfTweetsFut: Future[Option[Int]] = - creatorTweetCountStore.get(UserId(creatorId)) - - lazy val userProfile = hydratedCreator - .flatMap(_.profile).getOrElse( - throw new MagicFanoutCreatorEventPushCandidateHydratorException( - s"Unable to get user profile to generate tapThrough for userId: $creatorId")) - - override val frigateNotification: FrigateNotification = candidate.frigateNotification - - override def subscriberId: Option[Long] = candidate.subscriberId - - override def creatorFanoutType: CreatorFanoutType = candidate.creatorFanoutType - - override def target: PushTypes.Target = candidate.target - - override def pushId: Long = candidate.pushId - - override def candidateMagicEventsReasons: Seq[MagicEventsReason] = - candidate.candidateMagicEventsReasons - - override def statsReceiver: StatsReceiver = statsScoped - - override def pushCopyId: Option[Int] = copyIds.pushCopyId - - override def ntabCopyId: Option[Int] = copyIds.ntabCopyId - - override def copyAggregationId: Option[String] = copyIds.aggregationId - - override def commonRecType: CommonRecommendationType = candidate.commonRecType - - override def weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - -} - -case class MagicFanouCreatorSubscriptionEventPushPredicates(config: Config) - extends BasicSendHandlerPredicates[MagicFanoutCreatorEventPushCandidate] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override val preCandidateSpecificPredicates: List[ - NamedPredicate[MagicFanoutCreatorEventPushCandidate] - ] = - List( - PredicatesForCandidate.paramPredicate( - PushFeatureSwitchParams.EnableCreatorSubscriptionPush - ), - PredicatesForCandidate.isDeviceEligibleForCreatorPush, - MagicFanoutPredicatesForCandidate.creatorPushTargetIsNotCreator(), - MagicFanoutPredicatesForCandidate.duplicateCreatorPredicate, - MagicFanoutPredicatesForCandidate.magicFanoutCreatorPushFatiguePredicate(), - ) - - override val postCandidateSpecificPredicates: List[ - NamedPredicate[MagicFanoutCreatorEventPushCandidate] - ] = - List( - MagicFanoutNtabCaretFatiguePredicate(), - MagicFanoutPredicatesForCandidate.isSuperFollowingCreator()(config, statsReceiver).flip - ) -} - -case class MagicFanoutNewCreatorEventPushPredicates(config: Config) - extends BasicSendHandlerPredicates[MagicFanoutCreatorEventPushCandidate] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override val preCandidateSpecificPredicates: List[ - NamedPredicate[MagicFanoutCreatorEventPushCandidate] - ] = - List( - PredicatesForCandidate.paramPredicate( - PushFeatureSwitchParams.EnableNewCreatorPush - ), - PredicatesForCandidate.isDeviceEligibleForCreatorPush, - MagicFanoutPredicatesForCandidate.duplicateCreatorPredicate, - MagicFanoutPredicatesForCandidate.magicFanoutCreatorPushFatiguePredicate, - ) - - override val postCandidateSpecificPredicates: List[ - NamedPredicate[MagicFanoutCreatorEventPushCandidate] - ] = - List( - MagicFanoutNtabCaretFatiguePredicate(), - MagicFanoutPredicatesForCandidate.isSuperFollowingCreator()(config, statsReceiver).flip - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutEventPushCandidate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutEventPushCandidate.docx new file mode 100644 index 000000000..8ae138755 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutEventPushCandidate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutEventPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutEventPushCandidate.scala deleted file mode 100644 index e0a5f5386..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutEventPushCandidate.scala +++ /dev/null @@ -1,303 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.escherbird.metadata.thriftscala.EntityMegadata -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.MagicFanoutEventCandidate -import com.twitter.frigate.common.base.RecommendationType -import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext -import com.twitter.frigate.common.util.HighPriorityLocaleUtil -import com.twitter.frigate.magic_events.thriftscala.FanoutEvent -import com.twitter.frigate.magic_events.thriftscala.FanoutMetadata -import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason -import com.twitter.frigate.magic_events.thriftscala.NewsForYouMetadata -import com.twitter.frigate.magic_events.thriftscala.ReasonSource -import com.twitter.frigate.magic_events.thriftscala.TargetID -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.Ibis2HydratorForCandidate -import com.twitter.frigate.pushservice.model.ntab.EventNTabRequestHydrator -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutPredicatesUtil -import com.twitter.frigate.pushservice.store.EventRequest -import com.twitter.frigate.pushservice.store.UttEntityHydrationStore -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.frigate.pushservice.util.TopicsUtil -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.frigate.thriftscala.MagicFanoutEventNotificationDetails -import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery -import com.twitter.interests.thriftscala.InterestId.SemanticCore -import com.twitter.interests.thriftscala.UserInterests -import com.twitter.livevideo.common.ids.CountryId -import com.twitter.livevideo.common.ids.UserId -import com.twitter.livevideo.timeline.domain.v2.Event -import com.twitter.livevideo.timeline.domain.v2.HydrationOptions -import com.twitter.livevideo.timeline.domain.v2.LookupContext -import com.twitter.simclusters_v2.thriftscala.SimClustersInferredEntities -import com.twitter.storehaus.ReadableStore -import com.twitter.topiclisting.utt.LocalizedEntity -import com.twitter.util.Future - -abstract class MagicFanoutEventPushCandidate( - candidate: RawCandidate with MagicFanoutEventCandidate with RecommendationType, - copyIds: CopyIds, - override val fanoutEvent: Option[FanoutEvent], - override val semanticEntityResults: Map[SemanticEntityForQuery, Option[EntityMegadata]], - simClusterToEntities: Map[Int, Option[SimClustersInferredEntities]], - lexServiceStore: ReadableStore[EventRequest, Event], - interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests], - uttEntityHydrationStore: UttEntityHydrationStore -)( - implicit statsScoped: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with MagicFanoutEventHydratedCandidate - with MagicFanoutEventCandidate - with EventNTabRequestHydrator - with RecommendationType - with Ibis2HydratorForCandidate { - - override lazy val eventFut: Future[Option[Event]] = { - eventRequestFut.flatMap { - case Some(eventRequest) => lexServiceStore.get(eventRequest) - case _ => Future.None - } - } - - override val frigateNotification: FrigateNotification = candidate.frigateNotification - - override val pushId: Long = candidate.pushId - - override val candidateMagicEventsReasons: Seq[MagicEventsReason] = - candidate.candidateMagicEventsReasons - - override val eventId: Long = candidate.eventId - - override val momentId: Option[Long] = candidate.momentId - - override val target: Target = candidate.target - - override val eventLanguage: Option[String] = candidate.eventLanguage - - override val details: Option[MagicFanoutEventNotificationDetails] = candidate.details - - override lazy val stats: StatsReceiver = statsScoped.scope("MagicFanoutEventPushCandidate") - - override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - - override val pushCopyId: Option[Int] = copyIds.pushCopyId - - override val ntabCopyId: Option[Int] = copyIds.ntabCopyId - - override val copyAggregationId: Option[String] = copyIds.aggregationId - - override val statsReceiver: StatsReceiver = statsScoped.scope("MagicFanoutEventPushCandidate") - - override val effectiveMagicEventsReasons: Option[Seq[MagicEventsReason]] = Some( - candidateMagicEventsReasons) - - lazy val newsForYouMetadata: Option[NewsForYouMetadata] = - fanoutEvent.flatMap { event => - { - event.fanoutMetadata.collect { - case FanoutMetadata.NewsForYouMetadata(nfyMetadata) => nfyMetadata - } - } - } - - val reverseIndexedTopicIds = candidate.candidateMagicEventsReasons - .filter(_.source.contains(ReasonSource.UttTopicFollowGraph)) - .map(_.reason).collect { - case TargetID.SemanticCoreID(semanticCoreID) => semanticCoreID.entityId - }.toSet - - val ergSemanticCoreIds = candidate.candidateMagicEventsReasons - .filter(_.source.contains(ReasonSource.ErgShortTermInterestSemanticCore)).map( - _.reason).collect { - case TargetID.SemanticCoreID(semanticCoreID) => semanticCoreID.entityId - }.toSet - - override lazy val ergLocalizedEntities = TopicsUtil - .getLocalizedEntityMap(target, ergSemanticCoreIds, uttEntityHydrationStore) - .map { localizedEntityMap => - ergSemanticCoreIds.collect { - case topicId if localizedEntityMap.contains(topicId) => localizedEntityMap(topicId) - } - } - - val eventSemanticCoreEntityIds: Seq[Long] = { - val entityIds = for { - event <- fanoutEvent - targets <- event.targets - } yield { - targets.flatMap { - _.whitelist.map { - _.collect { - case TargetID.SemanticCoreID(semanticCoreID) => semanticCoreID.entityId - } - } - } - } - - entityIds.map(_.flatten).getOrElse(Seq.empty) - } - - val eventSemanticCoreDomainIds: Seq[Long] = { - val domainIds = for { - event <- fanoutEvent - targets <- event.targets - } yield { - targets.flatMap { - _.whitelist.map { - _.collect { - case TargetID.SemanticCoreID(semanticCoreID) => semanticCoreID.domainId - } - } - } - } - - domainIds.map(_.flatten).getOrElse(Seq.empty) - } - - override lazy val followedTopicLocalizedEntities: Future[Set[LocalizedEntity]] = { - - val isNewSignupTargetingReason = candidateMagicEventsReasons.size == 1 && - candidateMagicEventsReasons.headOption.exists(_.source.contains(ReasonSource.NewSignup)) - - val shouldFetchTopicFollows = reverseIndexedTopicIds.nonEmpty || isNewSignupTargetingReason - - val topicFollows = if (shouldFetchTopicFollows) { - TopicsUtil - .getTopicsFollowedByUser( - candidate.target, - interestsLookupStore, - stats.stat("followed_topics") - ).map { _.getOrElse(Seq.empty) }.map { - _.flatMap { - _.interestId match { - case SemanticCore(semanticCore) => Some(semanticCore.id) - case _ => None - } - } - } - } else Future.Nil - - topicFollows.flatMap { followedTopicIds => - val topicIds = if (isNewSignupTargetingReason) { - // if new signup is the only targeting reason then we check the event targeting reason - // against realtime topic follows. - eventSemanticCoreEntityIds.toSet.intersect(followedTopicIds.toSet) - } else { - // check against the fanout reason of topics - followedTopicIds.toSet.intersect(reverseIndexedTopicIds) - } - - TopicsUtil - .getLocalizedEntityMap(target, topicIds, uttEntityHydrationStore) - .map { localizedEntityMap => - topicIds.collect { - case topicId if localizedEntityMap.contains(topicId) => localizedEntityMap(topicId) - } - } - } - } - - lazy val simClusterToEntityMapping: Map[Int, Seq[Long]] = - simClusterToEntities.flatMap { - case (clusterId, Some(inferredEntities)) => - statsReceiver.counter("with_cluster_to_entity_mapping").incr() - Some( - ( - clusterId, - inferredEntities.entities - .map(_.entityId))) - case _ => - statsReceiver.counter("without_cluster_to_entity_mapping").incr() - None - } - - lazy val annotatedAndInferredSemanticCoreEntities: Seq[Long] = - (simClusterToEntityMapping, eventFanoutReasonEntities) match { - case (entityMapping, eventFanoutReasons) => - entityMapping.values.flatten.toSeq ++ - eventFanoutReasons.semanticCoreIds.map(_.entityId) - } - - lazy val shouldHydrateSquareImage = target.deviceInfo.map { deviceInfo => - (PushDeviceUtil.isPrimaryDeviceIOS(deviceInfo) && - target.params(PushFeatureSwitchParams.EnableEventSquareMediaIosMagicFanoutNewsEvent)) || - (PushDeviceUtil.isPrimaryDeviceAndroid(deviceInfo) && - target.params(PushFeatureSwitchParams.EnableEventSquareMediaAndroid)) - } - - lazy val shouldHydratePrimaryImage: Future[Boolean] = target.deviceInfo.map { deviceInfo => - (PushDeviceUtil.isPrimaryDeviceAndroid(deviceInfo) && - target.params(PushFeatureSwitchParams.EnableEventPrimaryMediaAndroid)) - } - - lazy val eventRequestFut: Future[Option[EventRequest]] = - Future - .join( - target.inferredUserDeviceLanguage, - target.accountCountryCode, - shouldHydrateSquareImage, - shouldHydratePrimaryImage).map { - case ( - inferredUserDeviceLanguage, - accountCountryCode, - shouldHydrateSquareImage, - shouldHydratePrimaryImage) => - if (shouldHydrateSquareImage || shouldHydratePrimaryImage) { - Some( - EventRequest( - eventId, - lookupContext = LookupContext( - hydrationOptions = HydrationOptions( - includeSquareImage = shouldHydrateSquareImage, - includePrimaryImage = shouldHydratePrimaryImage - ), - language = inferredUserDeviceLanguage, - countryCode = accountCountryCode, - userId = Some(UserId(target.targetId)) - ) - )) - } else { - Some( - EventRequest( - eventId, - lookupContext = LookupContext( - language = inferredUserDeviceLanguage, - countryCode = accountCountryCode - ) - )) - } - case _ => None - } - - lazy val isHighPriorityEvent: Future[Boolean] = target.accountCountryCode.map { countryCodeOpt => - val isHighPriorityPushOpt = for { - countryCode <- countryCodeOpt - nfyMetadata <- newsForYouMetadata - eventContext <- nfyMetadata.eventContextScribe - } yield { - val highPriorityLocales = HighPriorityLocaleUtil.getHighPriorityLocales( - eventContext = eventContext, - defaultLocalesOpt = nfyMetadata.locales) - val highPriorityGeos = HighPriorityLocaleUtil.getHighPriorityGeos( - eventContext = eventContext, - defaultGeoPlaceIdsOpt = nfyMetadata.placeIds) - val isHighPriorityLocalePush = - highPriorityLocales.flatMap(_.country).map(CountryId(_)).contains(CountryId(countryCode)) - val isHighPriorityGeoPush = MagicFanoutPredicatesUtil - .geoPlaceIdsFromReasons(candidateMagicEventsReasons) - .intersect(highPriorityGeos.toSet) - .nonEmpty - stats.scope("is_high_priority_locale_push").counter(s"$isHighPriorityLocalePush").incr() - stats.scope("is_high_priority_geo_push").counter(s"$isHighPriorityGeoPush").incr() - isHighPriorityLocalePush || isHighPriorityGeoPush - } - isHighPriorityPushOpt.getOrElse(false) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutHydratedCandidate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutHydratedCandidate.docx new file mode 100644 index 000000000..b407e639c Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutHydratedCandidate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutHydratedCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutHydratedCandidate.scala deleted file mode 100644 index 36196120b..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutHydratedCandidate.scala +++ /dev/null @@ -1,147 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.escherbird.common.thriftscala.QualifiedId -import com.twitter.escherbird.metadata.thriftscala.BasicMetadata -import com.twitter.escherbird.metadata.thriftscala.EntityIndexFields -import com.twitter.escherbird.metadata.thriftscala.EntityMegadata -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.MagicFanoutCandidate -import com.twitter.frigate.common.base.MagicFanoutEventCandidate -import com.twitter.frigate.common.base.RichEventFutCandidate -import com.twitter.frigate.magic_events.thriftscala -import com.twitter.frigate.magic_events.thriftscala.AnnotationAlg -import com.twitter.frigate.magic_events.thriftscala.FanoutEvent -import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason -import com.twitter.frigate.magic_events.thriftscala.SemanticCoreID -import com.twitter.frigate.magic_events.thriftscala.SimClusterID -import com.twitter.frigate.magic_events.thriftscala.TargetID -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery -import com.twitter.livevideo.timeline.domain.v2.Event -import com.twitter.topiclisting.utt.LocalizedEntity -import com.twitter.util.Future - -case class FanoutReasonEntities( - userIds: Set[Long], - placeIds: Set[Long], - semanticCoreIds: Set[SemanticCoreID], - simclusterIds: Set[SimClusterID]) { - val qualifiedIds: Set[QualifiedId] = - semanticCoreIds.map(e => QualifiedId(e.domainId, e.entityId)) -} - -object FanoutReasonEntities { - val empty = FanoutReasonEntities( - userIds = Set.empty, - placeIds = Set.empty, - semanticCoreIds = Set.empty, - simclusterIds = Set.empty - ) - - def from(reasons: Seq[TargetID]): FanoutReasonEntities = { - val userIds: Set[Long] = reasons.collect { - case TargetID.UserID(userId) => userId.id - }.toSet - val placeIds: Set[Long] = reasons.collect { - case TargetID.PlaceID(placeId) => placeId.id - }.toSet - val semanticCoreIds: Set[SemanticCoreID] = reasons.collect { - case TargetID.SemanticCoreID(semanticCoreID) => semanticCoreID - }.toSet - val simclusterIds: Set[SimClusterID] = reasons.collect { - case TargetID.SimClusterID(simClusterID) => simClusterID - }.toSet - - FanoutReasonEntities( - userIds = userIds, - placeIds, - semanticCoreIds = semanticCoreIds, - simclusterIds = simclusterIds - ) - } -} - -trait MagicFanoutHydratedCandidate extends PushCandidate with MagicFanoutCandidate { - lazy val fanoutReasonEntities: FanoutReasonEntities = - FanoutReasonEntities.from(candidateMagicEventsReasons.map(_.reason)) -} - -trait MagicFanoutEventHydratedCandidate - extends MagicFanoutHydratedCandidate - with MagicFanoutEventCandidate - with RichEventFutCandidate { - - def target: PushTypes.Target - - def stats: StatsReceiver - - def fanoutEvent: Option[FanoutEvent] - - def eventFut: Future[Option[Event]] - - def semanticEntityResults: Map[SemanticEntityForQuery, Option[EntityMegadata]] - - def effectiveMagicEventsReasons: Option[Seq[MagicEventsReason]] - - def followedTopicLocalizedEntities: Future[Set[LocalizedEntity]] - - def ergLocalizedEntities: Future[Set[LocalizedEntity]] - - lazy val entityAnnotationAlg: Map[TargetID, Set[AnnotationAlg]] = - fanoutEvent - .flatMap { metadata => - metadata.eventAnnotationInfo.map { eventAnnotationInfo => - eventAnnotationInfo.map { - case (target, annotationInfoSet) => target -> annotationInfoSet.map(_.alg).toSet - }.toMap - } - }.getOrElse(Map.empty) - - lazy val eventSource: Option[String] = fanoutEvent.map { metadata => - val source = metadata.eventSource.getOrElse("undefined") - stats.scope("eventSource").counter(source).incr() - source - } - - lazy val semanticCoreEntityTags: Map[(Long, Long), Set[String]] = - semanticEntityResults.flatMap { - case (semanticEntityForQuery, entityMegadataOpt: Option[EntityMegadata]) => - for { - entityMegadata <- entityMegadataOpt - basicMetadata: BasicMetadata <- entityMegadata.basicMetadata - indexableFields: EntityIndexFields <- basicMetadata.indexableFields - tags <- indexableFields.tags - } yield { - ((semanticEntityForQuery.domainId, semanticEntityForQuery.entityId), tags.toSet) - } - } - - lazy val owningTwitterUserIds: Seq[Long] = semanticEntityResults.values.flatten - .flatMap { - _.basicMetadata.flatMap(_.twitter.flatMap(_.owningTwitterUserIds)) - }.flatten - .toSeq - .distinct - - lazy val eventFanoutReasonEntities: FanoutReasonEntities = - fanoutEvent match { - case Some(fanout) => - fanout.targets - .map { targets: Seq[thriftscala.Target] => - FanoutReasonEntities.from(targets.flatMap(_.whitelist).flatten) - }.getOrElse(FanoutReasonEntities.empty) - case _ => FanoutReasonEntities.empty - } - - override lazy val eventResultFut: Future[Event] = eventFut.map { - case Some(eventResult) => eventResult - case _ => - throw new IllegalArgumentException("event is None for MagicFanoutEventHydratedCandidate") - } - override val rankScore: Option[Double] = None - override val predictionScore: Option[Double] = None -} - -case class MagicFanoutEventHydratedInfo( - fanoutEvent: Option[FanoutEvent], - semanticEntityResults: Map[SemanticEntityForQuery, Option[EntityMegadata]]) diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutNewsEvent.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutNewsEvent.docx new file mode 100644 index 000000000..438d0876d Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutNewsEvent.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutNewsEvent.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutNewsEvent.scala deleted file mode 100644 index 61ead1f22..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutNewsEvent.scala +++ /dev/null @@ -1,99 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.escherbird.metadata.thriftscala.EntityMegadata -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.MagicFanoutNewsEventCandidate -import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext -import com.twitter.frigate.magic_events.thriftscala.FanoutEvent -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.MagicFanoutNewsEventIbis2Hydrator -import com.twitter.frigate.pushservice.model.ntab.MagicFanoutNewsEventNTabRequestHydrator -import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate -import com.twitter.frigate.pushservice.predicate.event.EventPredicatesForCandidate -import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutPredicatesForCandidate -import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutTargetingPredicateWrappersForCandidate -import com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue.MagicFanoutNtabCaretFatiguePredicate -import com.twitter.frigate.pushservice.store.EventRequest -import com.twitter.frigate.pushservice.store.UttEntityHydrationStore -import com.twitter.frigate.pushservice.take.predicates.BasicSendHandlerPredicates -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery -import com.twitter.interests.thriftscala.UserInterests -import com.twitter.livevideo.timeline.domain.v2.Event -import com.twitter.simclusters_v2.thriftscala.SimClustersInferredEntities -import com.twitter.storehaus.ReadableStore - -class MagicFanoutNewsEventPushCandidate( - candidate: RawCandidate with MagicFanoutNewsEventCandidate, - copyIds: CopyIds, - override val fanoutEvent: Option[FanoutEvent], - override val semanticEntityResults: Map[SemanticEntityForQuery, Option[EntityMegadata]], - simClusterToEntities: Map[Int, Option[SimClustersInferredEntities]], - lexServiceStore: ReadableStore[EventRequest, Event], - interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests], - uttEntityHydrationStore: UttEntityHydrationStore -)( - implicit statsScoped: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends MagicFanoutEventPushCandidate( - candidate, - copyIds, - fanoutEvent, - semanticEntityResults, - simClusterToEntities, - lexServiceStore, - interestsLookupStore, - uttEntityHydrationStore - )(statsScoped, pushModelScorer) - with MagicFanoutNewsEventCandidate - with MagicFanoutNewsEventIbis2Hydrator - with MagicFanoutNewsEventNTabRequestHydrator { - - override lazy val stats: StatsReceiver = statsScoped.scope("MagicFanoutNewsEventPushCandidate") - override val statsReceiver: StatsReceiver = statsScoped.scope("MagicFanoutNewsEventPushCandidate") -} - -case class MagicFanoutNewsEventCandidatePredicates(config: Config) - extends BasicSendHandlerPredicates[MagicFanoutNewsEventPushCandidate] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override val preCandidateSpecificPredicates: List[ - NamedPredicate[MagicFanoutNewsEventPushCandidate] - ] = - List( - EventPredicatesForCandidate.accountCountryPredicateWithAllowlist, - PredicatesForCandidate.isDeviceEligibleForNewsOrSports, - MagicFanoutPredicatesForCandidate.inferredUserDeviceLanguagePredicate, - PredicatesForCandidate.secondaryDormantAccountPredicate(statsReceiver), - MagicFanoutPredicatesForCandidate.highPriorityNewsEventExceptedPredicate( - MagicFanoutTargetingPredicateWrappersForCandidate - .magicFanoutTargetingPredicate(statsReceiver, config) - )(config), - MagicFanoutPredicatesForCandidate.geoOptOutPredicate(config.safeUserStore), - EventPredicatesForCandidate.isNotDuplicateWithEventId, - MagicFanoutPredicatesForCandidate.highPriorityNewsEventExceptedPredicate( - MagicFanoutPredicatesForCandidate.newsNotificationFatigue() - )(config), - MagicFanoutPredicatesForCandidate.highPriorityNewsEventExceptedPredicate( - MagicFanoutNtabCaretFatiguePredicate() - )(config), - MagicFanoutPredicatesForCandidate.escherbirdMagicfanoutEventParam()(statsReceiver), - MagicFanoutPredicatesForCandidate.hasCustomTargetingForNewsEventsParam( - statsReceiver - ) - ) - - override val postCandidateSpecificPredicates: List[ - NamedPredicate[MagicFanoutNewsEventPushCandidate] - ] = - List( - MagicFanoutPredicatesForCandidate.magicFanoutNoOptoutInterestPredicate, - MagicFanoutPredicatesForCandidate.geoTargetingHoldback(), - MagicFanoutPredicatesForCandidate.userGeneratedEventsPredicate, - EventPredicatesForCandidate.hasTitle, - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutProductLaunchPushCandidate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutProductLaunchPushCandidate.docx new file mode 100644 index 000000000..8fe0d0c16 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutProductLaunchPushCandidate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutProductLaunchPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutProductLaunchPushCandidate.scala deleted file mode 100644 index 4dc569e2b..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutProductLaunchPushCandidate.scala +++ /dev/null @@ -1,95 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.MagicFanoutProductLaunchCandidate -import com.twitter.frigate.common.util.{FeatureSwitchParams => FS} -import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason -import com.twitter.frigate.magic_events.thriftscala.ProductType -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutPredicatesUtil -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.MagicFanoutProductLaunchIbis2Hydrator -import com.twitter.frigate.pushservice.model.ntab.MagicFanoutProductLaunchNtabRequestHydrator -import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate -import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutPredicatesForCandidate -import com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue.MagicFanoutNtabCaretFatiguePredicate -import com.twitter.frigate.pushservice.take.predicates.BasicSendHandlerPredicates -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.hermit.predicate.NamedPredicate - -class MagicFanoutProductLaunchPushCandidate( - candidate: RawCandidate with MagicFanoutProductLaunchCandidate, - copyIds: CopyIds -)( - implicit val statsScoped: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with MagicFanoutProductLaunchCandidate - with MagicFanoutProductLaunchIbis2Hydrator - with MagicFanoutProductLaunchNtabRequestHydrator { - - override val frigateNotification: FrigateNotification = candidate.frigateNotification - - override val pushCopyId: Option[Int] = copyIds.pushCopyId - - override val ntabCopyId: Option[Int] = copyIds.ntabCopyId - - override val pushId: Long = candidate.pushId - - override val productLaunchType: ProductType = candidate.productLaunchType - - override val candidateMagicEventsReasons: Seq[MagicEventsReason] = - candidate.candidateMagicEventsReasons - - override val copyAggregationId: Option[String] = copyIds.aggregationId - - override val target: Target = candidate.target - - override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - - override val statsReceiver: StatsReceiver = - statsScoped.scope("MagicFanoutProductLaunchPushCandidate") -} - -case class MagicFanoutProductLaunchPushCandidatePredicates(config: Config) - extends BasicSendHandlerPredicates[MagicFanoutProductLaunchPushCandidate] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override val preCandidateSpecificPredicates: List[ - NamedPredicate[MagicFanoutProductLaunchPushCandidate] - ] = - List( - PredicatesForCandidate.isDeviceEligibleForCreatorPush, - PredicatesForCandidate.exceptedPredicate( - "excepted_is_target_blue_verified", - MagicFanoutPredicatesUtil.shouldSkipBlueVerifiedCheckForCandidate, - PredicatesForCandidate.isTargetBlueVerified.flip - ), // no need to send if target is already Blue Verified - PredicatesForCandidate.exceptedPredicate( - "excepted_is_target_legacy_verified", - MagicFanoutPredicatesUtil.shouldSkipLegacyVerifiedCheckForCandidate, - PredicatesForCandidate.isTargetLegacyVerified.flip - ), // no need to send if target is already Legacy Verified - PredicatesForCandidate.exceptedPredicate( - "excepted_is_target_super_follow_creator", - MagicFanoutPredicatesUtil.shouldSkipSuperFollowCreatorCheckForCandidate, - PredicatesForCandidate.isTargetSuperFollowCreator.flip - ), // no need to send if target is already Super Follow Creator - PredicatesForCandidate.paramPredicate( - FS.EnableMagicFanoutProductLaunch - ), - MagicFanoutPredicatesForCandidate.magicFanoutProductLaunchFatigue(), - ) - - override val postCandidateSpecificPredicates: List[ - NamedPredicate[MagicFanoutProductLaunchPushCandidate] - ] = - List( - MagicFanoutNtabCaretFatiguePredicate(), - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutSportsPushCandidate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutSportsPushCandidate.docx new file mode 100644 index 000000000..8e31cdc3f Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutSportsPushCandidate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutSportsPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutSportsPushCandidate.scala deleted file mode 100644 index 84535e4c2..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutSportsPushCandidate.scala +++ /dev/null @@ -1,119 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.escherbird.metadata.thriftscala.EntityMegadata -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.BaseGameScore -import com.twitter.frigate.common.base.MagicFanoutSportsEventCandidate -import com.twitter.frigate.common.base.MagicFanoutSportsScoreInformation -import com.twitter.frigate.common.base.TeamInfo -import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext -import com.twitter.frigate.magic_events.thriftscala.FanoutEvent -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.MagicFanoutSportsEventIbis2Hydrator -import com.twitter.frigate.pushservice.model.ntab.MagicFanoutSportsEventNTabRequestHydrator -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate -import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutPredicatesForCandidate -import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutTargetingPredicateWrappersForCandidate -import com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue.MagicFanoutNtabCaretFatiguePredicate -import com.twitter.frigate.pushservice.store.EventRequest -import com.twitter.frigate.pushservice.store.UttEntityHydrationStore -import com.twitter.frigate.pushservice.take.predicates.BasicSendHandlerPredicates -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery -import com.twitter.interests.thriftscala.UserInterests -import com.twitter.livevideo.timeline.domain.v2.Event -import com.twitter.livevideo.timeline.domain.v2.HydrationOptions -import com.twitter.livevideo.timeline.domain.v2.LookupContext -import com.twitter.simclusters_v2.thriftscala.SimClustersInferredEntities -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -class MagicFanoutSportsPushCandidate( - candidate: RawCandidate - with MagicFanoutSportsEventCandidate - with MagicFanoutSportsScoreInformation, - copyIds: CopyIds, - override val fanoutEvent: Option[FanoutEvent], - override val semanticEntityResults: Map[SemanticEntityForQuery, Option[EntityMegadata]], - simClusterToEntities: Map[Int, Option[SimClustersInferredEntities]], - lexServiceStore: ReadableStore[EventRequest, Event], - interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests], - uttEntityHydrationStore: UttEntityHydrationStore -)( - implicit statsScoped: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends MagicFanoutEventPushCandidate( - candidate, - copyIds, - fanoutEvent, - semanticEntityResults, - simClusterToEntities, - lexServiceStore, - interestsLookupStore, - uttEntityHydrationStore)(statsScoped, pushModelScorer) - with MagicFanoutSportsEventCandidate - with MagicFanoutSportsScoreInformation - with MagicFanoutSportsEventNTabRequestHydrator - with MagicFanoutSportsEventIbis2Hydrator { - - override val isScoreUpdate: Boolean = candidate.isScoreUpdate - override val gameScores: Future[Option[BaseGameScore]] = candidate.gameScores - override val homeTeamInfo: Future[Option[TeamInfo]] = candidate.homeTeamInfo - override val awayTeamInfo: Future[Option[TeamInfo]] = candidate.awayTeamInfo - - override lazy val stats: StatsReceiver = statsScoped.scope("MagicFanoutSportsPushCandidate") - override val statsReceiver: StatsReceiver = statsScoped.scope("MagicFanoutSportsPushCandidate") - - override lazy val eventRequestFut: Future[Option[EventRequest]] = { - Future.join(target.inferredUserDeviceLanguage, target.accountCountryCode).map { - case (inferredUserDeviceLanguage, accountCountryCode) => - Some( - EventRequest( - eventId, - lookupContext = LookupContext( - hydrationOptions = HydrationOptions( - includeSquareImage = true, - includePrimaryImage = true - ), - language = inferredUserDeviceLanguage, - countryCode = accountCountryCode - ) - )) - } - } -} - -case class MagicFanoutSportsEventCandidatePredicates(config: Config) - extends BasicSendHandlerPredicates[MagicFanoutSportsPushCandidate] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override val preCandidateSpecificPredicates: List[ - NamedPredicate[MagicFanoutSportsPushCandidate] - ] = - List( - PredicatesForCandidate.paramPredicate(PushFeatureSwitchParams.EnableScoreFanoutNotification) - ) - - override val postCandidateSpecificPredicates: List[ - NamedPredicate[MagicFanoutSportsPushCandidate] - ] = - List( - PredicatesForCandidate.isDeviceEligibleForNewsOrSports, - MagicFanoutPredicatesForCandidate.inferredUserDeviceLanguagePredicate, - MagicFanoutPredicatesForCandidate.highPriorityEventExceptedPredicate( - MagicFanoutTargetingPredicateWrappersForCandidate - .magicFanoutTargetingPredicate(statsReceiver, config) - )(config), - PredicatesForCandidate.secondaryDormantAccountPredicate( - statsReceiver - ), - MagicFanoutPredicatesForCandidate.highPriorityEventExceptedPredicate( - MagicFanoutNtabCaretFatiguePredicate() - )(config), - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/OutOfNetworkTweetPushCandidate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/OutOfNetworkTweetPushCandidate.docx new file mode 100644 index 000000000..512a26b7f Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/OutOfNetworkTweetPushCandidate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/OutOfNetworkTweetPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/OutOfNetworkTweetPushCandidate.scala deleted file mode 100644 index 0b8c533ea..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/OutOfNetworkTweetPushCandidate.scala +++ /dev/null @@ -1,68 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.contentrecommender.thriftscala.MetricTag -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.OutOfNetworkTweetCandidate -import com.twitter.frigate.common.base.TopicCandidate -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.OutOfNetworkTweetIbis2HydratorForCandidate -import com.twitter.frigate.pushservice.model.ntab.OutOfNetworkTweetNTabRequestHydrator -import com.twitter.frigate.pushservice.predicate.HealthPredicates -import com.twitter.frigate.pushservice.take.predicates.OutOfNetworkTweetPredicates -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.stitch.tweetypie.TweetyPie -import com.twitter.topiclisting.utt.LocalizedEntity -import com.twitter.util.Future - -class OutOfNetworkTweetPushCandidate( - candidate: RawCandidate with OutOfNetworkTweetCandidate with TopicCandidate, - author: Future[Option[User]], - copyIds: CopyIds -)( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with OutOfNetworkTweetCandidate - with TopicCandidate - with TweetAuthorDetails - with OutOfNetworkTweetNTabRequestHydrator - with OutOfNetworkTweetIbis2HydratorForCandidate { - override val statsReceiver: StatsReceiver = stats - override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - override val tweetId: Long = candidate.tweetId - override lazy val tweetyPieResult: Option[TweetyPie.TweetyPieResult] = - candidate.tweetyPieResult - override lazy val tweetAuthor: Future[Option[User]] = author - override val target: PushTypes.Target = candidate.target - override lazy val commonRecType: CommonRecommendationType = - candidate.commonRecType - override val pushCopyId: Option[Int] = copyIds.pushCopyId - override val ntabCopyId: Option[Int] = copyIds.ntabCopyId - override val copyAggregationId: Option[String] = copyIds.aggregationId - override lazy val semanticCoreEntityId: Option[Long] = candidate.semanticCoreEntityId - override lazy val localizedUttEntity: Option[LocalizedEntity] = candidate.localizedUttEntity - override lazy val algorithmCR: Option[String] = candidate.algorithmCR - override lazy val isMrBackfillCR: Option[Boolean] = candidate.isMrBackfillCR - override lazy val tagsCR: Option[Seq[MetricTag]] = candidate.tagsCR -} - -case class OutOfNetworkTweetCandidatePredicates(override val config: Config) - extends OutOfNetworkTweetPredicates[OutOfNetworkTweetPushCandidate] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override def postCandidateSpecificPredicates: List[ - NamedPredicate[OutOfNetworkTweetPushCandidate] - ] = - List( - HealthPredicates.agathaAbusiveTweetAuthorPredicateMrTwistly(), - ) - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/PushTypes.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/PushTypes.docx new file mode 100644 index 000000000..345a06441 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/PushTypes.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/PushTypes.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/PushTypes.scala deleted file mode 100644 index 83d5b67c3..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/PushTypes.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.candidate.UserLanguage -import com.twitter.frigate.common.candidate._ -import com.twitter.frigate.data_pipeline.features_common.RequestContextForFeatureStore -import com.twitter.frigate.pushservice.model.candidate.CopyInfo -import com.twitter.frigate.pushservice.model.candidate.MLScores -import com.twitter.frigate.pushservice.model.candidate.QualityScribing -import com.twitter.frigate.pushservice.model.candidate.Scriber -import com.twitter.frigate.pushservice.model.ibis.Ibis2HydratorForCandidate -import com.twitter.frigate.pushservice.model.ntab.NTabRequest -import com.twitter.frigate.pushservice.take.ChannelForCandidate -import com.twitter.frigate.pushservice.target._ -import com.twitter.util.Time - -object PushTypes { - - trait Target - extends TargetUser - with UserDetails - with TargetWithPushContext - with TargetDecider - with TargetABDecider - with FrigateHistory - with PushTargeting - with TargetScoringDetails - with TweetImpressionHistory - with CustomConfigForExpt - with CaretFeedbackHistory - with NotificationFeedbackHistory - with PromptFeedbackHistory - with HTLVisitHistory - with MaxTweetAge - with NewUserDetails - with ResurrectedUserDetails - with TargetWithSeedUsers - with MagicFanoutHistory - with OptOutUserInterests - with RequestContextForFeatureStore - with TargetAppPermissions - with UserLanguage - with InlineActionHistory - with TargetPlaces - - trait RawCandidate extends Candidate with TargetInfo[PushTypes.Target] with RecommendationType { - - val createdAt: Time = Time.now - } - - trait PushCandidate - extends RawCandidate - with CandidateScoringDetails - with MLScores - with QualityScribing - with CopyInfo - with Scriber - with Ibis2HydratorForCandidate - with NTabRequest - with ChannelForCandidate -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ScheduledSpaceSpeaker.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ScheduledSpaceSpeaker.docx new file mode 100644 index 000000000..3fba9a65c Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ScheduledSpaceSpeaker.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ScheduledSpaceSpeaker.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ScheduledSpaceSpeaker.scala deleted file mode 100644 index 6da10ed77..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ScheduledSpaceSpeaker.scala +++ /dev/null @@ -1,85 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.ScheduledSpaceSpeakerCandidate -import com.twitter.frigate.common.base.SpaceCandidateFanoutDetails -import com.twitter.frigate.common.util.FeatureSwitchParams -import com.twitter.frigate.magic_events.thriftscala.SpaceMetadata -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.ScheduledSpaceSpeakerIbis2Hydrator -import com.twitter.frigate.pushservice.model.ntab.ScheduledSpaceSpeakerNTabRequestHydrator -import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate -import com.twitter.frigate.pushservice.predicate.SpacePredicate -import com.twitter.frigate.pushservice.take.predicates.BasicSendHandlerPredicates -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.storehaus.ReadableStore -import com.twitter.ubs.thriftscala.AudioSpace -import com.twitter.util.Future - -class ScheduledSpaceSpeakerPushCandidate( - candidate: RawCandidate with ScheduledSpaceSpeakerCandidate, - hostUser: Option[User], - copyIds: CopyIds, - audioSpaceStore: ReadableStore[String, AudioSpace] -)( - implicit val statsScoped: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with ScheduledSpaceSpeakerCandidate - with ScheduledSpaceSpeakerIbis2Hydrator - with SpaceCandidateFanoutDetails - with ScheduledSpaceSpeakerNTabRequestHydrator { - - override val startTime: Long = candidate.startTime - - override val hydratedHost: Option[User] = hostUser - - override val spaceId: String = candidate.spaceId - - override val hostId: Option[Long] = candidate.hostId - - override val speakerIds: Option[Seq[Long]] = candidate.speakerIds - - override val listenerIds: Option[Seq[Long]] = candidate.listenerIds - - override val frigateNotification: FrigateNotification = candidate.frigateNotification - - override val pushCopyId: Option[Int] = copyIds.pushCopyId - - override val ntabCopyId: Option[Int] = copyIds.ntabCopyId - - override val copyAggregationId: Option[String] = copyIds.aggregationId - - override val target: Target = candidate.target - - override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - - override lazy val audioSpaceFut: Future[Option[AudioSpace]] = audioSpaceStore.get(spaceId) - - override val spaceFanoutMetadata: Option[SpaceMetadata] = None - - override val statsReceiver: StatsReceiver = - statsScoped.scope("ScheduledSpaceSpeakerCandidate") -} - -case class ScheduledSpaceSpeakerCandidatePredicates(config: Config) - extends BasicSendHandlerPredicates[ScheduledSpaceSpeakerPushCandidate] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override val preCandidateSpecificPredicates: List[ - NamedPredicate[ScheduledSpaceSpeakerPushCandidate] - ] = List( - SpacePredicate.scheduledSpaceStarted( - config.audioSpaceStore - ), - PredicatesForCandidate.paramPredicate(FeatureSwitchParams.EnableScheduledSpaceSpeakers) - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ScheduledSpaceSubscriber.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ScheduledSpaceSubscriber.docx new file mode 100644 index 000000000..e42942eae Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ScheduledSpaceSubscriber.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ScheduledSpaceSubscriber.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ScheduledSpaceSubscriber.scala deleted file mode 100644 index 78977ab5d..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ScheduledSpaceSubscriber.scala +++ /dev/null @@ -1,86 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.ScheduledSpaceSubscriberCandidate -import com.twitter.frigate.common.base.SpaceCandidateFanoutDetails -import com.twitter.frigate.common.util.FeatureSwitchParams -import com.twitter.frigate.magic_events.thriftscala.SpaceMetadata -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.ScheduledSpaceSubscriberIbis2Hydrator -import com.twitter.frigate.pushservice.model.ntab.ScheduledSpaceSubscriberNTabRequestHydrator -import com.twitter.frigate.pushservice.predicate._ -import com.twitter.frigate.pushservice.take.predicates.BasicSendHandlerPredicates -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.storehaus.ReadableStore -import com.twitter.ubs.thriftscala.AudioSpace -import com.twitter.util.Future - -class ScheduledSpaceSubscriberPushCandidate( - candidate: RawCandidate with ScheduledSpaceSubscriberCandidate, - hostUser: Option[User], - copyIds: CopyIds, - audioSpaceStore: ReadableStore[String, AudioSpace] -)( - implicit val statsScoped: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with ScheduledSpaceSubscriberCandidate - with SpaceCandidateFanoutDetails - with ScheduledSpaceSubscriberIbis2Hydrator - with ScheduledSpaceSubscriberNTabRequestHydrator { - - override val startTime: Long = candidate.startTime - - override val hydratedHost: Option[User] = hostUser - - override val spaceId: String = candidate.spaceId - - override val hostId: Option[Long] = candidate.hostId - - override val speakerIds: Option[Seq[Long]] = candidate.speakerIds - - override val listenerIds: Option[Seq[Long]] = candidate.listenerIds - - override val frigateNotification: FrigateNotification = candidate.frigateNotification - - override val pushCopyId: Option[Int] = copyIds.pushCopyId - - override val ntabCopyId: Option[Int] = copyIds.ntabCopyId - - override val copyAggregationId: Option[String] = copyIds.aggregationId - - override val target: Target = candidate.target - - override lazy val audioSpaceFut: Future[Option[AudioSpace]] = audioSpaceStore.get(spaceId) - - override val spaceFanoutMetadata: Option[SpaceMetadata] = None - - override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - - override val statsReceiver: StatsReceiver = - statsScoped.scope("ScheduledSpaceSubscriberCandidate") -} - -case class ScheduledSpaceSubscriberCandidatePredicates(config: Config) - extends BasicSendHandlerPredicates[ScheduledSpaceSubscriberPushCandidate] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override val preCandidateSpecificPredicates: List[ - NamedPredicate[ScheduledSpaceSubscriberPushCandidate] - ] = - List( - PredicatesForCandidate.paramPredicate(FeatureSwitchParams.EnableScheduledSpaceSubscribers), - SpacePredicate.narrowCastSpace, - SpacePredicate.targetInSpace(config.audioSpaceParticipantsStore), - SpacePredicate.spaceHostTargetUserBlocking(config.edgeStore), - PredicatesForCandidate.duplicateSpacesPredicate - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/SubscribedSearchTweetPushCandidate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/SubscribedSearchTweetPushCandidate.docx new file mode 100644 index 000000000..2fa492cf3 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/SubscribedSearchTweetPushCandidate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/SubscribedSearchTweetPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/SubscribedSearchTweetPushCandidate.scala deleted file mode 100644 index 4d71a0c75..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/SubscribedSearchTweetPushCandidate.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.SubscribedSearchTweetCandidate -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.SubscribedSearchTweetIbis2Hydrator -import com.twitter.frigate.pushservice.model.ntab.SubscribedSearchTweetNtabRequestHydrator -import com.twitter.frigate.pushservice.take.predicates.BasicTweetPredicatesForRFPH -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.stitch.tweetypie.TweetyPie -import com.twitter.util.Future - -class SubscribedSearchTweetPushCandidate( - candidate: RawCandidate with SubscribedSearchTweetCandidate, - author: Option[User], - copyIds: CopyIds -)( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with SubscribedSearchTweetCandidate - with TweetAuthorDetails - with SubscribedSearchTweetIbis2Hydrator - with SubscribedSearchTweetNtabRequestHydrator { - override def tweetAuthor: Future[Option[User]] = Future.value(author) - - override def weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - - override def tweetId: Long = candidate.tweetId - - override def pushCopyId: Option[Int] = copyIds.pushCopyId - - override def ntabCopyId: Option[Int] = copyIds.ntabCopyId - - override def copyAggregationId: Option[String] = copyIds.aggregationId - - override def target: PushTypes.Target = candidate.target - - override def searchTerm: String = candidate.searchTerm - - override def timeBoundedLandingUrl: Option[String] = None - - override def statsReceiver: StatsReceiver = stats - - override def tweetyPieResult: Option[TweetyPie.TweetyPieResult] = candidate.tweetyPieResult -} - -case class SubscribedSearchTweetCandidatePredicates(override val config: Config) - extends BasicTweetPredicatesForRFPH[SubscribedSearchTweetPushCandidate] { - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TopTweetImpressionsPushCandidate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TopTweetImpressionsPushCandidate.docx new file mode 100644 index 000000000..e3232dffe Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TopTweetImpressionsPushCandidate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TopTweetImpressionsPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TopTweetImpressionsPushCandidate.scala deleted file mode 100644 index b04a16ac3..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TopTweetImpressionsPushCandidate.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.TopTweetImpressionsCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.TopTweetImpressionsCandidateIbis2Hydrator -import com.twitter.frigate.pushservice.model.ntab.TopTweetImpressionsNTabRequestHydrator -import com.twitter.frigate.pushservice.predicate.TopTweetImpressionsPredicates -import com.twitter.frigate.pushservice.take.predicates.BasicTweetPredicatesForRFPH -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.notificationservice.thriftscala.StoryContext -import com.twitter.notificationservice.thriftscala.StoryContextValue -import com.twitter.stitch.tweetypie.TweetyPie - -/** - * This class defines a hydrated [[TopTweetImpressionsCandidate]] - * - * @param candidate: [[TopTweetImpressionsCandidate]] for the candidate representing the user's Tweet with the most impressions - * @param copyIds: push and ntab notification copy - * @param stats: finagle scoped states receiver - * @param pushModelScorer: ML model score object for fetching prediction scores - */ -class TopTweetImpressionsPushCandidate( - candidate: RawCandidate with TopTweetImpressionsCandidate, - copyIds: CopyIds -)( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with TopTweetImpressionsCandidate - with TopTweetImpressionsNTabRequestHydrator - with TopTweetImpressionsCandidateIbis2Hydrator { - override val target: PushTypes.Target = candidate.target - override val commonRecType: CommonRecommendationType = candidate.commonRecType - override val tweetId: Long = candidate.tweetId - override lazy val tweetyPieResult: Option[TweetyPie.TweetyPieResult] = - candidate.tweetyPieResult - override val impressionsCount: Long = candidate.impressionsCount - - override val statsReceiver: StatsReceiver = stats.scope(getClass.getSimpleName) - override val pushCopyId: Option[Int] = copyIds.pushCopyId - override val ntabCopyId: Option[Int] = copyIds.ntabCopyId - override val copyAggregationId: Option[String] = copyIds.aggregationId - override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - override val storyContext: Option[StoryContext] = - Some(StoryContext(altText = "", value = Some(StoryContextValue.Tweets(Seq(tweetId))))) -} - -case class TopTweetImpressionsPushCandidatePredicates(config: Config) - extends BasicTweetPredicatesForRFPH[TopTweetImpressionsPushCandidate] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override val preCandidateSpecificPredicates: List[ - NamedPredicate[TopTweetImpressionsPushCandidate] - ] = List( - TopTweetImpressionsPredicates.topTweetImpressionsFatiguePredicate - ) - - override val postCandidateSpecificPredicates: List[ - NamedPredicate[TopTweetImpressionsPushCandidate] - ] = List( - TopTweetImpressionsPredicates.topTweetImpressionsThreshold() - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TopicProofTweetPushCandidate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TopicProofTweetPushCandidate.docx new file mode 100644 index 000000000..0b3eee060 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TopicProofTweetPushCandidate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TopicProofTweetPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TopicProofTweetPushCandidate.scala deleted file mode 100644 index f89eb28bf..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TopicProofTweetPushCandidate.scala +++ /dev/null @@ -1,71 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.TopicProofTweetCandidate -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.TopicProofTweetIbis2Hydrator -import com.twitter.frigate.pushservice.model.ntab.TopicProofTweetNtabRequestHydrator -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate -import com.twitter.frigate.pushservice.take.predicates.BasicTweetPredicatesForRFPH -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.stitch.tweetypie.TweetyPie -import com.twitter.util.Future - -/** - * This class defines a hydrated [[TopicProofTweetCandidate]] - * - * @param candidate : [[TopicProofTweetCandidate]] for the candidate representint a Tweet recommendation for followed Topic - * @param author : Tweet author representated as Gizmoduck user object - * @param copyIds : push and ntab notification copy - * @param stats : finagle scoped states receiver - * @param pushModelScorer : ML model score object for fetching prediction scores - */ -class TopicProofTweetPushCandidate( - candidate: RawCandidate with TopicProofTweetCandidate, - author: Option[User], - copyIds: CopyIds -)( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with TopicProofTweetCandidate - with TweetAuthorDetails - with TopicProofTweetNtabRequestHydrator - with TopicProofTweetIbis2Hydrator { - override val statsReceiver: StatsReceiver = stats - override val target: PushTypes.Target = candidate.target - override val tweetId: Long = candidate.tweetId - override lazy val tweetyPieResult: Option[TweetyPie.TweetyPieResult] = candidate.tweetyPieResult - override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - override val pushCopyId: Option[Int] = copyIds.pushCopyId - override val ntabCopyId: Option[Int] = copyIds.ntabCopyId - override val copyAggregationId: Option[String] = copyIds.aggregationId - override val semanticCoreEntityId = candidate.semanticCoreEntityId - override val localizedUttEntity = candidate.localizedUttEntity - override val tweetAuthor = Future.value(author) - override val topicListingSetting = candidate.topicListingSetting - override val algorithmCR = candidate.algorithmCR - override val commonRecType: CommonRecommendationType = candidate.commonRecType - override val tagsCR = candidate.tagsCR - override val isOutOfNetwork = candidate.isOutOfNetwork -} - -case class TopicProofTweetCandidatePredicates(override val config: Config) - extends BasicTweetPredicatesForRFPH[TopicProofTweetPushCandidate] { - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override val preCandidateSpecificPredicates: List[NamedPredicate[TopicProofTweetPushCandidate]] = - List( - PredicatesForCandidate.paramPredicate( - PushFeatureSwitchParams.EnableTopicProofTweetRecs - ), - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TrendTweetPushCandidate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TrendTweetPushCandidate.docx new file mode 100644 index 000000000..eff21436a Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TrendTweetPushCandidate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TrendTweetPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TrendTweetPushCandidate.scala deleted file mode 100644 index ec580f629..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TrendTweetPushCandidate.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.events.recos.thriftscala.TrendsContext -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.TrendTweetCandidate -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.TrendTweetIbis2Hydrator -import com.twitter.frigate.pushservice.model.ntab.TrendTweetNtabHydrator -import com.twitter.frigate.pushservice.take.predicates.BasicTweetPredicatesForRFPH -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.stitch.tweetypie.TweetyPie -import com.twitter.util.Future - -class TrendTweetPushCandidate( - candidate: RawCandidate with TrendTweetCandidate, - author: Option[User], - copyIds: CopyIds -)( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with TrendTweetCandidate - with TweetAuthorDetails - with TrendTweetIbis2Hydrator - with TrendTweetNtabHydrator { - override val statsReceiver: StatsReceiver = stats - override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - override val tweetId: Long = candidate.tweetId - override lazy val tweetyPieResult: Option[TweetyPie.TweetyPieResult] = candidate.tweetyPieResult - override lazy val tweetAuthor: Future[Option[User]] = Future.value(author) - override val target: PushTypes.Target = candidate.target - override val landingUrl: String = candidate.landingUrl - override val timeBoundedLandingUrl: Option[String] = candidate.timeBoundedLandingUrl - override val pushCopyId: Option[Int] = copyIds.pushCopyId - override val ntabCopyId: Option[Int] = copyIds.ntabCopyId - override val trendId: String = candidate.trendId - override val trendName: String = candidate.trendName - override val copyAggregationId: Option[String] = copyIds.aggregationId - override val context: TrendsContext = candidate.context -} - -case class TrendTweetPredicates(override val config: Config) - extends BasicTweetPredicatesForRFPH[TrendTweetPushCandidate] { - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TripTweetPushCandidate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TripTweetPushCandidate.docx new file mode 100644 index 000000000..729f36702 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TripTweetPushCandidate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TripTweetPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TripTweetPushCandidate.scala deleted file mode 100644 index 1981e7bb5..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TripTweetPushCandidate.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.OutOfNetworkTweetCandidate -import com.twitter.frigate.common.base.TopicCandidate -import com.twitter.frigate.common.base.TripCandidate -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.OutOfNetworkTweetIbis2HydratorForCandidate -import com.twitter.frigate.pushservice.model.ntab.OutOfNetworkTweetNTabRequestHydrator -import com.twitter.frigate.pushservice.take.predicates.OutOfNetworkTweetPredicates -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.stitch.tweetypie.TweetyPie -import com.twitter.topiclisting.utt.LocalizedEntity -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain -import com.twitter.util.Future - -class TripTweetPushCandidate( - candidate: RawCandidate with OutOfNetworkTweetCandidate with TripCandidate, - author: Future[Option[User]], - copyIds: CopyIds -)( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with TripCandidate - with TopicCandidate - with OutOfNetworkTweetCandidate - with TweetAuthorDetails - with OutOfNetworkTweetNTabRequestHydrator - with OutOfNetworkTweetIbis2HydratorForCandidate { - override val statsReceiver: StatsReceiver = stats - override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - override val tweetId: Long = candidate.tweetId - override lazy val tweetyPieResult: Option[TweetyPie.TweetyPieResult] = - candidate.tweetyPieResult - override lazy val tweetAuthor: Future[Option[User]] = author - override val target: PushTypes.Target = candidate.target - override lazy val commonRecType: CommonRecommendationType = - candidate.commonRecType - override val pushCopyId: Option[Int] = copyIds.pushCopyId - override val ntabCopyId: Option[Int] = copyIds.ntabCopyId - override val copyAggregationId: Option[String] = copyIds.aggregationId - override lazy val semanticCoreEntityId: Option[Long] = None - override lazy val localizedUttEntity: Option[LocalizedEntity] = None - override lazy val algorithmCR: Option[String] = None - override val tripDomain: Option[collection.Set[TripDomain]] = candidate.tripDomain -} - -case class TripTweetCandidatePredicates(override val config: Config) - extends OutOfNetworkTweetPredicates[TripTweetPushCandidate] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetAction.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetAction.docx new file mode 100644 index 000000000..21a8188c8 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetAction.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetAction.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetAction.scala deleted file mode 100644 index 72453224d..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetAction.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.SocialContextActions -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.common.base.TweetDetails -import com.twitter.frigate.pushservice.model.PushTypes._ -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.predicate._ -import com.twitter.frigate.pushservice.take.predicates.BasicTweetPredicatesForRFPH - -case class TweetActionCandidatePredicates(override val config: Config) - extends BasicTweetPredicatesForRFPH[ - PushCandidate with TweetCandidate with TweetDetails with SocialContextActions - ] { - - implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) - - override val preCandidateSpecificPredicates = List(PredicatesForCandidate.minSocialContext(1)) - - override val postCandidateSpecificPredicates = List( - PredicatesForCandidate.socialContextBeingFollowed(config.edgeStore), - PredicatesForCandidate.socialContextBlockingOrMuting(config.edgeStore), - PredicatesForCandidate.socialContextNotRetweetFollowing(config.edgeStore) - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetFavorite.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetFavorite.docx new file mode 100644 index 000000000..3c40ccaa8 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetFavorite.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetFavorite.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetFavorite.scala deleted file mode 100644 index ae31ddc6c..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetFavorite.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.SocialContextAction -import com.twitter.frigate.common.base.SocialContextUserDetails -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.base.TweetFavoriteCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.TweetFavoriteCandidateIbis2Hydrator -import com.twitter.frigate.pushservice.model.ntab.TweetFavoriteNTabRequestHydrator -import com.twitter.frigate.pushservice.util.CandidateHydrationUtil.TweetWithSocialContextTraits -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.stitch.tweetypie.TweetyPie -import com.twitter.util.Future - -class TweetFavoritePushCandidate( - candidate: RawCandidate with TweetWithSocialContextTraits, - socialContextUserMap: Future[Map[Long, Option[User]]], - author: Future[Option[User]], - copyIds: CopyIds -)( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with TweetFavoriteCandidate - with SocialContextUserDetails - with TweetAuthorDetails - with TweetFavoriteNTabRequestHydrator - with TweetFavoriteCandidateIbis2Hydrator { - override val statsReceiver: StatsReceiver = stats - override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - override val tweetId: Long = candidate.tweetId - override val socialContextActions: Seq[SocialContextAction] = - candidate.socialContextActions - - override val socialContextAllTypeActions: Seq[SocialContextAction] = - candidate.socialContextAllTypeActions - - override lazy val scUserMap: Future[Map[Long, Option[User]]] = socialContextUserMap - override lazy val tweetAuthor: Future[Option[User]] = author - override lazy val commonRecType: CommonRecommendationType = - candidate.commonRecType - override val target: PushTypes.Target = candidate.target - override lazy val tweetyPieResult: Option[TweetyPie.TweetyPieResult] = - candidate.tweetyPieResult - override val pushCopyId: Option[Int] = copyIds.pushCopyId - override val ntabCopyId: Option[Int] = copyIds.ntabCopyId - override val copyAggregationId: Option[String] = copyIds.aggregationId -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetRetweet.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetRetweet.docx new file mode 100644 index 000000000..39ea30c99 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetRetweet.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetRetweet.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetRetweet.scala deleted file mode 100644 index 61c8c6526..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetRetweet.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.frigate.pushservice.model - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.SocialContextAction -import com.twitter.frigate.common.base.SocialContextUserDetails -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.base.TweetRetweetCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.model.ibis.TweetRetweetCandidateIbis2Hydrator -import com.twitter.frigate.pushservice.model.ntab.TweetRetweetNTabRequestHydrator -import com.twitter.frigate.pushservice.util.CandidateHydrationUtil.TweetWithSocialContextTraits -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.stitch.tweetypie.TweetyPie -import com.twitter.util.Future - -class TweetRetweetPushCandidate( - candidate: RawCandidate with TweetWithSocialContextTraits, - socialContextUserMap: Future[Map[Long, Option[User]]], - author: Future[Option[User]], - copyIds: CopyIds -)( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer) - extends PushCandidate - with TweetRetweetCandidate - with SocialContextUserDetails - with TweetAuthorDetails - with TweetRetweetNTabRequestHydrator - with TweetRetweetCandidateIbis2Hydrator { - override val statsReceiver: StatsReceiver = stats - override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer - override val tweetId: Long = candidate.tweetId - override val socialContextActions: Seq[SocialContextAction] = - candidate.socialContextActions - - override val socialContextAllTypeActions: Seq[SocialContextAction] = - candidate.socialContextAllTypeActions - - override lazy val scUserMap: Future[Map[Long, Option[User]]] = socialContextUserMap - override lazy val tweetAuthor: Future[Option[User]] = author - override lazy val commonRecType: CommonRecommendationType = candidate.commonRecType - override val target: PushTypes.Target = candidate.target - override lazy val tweetyPieResult: Option[TweetyPie.TweetyPieResult] = candidate.tweetyPieResult - override val pushCopyId: Option[Int] = copyIds.pushCopyId - override val ntabCopyId: Option[Int] = copyIds.ntabCopyId - override val copyAggregationId: Option[String] = copyIds.aggregationId -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/CopyInfo.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/CopyInfo.docx new file mode 100644 index 000000000..f303b4d8b Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/CopyInfo.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/CopyInfo.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/CopyInfo.scala deleted file mode 100644 index 11cc0617a..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/CopyInfo.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.frigate.pushservice.model.candidate - -import com.twitter.frigate.common.util.MRPushCopy -import com.twitter.frigate.common.util.MrPushCopyObjects -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.util.CandidateUtil - -case class CopyIds( - pushCopyId: Option[Int] = None, - ntabCopyId: Option[Int] = None, - aggregationId: Option[String] = None) - -trait CopyInfo { - self: PushCandidate => - - import com.twitter.frigate.data_pipeline.common.FrigateNotificationUtil._ - - def getPushCopy: Option[MRPushCopy] = - pushCopyId match { - case Some(pushCopyId) => MrPushCopyObjects.getCopyFromId(pushCopyId) - case _ => - crt2PushCopy( - commonRecType, - CandidateUtil.getSocialContextActionsFromCandidate(self).size - ) - } - - def pushCopyId: Option[Int] - - def ntabCopyId: Option[Int] - - def copyAggregationId: Option[String] -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/MLScores.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/MLScores.docx new file mode 100644 index 000000000..dce329715 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/MLScores.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/MLScores.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/MLScores.scala deleted file mode 100644 index 4ba79f485..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/MLScores.scala +++ /dev/null @@ -1,307 +0,0 @@ -package com.twitter.frigate.pushservice.model.candidate - -import com.twitter.frigate.common.base.FeatureMap -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.ml.HydrationContextBuilder -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushMLModel -import com.twitter.frigate.pushservice.params.WeightedOpenOrNtabClickModel -import com.twitter.nrel.hydration.push.HydrationContext -import com.twitter.timelines.configapi.FSParam -import com.twitter.util.Future -import java.util.concurrent.ConcurrentHashMap -import scala.collection.concurrent.{Map => CMap} -import scala.collection.convert.decorateAsScala._ - -trait MLScores { - - self: PushCandidate => - - lazy val candidateHydrationContext: Future[HydrationContext] = HydrationContextBuilder.build(self) - - def weightedOpenOrNtabClickModelScorer: PushMLModelScorer - - // Used to store the scores and avoid duplicate prediction - private val qualityModelScores: CMap[ - (PushMLModel.Value, WeightedOpenOrNtabClickModel.ModelNameType), - Future[Option[Double]] - ] = - new ConcurrentHashMap[(PushMLModel.Value, WeightedOpenOrNtabClickModel.ModelNameType), Future[ - Option[Double] - ]]().asScala - - def populateQualityModelScore( - pushMLModel: PushMLModel.Value, - modelVersion: WeightedOpenOrNtabClickModel.ModelNameType, - prob: Future[Option[Double]] - ) = { - val modelAndVersion = (pushMLModel, modelVersion) - if (!qualityModelScores.contains(modelAndVersion)) { - qualityModelScores += modelAndVersion -> prob - } - } - - // The ML scores that also depend on other candidates and are only available after all candidates are processed - // For example, the likelihood info for Importance Sampling - private lazy val crossCandidateMlScores: CMap[String, Double] = - new ConcurrentHashMap[String, Double]().asScala - - def populateCrossCandidateMlScores(scoreName: String, score: Double): Unit = { - if (crossCandidateMlScores.contains(scoreName)) { - throw new Exception( - s"$scoreName has been populated in the CrossCandidateMlScores!\n" + - s"Existing crossCandidateMlScores are ${crossCandidateMlScores}\n" - ) - } - crossCandidateMlScores += scoreName -> score - } - - def getMLModelScore( - pushMLModel: PushMLModel.Value, - modelVersion: WeightedOpenOrNtabClickModel.ModelNameType - ): Future[Option[Double]] = { - qualityModelScores.getOrElseUpdate( - (pushMLModel, modelVersion), - weightedOpenOrNtabClickModelScorer - .singlePredicationForModelVersion(modelVersion, self, Some(pushMLModel)) - ) - } - - def getMLModelScoreWithoutUpdate( - pushMLModel: PushMLModel.Value, - modelVersion: WeightedOpenOrNtabClickModel.ModelNameType - ): Future[Option[Double]] = { - qualityModelScores.getOrElse( - (pushMLModel, modelVersion), - Future.None - ) - } - - def getWeightedOpenOrNtabClickModelScore( - weightedOONCModelParam: FSParam[WeightedOpenOrNtabClickModel.ModelNameType] - ): Future[Option[Double]] = { - getMLModelScore( - PushMLModel.WeightedOpenOrNtabClickProbability, - target.params(weightedOONCModelParam) - ) - } - - /* After we unify the ranking and filtering models, we follow the iteration process below - When improving the WeightedOONC model, - 1) Run experiment which only replace the ranking model - 2) Make decisions according to the experiment results - 3) Use the ranking model for filtering - 4) Adjust percentile thresholds if necessary - */ - lazy val mrWeightedOpenOrNtabClickRankingProbability: Future[Option[Double]] = - target.rankingModelParam.flatMap { modelParam => - getWeightedOpenOrNtabClickModelScore(modelParam) - } - - def getBigFilteringScore( - pushMLModel: PushMLModel.Value, - modelVersion: WeightedOpenOrNtabClickModel.ModelNameType - ): Future[Option[Double]] = { - mrWeightedOpenOrNtabClickRankingProbability.flatMap { - case Some(rankingScore) => - // Adds ranking score to feature map (we must ensure the feature key is also in the feature context) - mergeFeatures( - FeatureMap( - numericFeatures = Map("scribe.WeightedOpenOrNtabClickProbability" -> rankingScore) - ) - ) - getMLModelScore(pushMLModel, modelVersion) - case _ => Future.None - } - } - - def getWeightedOpenOrNtabClickScoreForScribing(): Seq[Future[Map[String, Double]]] = { - Seq( - mrWeightedOpenOrNtabClickRankingProbability.map { - case Some(score) => Map(PushMLModel.WeightedOpenOrNtabClickProbability.toString -> score) - case _ => Map.empty[String, Double] - }, - Future - .join( - target.rankingModelParam, - mrWeightedOpenOrNtabClickRankingProbability - ).map { - case (rankingModelParam, Some(score)) => - Map(target.params(rankingModelParam).toString -> score) - case _ => Map.empty[String, Double] - } - ) - } - - def getNsfwScoreForScribing(): Seq[Future[Map[String, Double]]] = { - val nsfwScoreFut = getMLModelScoreWithoutUpdate( - PushMLModel.HealthNsfwProbability, - target.params(PushFeatureSwitchParams.BqmlHealthModelTypeParam)) - Seq(nsfwScoreFut.map { nsfwScoreOpt => - nsfwScoreOpt - .map(nsfwScore => Map(PushMLModel.HealthNsfwProbability.toString -> nsfwScore)).getOrElse( - Map.empty[String, Double]) - }) - } - - def getBigFilteringSupervisedScoresForScribing(): Seq[Future[Map[String, Double]]] = { - if (target.params( - PushFeatureSwitchParams.EnableMrRequestScribingBigFilteringSupervisedScores)) { - Seq( - mrBigFilteringSupervisedSendingScore.map { - case Some(score) => - Map(PushMLModel.BigFilteringSupervisedSendingModel.toString -> score) - case _ => Map.empty[String, Double] - }, - mrBigFilteringSupervisedWithoutSendingScore.map { - case Some(score) => - Map(PushMLModel.BigFilteringSupervisedWithoutSendingModel.toString -> score) - case _ => Map.empty[String, Double] - } - ) - } else Seq.empty[Future[Map[String, Double]]] - } - - def getBigFilteringRLScoresForScribing(): Seq[Future[Map[String, Double]]] = { - if (target.params(PushFeatureSwitchParams.EnableMrRequestScribingBigFilteringRLScores)) { - Seq( - mrBigFilteringRLSendingScore.map { - case Some(score) => Map(PushMLModel.BigFilteringRLSendingModel.toString -> score) - case _ => Map.empty[String, Double] - }, - mrBigFilteringRLWithoutSendingScore.map { - case Some(score) => Map(PushMLModel.BigFilteringRLWithoutSendingModel.toString -> score) - case _ => Map.empty[String, Double] - } - ) - } else Seq.empty[Future[Map[String, Double]]] - } - - def buildModelScoresSeqForScribing(): Seq[Future[Map[String, Double]]] = { - getWeightedOpenOrNtabClickScoreForScribing() ++ - getBigFilteringSupervisedScoresForScribing() ++ - getBigFilteringRLScoresForScribing() ++ - getNsfwScoreForScribing() - } - - lazy val mrBigFilteringSupervisedSendingScore: Future[Option[Double]] = - getBigFilteringScore( - PushMLModel.BigFilteringSupervisedSendingModel, - target.params(PushFeatureSwitchParams.BigFilteringSupervisedSendingModelParam) - ) - - lazy val mrBigFilteringSupervisedWithoutSendingScore: Future[Option[Double]] = - getBigFilteringScore( - PushMLModel.BigFilteringSupervisedWithoutSendingModel, - target.params(PushFeatureSwitchParams.BigFilteringSupervisedWithoutSendingModelParam) - ) - - lazy val mrBigFilteringRLSendingScore: Future[Option[Double]] = - getBigFilteringScore( - PushMLModel.BigFilteringRLSendingModel, - target.params(PushFeatureSwitchParams.BigFilteringRLSendingModelParam) - ) - - lazy val mrBigFilteringRLWithoutSendingScore: Future[Option[Double]] = - getBigFilteringScore( - PushMLModel.BigFilteringRLWithoutSendingModel, - target.params(PushFeatureSwitchParams.BigFilteringRLWithoutSendingModelParam) - ) - - lazy val mrWeightedOpenOrNtabClickFilteringProbability: Future[Option[Double]] = - getWeightedOpenOrNtabClickModelScore( - target.filteringModelParam - ) - - lazy val mrQualityUprankingProbability: Future[Option[Double]] = - getMLModelScore( - PushMLModel.FilteringProbability, - target.params(PushFeatureSwitchParams.QualityUprankingModelTypeParam) - ) - - lazy val mrNsfwScore: Future[Option[Double]] = - getMLModelScoreWithoutUpdate( - PushMLModel.HealthNsfwProbability, - target.params(PushFeatureSwitchParams.BqmlHealthModelTypeParam) - ) - - // MR quality upranking param - private val qualityUprankingBoost: String = "QualityUprankingBoost" - private val producerQualityUprankingBoost: String = "ProducerQualityUprankingBoost" - private val qualityUprankingInfo: CMap[String, Double] = - new ConcurrentHashMap[String, Double]().asScala - - lazy val mrQualityUprankingBoost: Option[Double] = - qualityUprankingInfo.get(qualityUprankingBoost) - lazy val mrProducerQualityUprankingBoost: Option[Double] = - qualityUprankingInfo.get(producerQualityUprankingBoost) - - def setQualityUprankingBoost(boost: Double) = - if (qualityUprankingInfo.contains(qualityUprankingBoost)) { - qualityUprankingInfo(qualityUprankingBoost) = boost - } else { - qualityUprankingInfo += qualityUprankingBoost -> boost - } - def setProducerQualityUprankingBoost(boost: Double) = - if (qualityUprankingInfo.contains(producerQualityUprankingBoost)) { - qualityUprankingInfo(producerQualityUprankingBoost) = boost - } else { - qualityUprankingInfo += producerQualityUprankingBoost -> boost - } - - private lazy val mrModelScoresFut: Future[Map[String, Double]] = { - if (self.target.isLoggedOutUser) { - Future.value(Map.empty[String, Double]) - } else { - Future - .collectToTry { - buildModelScoresSeqForScribing() - }.map { scoreTrySeq => - scoreTrySeq - .collect { - case result if result.isReturn => result.get() - }.reduce(_ ++ _) - } - } - } - - // Internal model scores (scores that are independent of other candidates) for scribing - lazy val modelScores: Future[Map[String, Double]] = - target.dauProbability.flatMap { dauProbabilityOpt => - val dauProbScoreMap = dauProbabilityOpt - .map(_.probability).map { dauProb => - PushMLModel.DauProbability.toString -> dauProb - }.toMap - - // Avoid unnecessary MR model scribing - if (target.isDarkWrite) { - mrModelScoresFut.map(dauProbScoreMap ++ _) - } else if (RecTypes.isSendHandlerType(commonRecType) && !RecTypes - .sendHandlerTypesUsingMrModel(commonRecType)) { - Future.value(dauProbScoreMap) - } else { - mrModelScoresFut.map(dauProbScoreMap ++ _) - } - } - - // We will scribe both internal ML scores and cross-Candidate scores - def getModelScoresforScribing(): Future[Map[String, Double]] = { - if (RecTypes.notEligibleForModelScoreTracking(commonRecType) || self.target.isLoggedOutUser) { - Future.value(Map.empty[String, Double]) - } else { - modelScores.map { internalScores => - if (internalScores.keySet.intersect(crossCandidateMlScores.keySet).nonEmpty) { - throw new Exception( - "crossCandidateMlScores overlap internalModelScores\n" + - s"internalScores keySet: ${internalScores.keySet}\n" + - s"crossCandidateScores keySet: ${crossCandidateMlScores.keySet}\n" - ) - } - - internalScores ++ crossCandidateMlScores - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/QualityScribing.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/QualityScribing.docx new file mode 100644 index 000000000..0b31be39a Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/QualityScribing.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/QualityScribing.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/QualityScribing.scala deleted file mode 100644 index 283c3d97c..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/QualityScribing.scala +++ /dev/null @@ -1,104 +0,0 @@ -package com.twitter.frigate.pushservice.model.candidate - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.HighQualityScribingScores -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushMLModel -import com.twitter.util.Future -import java.util.concurrent.ConcurrentHashMap -import scala.collection.concurrent.{Map => CMap} -import scala.collection.convert.decorateAsScala._ - -trait QualityScribing { - self: PushCandidate with MLScores => - - // Use to store other scores (to avoid duplicate queries to other services, e.g. HSS) - private val externalCachedScores: CMap[String, Future[Option[Double]]] = - new ConcurrentHashMap[String, Future[Option[Double]]]().asScala - - /** - * Retrieves the model version as specified by the corresponding FS param. - * This model version will be used for getting the cached score or triggering - * a prediction request. - * - * @param modelName The score we will like to scribe - */ - private def getModelVersion( - modelName: HighQualityScribingScores.Name - ): String = { - modelName match { - case HighQualityScribingScores.HeavyRankingScore => - target.params(PushFeatureSwitchParams.HighQualityCandidatesHeavyRankingModel) - case HighQualityScribingScores.NonPersonalizedQualityScoreUsingCnn => - target.params(PushFeatureSwitchParams.HighQualityCandidatesNonPersonalizedQualityCnnModel) - case HighQualityScribingScores.BqmlNsfwScore => - target.params(PushFeatureSwitchParams.HighQualityCandidatesBqmlNsfwModel) - case HighQualityScribingScores.BqmlReportScore => - target.params(PushFeatureSwitchParams.HighQualityCandidatesBqmlReportModel) - } - } - - /** - * Retrieves the score for scribing either from a cached value or - * by generating a prediction request. This will increase model QPS - * - * @param pushMLModel This represents the prefix of the model name (i.e. [pushMLModel]_[version]) - * @param scoreName The name to be use when scribing this score - */ - def getScribingScore( - pushMLModel: PushMLModel.Value, - scoreName: HighQualityScribingScores.Name - ): Future[(String, Option[Double])] = { - getMLModelScore( - pushMLModel, - getModelVersion(scoreName) - ).map { scoreOpt => - scoreName.toString -> scoreOpt - } - } - - /** - * Retrieves the score for scribing if it has been computed/cached before otherwise - * it will return Future.None - * - * @param pushMLModel This represents the prefix of the model name (i.e. [pushMLModel]_[version]) - * @param scoreName The name to be use when scribing this score - */ - def getScribingScoreWithoutUpdate( - pushMLModel: PushMLModel.Value, - scoreName: HighQualityScribingScores.Name - ): Future[(String, Option[Double])] = { - getMLModelScoreWithoutUpdate( - pushMLModel, - getModelVersion(scoreName) - ).map { scoreOpt => - scoreName.toString -> scoreOpt - } - } - - /** - * Caches the given score future - * - * @param scoreName The name to be use when scribing this score - * @param scoreFut Future mapping scoreName -> scoreOpt - */ - def cacheExternalScore(scoreName: String, scoreFut: Future[Option[Double]]) = { - if (!externalCachedScores.contains(scoreName)) { - externalCachedScores += scoreName -> scoreFut - } - } - - /** - * Returns all external scores future cached as a sequence - */ - def getExternalCachedScores: Seq[Future[(String, Option[Double])]] = { - externalCachedScores.map { - case (modelName, scoreFut) => - scoreFut.map { scoreOpt => modelName -> scoreOpt } - }.toSeq - } - - def getExternalCachedScoreByName(name: String): Future[Option[Double]] = { - externalCachedScores.getOrElse(name, Future.None) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/Scriber.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/Scriber.docx new file mode 100644 index 000000000..761681f3e Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/Scriber.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/Scriber.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/Scriber.scala deleted file mode 100644 index a43530b44..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/Scriber.scala +++ /dev/null @@ -1,277 +0,0 @@ -package com.twitter.frigate.pushservice.model.candidate - -import com.twitter.frigate.data_pipeline.features_common.PushQualityModelFeatureContext.featureContext -import com.twitter.frigate.data_pipeline.features_common.PushQualityModelUtil -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.common.util.NotificationScribeUtil -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.OutOfNetworkTweetPushCandidate -import com.twitter.frigate.pushservice.model.TopicProofTweetPushCandidate -import com.twitter.frigate.pushservice.ml.HydrationContextBuilder -import com.twitter.frigate.pushservice.predicate.quality_model_predicate.PDauCohort -import com.twitter.frigate.pushservice.predicate.quality_model_predicate.PDauCohortUtil -import com.twitter.frigate.pushservice.util.Candidate2FrigateNotification -import com.twitter.frigate.pushservice.util.MediaAnnotationsUtil.sensitiveMediaCategoryFeatureName -import com.twitter.frigate.scribe.thriftscala.FrigateNotificationScribeType -import com.twitter.frigate.scribe.thriftscala.NotificationScribe -import com.twitter.frigate.scribe.thriftscala.PredicateDetailedInfo -import com.twitter.frigate.scribe.thriftscala.PushCapInfo -import com.twitter.frigate.thriftscala.ChannelName -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.frigate.thriftscala.OverrideInfo -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.hermit.model.user_state.UserState.UserState -import com.twitter.ibis2.service.thriftscala.Ibis2Response -import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions -import com.twitter.nrel.heavyranker.FeatureHydrator -import com.twitter.util.Future -import java.util.UUID -import java.util.concurrent.ConcurrentHashMap -import scala.collection.concurrent.{Map => CMap} -import scala.collection.Map -import scala.collection.convert.decorateAsScala._ - -trait Scriber { - self: PushCandidate => - - def statsReceiver: StatsReceiver - - def frigateNotification: FrigateNotification = Candidate2FrigateNotification - .getFrigateNotification(self)(statsReceiver) - .copy(copyAggregationId = self.copyAggregationId) - - lazy val impressionId: String = UUID.randomUUID.toString.replaceAll("-", "") - - // Used to store the score and threshold for predicates - // Map(predicate name, (score, threshold, filter?)) - private val predicateScoreAndThreshold: CMap[String, PredicateDetailedInfo] = - new ConcurrentHashMap[String, PredicateDetailedInfo]().asScala - - def cachePredicateInfo( - predName: String, - predScore: Double, - predThreshold: Double, - predResult: Boolean, - additionalInformation: Option[Map[String, Double]] = None - ) = { - if (!predicateScoreAndThreshold.contains(predName)) { - predicateScoreAndThreshold += predName -> PredicateDetailedInfo( - predName, - predScore, - predThreshold, - predResult, - additionalInformation) - } - } - - def getCachedPredicateInfo(): Seq[PredicateDetailedInfo] = predicateScoreAndThreshold.values.toSeq - - def frigateNotificationForPersistence( - channels: Seq[ChannelName], - isSilentPush: Boolean, - overrideInfoOpt: Option[OverrideInfo] = None, - copyFeaturesList: Set[String] - ): Future[FrigateNotification] = { - - // record display location for frigate notification - statsReceiver - .scope("FrigateNotificationForPersistence") - .scope("displayLocation") - .counter(frigateNotification.notificationDisplayLocation.name) - .incr() - - val getModelScores = self.getModelScoresforScribing() - - Future.join(getModelScores, self.target.targetMrUserState).map { - case (mlScores, mrUserState) => - frigateNotification.copy( - impressionId = Some(impressionId), - isSilentPush = Some(isSilentPush), - overrideInfo = overrideInfoOpt, - mlModelScores = Some(mlScores), - mrUserState = mrUserState.map(_.name), - copyFeatures = Some(copyFeaturesList.toSeq) - ) - } - } - // scribe data - private def getNotificationScribe( - notifForPersistence: FrigateNotification, - userState: Option[UserState], - dauCohort: PDauCohort.Value, - ibis2Response: Option[Ibis2Response], - tweetAuthorId: Option[Long], - recUserId: Option[Long], - modelScoresMap: Option[Map[String, Double]], - primaryClient: Option[String], - isMrBackfillCR: Option[Boolean] = None, - tagsCR: Option[Seq[String]] = None, - gizmoduckTargetUser: Option[User], - predicateDetailedInfoList: Option[Seq[PredicateDetailedInfo]] = None, - pushCapInfoList: Option[Seq[PushCapInfo]] = None - ): NotificationScribe = { - NotificationScribe( - FrigateNotificationScribeType.SendMessage, - System.currentTimeMillis(), - targetUserId = Some(self.target.targetId), - timestampKeyForHistoryV2 = Some(createdAt.inSeconds), - sendType = NotificationScribeUtil.convertToScribeDisplayLocation( - self.frigateNotification.notificationDisplayLocation - ), - recommendationType = NotificationScribeUtil.convertToScribeRecommendationType( - self.frigateNotification.commonRecommendationType - ), - commonRecommendationType = Some(self.frigateNotification.commonRecommendationType), - fromPushService = Some(true), - frigateNotification = Some(notifForPersistence), - impressionId = Some(impressionId), - skipModelInfo = target.skipModelInfo, - ibis2Response = ibis2Response, - tweetAuthorId = tweetAuthorId, - scribeFeatures = Some(target.noSkipButScribeFeatures), - userState = userState.map(_.toString), - pDauCohort = Some(dauCohort.toString), - recommendedUserId = recUserId, - modelScores = modelScoresMap, - primaryClient = primaryClient, - isMrBackfillCR = isMrBackfillCR, - tagsCR = tagsCR, - targetUserType = gizmoduckTargetUser.map(_.userType), - predicateDetailedInfoList = predicateDetailedInfoList, - pushCapInfoList = pushCapInfoList - ) - } - - def scribeData( - ibis2Response: Option[Ibis2Response] = None, - isSilentPush: Boolean = false, - overrideInfoOpt: Option[OverrideInfo] = None, - copyFeaturesList: Set[String] = Set.empty, - channels: Seq[ChannelName] = Seq.empty - ): Future[NotificationScribe] = { - - val recTweetAuthorId = self match { - case t: TweetCandidate with TweetAuthor => t.authorId - case _ => None - } - - val recUserId = self match { - case u: UserCandidate => Some(u.userId) - case _ => None - } - - val isMrBackfillCR = self match { - case t: OutOfNetworkTweetPushCandidate => t.isMrBackfillCR - case _ => None - } - - val tagsCR = self match { - case t: OutOfNetworkTweetPushCandidate => - t.tagsCR.map { tags => - tags.map(_.toString) - } - case t: TopicProofTweetPushCandidate => - t.tagsCR.map { tags => - tags.map(_.toString) - } - case _ => None - } - - Future - .join( - frigateNotificationForPersistence( - channels = channels, - isSilentPush = isSilentPush, - overrideInfoOpt = overrideInfoOpt, - copyFeaturesList = copyFeaturesList - ), - target.targetUserState, - PDauCohortUtil.getPDauCohort(target), - target.deviceInfo, - target.targetUser - ) - .flatMap { - case (notifForPersistence, userState, dauCohort, deviceInfo, gizmoduckTargetUserOpt) => - val primaryClient = deviceInfo.flatMap(_.guessedPrimaryClient).map(_.toString) - val cachedPredicateInfo = - if (self.target.params(PushParams.EnablePredicateDetailedInfoScribing)) { - Some(getCachedPredicateInfo()) - } else None - - val cachedPushCapInfo = - if (self.target - .params(PushParams.EnablePushCapInfoScribing)) { - Some(target.finalPushcapAndFatigue.values.toSeq) - } else None - - val data = getNotificationScribe( - notifForPersistence, - userState, - dauCohort, - ibis2Response, - recTweetAuthorId, - recUserId, - notifForPersistence.mlModelScores, - primaryClient, - isMrBackfillCR, - tagsCR, - gizmoduckTargetUserOpt, - cachedPredicateInfo, - cachedPushCapInfo - ) - //Don't scribe features for CRTs not eligible for ML Layer - if ((target.isModelTrainingData || target.scribeFeatureWithoutHydratingNewFeatures) - && !RecTypes.notEligibleForModelScoreTracking(self.commonRecType)) { - // scribe all the features for the model training data - self.getFeaturesForScribing.map { scribedFeatureMap => - if (target.params(PushParams.EnableScribingMLFeaturesAsDataRecord) && !target.params( - PushFeatureSwitchParams.EnableMrScribingMLFeaturesAsFeatureMapForStaging)) { - val scribedFeatureDataRecord = - ScalaToJavaDataRecordConversions.javaDataRecord2ScalaDataRecord( - PushQualityModelUtil.adaptToDataRecord(scribedFeatureMap, featureContext)) - data.copy( - featureDataRecord = Some(scribedFeatureDataRecord) - ) - } else { - data.copy(features = - Some(PushQualityModelUtil.convertFeatureMapToFeatures(scribedFeatureMap))) - } - } - } else Future.value(data) - } - } - - def getFeaturesForScribing: Future[FeatureMap] = { - target.featureMap - .flatMap { targetFeatureMap => - val onlineFeatureMap = targetFeatureMap ++ self - .candidateFeatureMap() // targetFeatureMap includes target core user history features - - val filteredFeatureMap = { - onlineFeatureMap.copy( - sparseContinuousFeatures = onlineFeatureMap.sparseContinuousFeatures.filterKeys( - !_.equals(sensitiveMediaCategoryFeatureName)) - ) - } - - val targetHydrationContext = HydrationContextBuilder.build(self.target) - val candidateHydrationContext = HydrationContextBuilder.build(self) - - val featureMapFut = targetHydrationContext.join(candidateHydrationContext).flatMap { - case (targetContext, candidateContext) => - FeatureHydrator.getFeatures( - candidateHydrationContext = candidateContext, - targetHydrationContext = targetContext, - onlineFeatures = filteredFeatureMap, - statsReceiver = statsReceiver) - } - - featureMapFut - } - } - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/CustomConfigurationMapForIbis.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/CustomConfigurationMapForIbis.docx new file mode 100644 index 000000000..eecd049dc Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/CustomConfigurationMapForIbis.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/CustomConfigurationMapForIbis.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/CustomConfigurationMapForIbis.scala deleted file mode 100644 index 75b00e346..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/CustomConfigurationMapForIbis.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.ibis2.lib.util.JsonMarshal -import com.twitter.util.Future - -trait CustomConfigurationMapForIbis { - self: PushCandidate => - - lazy val customConfigMapsJsonFut: Future[String] = { - customFieldsMapFut.map { customFields => - JsonMarshal.toJson(customFields) - } - } - - lazy val customConfigMapsFut: Future[Map[String, String]] = { - if (self.target.isLoggedOutUser) { - Future.value(Map.empty[String, String]) - } else { - customConfigMapsJsonFut.map { customConfigMapsJson => - Map("custom_config" -> customConfigMapsJson) - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/DiscoverTwitterPushIbis2Hydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/DiscoverTwitterPushIbis2Hydrator.docx new file mode 100644 index 000000000..ccae5a757 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/DiscoverTwitterPushIbis2Hydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/DiscoverTwitterPushIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/DiscoverTwitterPushIbis2Hydrator.scala deleted file mode 100644 index a3a48ff28..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/DiscoverTwitterPushIbis2Hydrator.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.common.base.DiscoverTwitterCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.util.PushIbisUtil.mergeFutModelValues -import com.twitter.util.Future - -trait DiscoverTwitterPushIbis2Hydrator extends Ibis2HydratorForCandidate { - self: PushCandidate with DiscoverTwitterCandidate => - - private lazy val targetModelValues: Map[String, String] = Map( - "target_user" -> target.targetId.toString - ) - - override lazy val modelValues: Future[Map[String, String]] = - mergeFutModelValues(super.modelValues, Future.value(targetModelValues)) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/F1FirstDegreeTweetIbis2Hydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/F1FirstDegreeTweetIbis2Hydrator.docx new file mode 100644 index 000000000..12334797b Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/F1FirstDegreeTweetIbis2Hydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/F1FirstDegreeTweetIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/F1FirstDegreeTweetIbis2Hydrator.scala deleted file mode 100644 index 6ddaa49d1..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/F1FirstDegreeTweetIbis2Hydrator.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.F1FirstDegree -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.util.Future - -trait F1FirstDegreeTweetIbis2HydratorForCandidate - extends TweetCandidateIbis2Hydrator - with RankedSocialContextIbis2Hydrator { - self: PushCandidate with F1FirstDegree with TweetAuthorDetails => - - override lazy val scopedStats: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) - - override lazy val tweetModelValues: Future[Map[String, String]] = { - for { - superModelValues <- super.tweetModelValues - tweetInlineModelValues <- tweetInlineActionModelValue - } yield { - superModelValues ++ otherModelValues ++ mediaModelValue ++ tweetInlineModelValues ++ inlineVideoMediaMap - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/Ibis2Hydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/Ibis2Hydrator.docx new file mode 100644 index 000000000..72f833871 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/Ibis2Hydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/Ibis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/Ibis2Hydrator.scala deleted file mode 100644 index fd7d26186..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/Ibis2Hydrator.scala +++ /dev/null @@ -1,127 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.common.util.MRPushCopy -import com.twitter.frigate.common.util.MrPushCopyObjects -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} -import com.twitter.ibis2.service.thriftscala.Flags -import com.twitter.ibis2.service.thriftscala.Ibis2Request -import com.twitter.ibis2.service.thriftscala.RecipientSelector -import com.twitter.ibis2.service.thriftscala.ResponseFlags -import com.twitter.util.Future -import scala.util.control.NoStackTrace -import com.twitter.ni.lib.logged_out_transform.Ibis2RequestTransform - -class PushCopyIdNotFoundException(private val message: String) - extends Exception(message) - with NoStackTrace - -class InvalidPushCopyIdException(private val message: String) - extends Exception(message) - with NoStackTrace - -trait Ibis2HydratorForCandidate - extends CandidatePushCopy - with OverrideForIbis2Request - with CustomConfigurationMapForIbis { - self: PushCandidate => - - lazy val silentPushModelValue: Map[String, String] = - if (RecTypes.silentPushDefaultEnabledCrts.contains(commonRecType)) { - Map.empty - } else { - Map("is_silent_push" -> "true") - } - - private def transformRelevanceScore( - mlScore: Double, - scoreRange: Seq[Double] - ): Double = { - val (lowerBound, upperBound) = (scoreRange.head, scoreRange.last) - (mlScore * (upperBound - lowerBound)) + lowerBound - } - - private def getBoundedMlScore(mlScore: Double): Double = { - if (RecTypes.isMagicFanoutEventType(commonRecType)) { - val mfScoreRange = target.params(FS.MagicFanoutRelevanceScoreRange) - transformRelevanceScore(mlScore, mfScoreRange) - } else { - val mrScoreRange = target.params(FS.MagicRecsRelevanceScoreRange) - transformRelevanceScore(mlScore, mrScoreRange) - } - } - - lazy val relevanceScoreMapFut: Future[Map[String, String]] = { - mrWeightedOpenOrNtabClickRankingProbability.map { - case Some(mlScore) if target.params(FS.IncludeRelevanceScoreInIbis2Payload) => - val boundedMlScore = getBoundedMlScore(mlScore) - Map("relevance_score" -> boundedMlScore.toString) - case _ => Map.empty[String, String] - } - } - - def customFieldsMapFut: Future[Map[String, String]] = relevanceScoreMapFut - - //override is only enabled for RFPH CRT - def modelValues: Future[Map[String, String]] = { - Future.join(overrideModelValueFut, customConfigMapsFut).map { - case (overrideModelValue, customConfig) => - overrideModelValue ++ silentPushModelValue ++ customConfig - } - } - - def modelName: String = pushCopy.ibisPushModelName - - def senderId: Option[Long] = None - - def ibis2Request: Future[Option[Ibis2Request]] = { - Future.join(self.target.loggedOutMetadata, modelValues).map { - case (Some(metadata), modelVals) => - Some( - Ibis2RequestTransform - .apply(metadata, modelName, modelVals).copy( - senderId = senderId, - flags = Some(Flags( - darkWrite = Some(target.isDarkWrite), - skipDupcheck = target.pushContext.flatMap(_.useDebugHandler), - responseFlags = Some(ResponseFlags(stringTelemetry = Some(true))) - )) - )) - case (None, modelVals) => - Some( - Ibis2Request( - recipientSelector = RecipientSelector(Some(target.targetId)), - modelName = modelName, - modelValues = Some(modelVals), - senderId = senderId, - flags = Some( - Flags( - darkWrite = Some(target.isDarkWrite), - skipDupcheck = target.pushContext.flatMap(_.useDebugHandler), - responseFlags = Some(ResponseFlags(stringTelemetry = Some(true))) - ) - ) - )) - } - } -} - -trait CandidatePushCopy { - self: PushCandidate => - - final lazy val pushCopy: MRPushCopy = - pushCopyId match { - case Some(pushCopyId) => - MrPushCopyObjects - .getCopyFromId(pushCopyId) - .getOrElse( - throw new InvalidPushCopyIdException( - s"Invalid push copy id: $pushCopyId for ${self.commonRecType}")) - - case None => - throw new PushCopyIdNotFoundException( - s"PushCopy not found in frigateNotification for ${self.commonRecType}" - ) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/InlineActionIbis2Hydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/InlineActionIbis2Hydrator.docx new file mode 100644 index 000000000..722709ba8 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/InlineActionIbis2Hydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/InlineActionIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/InlineActionIbis2Hydrator.scala deleted file mode 100644 index 8e927254a..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/InlineActionIbis2Hydrator.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.util.InlineActionUtil -import com.twitter.util.Future - -trait InlineActionIbis2Hydrator { - self: PushCandidate => - - lazy val tweetInlineActionModelValue: Future[Map[String, String]] = - InlineActionUtil.getTweetInlineActionValue(target) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ListIbis2Hydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ListIbis2Hydrator.docx new file mode 100644 index 000000000..a738cd9f2 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ListIbis2Hydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ListIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ListIbis2Hydrator.scala deleted file mode 100644 index 57483c8ba..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ListIbis2Hydrator.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.pushservice.model.ListRecommendationPushCandidate -import com.twitter.util.Future - -trait ListIbis2Hydrator extends Ibis2HydratorForCandidate { - self: ListRecommendationPushCandidate => - - override lazy val senderId: Option[Long] = Some(0L) - - override lazy val modelValues: Future[Map[String, String]] = - Future.join(listName, listOwnerId).map { - case (nameOpt, authorId) => - Map( - "list" -> listId.toString, - "list_name" -> nameOpt - .getOrElse(""), - "list_author" -> s"${authorId.getOrElse(0L)}" - ) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutCreatorEventIbis2Hydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutCreatorEventIbis2Hydrator.docx new file mode 100644 index 000000000..7a1551097 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutCreatorEventIbis2Hydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutCreatorEventIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutCreatorEventIbis2Hydrator.scala deleted file mode 100644 index edb0aa51e..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutCreatorEventIbis2Hydrator.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.magic_events.thriftscala.CreatorFanoutType -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.MagicFanoutCreatorEventPushCandidate -import com.twitter.frigate.pushservice.util.PushIbisUtil.mergeModelValues -import com.twitter.util.Future - -trait MagicFanoutCreatorEventIbis2Hydrator - extends CustomConfigurationMapForIbis - with Ibis2HydratorForCandidate { - self: PushCandidate with MagicFanoutCreatorEventPushCandidate => - - val userMap = Map( - "handle" -> userProfile.screenName, - "display_name" -> userProfile.name - ) - - override val senderId = hydratedCreator.map(_.id) - - override lazy val modelValues: Future[Map[String, String]] = - mergeModelValues(super.modelValues, userMap) - - override val ibis2Request = creatorFanoutType match { - case CreatorFanoutType.UserSubscription => Future.None - case CreatorFanoutType.NewCreator => super.ibis2Request - case _ => super.ibis2Request - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutNewsEventIbis2Hydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutNewsEventIbis2Hydrator.docx new file mode 100644 index 000000000..57854af95 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutNewsEventIbis2Hydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutNewsEventIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutNewsEventIbis2Hydrator.scala deleted file mode 100644 index a1b073a38..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutNewsEventIbis2Hydrator.scala +++ /dev/null @@ -1,103 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.MagicFanoutEventHydratedCandidate -import com.twitter.frigate.pushservice.params.PushConstants -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutPredicatesUtil -import com.twitter.frigate.pushservice.util.PushIbisUtil._ -import com.twitter.util.Future - -trait MagicFanoutNewsEventIbis2Hydrator extends Ibis2HydratorForCandidate { - self: PushCandidate with MagicFanoutEventHydratedCandidate => - - override lazy val senderId: Option[Long] = { - val isUgmMoment = self.semanticCoreEntityTags.values.flatten.toSet - .contains(MagicFanoutPredicatesUtil.UgmMomentTag) - - owningTwitterUserIds.headOption match { - case Some(owningTwitterUserId) - if isUgmMoment && target.params( - PushFeatureSwitchParams.MagicFanoutNewsUserGeneratedEventsEnable) => - Some(owningTwitterUserId) - case _ => None - } - } - - lazy val stats = self.statsReceiver.scope("MagicFanout") - lazy val defaultImageCounter = stats.counter("default_image") - lazy val requestImageCounter = stats.counter("request_num") - lazy val noneImageCounter = stats.counter("none_num") - - private def getModelValueMediaUrl( - urlOpt: Option[String], - mapKey: String - ): Option[(String, String)] = { - requestImageCounter.incr() - urlOpt match { - case Some(PushConstants.DefaultEventMediaUrl) => - defaultImageCounter.incr() - None - case Some(url) => Some(mapKey -> url) - case None => - noneImageCounter.incr() - None - } - } - - private lazy val eventModelValuesFut: Future[Map[String, String]] = { - for { - title <- eventTitleFut - squareImageUrl <- squareImageUrlFut - primaryImageUrl <- primaryImageUrlFut - eventDescriptionOpt <- eventDescriptionFut - } yield { - - val authorId = owningTwitterUserIds.headOption match { - case Some(author) - if target.params(PushFeatureSwitchParams.MagicFanoutNewsUserGeneratedEventsEnable) => - Some("author" -> author.toString) - case _ => None - } - - val eventDescription = eventDescriptionOpt match { - case Some(description) - if target.params(PushFeatureSwitchParams.MagicFanoutNewsEnableDescriptionCopy) => - Some("event_description" -> description) - case _ => - None - } - - Map( - "event_id" -> s"$eventId", - "event_title" -> title - ) ++ - getModelValueMediaUrl(squareImageUrl, "square_media_url") ++ - getModelValueMediaUrl(primaryImageUrl, "media_url") ++ - authorId ++ - eventDescription - } - } - - private lazy val topicValuesFut: Future[Map[String, String]] = { - if (target.params(PushFeatureSwitchParams.EnableTopicCopyForMF)) { - followedTopicLocalizedEntities.map(_.headOption).flatMap { - case Some(localizedEntity) => - Future.value(Map("topic_name" -> localizedEntity.localizedNameForDisplay)) - case _ => - ergLocalizedEntities.map(_.headOption).map { - case Some(localizedEntity) - if target.params(PushFeatureSwitchParams.EnableTopicCopyForImplicitTopics) => - Map("topic_name" -> localizedEntity.localizedNameForDisplay) - case _ => Map.empty[String, String] - } - } - } else { - Future.value(Map.empty[String, String]) - } - } - - override lazy val modelValues: Future[Map[String, String]] = - mergeFutModelValues(super.modelValues, mergeFutModelValues(eventModelValuesFut, topicValuesFut)) - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutProductLaunchIbis2Hydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutProductLaunchIbis2Hydrator.docx new file mode 100644 index 000000000..a419a8bb4 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutProductLaunchIbis2Hydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutProductLaunchIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutProductLaunchIbis2Hydrator.scala deleted file mode 100644 index 3062a66d0..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutProductLaunchIbis2Hydrator.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.common.base.MagicFanoutProductLaunchCandidate -import com.twitter.frigate.magic_events.thriftscala.ProductInfo -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.util.PushIbisUtil.mergeModelValues -import com.twitter.util.Future - -trait MagicFanoutProductLaunchIbis2Hydrator - extends CustomConfigurationMapForIbis - with Ibis2HydratorForCandidate { - self: PushCandidate with MagicFanoutProductLaunchCandidate => - - private def getProductInfoMap(productInfo: ProductInfo): Map[String, String] = { - val titleMap = productInfo.title - .map { title => - Map("title" -> title) - }.getOrElse(Map.empty) - val bodyMap = productInfo.body - .map { body => - Map("body" -> body) - }.getOrElse(Map.empty) - val deeplinkMap = productInfo.deeplink - .map { deeplink => - Map("deeplink" -> deeplink) - }.getOrElse(Map.empty) - - titleMap ++ bodyMap ++ deeplinkMap - } - - private lazy val landingPage: Map[String, String] = { - val urlFromFS = target.params(PushFeatureSwitchParams.ProductLaunchLandingPageDeepLink) - Map("push_land_url" -> urlFromFS) - } - - private lazy val customProductLaunchPushDetails: Map[String, String] = { - frigateNotification.magicFanoutProductLaunchNotification match { - case Some(productLaunchNotif) => - productLaunchNotif.productInfo match { - case Some(productInfo) => - getProductInfoMap(productInfo) - case _ => Map.empty - } - case _ => Map.empty - } - } - - override lazy val customFieldsMapFut: Future[Map[String, String]] = - mergeModelValues(super.customFieldsMapFut, customProductLaunchPushDetails) - - override lazy val modelValues: Future[Map[String, String]] = - mergeModelValues(super.modelValues, landingPage) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutSportsEventIbis2Hydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutSportsEventIbis2Hydrator.docx new file mode 100644 index 000000000..1b9f3adc9 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutSportsEventIbis2Hydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutSportsEventIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutSportsEventIbis2Hydrator.scala deleted file mode 100644 index 811caa993..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutSportsEventIbis2Hydrator.scala +++ /dev/null @@ -1,89 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.common.base.BaseGameScore -import com.twitter.frigate.common.base.MagicFanoutSportsEventCandidate -import com.twitter.frigate.common.base.MagicFanoutSportsScoreInformation -import com.twitter.frigate.common.base.NflGameScore -import com.twitter.frigate.common.base.SoccerGameScore -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.MagicFanoutEventHydratedCandidate -import com.twitter.frigate.pushservice.params.PushConstants -import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutSportsUtil -import com.twitter.frigate.pushservice.util.PushIbisUtil._ -import com.twitter.util.Future - -trait MagicFanoutSportsEventIbis2Hydrator extends Ibis2HydratorForCandidate { - self: PushCandidate - with MagicFanoutEventHydratedCandidate - with MagicFanoutSportsEventCandidate - with MagicFanoutSportsScoreInformation => - - lazy val stats = self.statsReceiver.scope("MagicFanoutSportsEvent") - lazy val defaultImageCounter = stats.counter("default_image") - lazy val requestImageCounter = stats.counter("request_num") - lazy val noneImageCounter = stats.counter("none_num") - - override lazy val relevanceScoreMapFut = Future.value(Map.empty[String, String]) - - private def getModelValueMediaUrl( - urlOpt: Option[String], - mapKey: String - ): Option[(String, String)] = { - requestImageCounter.incr() - urlOpt match { - case Some(PushConstants.DefaultEventMediaUrl) => - defaultImageCounter.incr() - None - case Some(url) => Some(mapKey -> url) - case None => - noneImageCounter.incr() - None - } - } - - private lazy val eventModelValuesFut: Future[Map[String, String]] = { - for { - title <- eventTitleFut - squareImageUrl <- squareImageUrlFut - primaryImageUrl <- primaryImageUrlFut - } yield { - Map( - "event_id" -> s"$eventId", - "event_title" -> title - ) ++ - getModelValueMediaUrl(squareImageUrl, "square_media_url") ++ - getModelValueMediaUrl(primaryImageUrl, "media_url") - } - } - - private lazy val sportsScoreValues: Future[Map[String, String]] = { - for { - scores <- gameScores - homeName <- homeTeamInfo.map(_.map(_.name)) - awayName <- awayTeamInfo.map(_.map(_.name)) - } yield { - if (awayName.isDefined && homeName.isDefined && scores.isDefined) { - scores.get match { - case game: SoccerGameScore => - MagicFanoutSportsUtil.getSoccerIbisMap(game) ++ Map( - "away_team" -> awayName.get, - "home_team" -> homeName.get - ) - case game: NflGameScore => - MagicFanoutSportsUtil.getNflIbisMap(game) ++ Map( - "away_team" -> MagicFanoutSportsUtil.getNFLReadableName(awayName.get), - "home_team" -> MagicFanoutSportsUtil.getNFLReadableName(homeName.get) - ) - case baseGameScore: BaseGameScore => - Map.empty[String, String] - } - } else Map.empty[String, String] - } - } - - override lazy val customFieldsMapFut: Future[Map[String, String]] = - mergeFutModelValues(super.customFieldsMapFut, sportsScoreValues) - - override lazy val modelValues: Future[Map[String, String]] = - mergeFutModelValues(super.modelValues, eventModelValuesFut) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/OutOfNetworkTweetIbis2Hydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/OutOfNetworkTweetIbis2Hydrator.docx new file mode 100644 index 000000000..fa8c6a79a Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/OutOfNetworkTweetIbis2Hydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/OutOfNetworkTweetIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/OutOfNetworkTweetIbis2Hydrator.scala deleted file mode 100644 index c7bd051b7..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/OutOfNetworkTweetIbis2Hydrator.scala +++ /dev/null @@ -1,90 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.common.base.OutOfNetworkTweetCandidate -import com.twitter.frigate.common.base.TopicCandidate -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.rec_types.RecTypes._ -import com.twitter.frigate.common.util.MrPushCopyObjects -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.util.InlineActionUtil -import com.twitter.frigate.pushservice.util.PushIbisUtil.mergeModelValues -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.util.Future - -trait OutOfNetworkTweetIbis2HydratorForCandidate extends TweetCandidateIbis2Hydrator { - self: PushCandidate with OutOfNetworkTweetCandidate with TopicCandidate with TweetAuthorDetails => - - private lazy val useNewOonCopyValue = - if (target.params(PushFeatureSwitchParams.EnableNewMROONCopyForPush)) { - Map( - "use_new_oon_copy" -> "true" - ) - } else Map.empty[String, String] - - override lazy val tweetDynamicInlineActionsModelValues = - if (target.params(PushFeatureSwitchParams.EnableOONGeneratedInlineActions)) { - val actions = target.params(PushFeatureSwitchParams.OONTweetDynamicInlineActionsList) - InlineActionUtil.getGeneratedTweetInlineActions(target, statsReceiver, actions) - } else Map.empty[String, String] - - private lazy val ibtModelValues: Map[String, String] = - Map( - "is_tweet" -> s"${!(hasPhoto || hasVideo)}", - "is_photo" -> s"$hasPhoto", - "is_video" -> s"$hasVideo" - ) - - private lazy val launchVideosInImmersiveExploreValue = - Map( - "launch_videos_in_immersive_explore" -> s"${hasVideo && target.params(PushFeatureSwitchParams.EnableLaunchVideosInImmersiveExplore)}" - ) - - private lazy val oonTweetModelValues = - useNewOonCopyValue ++ ibtModelValues ++ tweetDynamicInlineActionsModelValues ++ launchVideosInImmersiveExploreValue - - lazy val useTopicCopyForMBCGIbis = mrModelingBasedTypes.contains(commonRecType) && target.params( - PushFeatureSwitchParams.EnableMrModelingBasedCandidatesTopicCopy) - lazy val useTopicCopyForFrsIbis = frsTypes.contains(commonRecType) && target.params( - PushFeatureSwitchParams.EnableFrsTweetCandidatesTopicCopy) - lazy val useTopicCopyForTagspaceIbis = tagspaceTypes.contains(commonRecType) && target.params( - PushFeatureSwitchParams.EnableHashspaceCandidatesTopicCopy) - - override lazy val modelName: String = { - if (localizedUttEntity.isDefined && - (useTopicCopyForMBCGIbis || useTopicCopyForFrsIbis || useTopicCopyForTagspaceIbis)) { - MrPushCopyObjects.TopicTweet.ibisPushModelName // uses topic copy - } else super.modelName - } - - lazy val exploreVideoParams: Map[String, String] = { - if (self.commonRecType == CommonRecommendationType.ExploreVideoTweet) { - Map( - "is_explore_video" -> "true" - ) - } else Map.empty[String, String] - } - - override lazy val customFieldsMapFut: Future[Map[String, String]] = - mergeModelValues(super.customFieldsMapFut, exploreVideoParams) - - override lazy val tweetModelValues: Future[Map[String, String]] = - if (localizedUttEntity.isDefined && - (useTopicCopyForMBCGIbis || useTopicCopyForFrsIbis || useTopicCopyForTagspaceIbis)) { - lazy val topicTweetModelValues: Map[String, String] = - Map("topic_name" -> s"${localizedUttEntity.get.localizedNameForDisplay}") - for { - superModelValues <- super.tweetModelValues - tweetInlineModelValue <- tweetInlineActionModelValue - } yield { - superModelValues ++ topicTweetModelValues ++ tweetInlineModelValue - } - } else { - for { - superModelValues <- super.tweetModelValues - tweetInlineModelValues <- tweetInlineActionModelValue - } yield { - superModelValues ++ mediaModelValue ++ oonTweetModelValues ++ tweetInlineModelValues ++ inlineVideoMediaMap - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/OverrideForIbis2Request.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/OverrideForIbis2Request.docx new file mode 100644 index 000000000..45deb3146 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/OverrideForIbis2Request.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/OverrideForIbis2Request.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/OverrideForIbis2Request.scala deleted file mode 100644 index e802a6421..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/OverrideForIbis2Request.scala +++ /dev/null @@ -1,210 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.common.store.deviceinfo.DeviceInfo -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FSParams} -import com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue.ContinuousFunction -import com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue.ContinuousFunctionParam -import com.twitter.frigate.pushservice.util.OverrideNotificationUtil -import com.twitter.frigate.pushservice.util.PushCapUtil -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.frigate.thriftscala.CommonRecommendationType.MagicFanoutSportsEvent -import com.twitter.ibis2.lib.util.JsonMarshal -import com.twitter.util.Future - -trait OverrideForIbis2Request { - self: PushCandidate => - - private lazy val overrideStats = self.statsReceiver.scope("override_for_ibis2") - - private lazy val addedOverrideAndroidCounter = - overrideStats.scope("android").counter("added_override_for_ibis2_request") - private lazy val addedSmartPushConfigAndroidCounter = - overrideStats.scope("android").counter("added_smart_push_config_for_ibis2_request") - private lazy val addedOverrideIosCounter = - overrideStats.scope("ios").counter("added_override_for_ibis2_request") - private lazy val noOverrideCounter = overrideStats.counter("no_override_for_ibis2_request") - private lazy val noOverrideDueToDeviceInfoCounter = - overrideStats.counter("no_override_due_to_device_info") - private lazy val addedMlScoreToPayloadAndroid = - overrideStats.scope("android").counter("added_ml_score") - private lazy val noMlScoreAddedToPayload = - overrideStats.counter("no_ml_score") - private lazy val addedNSlotsToPayload = - overrideStats.counter("added_n_slots") - private lazy val noNSlotsAddedToPayload = - overrideStats.counter("no_n_slots") - private lazy val addedCustomThreadIdToPayload = - overrideStats.counter("added_custom_thread_id") - private lazy val noCustomThreadIdAddedToPayload = - overrideStats.counter("no_custom_thread_id") - private lazy val enableTargetIdOverrideForMagicFanoutSportsEventCounter = - overrideStats.counter("enable_target_id_override_for_mf_sports_event") - - lazy val candidateModelScoreFut: Future[Option[Double]] = { - if (RecTypes.notEligibleForModelScoreTracking(commonRecType)) Future.None - else mrWeightedOpenOrNtabClickRankingProbability - } - - lazy val overrideModelValueFut: Future[Map[String, String]] = { - if (self.target.isLoggedOutUser) { - Future.value(Map.empty[String, String]) - } else { - Future - .join( - target.deviceInfo, - target.accountCountryCode, - OverrideNotificationUtil.getCollapseAndImpressionIdForOverride(self), - candidateModelScoreFut, - target.dynamicPushcap, - target.optoutAdjustedPushcap, - PushCapUtil.getDefaultPushCap(target) - ).map { - case ( - deviceInfoOpt, - countryCodeOpt, - Some((collapseId, impressionIds)), - mlScore, - dynamicPushcapOpt, - optoutAdjustedPushcapOpt, - defaultPushCap) => - val pushCap: Int = (dynamicPushcapOpt, optoutAdjustedPushcapOpt) match { - case (_, Some(optoutAdjustedPushcap)) => optoutAdjustedPushcap - case (Some(pushcapInfo), _) => pushcapInfo.pushcap - case _ => defaultPushCap - } - getClientSpecificOverrideModelValues( - target, - deviceInfoOpt, - countryCodeOpt, - collapseId, - impressionIds, - mlScore, - pushCap) - case _ => - noOverrideCounter.incr() - Map.empty[String, String] - } - } - } - - /** - * Determines the appropriate Override Notification model values based on the client - * @param target Target that will be receiving the push recommendation - * @param deviceInfoOpt Target's Device Info - * @param collapseId Collapse ID determined by OverrideNotificationUtil - * @param impressionIds Impression IDs of previously sent Override Notifications - * @param mlScore Open/NTab click ranking score of the current push candidate - * @param pushCap Push cap for the target - * @return Map consisting of the model values that need to be added to the Ibis2 Request - */ - def getClientSpecificOverrideModelValues( - target: Target, - deviceInfoOpt: Option[DeviceInfo], - countryCodeOpt: Option[String], - collapseId: String, - impressionIds: Seq[String], - mlScoreOpt: Option[Double], - pushCap: Int - ): Map[String, String] = { - - val primaryDeviceIos = PushDeviceUtil.isPrimaryDeviceIOS(deviceInfoOpt) - val primaryDeviceAndroid = PushDeviceUtil.isPrimaryDeviceAndroid(deviceInfoOpt) - - if (primaryDeviceIos || - (primaryDeviceAndroid && - target.params(FSParams.EnableOverrideNotificationsSmartPushConfigForAndroid))) { - - if (primaryDeviceIos) addedOverrideIosCounter.incr() - else addedSmartPushConfigAndroidCounter.incr() - - val impressionIdsSeq = { - if (target.params(FSParams.EnableTargetIdsInSmartPushPayload)) { - if (target.params(FSParams.EnableOverrideNotificationsMultipleTargetIds)) - impressionIds - else Seq(impressionIds.head) - } - // Explicitly enable targetId-based override for MagicFanoutSportsEvent candidates (live sport update notifications) - else if (self.commonRecType == MagicFanoutSportsEvent && target.params( - FSParams.EnableTargetIdInSmartPushPayloadForMagicFanoutSportsEvent)) { - enableTargetIdOverrideForMagicFanoutSportsEventCounter.incr() - Seq(impressionIds.head) - } else Seq.empty[String] - } - - val mlScoreMap = mlScoreOpt match { - case Some(mlScore) - if target.params(FSParams.EnableOverrideNotificationsScoreBasedOverride) => - addedMlScoreToPayloadAndroid.incr() - Map("score" -> mlScore) - case _ => - noMlScoreAddedToPayload.incr() - Map.empty - } - - val nSlotsMap = { - if (target.params(FSParams.EnableOverrideNotificationsNSlots)) { - if (target.params(FSParams.EnableOverrideMaxSlotFn)) { - val nslotFnParam = ContinuousFunctionParam( - target - .params(PushFeatureSwitchParams.OverrideMaxSlotFnPushCapKnobs), - target - .params(PushFeatureSwitchParams.OverrideMaxSlotFnNSlotKnobs), - target - .params(PushFeatureSwitchParams.OverrideMaxSlotFnPowerKnobs), - target - .params(PushFeatureSwitchParams.OverrideMaxSlotFnWeight), - target.params(FSParams.OverrideNotificationsMaxNumOfSlots) - ) - val numOfSlots = ContinuousFunction.safeEvaluateFn( - pushCap, - nslotFnParam, - overrideStats.scope("max_nslot_fn")) - overrideStats.counter("max_notification_slots_num_" + numOfSlots.toString).incr() - addedNSlotsToPayload.incr() - Map("max_notification_slots" -> numOfSlots) - } else { - addedNSlotsToPayload.incr() - val numOfSlots = target.params(FSParams.OverrideNotificationsMaxNumOfSlots) - Map("max_notification_slots" -> numOfSlots) - } - } else { - noNSlotsAddedToPayload.incr() - Map.empty - } - } - - val baseActionDetailsMap = Map("target_ids" -> impressionIdsSeq) - - val actionDetailsMap = - Map("action_details" -> (baseActionDetailsMap ++ nSlotsMap)) - - val baseSmartPushConfigMap = Map("notification_action" -> "REPLACE") - - val customThreadId = { - if (target.params(FSParams.EnableCustomThreadIdForOverride)) { - addedCustomThreadIdToPayload.incr() - Map("custom_thread_id" -> impressionId) - } else { - noCustomThreadIdAddedToPayload.incr() - Map.empty - } - } - - val smartPushConfigMap = - JsonMarshal.toJson( - baseSmartPushConfigMap ++ actionDetailsMap ++ mlScoreMap ++ customThreadId) - - Map("smart_notification_configuration" -> smartPushConfigMap) - } else if (primaryDeviceAndroid) { - addedOverrideAndroidCounter.incr() - Map("notification_id" -> collapseId, "overriding_impression_id" -> impressionIds.head) - } else { - noOverrideDueToDeviceInfoCounter.incr() - Map.empty[String, String] - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/PushOverrideInfo.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/PushOverrideInfo.docx new file mode 100644 index 000000000..312cf6cbe Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/PushOverrideInfo.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/PushOverrideInfo.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/PushOverrideInfo.scala deleted file mode 100644 index 359b876a1..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/PushOverrideInfo.scala +++ /dev/null @@ -1,246 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.history.History -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.frigate.thriftscala.OverrideInfo -import com.twitter.util.Duration -import com.twitter.util.Time - -object PushOverrideInfo { - - private val name: String = this.getClass.getSimpleName - - /** - * Gets all eligible time + override push notification pairs from a target's History - * - * @param history: history of push notifications - * @param lookbackDuration: duration to look back up in history for overriding notifications - * @return: list of notifications with send timestamps which are eligible for overriding - */ - def getOverrideEligibleHistory( - history: History, - lookbackDuration: Duration, - ): Seq[(Time, FrigateNotification)] = { - history.sortedHistory - .takeWhile { case (notifTimestamp, _) => lookbackDuration.ago < notifTimestamp } - .filter { - case (_, notification) => notification.overrideInfo.isDefined - } - } - - /** - * Gets all eligible override push notifications from a target's History - * - * @param history Target's History - * @param lookbackDuration Duration in which we would like to obtain the eligible push notifications - * @param stats StatsReceiver to track stats for this function - * @return Returns a list of FrigateNotification - */ - def getOverrideEligiblePushNotifications( - history: History, - lookbackDuration: Duration, - stats: StatsReceiver, - ): Seq[FrigateNotification] = { - val eligibleNotificationsDistribution = - stats.scope(name).stat("eligible_notifications_size_distribution") - val eligibleNotificationsSeq = - getOverrideEligibleHistory(history, lookbackDuration) - .collect { - case (_, notification) => notification - } - - eligibleNotificationsDistribution.add(eligibleNotificationsSeq.size) - eligibleNotificationsSeq - } - - /** - * Gets the OverrideInfo for the last eligible Override Notification FrigateNotification, if it exists - * @param history Target's History - * @param lookbackDuration Duration in which we would like to obtain the last override notification - * @param stats StatsReceiver to track stats for this function - * @return Returns OverrideInfo of the last MR push, else None - */ - def getOverrideInfoOfLastEligiblePushNotif( - history: History, - lookbackDuration: Duration, - stats: StatsReceiver - ): Option[OverrideInfo] = { - val overrideInfoEmptyOfLastPush = stats.scope(name).counter("override_info_empty_of_last_push") - val overrideInfoExistsForLastPush = - stats.scope(name).counter("override_info_exists_for_last_push") - val overrideHistory = - getOverrideEligiblePushNotifications(history, lookbackDuration, stats) - if (overrideHistory.isEmpty) { - overrideInfoEmptyOfLastPush.incr() - None - } else { - overrideInfoExistsForLastPush.incr() - overrideHistory.head.overrideInfo - } - } - - /** - * Gets all the MR Push Notifications in the specified override chain - * @param history Target's History - * @param overrideChainId Override Chain Identifier - * @param stats StatsReceiver to track stats for this function - * @return Returns a sequence of FrigateNotification that exist in the override chain - */ - def getMrPushNotificationsInOverrideChain( - history: History, - overrideChainId: String, - stats: StatsReceiver - ): Seq[FrigateNotification] = { - val notificationInOverrideChain = stats.scope(name).counter("notification_in_override_chain") - val notificationNotInOverrideChain = - stats.scope(name).counter("notification_not_in_override_chain") - history.sortedHistory.flatMap { - case (_, notification) - if isNotificationInOverrideChain(notification, overrideChainId, stats) => - notificationInOverrideChain.incr() - Some(notification) - case _ => - notificationNotInOverrideChain.incr() - None - } - } - - /** - * Gets the timestamp (in milliseconds) for the specified FrigateNotification - * @param notification The FrigateNotification that we would like the timestamp for - * @param history Target's History - * @param stats StatsReceiver to track stats for this function - * @return Returns the timestamp in milliseconds for the specified notification - * if it exists History, else None - */ - def getTimestampInMillisForFrigateNotification( - notification: FrigateNotification, - history: History, - stats: StatsReceiver - ): Option[Long] = { - val foundTimestampOfNotificationInHistory = - stats.scope(name).counter("found_timestamp_of_notification_in_history") - history.sortedHistory - .find(_._2.equals(notification)).map { - case (time, _) => - foundTimestampOfNotificationInHistory.incr() - time.inMilliseconds - } - } - - /** - * Gets the oldest frigate notification based on the user's NTab last read position - * @param overrideCandidatesMap All the NTab Notifications in the override chain - * @return Returns the oldest frigate notification in the chain - */ - def getOldestFrigateNotification( - overrideCandidatesMap: Map[Long, FrigateNotification], - ): FrigateNotification = { - overrideCandidatesMap.minBy(_._1)._2 - } - - /** - * Gets the impression ids of previous eligible push notification. - * @param history Target's History - * @param lookbackDuration Duration in which we would like to obtain previous impression ids - * @param stats StatsReceiver to track stats for this function - * @return Returns the impression identifier for the last eligible push notif. - * if it exists in the target's History, else None. - */ - def getImpressionIdsOfPrevEligiblePushNotif( - history: History, - lookbackDuration: Duration, - stats: StatsReceiver - ): Seq[String] = { - val foundImpressionIdOfLastEligiblePushNotif = - stats.scope(name).counter("found_impression_id_of_last_eligible_push_notif") - val overrideHistoryEmptyWhenFetchingImpressionId = - stats.scope(name).counter("override_history_empty_when_fetching_impression_id") - val overrideHistory = getOverrideEligiblePushNotifications(history, lookbackDuration, stats) - .filter(frigateNotification => - // Exclude notifications of nonGenericOverrideTypes from being overridden - !RecTypes.nonGenericOverrideTypes.contains(frigateNotification.commonRecommendationType)) - - if (overrideHistory.isEmpty) { - overrideHistoryEmptyWhenFetchingImpressionId.incr() - Seq.empty - } else { - foundImpressionIdOfLastEligiblePushNotif.incr() - overrideHistory.flatMap(_.impressionId) - } - } - - /** - * Gets the impressions ids by eventId, for MagicFanoutEvent candidates. - * - * @param history Target's History - * @param lookbackDuration Duration in which we would like to obtain previous impression ids - * @param stats StatsReceiver to track stats for this function - * @param overridableType Specific MagicFanoutEvent CRT - * @param eventId Event identifier for MagicFanoutEventCandidate. - * @return Returns the impression identifiers for the last eligible, eventId-matching - * MagicFanoutEvent push notifications if they exist in the target's history, else None. - */ - def getImpressionIdsForPrevEligibleMagicFanoutEventCandidates( - history: History, - lookbackDuration: Duration, - stats: StatsReceiver, - overridableType: CommonRecommendationType, - eventId: Long - ): Seq[String] = { - val foundImpressionIdOfMagicFanoutEventNotif = - stats.scope(name).counter("found_impression_id_of_magic_fanout_event_notif") - val overrideHistoryEmptyWhenFetchingImpressionId = - stats - .scope(name).counter( - "override_history_empty_when_fetching_impression_id_for_magic_fanout_event_notif") - - val overrideHistory = - getOverrideEligiblePushNotifications(history, lookbackDuration, stats) - .filter(frigateNotification => - // Only override notifications with same CRT and eventId - frigateNotification.commonRecommendationType == overridableType && - frigateNotification.magicFanoutEventNotification.exists(_.eventId == eventId)) - - if (overrideHistory.isEmpty) { - overrideHistoryEmptyWhenFetchingImpressionId.incr() - Seq.empty - } else { - foundImpressionIdOfMagicFanoutEventNotif.incr() - overrideHistory.flatMap(_.impressionId) - } - } - - /** - * Determines if the provided notification is part of the specified override chain - * @param notification FrigateNotification that we're trying to identify as within the override chain - * @param overrideChainId Override Chain Identifier - * @param stats StatsReceiver to track stats for this function - * @return Returns true if the provided FrigateNotification is within the override chain, else false - */ - private def isNotificationInOverrideChain( - notification: FrigateNotification, - overrideChainId: String, - stats: StatsReceiver - ): Boolean = { - val notifIsInOverrideChain = stats.scope(name).counter("notif_is_in_override_chain") - val notifNotInOverrideChain = stats.scope(name).counter("notif_not_in_override_chain") - notification.overrideInfo match { - case Some(overrideInfo) => - val isNotifInOverrideChain = overrideInfo.collapseInfo.overrideChainId == overrideChainId - if (isNotifInOverrideChain) { - notifIsInOverrideChain.incr() - true - } else { - notifNotInOverrideChain.incr() - false - } - case _ => - notifNotInOverrideChain.incr() - false - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/RankedSocialContextIbis2Hydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/RankedSocialContextIbis2Hydrator.docx new file mode 100644 index 000000000..d4beb60d4 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/RankedSocialContextIbis2Hydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/RankedSocialContextIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/RankedSocialContextIbis2Hydrator.scala deleted file mode 100644 index 479c230eb..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/RankedSocialContextIbis2Hydrator.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.common.base.SocialContextAction -import com.twitter.frigate.common.base.SocialContextActions -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.util.CandidateUtil -import com.twitter.frigate.pushservice.util.PushIbisUtil -import com.twitter.util.Future - -trait RankedSocialContextIbis2Hydrator { - self: PushCandidate with SocialContextActions => - - lazy val socialContextModelValues: Future[Map[String, String]] = - rankedSocialContextActionsFut.map(rankedSocialContextActions => - PushIbisUtil.getSocialContextModelValues(rankedSocialContextActions.map(_.userId))) - - lazy val rankedSocialContextActionsFut: Future[Seq[SocialContextAction]] = - CandidateUtil.getRankedSocialContext( - socialContextActions, - target.seedsWithWeight, - defaultToRecency = false) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ScheduledSpaceSpeakerIbis2Hydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ScheduledSpaceSpeakerIbis2Hydrator.docx new file mode 100644 index 000000000..d2f5bc485 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ScheduledSpaceSpeakerIbis2Hydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ScheduledSpaceSpeakerIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ScheduledSpaceSpeakerIbis2Hydrator.scala deleted file mode 100644 index d1a439972..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ScheduledSpaceSpeakerIbis2Hydrator.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.pushservice.model.ScheduledSpaceSpeakerPushCandidate -import com.twitter.frigate.pushservice.util.PushIbisUtil._ -import com.twitter.frigate.thriftscala.SpaceNotificationType -import com.twitter.util.Future - -trait ScheduledSpaceSpeakerIbis2Hydrator extends Ibis2HydratorForCandidate { - self: ScheduledSpaceSpeakerPushCandidate => - - override lazy val senderId: Option[Long] = None - - private lazy val targetModelValues: Future[Map[String, String]] = { - hostId match { - case Some(spaceHostId) => - audioSpaceFut.map { audioSpace => - val isStartNow = frigateNotification.spaceNotification.exists( - _.spaceNotificationType.contains(SpaceNotificationType.AtSpaceBroadcast)) - - Map( - "host_id" -> s"$spaceHostId", - "space_id" -> spaceId, - "is_start_now" -> s"$isStartNow" - ) ++ audioSpace.flatMap(_.title.map("space_title" -> _)) - } - case _ => - Future.exception( - new IllegalStateException("Unable to get host id for ScheduledSpaceSpeakerIbis2Hydrator")) - } - } - - override lazy val modelValues: Future[Map[String, String]] = - mergeFutModelValues(super.modelValues, targetModelValues) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ScheduledSpaceSubscriberIbis2Hydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ScheduledSpaceSubscriberIbis2Hydrator.docx new file mode 100644 index 000000000..d23497843 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ScheduledSpaceSubscriberIbis2Hydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ScheduledSpaceSubscriberIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ScheduledSpaceSubscriberIbis2Hydrator.scala deleted file mode 100644 index b1486de3f..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ScheduledSpaceSubscriberIbis2Hydrator.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.pushservice.model.ScheduledSpaceSubscriberPushCandidate -import com.twitter.frigate.pushservice.util.PushIbisUtil._ -import com.twitter.util.Future - -trait ScheduledSpaceSubscriberIbis2Hydrator extends Ibis2HydratorForCandidate { - self: ScheduledSpaceSubscriberPushCandidate => - - override lazy val senderId: Option[Long] = hostId - - private lazy val targetModelValues: Future[Map[String, String]] = { - hostId match { - case Some(spaceHostId) => - audioSpaceFut.map { audioSpace => - Map( - "host_id" -> s"$spaceHostId", - "space_id" -> spaceId, - ) ++ audioSpace.flatMap(_.title.map("space_title" -> _)) - } - case _ => - Future.exception( - new RuntimeException("Unable to get host id for ScheduledSpaceSubscriberIbis2Hydrator")) - } - } - - override lazy val modelValues: Future[Map[String, String]] = - mergeFutModelValues(super.modelValues, targetModelValues) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/SubscribedSearchTweetIbis2Hydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/SubscribedSearchTweetIbis2Hydrator.docx new file mode 100644 index 000000000..ad7241ee6 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/SubscribedSearchTweetIbis2Hydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/SubscribedSearchTweetIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/SubscribedSearchTweetIbis2Hydrator.scala deleted file mode 100644 index a61edc509..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/SubscribedSearchTweetIbis2Hydrator.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.pushservice.model.SubscribedSearchTweetPushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.util.InlineActionUtil -import com.twitter.util.Future - -trait SubscribedSearchTweetIbis2Hydrator extends TweetCandidateIbis2Hydrator { - self: SubscribedSearchTweetPushCandidate => - - override lazy val tweetDynamicInlineActionsModelValues = { - if (target.params(PushFeatureSwitchParams.EnableOONGeneratedInlineActions)) { - val actions = target.params(PushFeatureSwitchParams.TweetDynamicInlineActionsList) - InlineActionUtil.getGeneratedTweetInlineActions(target, statsReceiver, actions) - } else Map.empty[String, String] - } - - private lazy val searchTermValue: Map[String, String] = - Map( - "search_term" -> searchTerm, - "search_url" -> pushLandingUrl - ) - - private lazy val searchModelValues = searchTermValue ++ tweetDynamicInlineActionsModelValues - - override lazy val tweetModelValues: Future[Map[String, String]] = - for { - superModelValues <- super.tweetModelValues - tweetInlineModelValues <- tweetInlineActionModelValue - } yield { - superModelValues ++ mediaModelValue ++ searchModelValues ++ tweetInlineModelValues ++ inlineVideoMediaMap - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TopTweetImpressionsCandidateIbis2Hydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TopTweetImpressionsCandidateIbis2Hydrator.docx new file mode 100644 index 000000000..56286e905 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TopTweetImpressionsCandidateIbis2Hydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TopTweetImpressionsCandidateIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TopTweetImpressionsCandidateIbis2Hydrator.scala deleted file mode 100644 index e12733fb2..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TopTweetImpressionsCandidateIbis2Hydrator.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.common.base.TopTweetImpressionsCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.util.PushIbisUtil.mergeFutModelValues -import com.twitter.util.Future - -trait TopTweetImpressionsCandidateIbis2Hydrator extends Ibis2HydratorForCandidate { - self: PushCandidate with TopTweetImpressionsCandidate => - - private lazy val targetModelValues: Map[String, String] = { - Map( - "target_user" -> target.targetId.toString, - "tweet" -> tweetId.toString, - "impressions_count" -> impressionsCount.toString - ) - } - - override lazy val modelValues: Future[Map[String, String]] = - mergeFutModelValues(super.modelValues, Future.value(targetModelValues)) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TopicProofTweetIbis2Hydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TopicProofTweetIbis2Hydrator.docx new file mode 100644 index 000000000..7f6fd1616 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TopicProofTweetIbis2Hydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TopicProofTweetIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TopicProofTweetIbis2Hydrator.scala deleted file mode 100644 index 6a187dfeb..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TopicProofTweetIbis2Hydrator.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.pushservice.model.TopicProofTweetPushCandidate -import com.twitter.frigate.pushservice.exception.UttEntityNotFoundException -import com.twitter.util.Future - -trait TopicProofTweetIbis2Hydrator extends TweetCandidateIbis2Hydrator { - self: TopicProofTweetPushCandidate => - - private lazy val implicitTopicTweetModelValues: Map[String, String] = { - val uttEntity = localizedUttEntity.getOrElse( - throw new UttEntityNotFoundException( - s"${getClass.getSimpleName} UttEntity missing for $tweetId")) - - Map( - "topic_name" -> uttEntity.localizedNameForDisplay, - "topic_id" -> uttEntity.entityId.toString - ) - } - - override lazy val modelName: String = pushCopy.ibisPushModelName - - override lazy val tweetModelValues: Future[Map[String, String]] = - for { - superModelValues <- super.tweetModelValues - tweetInlineModelValues <- tweetInlineActionModelValue - } yield { - superModelValues ++ - tweetInlineModelValues ++ - implicitTopicTweetModelValues - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TrendTweetIbis2Hydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TrendTweetIbis2Hydrator.docx new file mode 100644 index 000000000..43c30dedf Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TrendTweetIbis2Hydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TrendTweetIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TrendTweetIbis2Hydrator.scala deleted file mode 100644 index 1c3420df4..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TrendTweetIbis2Hydrator.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.common.base.TrendTweetCandidate -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate - -trait TrendTweetIbis2Hydrator extends TweetCandidateIbis2Hydrator { - self: PushCandidate with TrendTweetCandidate with TweetAuthorDetails => - - lazy val trendNameModelValue = Map("trend_name" -> trendName) - - override lazy val tweetModelValues = for { - tweetValues <- super.tweetModelValues - inlineActionValues <- tweetInlineActionModelValue - } yield tweetValues ++ inlineActionValues ++ trendNameModelValue -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetCandidateIbis2Hydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetCandidateIbis2Hydrator.docx new file mode 100644 index 000000000..788787132 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetCandidateIbis2Hydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetCandidateIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetCandidateIbis2Hydrator.scala deleted file mode 100644 index 0b0a5db05..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetCandidateIbis2Hydrator.scala +++ /dev/null @@ -1,166 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.common.base.TweetDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.SubtextForAndroidPushHeader -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} -import com.twitter.frigate.pushservice.util.CopyUtil -import com.twitter.frigate.pushservice.util.EmailLandingPageExperimentUtil -import com.twitter.frigate.pushservice.util.InlineActionUtil -import com.twitter.frigate.pushservice.util.PushToHomeUtil -import com.twitter.frigate.pushservice.util.PushIbisUtil.mergeFutModelValues -import com.twitter.util.Future - -trait TweetCandidateIbis2Hydrator - extends Ibis2HydratorForCandidate - with InlineActionIbis2Hydrator - with CustomConfigurationMapForIbis { - self: PushCandidate with TweetCandidate with TweetDetails with TweetAuthorDetails => - - lazy val scopedStats: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) - - lazy val tweetIdModelValue: Map[String, String] = - Map( - "tweet" -> tweetId.toString - ) - - lazy val authorModelValue: Map[String, String] = { - assert(authorId.isDefined) - Map( - "author" -> authorId.getOrElse(0L).toString - ) - } - - lazy val otherModelValues: Map[String, String] = - Map( - "show_explanatory_text" -> "true", - "show_negative_feedback" -> "true" - ) - - lazy val mediaModelValue: Map[String, String] = - Map( - "show_media" -> "true" - ) - - lazy val inlineVideoMediaMap: Map[String, String] = { - if (hasVideo) { - val isInlineVideoEnabled = target.params(FS.EnableInlineVideo) - val isAutoplayEnabled = target.params(FS.EnableAutoplayForInlineVideo) - Map( - "enable_inline_video_for_ios" -> isInlineVideoEnabled.toString, - "enable_autoplay_for_inline_video_ios" -> isAutoplayEnabled.toString - ) - } else Map.empty - } - - lazy val landingPageModelValues: Future[Map[String, String]] = { - for { - deviceInfoOpt <- target.deviceInfo - } yield { - PushToHomeUtil.getIbis2ModelValue(deviceInfoOpt, target, scopedStats) match { - case Some(pushToHomeModelValues) => pushToHomeModelValues - case _ => - EmailLandingPageExperimentUtil.getIbis2ModelValue( - deviceInfoOpt, - target, - tweetId - ) - } - } - } - - lazy val tweetDynamicInlineActionsModelValues = { - if (target.params(PushFeatureSwitchParams.EnableTweetDynamicInlineActions)) { - val actions = target.params(PushFeatureSwitchParams.TweetDynamicInlineActionsList) - InlineActionUtil.getGeneratedTweetInlineActions(target, statsReceiver, actions) - } else Map.empty[String, String] - } - - lazy val tweetDynamicInlineActionsModelValuesForWeb: Map[String, String] = { - if (target.isLoggedOutUser) { - Map.empty[String, String] - } else { - InlineActionUtil.getGeneratedTweetInlineActionsForWeb( - actions = target.params(PushFeatureSwitchParams.TweetDynamicInlineActionsListForWeb), - enableForDesktopWeb = - target.params(PushFeatureSwitchParams.EnableDynamicInlineActionsForDesktopWeb), - enableForMobileWeb = - target.params(PushFeatureSwitchParams.EnableDynamicInlineActionsForMobileWeb) - ) - } - } - - lazy val copyFeaturesFut: Future[Map[String, String]] = - CopyUtil.getCopyFeatures(self, scopedStats) - - private def getVerifiedSymbolModelValue: Future[Map[String, String]] = { - self.tweetAuthor.map { - case Some(author) => - if (author.safety.exists(_.verified)) { - scopedStats.counter("is_verified").incr() - if (target.params(FS.EnablePushPresentationVerifiedSymbol)) { - scopedStats.counter("is_verified_and_add").incr() - Map("is_author_verified" -> "true") - } else { - scopedStats.counter("is_verified_and_NOT_add").incr() - Map.empty - } - } else { - scopedStats.counter("is_NOT_verified").incr() - Map.empty - } - case _ => - scopedStats.counter("none_author").incr() - Map.empty - } - } - - private def subtextAndroidPushHeader: Map[String, String] = { - self.target.params(PushFeatureSwitchParams.SubtextInAndroidPushHeaderParam) match { - case SubtextForAndroidPushHeader.None => - Map.empty - case SubtextForAndroidPushHeader.TargetHandler => - Map("subtext_target_handler" -> "true") - case SubtextForAndroidPushHeader.TargetTagHandler => - Map("subtext_target_tag_handler" -> "true") - case SubtextForAndroidPushHeader.TargetName => - Map("subtext_target_name" -> "true") - case SubtextForAndroidPushHeader.AuthorTagHandler => - Map("subtext_author_tag_handler" -> "true") - case SubtextForAndroidPushHeader.AuthorName => - Map("subtext_author_name" -> "true") - case _ => - Map.empty - } - } - - lazy val bodyPushMap: Map[String, String] = { - if (self.target.params(PushFeatureSwitchParams.EnableEmptyBody)) { - Map("enable_empty_body" -> "true") - } else Map.empty[String, String] - } - - override def customFieldsMapFut: Future[Map[String, String]] = - for { - superModelValues <- super.customFieldsMapFut - copyFeaturesModelValues <- copyFeaturesFut - verifiedSymbolModelValue <- getVerifiedSymbolModelValue - } yield { - superModelValues ++ copyFeaturesModelValues ++ - verifiedSymbolModelValue ++ subtextAndroidPushHeader ++ bodyPushMap - } - - override lazy val senderId: Option[Long] = authorId - - def tweetModelValues: Future[Map[String, String]] = - landingPageModelValues.map { landingPageModelValues => - tweetIdModelValue ++ authorModelValue ++ landingPageModelValues ++ tweetDynamicInlineActionsModelValues ++ tweetDynamicInlineActionsModelValuesForWeb - } - - override lazy val modelValues: Future[Map[String, String]] = - mergeFutModelValues(super.modelValues, tweetModelValues) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetFavoriteIbis2Hydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetFavoriteIbis2Hydrator.docx new file mode 100644 index 000000000..0526acd59 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetFavoriteIbis2Hydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetFavoriteIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetFavoriteIbis2Hydrator.scala deleted file mode 100644 index ae4cd9174..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetFavoriteIbis2Hydrator.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.base.TweetFavoriteCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.util.Future - -trait TweetFavoriteCandidateIbis2Hydrator - extends TweetCandidateIbis2Hydrator - with RankedSocialContextIbis2Hydrator { - self: PushCandidate with TweetFavoriteCandidate with TweetAuthorDetails => - - override lazy val tweetModelValues: Future[Map[String, String]] = - for { - socialContextModelValues <- socialContextModelValues - superModelValues <- super.tweetModelValues - tweetInlineModelValues <- tweetInlineActionModelValue - } yield { - superModelValues ++ mediaModelValue ++ otherModelValues ++ socialContextModelValues ++ tweetInlineModelValues - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetRetweetIbis2Hydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetRetweetIbis2Hydrator.docx new file mode 100644 index 000000000..d5f7860a4 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetRetweetIbis2Hydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetRetweetIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetRetweetIbis2Hydrator.scala deleted file mode 100644 index 2b665a8fa..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetRetweetIbis2Hydrator.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.frigate.pushservice.model.ibis - -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.base.TweetRetweetCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.util.PushIbisUtil.mergeModelValues - -import com.twitter.util.Future - -trait TweetRetweetCandidateIbis2Hydrator - extends TweetCandidateIbis2Hydrator - with RankedSocialContextIbis2Hydrator { - self: PushCandidate with TweetRetweetCandidate with TweetAuthorDetails => - - override lazy val tweetModelValues: Future[Map[String, String]] = - for { - socialContextModelValues <- socialContextModelValues - superModelValues <- super.tweetModelValues - tweetInlineModelValues <- tweetInlineActionModelValue - } yield { - superModelValues ++ mediaModelValue ++ otherModelValues ++ socialContextModelValues ++ tweetInlineModelValues ++ inlineVideoMediaMap - } - - lazy val socialContextForRetweetMap: Map[String, String] = - if (self.target.params(PushFeatureSwitchParams.EnableSocialContextForRetweet)) { - Map("enable_social_context_retweet" -> "true") - } else Map.empty[String, String] - - override lazy val customFieldsMapFut: Future[Map[String, String]] = - mergeModelValues(super.customFieldsMapFut, socialContextForRetweetMap) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/CandidateNTabCopy.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/CandidateNTabCopy.docx new file mode 100644 index 000000000..eb73c77ba Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/CandidateNTabCopy.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/CandidateNTabCopy.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/CandidateNTabCopy.scala deleted file mode 100644 index ef80db5b5..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/CandidateNTabCopy.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.common.util.MRNtabCopy -import com.twitter.frigate.common.util.MrNtabCopyObjects -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.take.InvalidNtabCopyIdException -import com.twitter.frigate.pushservice.take.NtabCopyIdNotFoundException - -trait CandidateNTabCopy { - self: PushCandidate => - - def ntabCopy: MRNtabCopy = - ntabCopyId - .map(getNtabCopyFromCopyId).getOrElse( - throw new NtabCopyIdNotFoundException(s"NtabCopyId not found for $commonRecType")) - - private def getNtabCopyFromCopyId(ntabCopyId: Int): MRNtabCopy = - MrNtabCopyObjects - .getCopyFromId(ntabCopyId).getOrElse( - throw new InvalidNtabCopyIdException(s"Unknown NTab Copy ID: $ntabCopyId")) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/DiscoverTwitterNtabRequestHydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/DiscoverTwitterNtabRequestHydrator.docx new file mode 100644 index 000000000..75b836eb1 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/DiscoverTwitterNtabRequestHydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/DiscoverTwitterNtabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/DiscoverTwitterNtabRequestHydrator.scala deleted file mode 100644 index 4d6d67893..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/DiscoverTwitterNtabRequestHydrator.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushConstants -import com.twitter.frigate.thriftscala.{CommonRecommendationType => CRT} -import com.twitter.notificationservice.thriftscala._ -import com.twitter.util.Future -import com.twitter.util.Time - -trait DiscoverTwitterNtabRequestHydrator extends NTabRequestHydrator { - self: PushCandidate => - - override val senderIdFut: Future[Long] = Future.value(0L) - - override val tapThroughFut: Future[String] = - commonRecType match { - case CRT.AddressBookUploadPush => Future.value(PushConstants.AddressBookUploadTapThrough) - case CRT.InterestPickerPush => Future.value(PushConstants.InterestPickerTapThrough) - case CRT.CompleteOnboardingPush => - Future.value(PushConstants.CompleteOnboardingInterestAddressTapThrough) - case _ => - Future.value(PushConstants.ConnectTabPushTapThrough) - } - - override val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = Future.Nil - - override val facepileUsersFut: Future[Seq[Long]] = Future.Nil - - override val storyContext: Option[StoryContext] = None - - override val inlineCard: Option[InlineCard] = None - - override val socialProofDisplayText: Option[DisplayText] = Some(DisplayText()) - - override lazy val ntabRequest: Future[Option[CreateGenericNotificationRequest]] = - if (self.commonRecType == CRT.ConnectTabPush || RecTypes.isOnboardingFlowType( - self.commonRecType)) { - Future.join(senderIdFut, displayTextEntitiesFut, facepileUsersFut, tapThroughFut).map { - case (senderId, displayTextEntities, facepileUsers, tapThrough) => - Some( - CreateGenericNotificationRequest( - userId = target.targetId, - senderId = senderId, - genericType = GenericType.RefreshableNotification, - displayText = DisplayText(values = displayTextEntities), - facepileUsers = facepileUsers, - timestampMillis = Time.now.inMillis, - tapThroughAction = Some(TapThroughAction(Some(tapThrough))), - impressionId = Some(impressionId), - socialProofText = socialProofDisplayText, - context = storyContext, - inlineCard = inlineCard, - refreshableType = refreshableType - )) - } - } else Future.None -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/EventNTabRequestHydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/EventNTabRequestHydrator.docx new file mode 100644 index 000000000..de542d63c Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/EventNTabRequestHydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/EventNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/EventNTabRequestHydrator.scala deleted file mode 100644 index 082bc1742..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/EventNTabRequestHydrator.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.notificationservice.thriftscala.DisplayText -import com.twitter.notificationservice.thriftscala.InlineCard -import com.twitter.notificationservice.thriftscala.StoryContext -import com.twitter.util.Future - -trait EventNTabRequestHydrator extends NTabRequestHydrator { - self: PushCandidate => - - override def senderIdFut: Future[Long] = Future.value(0L) - - override def facepileUsersFut: Future[Seq[Long]] = Future.Nil - - override val storyContext: Option[StoryContext] = None - - override val inlineCard: Option[InlineCard] = None - - override val socialProofDisplayText: Option[DisplayText] = Some(DisplayText()) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/F1FirstDegreeTweetNTabRequestHydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/F1FirstDegreeTweetNTabRequestHydrator.docx new file mode 100644 index 000000000..a9645261c Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/F1FirstDegreeTweetNTabRequestHydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/F1FirstDegreeTweetNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/F1FirstDegreeTweetNTabRequestHydrator.scala deleted file mode 100644 index 18662e257..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/F1FirstDegreeTweetNTabRequestHydrator.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.take.NotificationServiceSender -import com.twitter.notificationservice.thriftscala.DisplayTextEntity -import com.twitter.util.Future - -trait F1FirstDegreeTweetNTabRequestHydrator extends TweetNTabRequestHydrator { - self: PushCandidate with TweetCandidate with TweetAuthorDetails => - - override val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = - NotificationServiceSender.getDisplayTextEntityFromUser(tweetAuthor, "author", true).map(_.toSeq) - - override lazy val facepileUsersFut: Future[Seq[Long]] = senderIdFut.map(Seq(_)) - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/ListCandidateNTabRequestHydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/ListCandidateNTabRequestHydrator.docx new file mode 100644 index 000000000..7de81413b Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/ListCandidateNTabRequestHydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/ListCandidateNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/ListCandidateNTabRequestHydrator.scala deleted file mode 100644 index 8475256ad..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/ListCandidateNTabRequestHydrator.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.pushservice.model.ListRecommendationPushCandidate -import com.twitter.notificationservice.thriftscala.DisplayText -import com.twitter.notificationservice.thriftscala.DisplayTextEntity -import com.twitter.notificationservice.thriftscala.InlineCard -import com.twitter.notificationservice.thriftscala.StoryContext -import com.twitter.notificationservice.thriftscala.TextValue -import com.twitter.util.Future - -trait ListCandidateNTabRequestHydrator extends NTabRequestHydrator { - - self: ListRecommendationPushCandidate => - - override lazy val senderIdFut: Future[Long] = - listOwnerId.map(_.getOrElse(0L)) - - override lazy val facepileUsersFut: Future[Seq[Long]] = Future.Nil - - override lazy val storyContext: Option[StoryContext] = None - - override lazy val inlineCard: Option[InlineCard] = None - - override lazy val tapThroughFut: Future[String] = Future.value(s"i/lists/${listId}") - - override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = listName.map { - listNameOpt => - listNameOpt.toSeq.map { name => - DisplayTextEntity(name = "title", value = TextValue.Text(name)) - } - } - - override val socialProofDisplayText: Option[DisplayText] = Some(DisplayText()) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutCreatorEventNtabRequestHydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutCreatorEventNtabRequestHydrator.docx new file mode 100644 index 000000000..a6c3db2b1 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutCreatorEventNtabRequestHydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutCreatorEventNtabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutCreatorEventNtabRequestHydrator.scala deleted file mode 100644 index a245769a6..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutCreatorEventNtabRequestHydrator.scala +++ /dev/null @@ -1,110 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.magic_events.thriftscala.CreatorFanoutType -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.MagicFanoutCreatorEventPushCandidate -import com.twitter.frigate.pushservice.take.NotificationServiceSender -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationRequest -import com.twitter.notificationservice.thriftscala.DisplayText -import com.twitter.notificationservice.thriftscala.DisplayTextEntity -import com.twitter.notificationservice.thriftscala.GenericType -import com.twitter.notificationservice.thriftscala.InlineCard -import com.twitter.notificationservice.thriftscala.StoryContext -import com.twitter.notificationservice.thriftscala.TextValue -import com.twitter.notificationservice.thriftscala.TapThroughAction -import com.twitter.util.Future -import com.twitter.util.Time - -trait MagicFanoutCreatorEventNtabRequestHydrator extends NTabRequestHydrator { - self: PushCandidate with MagicFanoutCreatorEventPushCandidate => - - override val senderIdFut: Future[Long] = Future.value(creatorId) - - override lazy val tapThroughFut: Future[String] = - Future.value(s"/${userProfile.screenName}/superfollows/subscribe") - - lazy val optionalTweetCountEntityFut: Future[Option[DisplayTextEntity]] = { - creatorFanoutType match { - case CreatorFanoutType.UserSubscription => - numberOfTweetsFut.map { - _.flatMap { - case numberOfTweets if numberOfTweets >= 10 => - Some( - DisplayTextEntity( - name = "tweet_count", - emphasis = true, - value = TextValue.Text(numberOfTweets.toString))) - case _ => None - } - } - case _ => Future.None - } - } - - override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = - optionalTweetCountEntityFut - .map { tweetCountOpt => - Seq( - NotificationServiceSender - .getDisplayTextEntityFromUser(hydratedCreator, "display_name", isBold = true), - tweetCountOpt).flatten - } - - override lazy val facepileUsersFut: Future[Seq[Long]] = Future.value(Seq(creatorId)) - - override val storyContext: Option[StoryContext] = None - - override val inlineCard: Option[InlineCard] = None - - lazy val refreshableTypeFut = { - creatorFanoutType match { - case CreatorFanoutType.UserSubscription => - numberOfTweetsFut.map { - _.flatMap { - case numberOfTweets if numberOfTweets >= 10 => - Some("MagicFanoutCreatorSubscriptionWithTweets") - case _ => super.refreshableType - } - } - case _ => Future.value(super.refreshableType) - } - } - - override lazy val socialProofDisplayText: Option[DisplayText] = { - creatorFanoutType match { - case CreatorFanoutType.UserSubscription => - Some( - DisplayText(values = Seq( - DisplayTextEntity(name = "handle", value = TextValue.Text(userProfile.screenName))))) - case CreatorFanoutType.NewCreator => None - case _ => None - } - } - - override lazy val ntabRequest = { - Future - .join( - senderIdFut, - displayTextEntitiesFut, - facepileUsersFut, - tapThroughFut, - refreshableTypeFut).map { - case (senderId, displayTextEntities, facepileUsers, tapThrough, refreshableTypeOpt) => - Some( - CreateGenericNotificationRequest( - userId = target.targetId, - senderId = senderId, - genericType = GenericType.RefreshableNotification, - displayText = DisplayText(values = displayTextEntities), - facepileUsers = facepileUsers, - timestampMillis = Time.now.inMillis, - tapThroughAction = Some(TapThroughAction(Some(tapThrough))), - impressionId = Some(impressionId), - socialProofText = socialProofDisplayText, - context = storyContext, - inlineCard = inlineCard, - refreshableType = refreshableTypeOpt - )) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutNewsEventNTabRequestHydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutNewsEventNTabRequestHydrator.docx new file mode 100644 index 000000000..2d62165bd Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutNewsEventNTabRequestHydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutNewsEventNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutNewsEventNTabRequestHydrator.scala deleted file mode 100644 index 202533e3c..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutNewsEventNTabRequestHydrator.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.MagicFanoutEventHydratedCandidate -import com.twitter.notificationservice.thriftscala.DisplayTextEntity -import com.twitter.notificationservice.thriftscala.TextValue -import com.twitter.util.Future - -trait MagicFanoutNewsEventNTabRequestHydrator extends EventNTabRequestHydrator { - self: PushCandidate with MagicFanoutEventHydratedCandidate => - override lazy val tapThroughFut: Future[String] = Future.value(s"i/events/$eventId") - override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = - eventTitleFut.map { eventTitle => - Seq(DisplayTextEntity(name = "title", value = TextValue.Text(eventTitle))) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutProductLaunchNtabRequestHydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutProductLaunchNtabRequestHydrator.docx new file mode 100644 index 000000000..1329dd79a Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutProductLaunchNtabRequestHydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutProductLaunchNtabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutProductLaunchNtabRequestHydrator.scala deleted file mode 100644 index 797dbe890..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutProductLaunchNtabRequestHydrator.scala +++ /dev/null @@ -1,97 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.common.base.MagicFanoutProductLaunchCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.notificationservice.thriftscala._ -import com.twitter.util.Future -import com.twitter.util.Time - -trait MagicFanoutProductLaunchNtabRequestHydrator extends NTabRequestHydrator { - self: PushCandidate with MagicFanoutProductLaunchCandidate => - - override val senderIdFut: Future[Long] = Future.value(0L) - - override lazy val tapThroughFut: Future[String] = Future.value(getProductLaunchTapThrough()) - - override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = { - Future.value( - frigateNotification.magicFanoutProductLaunchNotification - .flatMap { - _.productInfo.flatMap { - _.body.map { body => - Seq( - DisplayTextEntity(name = "body", value = TextValue.Text(body)), - ) - } - } - }.getOrElse(Nil)) - } - - override lazy val facepileUsersFut: Future[Seq[Long]] = { - Future.value( - frigateNotification.magicFanoutProductLaunchNotification - .flatMap { - _.productInfo.flatMap { - _.facepileUsers - } - }.getOrElse(Nil)) - } - - override val storyContext: Option[StoryContext] = None - - override val inlineCard: Option[InlineCard] = None - - override lazy val socialProofDisplayText: Option[DisplayText] = { - frigateNotification.magicFanoutProductLaunchNotification.flatMap { - _.productInfo.flatMap { - _.title.map { title => - DisplayText(values = - Seq(DisplayTextEntity(name = "social_context", value = TextValue.Text(title)))) - } - } - } - } - - lazy val defaultTapThrough = target.params(PushFeatureSwitchParams.ProductLaunchTapThrough) - - private def getProductLaunchTapThrough(): String = { - frigateNotification.magicFanoutProductLaunchNotification match { - case Some(productLaunchNotif) => - productLaunchNotif.productInfo match { - case Some(productInfo) => productInfo.tapThrough.getOrElse(defaultTapThrough) - case _ => defaultTapThrough - } - case _ => defaultTapThrough - } - } - - private lazy val productLaunchNtabRequest: Future[Option[CreateGenericNotificationRequest]] = { - Future - .join(senderIdFut, displayTextEntitiesFut, facepileUsersFut, tapThroughFut) - .map { - case (senderId, displayTextEntities, facepileUsers, tapThrough) => - Some( - CreateGenericNotificationRequest( - userId = target.targetId, - senderId = senderId, - genericType = GenericType.RefreshableNotification, - displayText = DisplayText(values = displayTextEntities), - facepileUsers = facepileUsers, - timestampMillis = Time.now.inMillis, - tapThroughAction = Some(TapThroughAction(Some(tapThrough))), - impressionId = Some(impressionId), - socialProofText = socialProofDisplayText, - context = storyContext, - inlineCard = inlineCard, - refreshableType = refreshableType - )) - } - } - - override lazy val ntabRequest: Future[Option[CreateGenericNotificationRequest]] = { - if (target.params(PushFeatureSwitchParams.EnableNTabEntriesForProductLaunchNotifications)) { - productLaunchNtabRequest - } else Future.None - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutSportsEventNTabRequestHydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutSportsEventNTabRequestHydrator.docx new file mode 100644 index 000000000..97678f7ee Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutSportsEventNTabRequestHydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutSportsEventNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutSportsEventNTabRequestHydrator.scala deleted file mode 100644 index ca3d9faf0..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutSportsEventNTabRequestHydrator.scala +++ /dev/null @@ -1,95 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.common.base.MagicFanoutSportsEventCandidate -import com.twitter.frigate.common.base.MagicFanoutSportsScoreInformation -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.MagicFanoutEventHydratedCandidate -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationRequest -import com.twitter.notificationservice.thriftscala.DisplayText -import com.twitter.notificationservice.thriftscala.DisplayTextEntity -import com.twitter.notificationservice.thriftscala.GenericType -import com.twitter.notificationservice.thriftscala.TextValue -import com.twitter.notificationservice.thriftscala.TapThroughAction -import com.twitter.util.Future -import com.twitter.util.Time - -trait MagicFanoutSportsEventNTabRequestHydrator extends EventNTabRequestHydrator { - self: PushCandidate - with MagicFanoutEventHydratedCandidate - with MagicFanoutSportsEventCandidate - with MagicFanoutSportsScoreInformation => - - lazy val stats = self.statsReceiver.scope("MagicFanoutSportsEventNtabHydrator") - lazy val inNetworkOnlyCounter = stats.counter("in_network_only") - lazy val facePilesEnabledCounter = stats.counter("face_piles_enabled") - lazy val facePilesDisabledCounter = stats.counter("face_piles_disabled") - lazy val filterPeopleWhoDontFollowMeCounter = stats.counter("pepole_who_dont_follow_me_counter") - - override lazy val tapThroughFut: Future[String] = { - Future.value(s"i/events/$eventId") - } - override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = - eventTitleFut.map { eventTitle => - Seq(DisplayTextEntity(name = "title", value = TextValue.Text(eventTitle))) - } - - override lazy val facepileUsersFut: Future[Seq[Long]] = - if (target.params(FS.EnableNTabFacePileForSportsEventNotifications)) { - Future - .join( - target.notificationsFromOnlyPeopleIFollow, - target.filterNotificationsFromPeopleThatDontFollowMe, - awayTeamInfo, - homeTeamInfo).map { - case (inNetworkOnly, filterPeopleWhoDontFollowMe, away, home) - if !(inNetworkOnly || filterPeopleWhoDontFollowMe) => - val awayTeamId = away.flatMap(_.twitterUserId) - val homeTeamId = home.flatMap(_.twitterUserId) - facePilesEnabledCounter.incr - Seq(awayTeamId, homeTeamId).flatten - case (inNetworkOnly, filterPeopleWhoDontFollowMe, _, _) => - facePilesDisabledCounter.incr - if (inNetworkOnly) inNetworkOnlyCounter.incr - if (filterPeopleWhoDontFollowMe) filterPeopleWhoDontFollowMeCounter.incr - Seq.empty[Long] - } - } else Future.Nil - - private lazy val sportsNtabRequest: Future[Option[CreateGenericNotificationRequest]] = { - Future - .join(senderIdFut, displayTextEntitiesFut, facepileUsersFut, tapThroughFut) - .map { - case (senderId, displayTextEntities, facepileUsers, tapThrough) => - Some( - CreateGenericNotificationRequest( - userId = target.targetId, - senderId = senderId, - genericType = GenericType.RefreshableNotification, - displayText = DisplayText(values = displayTextEntities), - facepileUsers = facepileUsers, - timestampMillis = Time.now.inMillis, - tapThroughAction = Some(TapThroughAction(Some(tapThrough))), - impressionId = Some(impressionId), - socialProofText = socialProofDisplayText, - context = storyContext, - inlineCard = inlineCard, - refreshableType = refreshableType - )) - } - } - - override lazy val ntabRequest: Future[Option[CreateGenericNotificationRequest]] = { - if (target.params(FS.EnableNTabEntriesForSportsEventNotifications)) { - self.target.history.flatMap { pushHistory => - val prevEventHistoryExists = pushHistory.sortedHistory.exists { - case (_, notification) => - notification.magicFanoutEventNotification.exists(_.eventId == self.eventId) - } - if (prevEventHistoryExists) { - Future.None - } else sportsNtabRequest - } - } else Future.None - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabRequest.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabRequest.docx new file mode 100644 index 000000000..8adbcd66e Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabRequest.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabRequest.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabRequest.scala deleted file mode 100644 index ea99ea68d..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabRequest.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationRequest -import com.twitter.util.Future - -trait NTabRequest { - - def ntabRequest: Future[Option[CreateGenericNotificationRequest]] - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabRequestHydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabRequestHydrator.docx new file mode 100644 index 000000000..35d982f83 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabRequestHydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabRequestHydrator.scala deleted file mode 100644 index 01df5365f..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabRequestHydrator.scala +++ /dev/null @@ -1,64 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationRequest -import com.twitter.notificationservice.thriftscala.DisplayText -import com.twitter.notificationservice.thriftscala.DisplayTextEntity -import com.twitter.notificationservice.thriftscala.GenericType -import com.twitter.notificationservice.thriftscala.InlineCard -import com.twitter.notificationservice.thriftscala.StoryContext -import com.twitter.notificationservice.thriftscala.TapThroughAction -import com.twitter.util.Future -import com.twitter.util.Time - -trait NTabRequestHydrator extends NTabRequest with CandidateNTabCopy { - self: PushCandidate => - - // Represents the sender of the recommendation - def senderIdFut: Future[Long] - - // Consists of a sequence representing the social context user ids. - def facepileUsersFut: Future[Seq[Long]] - - // Story Context is required for Tweet Recommendations - // Contains the Tweet ID of the recommended Tweet - def storyContext: Option[StoryContext] - - // Inline card used to render a generic notification. - def inlineCard: Option[InlineCard] - - // Represents where the recommendation should land when clicked - def tapThroughFut: Future[String] - - // Hydration for fields that are used within the NTab copy - def displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] - - // Represents the social proof text that is needed for specific NTab copies - def socialProofDisplayText: Option[DisplayText] - - // MagicRecs NTab entries always use RefreshableType as the Generic Type - final val genericType: GenericType = GenericType.RefreshableNotification - - def refreshableType: Option[String] = ntabCopy.refreshableType - - lazy val ntabRequest: Future[Option[CreateGenericNotificationRequest]] = { - Future.join(senderIdFut, displayTextEntitiesFut, facepileUsersFut, tapThroughFut).map { - case (senderId, displayTextEntities, facepileUsers, tapThrough) => - Some( - CreateGenericNotificationRequest( - userId = target.targetId, - senderId = senderId, - genericType = GenericType.RefreshableNotification, - displayText = DisplayText(values = displayTextEntities), - facepileUsers = facepileUsers, - timestampMillis = Time.now.inMillis, - tapThroughAction = Some(TapThroughAction(Some(tapThrough))), - impressionId = Some(impressionId), - socialProofText = socialProofDisplayText, - context = storyContext, - inlineCard = inlineCard, - refreshableType = refreshableType - )) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabSocialContext.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabSocialContext.docx new file mode 100644 index 000000000..69f4d5cde Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabSocialContext.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabSocialContext.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabSocialContext.scala deleted file mode 100644 index 17b43f457..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabSocialContext.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.common.base.SocialContextActions -import com.twitter.frigate.common.base.SocialContextUserDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.util.CandidateUtil -import com.twitter.util.Future - -trait NTabSocialContext { - self: PushCandidate with SocialContextActions with SocialContextUserDetails => - - private def ntabDisplayUserIds: Seq[Long] = - socialContextUserIds.take(ntabDisplayUserIdsLength) - - def ntabDisplayUserIdsLength: Int = - if (socialContextUserIds.size == 2) 2 else 1 - - def ntabDisplayNamesAndIds: Future[Seq[(String, Long)]] = - scUserMap.map { userObjMap => - ntabDisplayUserIds.flatMap { id => - userObjMap(id).flatMap(_.profile.map(_.name)).map { name => (name, id) } - } - } - - def rankedNtabDisplayNamesAndIds(defaultToRecency: Boolean): Future[Seq[(String, Long)]] = - scUserMap.flatMap { userObjMap => - val rankedSocialContextActivityFut = - CandidateUtil.getRankedSocialContext( - socialContextActions, - target.seedsWithWeight, - defaultToRecency) - rankedSocialContextActivityFut.map { rankedSocialContextActivity => - val ntabDisplayUserIds = - rankedSocialContextActivity.map(_.userId).take(ntabDisplayUserIdsLength) - ntabDisplayUserIds.flatMap { id => - userObjMap(id).flatMap(_.profile.map(_.name)).map { name => (name, id) } - } - } - } - - def otherCount: Future[Int] = - ntabDisplayNamesAndIds.map { - case namesWithIdSeq => - Math.max(0, socialContextUserIds.length - namesWithIdSeq.size) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/OutOfNetworkTweetNTabRequestHydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/OutOfNetworkTweetNTabRequestHydrator.docx new file mode 100644 index 000000000..7475ca939 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/OutOfNetworkTweetNTabRequestHydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/OutOfNetworkTweetNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/OutOfNetworkTweetNTabRequestHydrator.scala deleted file mode 100644 index a2b99d1af..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/OutOfNetworkTweetNTabRequestHydrator.scala +++ /dev/null @@ -1,78 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.common.base.TopicCandidate -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.common.base.TweetDetails -import com.twitter.frigate.common.rec_types.RecTypes._ -import com.twitter.frigate.common.util.MrNtabCopyObjects -import com.twitter.frigate.pushservice.exception.TweetNTabRequestHydratorException -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.take.NotificationServiceSender -import com.twitter.notificationservice.thriftscala.DisplayText -import com.twitter.notificationservice.thriftscala.DisplayTextEntity -import com.twitter.notificationservice.thriftscala.TextValue -import com.twitter.util.Future - -trait OutOfNetworkTweetNTabRequestHydrator extends TweetNTabRequestHydrator { - self: PushCandidate - with TweetCandidate - with TweetAuthorDetails - with TopicCandidate - with TweetDetails => - - lazy val useTopicCopyForMBCGNtab = mrModelingBasedTypes.contains(commonRecType) && target.params( - PushFeatureSwitchParams.EnableMrModelingBasedCandidatesTopicCopy) - lazy val useTopicCopyForFrsNtab = frsTypes.contains(commonRecType) && target.params( - PushFeatureSwitchParams.EnableFrsTweetCandidatesTopicCopy) - lazy val useTopicCopyForTagspaceNtab = tagspaceTypes.contains(commonRecType) && target.params( - PushFeatureSwitchParams.EnableHashspaceCandidatesTopicCopy) - - override lazy val tapThroughFut: Future[String] = { - if (hasVideo && self.target.params( - PushFeatureSwitchParams.EnableLaunchVideosInImmersiveExplore)) { - Future.value( - s"i/immersive_timeline?display_location=notification&include_pinned_tweet=true&pinned_tweet_id=${tweetId}&tl_type=imv") - } else { - tweetAuthor.map { - case Some(author) => - val authorProfile = author.profile.getOrElse( - throw new TweetNTabRequestHydratorException( - s"Unable to obtain author profile for: ${author.id}")) - s"${authorProfile.screenName}/status/${tweetId.toString}" - case _ => - throw new TweetNTabRequestHydratorException( - s"Unable to obtain author and target details to generate tap through for Tweet: $tweetId") - } - } - } - - override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = - if (localizedUttEntity.isDefined && - (useTopicCopyForMBCGNtab || useTopicCopyForFrsNtab || useTopicCopyForTagspaceNtab)) { - NotificationServiceSender - .getDisplayTextEntityFromUser(tweetAuthor, "tweetAuthorName", isBold = true).map(_.toSeq) - } else { - NotificationServiceSender - .getDisplayTextEntityFromUser(tweetAuthor, "author", isBold = true).map(_.toSeq) - } - - override lazy val refreshableType: Option[String] = { - if (localizedUttEntity.isDefined && - (useTopicCopyForMBCGNtab || useTopicCopyForFrsNtab || useTopicCopyForTagspaceNtab)) { - MrNtabCopyObjects.TopicTweet.refreshableType - } else ntabCopy.refreshableType - } - - override def socialProofDisplayText: Option[DisplayText] = { - if (localizedUttEntity.isDefined && - (useTopicCopyForMBCGNtab || useTopicCopyForFrsNtab || useTopicCopyForTagspaceNtab)) { - localizedUttEntity.map(uttEntity => - DisplayText(values = - Seq(DisplayTextEntity("topic_name", TextValue.Text(uttEntity.localizedNameForDisplay))))) - } else None - } - - override lazy val facepileUsersFut: Future[Seq[Long]] = senderIdFut.map(Seq(_)) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/ScheduledSpaceNTabRequestHydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/ScheduledSpaceNTabRequestHydrator.docx new file mode 100644 index 000000000..b793420b6 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/ScheduledSpaceNTabRequestHydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/ScheduledSpaceNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/ScheduledSpaceNTabRequestHydrator.scala deleted file mode 100644 index 4673a001e..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/ScheduledSpaceNTabRequestHydrator.scala +++ /dev/null @@ -1,106 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.common.base.SpaceCandidate -import com.twitter.frigate.common.util.MrNtabCopyObjects -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.ScheduledSpaceSpeakerPushCandidate -import com.twitter.frigate.pushservice.model.ScheduledSpaceSubscriberPushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.take.NotificationServiceSender -import com.twitter.frigate.thriftscala.SpaceNotificationType -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.notificationservice.thriftscala._ -import com.twitter.util.Future -import com.twitter.util.Time - -trait ScheduledSpaceSpeakerNTabRequestHydrator extends ScheduledSpaceNTabRequestHydrator { - self: PushCandidate with ScheduledSpaceSpeakerPushCandidate => - - override def refreshableType: Option[String] = { - frigateNotification.spaceNotification.flatMap { spaceNotification => - spaceNotification.spaceNotificationType.flatMap { - case SpaceNotificationType.PreSpaceBroadcast => - MrNtabCopyObjects.ScheduledSpaceSpeakerSoon.refreshableType - case SpaceNotificationType.AtSpaceBroadcast => - MrNtabCopyObjects.ScheduledSpaceSpeakerNow.refreshableType - case _ => - throw new IllegalStateException(s"Unexpected SpaceNotificationType") - } - } - } - - override lazy val facepileUsersFut: Future[Seq[Long]] = Future.Nil - - override val socialProofDisplayText: Option[DisplayText] = Some(DisplayText()) -} - -trait ScheduledSpaceSubscriberNTabRequestHydrator extends ScheduledSpaceNTabRequestHydrator { - self: PushCandidate with ScheduledSpaceSubscriberPushCandidate => - - override lazy val facepileUsersFut: Future[Seq[Long]] = { - hostId match { - case Some(spaceHostId) => Future.value(Seq(spaceHostId)) - case _ => - Future.exception( - new IllegalStateException( - "Unable to get host id for ScheduledSpaceSubscriberNTabRequestHydrator")) - } - } - - override val socialProofDisplayText: Option[DisplayText] = None -} - -trait ScheduledSpaceNTabRequestHydrator extends NTabRequestHydrator { - self: PushCandidate with SpaceCandidate => - - def hydratedHost: Option[User] - - override lazy val senderIdFut: Future[Long] = { - hostId match { - case Some(spaceHostId) => Future.value(spaceHostId) - case _ => throw new IllegalStateException(s"No Space Host Id") - } - } - - override lazy val tapThroughFut: Future[String] = Future.value(s"i/spaces/$spaceId") - - override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = - NotificationServiceSender - .getDisplayTextEntityFromUser( - Future.value(hydratedHost), - fieldName = "space_host_name", - isBold = true - ).map(_.toSeq) - - override val storyContext: Option[StoryContext] = None - - override val inlineCard: Option[InlineCard] = None - - override lazy val ntabRequest: Future[Option[CreateGenericNotificationRequest]] = { - Future.join(senderIdFut, displayTextEntitiesFut, facepileUsersFut, tapThroughFut).map { - case (senderId, displayTextEntities, facepileUsers, tapThrough) => - val expiryTimeMillis = if (target.params(PushFeatureSwitchParams.EnableSpacesTtlForNtab)) { - Some( - (Time.now + target.params( - PushFeatureSwitchParams.SpaceNotificationsTTLDurationForNTab)).inMillis) - } else None - - Some( - CreateGenericNotificationRequest( - userId = target.targetId, - senderId = senderId, - genericType = GenericType.RefreshableNotification, - displayText = DisplayText(values = displayTextEntities), - facepileUsers = facepileUsers, - timestampMillis = Time.now.inMillis, - tapThroughAction = Some(TapThroughAction(Some(tapThrough))), - impressionId = Some(impressionId), - socialProofText = socialProofDisplayText, - context = storyContext, - inlineCard = inlineCard, - refreshableType = refreshableType, - expiryTimeMillis = expiryTimeMillis - )) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/SubscribedSearchTweetNtabRequestHydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/SubscribedSearchTweetNtabRequestHydrator.docx new file mode 100644 index 000000000..051ff38f0 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/SubscribedSearchTweetNtabRequestHydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/SubscribedSearchTweetNtabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/SubscribedSearchTweetNtabRequestHydrator.scala deleted file mode 100644 index caa2a8cd0..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/SubscribedSearchTweetNtabRequestHydrator.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.pushservice.model.SubscribedSearchTweetPushCandidate -import com.twitter.frigate.pushservice.take.NotificationServiceSender -import com.twitter.notificationservice.thriftscala.DisplayText -import com.twitter.notificationservice.thriftscala.DisplayTextEntity -import com.twitter.notificationservice.thriftscala.TextValue -import com.twitter.util.Future - -trait SubscribedSearchTweetNtabRequestHydrator extends TweetNTabRequestHydrator { - self: SubscribedSearchTweetPushCandidate => - override def displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = NotificationServiceSender - .getDisplayTextEntityFromUser(tweetAuthor, "tweetAuthor", isBold = true).map(_.toSeq) - - override def socialProofDisplayText: Option[DisplayText] = { - Some(DisplayText(values = Seq(DisplayTextEntity("search_query", TextValue.Text(searchTerm))))) - } - - override lazy val facepileUsersFut: Future[Seq[Long]] = senderIdFut.map(Seq(_)) - - override lazy val tapThroughFut: Future[String] = - Future.value(self.ntabLandingUrl) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TopTweetImpressionsNTabRequestHydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TopTweetImpressionsNTabRequestHydrator.docx new file mode 100644 index 000000000..4221c73a3 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TopTweetImpressionsNTabRequestHydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TopTweetImpressionsNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TopTweetImpressionsNTabRequestHydrator.scala deleted file mode 100644 index a67dee399..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TopTweetImpressionsNTabRequestHydrator.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.common.base.TopTweetImpressionsCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.notificationservice.thriftscala.DisplayText -import com.twitter.notificationservice.thriftscala.DisplayTextEntity -import com.twitter.notificationservice.thriftscala.InlineCard -import com.twitter.notificationservice.thriftscala.StoryContext -import com.twitter.notificationservice.thriftscala.StoryContextValue -import com.twitter.notificationservice.thriftscala.TextValue -import com.twitter.util.Future - -trait TopTweetImpressionsNTabRequestHydrator extends NTabRequestHydrator { - self: PushCandidate with TopTweetImpressionsCandidate => - - override lazy val tapThroughFut: Future[String] = - Future.value(s"${target.targetId}/status/$tweetId") - - override val senderIdFut: Future[Long] = Future.value(0L) - - override val facepileUsersFut: Future[Seq[Long]] = Future.Nil - - override val storyContext: Option[StoryContext] = - Some(StoryContext(altText = "", value = Some(StoryContextValue.Tweets(Seq(tweetId))))) - - override val inlineCard: Option[InlineCard] = None - - override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = { - Future.value( - Seq( - DisplayTextEntity(name = "num_impressions", value = TextValue.Number(self.impressionsCount)) - ) - ) - } - - override def socialProofDisplayText: Option[DisplayText] = None -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TopicProofTweetNtabRequestHydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TopicProofTweetNtabRequestHydrator.docx new file mode 100644 index 000000000..89e92d959 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TopicProofTweetNtabRequestHydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TopicProofTweetNtabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TopicProofTweetNtabRequestHydrator.scala deleted file mode 100644 index 17519efda..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TopicProofTweetNtabRequestHydrator.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.pushservice.model.TopicProofTweetPushCandidate -import com.twitter.frigate.pushservice.exception.TweetNTabRequestHydratorException -import com.twitter.frigate.pushservice.exception.UttEntityNotFoundException -import com.twitter.frigate.pushservice.take.NotificationServiceSender -import com.twitter.notificationservice.thriftscala.DisplayText -import com.twitter.notificationservice.thriftscala.DisplayTextEntity -import com.twitter.notificationservice.thriftscala.StoryContext -import com.twitter.notificationservice.thriftscala.StoryContextValue -import com.twitter.notificationservice.thriftscala.TextValue -import com.twitter.util.Future - -trait TopicProofTweetNtabRequestHydrator extends NTabRequestHydrator { - self: TopicProofTweetPushCandidate => - - override def displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = NotificationServiceSender - .getDisplayTextEntityFromUser(tweetAuthor, "tweetAuthorName", true) - .map(_.toSeq) - - private lazy val uttEntity = localizedUttEntity.getOrElse( - throw new UttEntityNotFoundException( - s"${getClass.getSimpleName} UttEntity missing for $tweetId") - ) - - override lazy val tapThroughFut: Future[String] = { - tweetAuthor.map { - case Some(author) => - val authorProfile = author.profile.getOrElse( - throw new TweetNTabRequestHydratorException( - s"Unable to obtain author profile for: ${author.id}")) - s"${authorProfile.screenName}/status/${tweetId.toString}" - case _ => - throw new TweetNTabRequestHydratorException( - s"Unable to obtain author and target details to generate tap through for Tweet: $tweetId") - } - } - - override lazy val socialProofDisplayText: Option[DisplayText] = { - Some( - DisplayText(values = - Seq(DisplayTextEntity("topic_name", TextValue.Text(uttEntity.localizedNameForDisplay)))) - ) - } - - override lazy val facepileUsersFut: Future[Seq[Long]] = senderIdFut.map(Seq(_)) - - override val inlineCard = None - - override def storyContext: Option[StoryContext] = Some( - StoryContext("", Some(StoryContextValue.Tweets(Seq(tweetId))))) - - override def senderIdFut: Future[Long] = - tweetAuthor.map { - case Some(author) => author.id - case _ => - throw new TweetNTabRequestHydratorException( - s"Unable to obtain Author ID for: $commonRecType") - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TrendTweetNtabHydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TrendTweetNtabHydrator.docx new file mode 100644 index 000000000..9fc5d5389 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TrendTweetNtabHydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TrendTweetNtabHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TrendTweetNtabHydrator.scala deleted file mode 100644 index 07946a220..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TrendTweetNtabHydrator.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.common.base.TrendTweetCandidate -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.pushservice.exception.TweetNTabRequestHydratorException -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.take.NotificationServiceSender -import com.twitter.frigate.pushservice.util.EmailLandingPageExperimentUtil -import com.twitter.notificationservice.thriftscala.DisplayText -import com.twitter.notificationservice.thriftscala.DisplayTextEntity -import com.twitter.notificationservice.thriftscala.TextValue -import com.twitter.util.Future - -trait TrendTweetNtabHydrator extends TweetNTabRequestHydrator { - self: PushCandidate with TrendTweetCandidate with TweetCandidate with TweetAuthorDetails => - - private lazy val trendTweetNtabStats = self.statsReceiver.scope("trend_tweet_ntab") - - private lazy val ruxLandingOnNtabCounter = - trendTweetNtabStats.counter("use_rux_landing_on_ntab") - - override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = - NotificationServiceSender - .getDisplayTextEntityFromUser(tweetAuthor, fieldName = "author_name", isBold = true) - .map( - _.toSeq :+ DisplayTextEntity( - name = "trend_name", - value = TextValue.Text(trendName), - emphasis = true) - ) - - override lazy val facepileUsersFut: Future[Seq[Long]] = senderIdFut.map(Seq(_)) - - override lazy val socialProofDisplayText: Option[DisplayText] = None - - override def refreshableType: Option[String] = ntabCopy.refreshableType - - override lazy val tapThroughFut: Future[String] = { - Future.join(tweetAuthor, target.deviceInfo).map { - case (Some(author), Some(deviceInfo)) => - val enableRuxLandingPage = deviceInfo.isRuxLandingPageEligible && target.params( - PushFeatureSwitchParams.EnableNTabRuxLandingPage) - val authorProfile = author.profile.getOrElse( - throw new TweetNTabRequestHydratorException( - s"Unable to obtain author profile for: ${author.id}")) - - if (enableRuxLandingPage) { - ruxLandingOnNtabCounter.incr() - EmailLandingPageExperimentUtil.createNTabRuxLandingURI(authorProfile.screenName, tweetId) - } else { - s"${authorProfile.screenName}/status/${tweetId.toString}" - } - - case _ => - throw new TweetNTabRequestHydratorException( - s"Unable to obtain author and target details to generate tap through for Tweet: $tweetId") - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetFavoriteNTabRequestHydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetFavoriteNTabRequestHydrator.docx new file mode 100644 index 000000000..3daf10713 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetFavoriteNTabRequestHydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetFavoriteNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetFavoriteNTabRequestHydrator.scala deleted file mode 100644 index 52a643b84..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetFavoriteNTabRequestHydrator.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.common.base.SocialContextActions -import com.twitter.frigate.common.base.SocialContextUserDetails -import com.twitter.frigate.common.base.TweetAuthor -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.take.NotificationServiceSender -import com.twitter.notificationservice.thriftscala.DisplayTextEntity -import com.twitter.util.Future - -trait TweetFavoriteNTabRequestHydrator extends TweetNTabRequestHydrator with NTabSocialContext { - self: PushCandidate - with TweetCandidate - with TweetAuthor - with TweetAuthorDetails - with SocialContextActions - with SocialContextUserDetails => - - override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = { - Future - .join( - NotificationServiceSender - .getDisplayTextEntityFromUser(tweetAuthor, "tweetAuthorName", isBold = false), - NotificationServiceSender - .generateSocialContextTextEntities( - rankedNtabDisplayNamesAndIds(defaultToRecency = false), - otherCount) - ) - .map { - case (authorDisplay, socialContextDisplay) => - socialContextDisplay ++ authorDisplay - } - } - - override lazy val facepileUsersFut: Future[Seq[Long]] = Future.value(socialContextUserIds) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetNTabRequestHydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetNTabRequestHydrator.docx new file mode 100644 index 000000000..d769d0b78 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetNTabRequestHydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetNTabRequestHydrator.scala deleted file mode 100644 index bfa8507f0..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetNTabRequestHydrator.scala +++ /dev/null @@ -1,55 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.pushservice.exception.TweetNTabRequestHydratorException -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.notificationservice.thriftscala.InlineCard -import com.twitter.notificationservice.thriftscala.StoryContext -import com.twitter.notificationservice.thriftscala.StoryContextValue -import com.twitter.frigate.pushservice.util.EmailLandingPageExperimentUtil -import com.twitter.notificationservice.thriftscala._ -import com.twitter.util.Future - -trait TweetNTabRequestHydrator extends NTabRequestHydrator { - self: PushCandidate with TweetCandidate with TweetAuthorDetails => - - override def senderIdFut: Future[Long] = - tweetAuthor.map { - case Some(author) => author.id - case _ => - throw new TweetNTabRequestHydratorException( - s"Unable to obtain Author ID for: $commonRecType") - } - - override def storyContext: Option[StoryContext] = Some( - StoryContext( - altText = "", - value = Some(StoryContextValue.Tweets(Seq(tweetId))), - details = None - )) - - override def inlineCard: Option[InlineCard] = Some(InlineCard.TweetCard(TweetCard(tweetId))) - - override lazy val tapThroughFut: Future[String] = { - Future.join(tweetAuthor, target.deviceInfo).map { - case (Some(author), Some(deviceInfo)) => - val enableRuxLandingPage = deviceInfo.isRuxLandingPageEligible && target.params( - PushFeatureSwitchParams.EnableNTabRuxLandingPage) - val authorProfile = author.profile.getOrElse( - throw new TweetNTabRequestHydratorException( - s"Unable to obtain author profile for: ${author.id}")) - if (enableRuxLandingPage) { - EmailLandingPageExperimentUtil.createNTabRuxLandingURI(authorProfile.screenName, tweetId) - } else { - s"${authorProfile.screenName}/status/${tweetId.toString}" - } - case _ => - throw new TweetNTabRequestHydratorException( - s"Unable to obtain author and target details to generate tap through for Tweet: $tweetId") - } - } - - override def socialProofDisplayText: Option[DisplayText] = None -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetRetweetNTabRequestHydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetRetweetNTabRequestHydrator.docx new file mode 100644 index 000000000..4ab0b2f99 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetRetweetNTabRequestHydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetRetweetNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetRetweetNTabRequestHydrator.scala deleted file mode 100644 index c142fbfba..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetRetweetNTabRequestHydrator.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.frigate.pushservice.model.ntab - -import com.twitter.frigate.common.base.SocialContextActions -import com.twitter.frigate.common.base.SocialContextUserDetails -import com.twitter.frigate.common.base.TweetAuthor -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.take.NotificationServiceSender -import com.twitter.notificationservice.thriftscala.DisplayTextEntity -import com.twitter.util.Future - -trait TweetRetweetNTabRequestHydrator extends TweetNTabRequestHydrator with NTabSocialContext { - self: PushCandidate - with TweetCandidate - with TweetAuthor - with TweetAuthorDetails - with SocialContextActions - with SocialContextUserDetails => - - override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = { - Future - .join( - NotificationServiceSender - .getDisplayTextEntityFromUser(tweetAuthor, "tweetAuthorName", isBold = false), - NotificationServiceSender - .generateSocialContextTextEntities( - rankedNtabDisplayNamesAndIds(defaultToRecency = false), - otherCount) - ) - .map { - case (authorDisplay, socialContextDisplay) => - socialContextDisplay ++ authorDisplay - } - } - - override lazy val facepileUsersFut: Future[Seq[Long]] = Future.value(socialContextUserIds) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/DeployConfigModule.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/DeployConfigModule.docx new file mode 100644 index 000000000..48f4a2896 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/DeployConfigModule.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/DeployConfigModule.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/DeployConfigModule.scala deleted file mode 100644 index 238efe0bb..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/DeployConfigModule.scala +++ /dev/null @@ -1,68 +0,0 @@ -package com.twitter.frigate.pushservice.module - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.abdecider.LoggingABDecider -import com.twitter.decider.Decider -import com.twitter.featureswitches.v2.FeatureSwitches -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.tunable.StandardTunableMap -import com.twitter.frigate.pushservice.config.DeployConfig -import com.twitter.frigate.pushservice.config.ProdConfig -import com.twitter.frigate.pushservice.config.StagingConfig -import com.twitter.frigate.pushservice.params.ShardParams -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.product_mixer.core.module.product_mixer_flags.ProductMixerFlagModule.ConfigRepoLocalPath -import com.twitter.product_mixer.core.module.product_mixer_flags.ProductMixerFlagModule.ServiceLocal - -object DeployConfigModule extends TwitterModule { - - @Provides - @Singleton - def providesDeployConfig( - @Flag(FlagName.numShards) numShards: Int, - @Flag(FlagName.shardId) shardId: Int, - @Flag(FlagName.isInMemCacheOff) inMemCacheOff: Boolean, - @Flag(ServiceLocal) isServiceLocal: Boolean, - @Flag(ConfigRepoLocalPath) localConfigRepoPath: String, - serviceIdentifier: ServiceIdentifier, - decider: Decider, - abDecider: LoggingABDecider, - featureSwitches: FeatureSwitches, - statsReceiver: StatsReceiver - ): DeployConfig = { - val tunableMap = if (serviceIdentifier.service.contains("canary")) { - StandardTunableMap(id = "frigate-pushservice-canary") - } else { StandardTunableMap(id = serviceIdentifier.service) } - val shardParams = ShardParams(numShards, shardId) - serviceIdentifier.environment match { - case "devel" | "staging" => - StagingConfig( - isServiceLocal = isServiceLocal, - localConfigRepoPath = localConfigRepoPath, - inMemCacheOff = inMemCacheOff, - decider = decider, - abDecider = abDecider, - featureSwitches = featureSwitches, - serviceIdentifier = serviceIdentifier, - tunableMap = tunableMap, - shardParams = shardParams - )(statsReceiver) - case "prod" => - ProdConfig( - isServiceLocal = isServiceLocal, - localConfigRepoPath = localConfigRepoPath, - inMemCacheOff = inMemCacheOff, - decider = decider, - abDecider = abDecider, - featureSwitches = featureSwitches, - serviceIdentifier = serviceIdentifier, - tunableMap = tunableMap, - shardParams = shardParams - )(statsReceiver) - case env => throw new Exception(s"Unknown environment $env") - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/FilterModule.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/FilterModule.docx new file mode 100644 index 000000000..c0a79de07 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/FilterModule.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/FilterModule.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/FilterModule.scala deleted file mode 100644 index 579f65acf..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/FilterModule.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.frigate.pushservice.module - -import com.google.inject.Provides -import javax.inject.Singleton -import com.twitter.discovery.common.nackwarmupfilter.NackWarmupFilter -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.util.Duration - -object FilterModule extends TwitterModule { - @Singleton - @Provides - def providesNackWarmupFilter( - @Flag(FlagName.nackWarmupDuration) warmupDuration: Duration - ): NackWarmupFilter = new NackWarmupFilter(warmupDuration) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/FlagModule.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/FlagModule.docx new file mode 100644 index 000000000..bf407be9e Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/FlagModule.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/FlagModule.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/FlagModule.scala deleted file mode 100644 index 4306e47ca..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/FlagModule.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.frigate.pushservice.module - -import com.twitter.app.Flag -import com.twitter.inject.TwitterModule -import com.twitter.util.Duration -import com.twitter.conversions.DurationOps._ - -object FlagName { - final val shardId = "service.shard" - final val numShards = "service.num_shards" - final val nackWarmupDuration = "service.nackWarmupDuration" - final val isInMemCacheOff = "service.isInMemCacheOff" -} - -object FlagModule extends TwitterModule { - - val shardId: Flag[Int] = flag[Int]( - name = FlagName.shardId, - help = "Service shard id" - ) - - val numShards: Flag[Int] = flag[Int]( - name = FlagName.numShards, - help = "Number of shards" - ) - - val mrLoggerIsTraceAll: Flag[Boolean] = flag[Boolean]( - name = "service.isTraceAll", - help = "atraceflag", - default = false - ) - - val mrLoggerNthLog: Flag[Boolean] = flag[Boolean]( - name = "service.nthLog", - help = "nthlog", - default = false - ) - - val inMemCacheOff: Flag[Boolean] = flag[Boolean]( - name = FlagName.isInMemCacheOff, - help = "is inMemCache Off (currently only applies for user_health_model_score_store_cache)", - default = false - ) - - val mrLoggerNthVal: Flag[Long] = flag[Long]( - name = "service.nthVal", - help = "nthlogval", - default = 0, - ) - - val nackWarmupDuration: Flag[Duration] = flag[Duration]( - name = FlagName.nackWarmupDuration, - help = "duration to nack at startup", - default = 0.seconds - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/LoggedOutPushTargetUserBuilderModule.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/LoggedOutPushTargetUserBuilderModule.docx new file mode 100644 index 000000000..946dec38c Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/LoggedOutPushTargetUserBuilderModule.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/LoggedOutPushTargetUserBuilderModule.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/LoggedOutPushTargetUserBuilderModule.scala deleted file mode 100644 index d4bceb549..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/LoggedOutPushTargetUserBuilderModule.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.frigate.pushservice.module - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.decider.Decider -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.target.LoggedOutPushTargetUserBuilder -import com.twitter.frigate.pushservice.config.DeployConfig -import com.twitter.inject.TwitterModule - -object LoggedOutPushTargetUserBuilderModule extends TwitterModule { - - @Provides - @Singleton - def providesLoggedOutPushTargetUserBuilder( - decider: Decider, - config: DeployConfig, - statsReceiver: StatsReceiver - ): LoggedOutPushTargetUserBuilder = { - LoggedOutPushTargetUserBuilder( - historyStore = config.loggedOutHistoryStore, - inputDecider = decider, - inputAbDecider = config.abDecider, - loggedOutPushInfoStore = config.loggedOutPushInfoStore - )(statsReceiver) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushHandlerModule.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushHandlerModule.docx new file mode 100644 index 000000000..40365ff78 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushHandlerModule.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushHandlerModule.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushHandlerModule.scala deleted file mode 100644 index c71ff24dd..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushHandlerModule.scala +++ /dev/null @@ -1,78 +0,0 @@ -package com.twitter.frigate.pushservice.module - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.target.LoggedOutPushTargetUserBuilder -import com.twitter.frigate.pushservice.refresh_handler.RefreshForPushHandler -import com.twitter.frigate.pushservice.config.DeployConfig -import com.twitter.frigate.pushservice.send_handler.SendHandler -import com.twitter.frigate.pushservice.take.candidate_validator.RFPHCandidateValidator -import com.twitter.frigate.pushservice.take.candidate_validator.SendHandlerPostCandidateValidator -import com.twitter.frigate.pushservice.take.candidate_validator.SendHandlerPreCandidateValidator -import com.twitter.frigate.pushservice.refresh_handler.LoggedOutRefreshForPushHandler -import com.twitter.frigate.pushservice.take.SendHandlerNotifier -import com.twitter.frigate.pushservice.target.PushTargetUserBuilder -import com.twitter.inject.TwitterModule - -object PushHandlerModule extends TwitterModule { - - @Provides - @Singleton - def providesRefreshForPushHandler( - pushTargetUserBuilder: PushTargetUserBuilder, - config: DeployConfig, - statsReceiver: StatsReceiver - ): RefreshForPushHandler = { - new RefreshForPushHandler( - pushTargetUserBuilder = pushTargetUserBuilder, - candSourceGenerator = config.candidateSourceGenerator, - rfphRanker = config.rfphRanker, - candidateHydrator = config.candidateHydrator, - candidateValidator = new RFPHCandidateValidator(config), - rfphTakeStepUtil = config.rfphTakeStepUtil, - rfphRestrictStep = config.rfphRestrictStep, - rfphNotifier = config.rfphNotifier, - rfphStatsRecorder = config.rfphStatsRecorder, - mrRequestScriberNode = config.mrRequestScriberNode, - rfphFeatureHydrator = config.rfphFeatureHydrator, - rfphPrerankFilter = config.rfphPrerankFilter, - rfphLightRanker = config.rfphLightRanker - )(statsReceiver) - } - - @Provides - @Singleton - def providesSendHandler( - pushTargetUserBuilder: PushTargetUserBuilder, - config: DeployConfig, - statsReceiver: StatsReceiver - ): SendHandler = { - new SendHandler( - pushTargetUserBuilder, - new SendHandlerPreCandidateValidator(config), - new SendHandlerPostCandidateValidator(config), - new SendHandlerNotifier(config.candidateNotifier, statsReceiver.scope("SendHandlerNotifier")), - config.sendHandlerCandidateHydrator, - config.featureHydrator, - config.sendHandlerPredicateUtil, - config.mrRequestScriberNode)(statsReceiver, config) - } - - @Provides - @Singleton - def providesLoggedOutRefreshForPushHandler( - loPushTargetUserBuilder: LoggedOutPushTargetUserBuilder, - config: DeployConfig, - statsReceiver: StatsReceiver - ): LoggedOutRefreshForPushHandler = { - new LoggedOutRefreshForPushHandler( - loPushTargetUserBuilder = loPushTargetUserBuilder, - loPushCandidateSourceGenerator = config.loCandidateSourceGenerator, - candidateHydrator = config.candidateHydrator, - loRanker = config.loggedOutRFPHRanker, - loRfphNotifier = config.loRfphNotifier, - loMrRequestScriberNode = config.loggedOutMrRequestScriberNode, - )(statsReceiver) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushServiceDarkTrafficModule.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushServiceDarkTrafficModule.docx new file mode 100644 index 000000000..06c7278eb Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushServiceDarkTrafficModule.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushServiceDarkTrafficModule.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushServiceDarkTrafficModule.scala deleted file mode 100644 index 97e484492..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushServiceDarkTrafficModule.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.frigate.pushservice.module - -import com.google.inject.Singleton -import com.twitter.decider.Decider -import com.twitter.decider.RandomRecipient -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.frigate.pushservice.thriftscala.PushService -import com.twitter.inject.Injector -import com.twitter.inject.thrift.modules.ReqRepDarkTrafficFilterModule - -/** - * The darkTraffic filter sample all requests by default - and set the diffy dest to nil for non prod environments - */ -@Singleton -object PushServiceDarkTrafficModule - extends ReqRepDarkTrafficFilterModule[PushService.ReqRepServicePerEndpoint] - with MtlsClient { - - override def label: String = "frigate-pushservice-diffy-proxy" - - /** - * Function to determine if the request should be "sampled", e.g. - * sent to the dark service. - * - * @param injector the [[com.twitter.inject.Injector]] for use in determining if a given request - * should be forwarded or not. - */ - override protected def enableSampling(injector: Injector): Any => Boolean = { - val decider = injector.instance[Decider] - _ => decider.isAvailable("frigate_pushservice_dark_traffic_percent", Some(RandomRecipient)) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushTargetUserBuilderModule.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushTargetUserBuilderModule.docx new file mode 100644 index 000000000..4833e94c9 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushTargetUserBuilderModule.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushTargetUserBuilderModule.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushTargetUserBuilderModule.scala deleted file mode 100644 index ccdd1f110..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushTargetUserBuilderModule.scala +++ /dev/null @@ -1,64 +0,0 @@ -package com.twitter.frigate.pushservice.module - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.decider.Decider -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.config.DeployConfig -import com.twitter.frigate.pushservice.target.PushTargetUserBuilder -import com.twitter.inject.TwitterModule - -object PushTargetUserBuilderModule extends TwitterModule { - - @Provides - @Singleton - def providesPushTargetUserBuilder( - decider: Decider, - config: DeployConfig, - statsReceiver: StatsReceiver - ): PushTargetUserBuilder = { - PushTargetUserBuilder( - historyStore = config.historyStore, - emailHistoryStore = config.emailHistoryStore, - labeledPushRecsStore = config.labeledPushRecsDecideredStore, - onlineUserHistoryStore = config.onlineUserHistoryStore, - pushRecItemsStore = config.pushRecItemStore, - userStore = config.safeUserStore, - pushInfoStore = config.pushInfoStore, - userCountryStore = config.userCountryStore, - userUtcOffsetStore = config.userUtcOffsetStore, - dauProbabilityStore = config.dauProbabilityStore, - nsfwConsumerStore = config.nsfwConsumerStore, - genericNotificationFeedbackStore = config.genericNotificationFeedbackStore, - userFeatureStore = config.userFeaturesStore, - mrUserStateStore = config.mrUserStatePredictionStore, - tweetImpressionStore = config.tweetImpressionStore, - timelinesUserSessionStore = config.timelinesUserSessionStore, - cachedTweetyPieStore = config.cachedTweetyPieStoreV2, - strongTiesStore = config.strongTiesStore, - userHTLLastVisitStore = config.userHTLLastVisitStore, - userLanguagesStore = config.userLanguagesStore, - inputDecider = decider, - inputAbDecider = config.abDecider, - realGraphScoresTop500InStore = config.realGraphScoresTop500InStore, - recentFollowsStore = config.recentFollowsStore, - resurrectedUserStore = config.reactivatedUserInfoStore, - configParamsBuilder = config.configParamsBuilder, - optOutUserInterestsStore = config.optOutUserInterestsStore, - deviceInfoStore = config.deviceInfoStore, - pushcapDynamicPredictionStore = config.pushcapDynamicPredictionStore, - appPermissionStore = config.appPermissionStore, - optoutModelScorer = config.optoutModelScorer, - userTargetingPropertyStore = config.userTargetingPropertyStore, - ntabCaretFeedbackStore = config.ntabCaretFeedbackStore, - genericFeedbackStore = config.genericFeedbackStore, - inlineActionHistoryStore = config.inlineActionHistoryStore, - featureHydrator = config.featureHydrator, - openAppUserStore = config.openAppUserStore, - openedPushByHourAggregatedStore = config.openedPushByHourAggregatedStore, - geoduckStoreV2 = config.geoDuckV2Store, - superFollowEligibilityUserStore = config.superFollowEligibilityUserStore, - superFollowApplicationStatusStore = config.superFollowApplicationStatusStore - )(statsReceiver) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/ThriftWebFormsModule.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/ThriftWebFormsModule.docx new file mode 100644 index 000000000..e88565b9b Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/ThriftWebFormsModule.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/ThriftWebFormsModule.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/ThriftWebFormsModule.scala deleted file mode 100644 index 049d731a3..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/ThriftWebFormsModule.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.frigate.pushservice.module - -import com.twitter.finatra.mtls.thriftmux.modules.MtlsThriftWebFormsModule -import com.twitter.finatra.thrift.ThriftServer -import com.twitter.frigate.pushservice.thriftscala.PushService - -class ThriftWebFormsModule(server: ThriftServer) - extends MtlsThriftWebFormsModule[PushService.MethodPerEndpoint](server) { -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/DeciderKey.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/DeciderKey.docx new file mode 100644 index 000000000..453fe1f1c Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/DeciderKey.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/DeciderKey.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/DeciderKey.scala deleted file mode 100644 index 9c17ea5f2..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/DeciderKey.scala +++ /dev/null @@ -1,210 +0,0 @@ -package com.twitter.frigate.pushservice.params - -import com.twitter.servo.decider.DeciderKeyEnum - -object DeciderKey extends DeciderKeyEnum { - val disableAllRelevance = Value("frigate_pushservice_disable_all_relevance") - val disableHeavyRanking = Value("frigate_pushservice_disable_heavy_ranking") - val restrictLightRanking = Value("frigate_pushservice_restrict_light_ranking") - val downSampleLightRankingScribeCandidates = Value( - "frigate_pushservice_down_sample_light_ranking_scribe_candidates") - val entityGraphTweetRecsDeciderKey = Value("user_tweet_entity_graph_tweet_recs") - val enablePushserviceWritesToNotificationServiceDeciderKey = Value( - "frigate_pushservice_enable_writes_to_notification_service") - val enablePushserviceWritesToNotificationServiceForAllEmployeesDeciderKey = Value( - "frigate_pushservice_enable_writes_to_notification_service_for_employees") - val enablePushserviceWritesToNotificationServiceForEveryoneDeciderKey = Value( - "frigate_pushservice_enable_writes_to_notification_service_for_everyone") - val enablePromptFeedbackFatigueResponseNoPredicateDeciderKey = Value( - "frigate_pushservice_enable_ntab_feedback_prompt_response_no_filter_predicate") - val enablePushserviceDeepbirdv2CanaryClusterDeciderKey = Value( - "frigate_pushservice_canary_enable_deepbirdv2_canary_cluster") - val enableUTEGSCForEarlybirdTweetsDecider = Value( - "frigate_pushservice_enable_uteg_sc_for_eb_tweets") - val enableTweetFavRecs = Value("frigate_pushservice_enable_tweet_fav_recs") - val enableTweetRetweetRecs = Value("frigate_pushservice_enable_tweet_retweet_recs") - val enablePushSendEventBus = Value("frigate_pushservice_enable_push_send_eventbus") - - val enableModelBasedPushcapAssignments = Value( - "frigate_pushservice_enable_model_based_pushcap_assignments") - - val enableTweetAnnotationFeatureHydration = Value( - "frigate_pushservice_enable_tweet_annotation_features_hydration") - val enableMrRequestScribing = Value("frigate_pushservice_enable_mr_request_scribing") - val enableHighQualityCandidateScoresScribing = Value( - "frigate_pushservice_enable_high_quality_candidate_scribing") - val enableHtlUserAuthorRealTimeAggregateFeatureHydration = Value( - "frigate_pushservice_enable_htl_new_user_user_author_rta_hydration") - val enableMrUserSemanticCoreFeaturesHydration = Value( - "frigate_pushservice_enable_mr_user_semantic_core_feature_hydration") - val enableMrUserSemanticCoreNoZeroFeaturesHydration = Value( - "frigate_pushservice_enable_mr_user_semantic_core_no_zero_feature_hydration") - val enableHtlOfflineUserAggregateExtendedFeaturesHydration = Value( - "frigate_pushservice_enable_htl_offline_user_aggregate_extended_features_hydration") - val enableNerErgFeaturesHydration = Value("frigate_pushservice_enable_ner_erg_features_hydration") - val enableDaysSinceRecentResurrectionFeatureHydration = Value( - "frigate_pushservice_enable_days_since_recent_resurrection_features_hydration") - val enableUserPastAggregatesFeatureHydration = Value( - "frigate_pushservice_enable_user_past_aggregates_features_hydration") - val enableUserSignalLanguageFeatureHydration = Value( - "frigate_pushservice_enable_user_signal_language_features_hydration") - val enableUserPreferredLanguageFeatureHydration = Value( - "frigate_pushservice_enable_user_preferred_language_features_hydration") - val enablePredicateDetailedInfoScribing = Value( - "frigate_pushservice_enable_predicate_detailed_info_scribing") - val enablePushCapInfoScribing = Value("frigate_pushservice_enable_push_cap_info_scribing") - val disableMLInFiltering = Value("frigate_pushservice_disable_ml_in_filtering") - val useHydratedLabeledSendsForFeaturesDeciderKey = Value( - "use_hydrated_labeled_sends_for_features") - val verifyHydratedLabeledSendsForFeaturesDeciderKey = Value( - "verify_hydrated_labeled_sends_for_features") - val trainingDataDeciderKey = Value("frigate_notifier_quality_model_training_data") - val skipMlModelPredicateDeciderKey = Value("skip_ml_model_predicate") - val scribeModelFeaturesDeciderKey = Value("scribe_model_features") - val scribeModelFeaturesWithoutHydratingNewFeaturesDeciderKey = Value( - "scribe_model_features_without_hydrating_new_features") - val scribeModelFeaturesForRequestScribe = Value("scribe_model_features_for_request_scribe") - val enableMrUserSimclusterV2020FeaturesHydration = Value( - "frigate_pushservice_enable_mr_user_simcluster_v2020_hydration") - val enableMrUserSimclusterV2020NoZeroFeaturesHydration = Value( - "frigate_pushservice_enable_mr_user_simcluster_v2020_no_zero_feature_hydration") - val enableMrUserEngagedTweetTokensFeaturesHydration = Value( - "frigate_pushservice_enable_mr_user_engaged_tweet_tokens_feature_hydration") - val enableMrCandidateTweetTokensFeaturesHydration = Value( - "frigate_pushservice_enable_mr_candidate_tweet_tokens_feature_hydration") - val enableTopicEngagementRealTimeAggregatesFeatureHydration = Value( - "frigate_pushservice_enable_topic_engagement_real_time_aggregates_feature_hydration" - ) - val enableUserTopicAggregatesFeatureHydration = Value( - "frigate_pushservice_enable_user_topic_aggregates_feature_hydration" - ) - val enableDurationSinceLastVisitFeatureHydration = Value( - "frigate_pushservice_enable_duration_since_last_visit_features_hydration" - ) - val enableTwistlyAggregatesFeatureHydration = Value( - "frigate_pushservice_enable_twistly_agg_feature_hydration" - ) - val enableTwHINUserEngagementFeaturesHydration = Value( - "frigate_pushservice_enable_twhin_user_engagement_features_hydration" - ) - val enableTwHINUserFollowFeaturesHydration = Value( - "frigate_pushservice_enable_twhin_user_follow_features_hydration" - ) - val enableTwHINAuthorFollowFeaturesHydration = Value( - "frigate_pushservice_enable_twhin_author_follow_features_hydration" - ) - val enableTweetTwHINFavFeaturesHydration = Value( - "frigate_pushservice_enable_tweet_twhin_fav_features_hydration" - ) - val enableSpaceVisibilityLibraryFiltering = Value( - "frigate_pushservice_enable_space_visibility_library_filtering" - ) - val enableVfFeatureHydrationSpaceShim = Value( - "frigate_pushservice_enable_visibility_filtering_feature_hydration_in_space_shim") - val enableUserTopicFollowFeatureSet = Value( - "frigate_pushservice_enable_user_topic_follow_feature_hydration") - val enableOnboardingNewUserFeatureSet = Value( - "frigate_pushservice_enable_onboarding_new_user_feature_hydration") - val enableMrUserTopicSparseContFeatureSet = Value( - "frigate_pushservice_enable_mr_user_topic_sparse_cont_feature_hydration" - ) - val enableUserPenguinLanguageFeatureSet = Value( - "frigate_pushservice_enable_user_penguin_language_feature_hydration") - val enableMrUserHashspaceEmbeddingFeatureSet = Value( - "frigate_pushservice_enable_mr_user_hashspace_embedding_feature_hydration") - val enableMrUserAuthorSparseContFeatureSet = Value( - "frigate_pushservice_enable_mr_user_author_sparse_cont_feature_hydration" - ) - val enableMrTweetSentimentFeatureSet = Value( - "frigate_pushservice_enable_mr_tweet_sentiment_feature_hydration" - ) - val enableMrTweetAuthorAggregatesFeatureSet = Value( - "frigate_pushservice_enable_mr_tweet_author_aggregates_feature_hydration" - ) - val enableUserGeoFeatureSet = Value("frigate_pushservice_enable_user_geo_feature_hydration") - val enableAuthorGeoFeatureSet = Value("frigate_pushservice_enable_author_geo_feature_hydration") - - val rampupUserGeoFeatureSet = Value("frigate_pushservice_ramp_up_user_geo_feature_hydration") - val rampupAuthorGeoFeatureSet = Value("frigate_pushservice_ramp_up_author_geo_feature_hydration") - - val enablePopGeoTweets = Value("frigate_pushservice_enable_pop_geo_tweets") - val enableTrendsTweets = Value("frigate_pushservice_enable_trends_tweets") - val enableTripGeoTweetCandidates = Value("frigate_pushservice_enable_trip_geo_tweets") - val enableContentRecommenderMixerAdaptor = Value( - "frigate_pushservice_enable_content_recommender_mixer_adaptor") - val enableGenericCandidateAdaptor = Value("frigate_pushservice_enable_generic_candidate_adaptor") - val enableTripGeoTweetContentMixerDarkTraffic = Value( - "frigate_pushservice_enable_trip_geo_tweets_content_mixer_dark_traffic") - - val enableInsTraffic = Value("frigate_pushservice_enable_ins_traffic") - val enableIsTweetTranslatable = Value("frigate_pushservice_enable_is_tweet_translatable") - - val enableMrTweetSimClusterFeatureSet = Value( - "frigate_pushservice_enable_mr_tweet_simcluster_feature_hydration") - - val enableMrOfflineUserTweetTopicAggregate = Value( - "frigate_pushservice_enable_mr_offline_user_tweet_topic_aggregate_hydration") - - val enableMrOfflineUserTweetSimClusterAggregate = Value( - "frigate_pushservice_enable_mr_offline_user_tweet_simcluster_aggregate_hydration" - ) - val enableRealGraphV2FeatureHydration = Value( - "frigate_pushservice_enable_real_graph_v2_features_hydration") - - val enableTweetBeTFeatureHydration = Value( - "frigate_pushservice_enable_tweet_bet_features_hydration") - - val enableInvalidatingCachedHistoryStoreAfterWrites = Value( - "frigate_pushservice_enable_invalidating_cached_history_store_after_writes") - - val enableInvalidatingCachedLoggedOutHistoryStoreAfterWrites = Value( - "frigate_pushservice_enable_invalidating_cached_logged_out_history_store_after_writes") - - val enableUserSendTimeFeatureHydration = Value( - "frigate_pushservice_enable_user_send_time_feature_hydration" - ) - - val enablePnegMultimodalPredictionForF1Tweets = Value( - "frigate_pushservice_enable_pneg_multimodal_prediction_for_f1_tweets" - ) - - val enableScribingOonFavScoreForF1Tweets = Value( - "frigate_pushservice_enable_oon_fav_scribe_for_f1_tweets" - ) - - val enableMrUserUtcSendTimeAggregateFeaturesHydration = Value( - "frigate_pushservice_enable_mr_user_utc_send_time_aggregate_hydration" - ) - - val enableMrUserLocalSendTimeAggregateFeaturesHydration = Value( - "frigate_pushservice_enable_mr_user_local_send_time_aggregate_hydration" - ) - - val enableBqmlReportModelPredictionForF1Tweets = Value( - "frigate_pushservice_enable_bqml_report_model_prediction_for_f1_tweets" - ) - - val enableUserTwhinEmbeddingFeatureHydration = Value( - "frigate_pushservice_enable_user_twhin_embedding_feature_hydration" - ) - - val enableAuthorFollowTwhinEmbeddingFeatureHydration = Value( - "frigate_pushservice_enable_author_follow_twhin_embedding_feature_hydration" - ) - - val enableScribingMLFeaturesAsDataRecord = Value( - "frigate_pushservice_enable_scribing_ml_features_as_datarecord" - ) - - val enableDirectHydrationForUserFeatures = Value( - "frigate_pushservice_enable_direct_hydration_for_user_features" - ) - - val enableAuthorVerifiedFeatureHydration = Value( - "frigate_pushservice_enable_author_verified_feature_hydration" - ) - - val enableAuthorCreatorSubscriptionFeatureHydration = Value( - "frigate_pushservice_enable_author_creator_subscription_feature_hydration" - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushConstants.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushConstants.docx new file mode 100644 index 000000000..2e15a0214 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushConstants.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushConstants.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushConstants.scala deleted file mode 100644 index cd96b4934..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushConstants.scala +++ /dev/null @@ -1,126 +0,0 @@ -package com.twitter.frigate.pushservice.params - -import com.twitter.conversions.DurationOps._ -import com.twitter.frigate.user_states.thriftscala.UserState -import java.util.Locale - -object PushConstants { - - final val ServiceProdEnvironmentName = "prod" - - final val RestrictLightRankingCandidatesThreshold = 1 - - final val DownSampleLightRankingScribeCandidatesRate = 1 - - final val NewUserLookbackWindow = 1.days - - final val PushCapInactiveUserAndroid = 1 - final val PushCapInactiveUserIos = 1 - final val PushCapLightOccasionalOpenerUserAndroid = 1 - final val PushCapLightOccasionalOpenerUserIos = 1 - - final val UserStateToPushCapIos = Map( - UserState.Inactive.name -> PushCapInactiveUserIos, - UserState.LightOccasionalOpener.name -> PushCapLightOccasionalOpenerUserIos - ) - final val UserStateToPushCapAndroid = Map( - UserState.Inactive.name -> PushCapInactiveUserAndroid, - UserState.LightOccasionalOpener.name -> PushCapLightOccasionalOpenerUserAndroid - ) - - final val AcceptableTimeSinceLastNegativeResponse = 1.days - - final val DefaultLookBackForHistory = 1.hours - - final val DefaultEventMediaUrl = "" - - final val ConnectTabPushTapThrough = "i/connect_people" - - final val AddressBookUploadTapThrough = "i/flow/mr-address-book-upload" - final val InterestPickerTapThrough = "i/flow/mr-interest-picker" - final val CompleteOnboardingInterestAddressTapThrough = "i/flow/mr-interest-address" - - final val IndiaCountryCode = "IN" - final val JapanCountryCode = Locale.JAPAN.getCountry.toUpperCase - final val UKCountryCode = Locale.UK.getCountry.toUpperCase - - final val IndiaTimeZoneCode = "Asia/Kolkata" - final val JapanTimeZoneCode = "Asia/Tokyo" - final val UKTimeZoneCode = "Europe/London" - - final val countryCodeToTimeZoneMap = Map( - IndiaCountryCode -> IndiaTimeZoneCode, - JapanCountryCode -> JapanTimeZoneCode, - UKCountryCode -> UKTimeZoneCode - ) - - final val AbuseStrike_Top2Percent_Id = "AbuseStrike_Top2Percent_Id" - final val AbuseStrike_Top1Percent_Id = "AbuseStrike_Top1Percent_Id" - final val AbuseStrike_Top05Percent_Id = "AbuseStrike_Top05Percent_Id" - final val AbuseStrike_Top025Percent_Id = "AbuseStrike_Top025Percent_Id" - final val AllSpamReportsPerFav_Top1Percent_Id = "AllSpamReportsPerFav_Top1Percent_Id" - final val ReportsPerFav_Top1Percent_Id = "ReportsPerFav_Top1Percent_Id" - final val ReportsPerFav_Top2Percent_Id = "ReportsPerFav_Top2Percent_Id" - final val MediaUnderstanding_Nudity_Id = "MediaUnderstanding_Nudity_Id" - final val MediaUnderstanding_Beauty_Id = "MediaUnderstanding_Beauty_Id" - final val MediaUnderstanding_SinglePerson_Id = "MediaUnderstanding_SinglePerson_Id" - final val PornList_Id = "PornList_Id" - final val PornographyAndNsfwContent_Id = "PornographyAndNsfwContent_Id" - final val SexLife_Id = "SexLife_Id" - final val SexLifeOrSexualOrientation_Id = "SexLifeOrSexualOrientation_Id" - final val ProfanityFilter_Id = "ProfanityFilter_Id" - final val TweetSemanticCoreIdFeature = "tweet.core.tweet.semantic_core_annotations" - final val targetUserGenderFeatureName = "Target.User.Gender" - final val targetUserAgeFeatureName = "Target.User.AgeBucket" - final val targetUserPreferredLanguage = "user.language.user.preferred_contents" - final val tweetAgeInHoursFeatureName = "RecTweet.TweetyPieResult.TweetAgeInHrs" - final val authorActiveFollowerFeatureName = "RecTweetAuthor.User.ActiveFollowers" - final val favFeatureName = "tweet.core.tweet_counts.favorite_count" - final val sentFeatureName = - "tweet.magic_recs_tweet_real_time_aggregates_v2.pair.v2.magicrecs.realtime.is_sent.any_feature.Duration.Top.count" - final val authorSendCountFeatureName = - "tweet_author_aggregate.pair.any_label.any_feature.28.days.count" - final val authorReportCountFeatureName = - "tweet_author_aggregate.pair.label.reportTweetDone.any_feature.28.days.count" - final val authorDislikeCountFeatureName = - "tweet_author_aggregate.pair.label.ntab.isDisliked.any_feature.28.days.count" - final val TweetLikesFeatureName = "tweet.core.tweet_counts.favorite_count" - final val TweetRepliesFeatureName = "tweet.core.tweet_counts.reply_count" - - final val EnableCopyFeaturesForIbis2ModelValues = "has_copy_features" - - final val EmojiFeatureNameForIbis2ModelValues = "emoji" - - final val TargetFeatureNameForIbis2ModelValues = "target" - - final val CopyBodyExpIbisModelValues = "enable_body_exp" - - final val TweetMediaEmbeddingBQKeyIds = Seq( - 230, 110, 231, 111, 232, 233, 112, 113, 234, 235, 114, 236, 115, 237, 116, 117, 238, 118, 239, - 119, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 240, 120, 241, 121, 242, 0, 1, 122, 243, 244, 123, - 2, 124, 245, 3, 4, 246, 125, 5, 126, 247, 127, 248, 6, 128, 249, 7, 8, 129, 9, 20, 21, 22, 23, - 24, 25, 26, 27, 28, 29, 250, 130, 251, 252, 131, 132, 253, 133, 254, 134, 255, 135, 136, 137, - 138, 139, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 140, 141, 142, 143, 144, 145, 146, 147, 148, - 149, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, - 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 60, - 61, 62, 63, 64, 65, 66, 67, 68, 69, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 70, 71, - 72, 73, 74, 75, 76, 77, 78, 79, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 80, 81, 82, - 83, 84, 85, 86, 87, 88, 89, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 90, 91, 92, 93, - 94, 95, 96, 97, 98, 99, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, - 214, 215, 216, 217, 218, 219, 220, 100, 221, 101, 222, 223, 102, 224, 103, 104, 225, 105, 226, - 227, 106, 107, 228, 108, 229, 109 - ) - - final val SportsEventDomainId = 6L - - final val OoncQualityCombinedScore = "OoncQualityCombinedScore" -} - -object PushQPSLimitConstants { - - final val PerspectiveStoreQPS = 100000 - - final val IbisOrNTabQPSForRFPH = 100000 - - final val SocialGraphServiceBatchSize = 100000 -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushEnums.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushEnums.docx new file mode 100644 index 000000000..8135a9f9e Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushEnums.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushEnums.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushEnums.scala deleted file mode 100644 index 370e40076..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushEnums.scala +++ /dev/null @@ -1,135 +0,0 @@ -package com.twitter.frigate.pushservice.params - -/** - * Enum for naming scores we will scribe for non-personalized high quality candidate generation - */ -object HighQualityScribingScores extends Enumeration { - type Name = Value - val HeavyRankingScore = Value - val NonPersonalizedQualityScoreUsingCnn = Value - val BqmlNsfwScore = Value - val BqmlReportScore = Value -} - -/** - * Enum for quality upranking transform - */ -object MrQualityUprankingTransformTypeEnum extends Enumeration { - val Linear = Value - val Sigmoid = Value -} - -/** - * Enum for quality partial upranking transform - */ -object MrQualityUprankingPartialTypeEnum extends Enumeration { - val All = Value - val Oon = Value -} - -/** - * Enum for bucket membership in DDG 10220 Mr Bold Title Favorite Retweet Notification experiment - */ -object MRBoldTitleFavoriteAndRetweetExperimentEnum extends Enumeration { - val ShortTitle = Value -} - -/** - * Enum for ML filtering predicates - */ -object QualityPredicateEnum extends Enumeration { - val WeightedOpenOrNtabClick = Value - val ExplicitOpenOrNtabClickFilter = Value - val AlwaysTrue = Value // Disable ML filtering -} - -/** - * Enum to specify normalization used in BigFiltering experiments - */ -object BigFilteringNormalizationEnum extends Enumeration { - val NormalizationDisabled = Value - val NormalizeByNotSendingScore = Value -} - -/** - * Enum for inline actions - */ -object InlineActionsEnum extends Enumeration { - val Favorite = Value - val Follow = Value - val Reply = Value - val Retweet = Value -} - -/** - * Enum for template format - */ -object IbisTemplateFormatEnum extends Enumeration { - val template1 = Value -} - -/** - * Enum for Store name for Top Tweets By Geo - */ -object TopTweetsForGeoCombination extends Enumeration { - val Default = Value - val AccountsTweetFavAsBackfill = Value - val AccountsTweetFavIntermixed = Value -} - -/** - * Enum for scoring function for Top Tweets By Geo - */ -object TopTweetsForGeoRankingFunction extends Enumeration { - val Score = Value - val GeohashLengthAndThenScore = Value -} - -/** - * Enum for which version of popgeo tweets to be using - */ -object PopGeoTweetVersion extends Enumeration { - val Prod = Value -} - -/** - * Enum for Subtext in Android header - */ -object SubtextForAndroidPushHeader extends Enumeration { - val None = Value - val TargetHandler = Value - val TargetTagHandler = Value - val TargetName = Value - val AuthorTagHandler = Value - val AuthorName = Value -} - -object NsfwTextDetectionModel extends Enumeration { - val ProdModel = Value - val RetrainedModel = Value -} - -object HighQualityCandidateGroupEnum extends Enumeration { - val AgeBucket = Value - val Language = Value - val Topic = Value - val Country = Value - val Admin0 = Value - val Admin1 = Value -} - -object CrtGroupEnum extends Enumeration { - val Twistly = Value - val Frs = Value - val F1 = Value - val Topic = Value - val Trip = Value - val GeoPop = Value - val Other = Value - val None = Value -} - -object SportGameEnum extends Enumeration { - val Soccer = Value - val Nfl = Value -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushFeatureSwitchParams.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushFeatureSwitchParams.docx new file mode 100644 index 000000000..a254b4ee1 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushFeatureSwitchParams.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushFeatureSwitchParams.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushFeatureSwitchParams.scala deleted file mode 100644 index 262b3a8b7..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushFeatureSwitchParams.scala +++ /dev/null @@ -1,5043 +0,0 @@ -package com.twitter.frigate.pushservice.params - -import com.twitter.conversions.DurationOps._ -import com.twitter.frigate.pushservice.params.InlineActionsEnum._ -import com.twitter.frigate.pushservice.params.HighQualityCandidateGroupEnum._ -import com.twitter.timelines.configapi.DurationConversion -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSEnumParam -import com.twitter.timelines.configapi.FSEnumSeqParam -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.util.Duration - -object PushFeatureSwitchParams { - - /** - * List of CRTs to uprank. Last CRT in sequence ends up on top of list - */ - object ListOfCrtsToUpRank - extends FSParam[Seq[String]]("rerank_candidates_crt_to_top", default = Seq.empty[String]) - - object ListOfCrtsForOpenApp - extends FSParam[Seq[String]]( - "open_app_allowed_crts", - default = Seq( - "f1firstdegreetweet", - "f1firstdegreephoto", - "f1firstdegreevideo", - "geopoptweet", - "frstweet", - "trendtweet", - "hermituser", - "triangularloopuser" - )) - - /** - * List of CRTs to downrank. Last CRT in sequence ends up on bottom of list - */ - object ListOfCrtsToDownRank - extends FSParam[Seq[String]]( - name = "rerank_candidates_crt_to_downrank", - default = Seq.empty[String]) - - /** - * Param to enable VF filtering in Tweetypie (vs using VisibilityLibrary) - */ - object EnableVFInTweetypie - extends FSParam[Boolean]( - name = "visibility_filtering_enable_vf_in_tweetypie", - default = true - ) - - /** - * Number of max earlybird candidates - */ - object NumberOfMaxEarlybirdInNetworkCandidatesParam - extends FSBoundedParam( - name = "frigate_push_max_earlybird_in_network_candidates", - default = 100, - min = 0, - max = 800 - ) - - /** - * Number of max UserTweetEntityGraph candidates to query - */ - object NumberOfMaxUTEGCandidatesQueriedParam - extends FSBoundedParam( - name = "frigate_push_max_uteg_candidates_queried", - default = 30, - min = 0, - max = 300 - ) - - /** - * Param to control the max tweet age for users - */ - object MaxTweetAgeParam - extends FSBoundedParam[Duration]( - name = "tweet_age_max_hours", - default = 24.hours, - min = 1.hours, - max = 72.hours - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to control the max tweet age for modeling-based candidates - */ - object ModelingBasedCandidateMaxTweetAgeParam - extends FSBoundedParam[Duration]( - name = "tweet_age_candidate_generation_model_max_hours", - default = 24.hours, - min = 1.hours, - max = 72.hours - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to control the max tweet age for simcluster-based candidates - */ - object GeoPopTweetMaxAgeInHours - extends FSBoundedParam[Duration]( - name = "tweet_age_geo_pop_max_hours", - default = 24.hours, - min = 1.hours, - max = 120.hours - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to control the max tweet age for simcluster-based candidates - */ - object SimclusterBasedCandidateMaxTweetAgeParam - extends FSBoundedParam[Duration]( - name = "tweet_age_simcluster_max_hours", - default = 24.hours, - min = 24.hours, - max = 48.hours - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to control the max tweet age for Detopic-based candidates - */ - object DetopicBasedCandidateMaxTweetAgeParam - extends FSBoundedParam[Duration]( - name = "tweet_age_detopic_max_hours", - default = 24.hours, - min = 24.hours, - max = 48.hours - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to control the max tweet age for F1 candidates - */ - object F1CandidateMaxTweetAgeParam - extends FSBoundedParam[Duration]( - name = "tweet_age_f1_max_hours", - default = 24.hours, - min = 1.hours, - max = 96.hours - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to control the max tweet age for Explore Video Tweet - */ - object ExploreVideoTweetAgeParam - extends FSBoundedParam[Duration]( - name = "explore_video_tweets_age_max_hours", - default = 48.hours, - min = 1.hours, - max = 336.hours // Two weeks - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to no send for new user playbook push if user login for past hours - */ - object NewUserPlaybookAllowedLastLoginHours - extends FSBoundedParam[Duration]( - name = "new_user_playbook_allowed_last_login_hours", - default = 0.hours, - min = 0.hours, - max = 72.hours - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * The batch size of RefreshForPushHandler's Take step - */ - object NumberOfMaxCandidatesToBatchInRFPHTakeStep - extends FSBoundedParam( - name = "frigate_push_rfph_batch_take_max_size", - default = 1, - min = 1, - max = 10 - ) - - /** - * The maximum number of candidates to batch for Importance Sampling - */ - object NumberOfMaxCandidatesToBatchForImportanceSampling - extends FSBoundedParam( - name = "frigate_push_rfph_max_candidates_to_batch_for_importance_sampling", - default = 65, - min = 1, - max = 500 - ) - - /** - * Maximum number of regular MR push in 24.hours/daytime/nighttime - */ - object MaxMrPushSends24HoursParam - extends FSBoundedParam( - name = "pushcap_max_sends_24hours", - default = 5, - min = 0, - max = 12 - ) - - /** - * Maximum number of regular MR ntab only channel in 24.hours/daytime/nighttime - */ - object MaxMrNtabOnlySends24HoursParamV3 - extends FSBoundedParam( - name = "pushcap_max_sends_24hours_ntabonly_v3", - default = 5, - min = 0, - max = 12 - ) - - /** - * Maximum number of regular MR ntab only in 24.hours/daytime/nighttime - */ - object MaxMrPushSends24HoursNtabOnlyUsersParam - extends FSBoundedParam( - name = "pushcap_max_sends_24hours_ntab_only", - default = 5, - min = 0, - max = 10 - ) - - /** - * Customized PushCap offset (e.g., to the predicted value) - */ - object CustomizedPushCapOffset - extends FSBoundedParam[Int]( - name = "pushcap_customized_offset", - default = 0, - min = -2, - max = 4 - ) - - /** - * Param to enable restricting minimum pushcap assigned with ML models - * */ - object EnableRestrictedMinModelPushcap - extends FSParam[Boolean]( - name = "pushcap_restricted_model_min_enable", - default = false - ) - - /** - * Param to specify the minimum pushcap allowed to be assigned with ML models - * */ - object RestrictedMinModelPushcap - extends FSBoundedParam[Int]( - name = "pushcap_restricted_model_min_value", - default = 1, - min = 0, - max = 9 - ) - - object EnablePushcapRefactor - extends FSParam[Boolean]( - name = "pushcap_enable_refactor", - default = false - ) - - /** - * Enables the restrict step in pushservice for a given user - * - * Setting this to false may cause a large number of candidates to be passed on to filtering/take - * step in RefreshForPushHandler, increasing the service latency significantly - */ - object EnableRestrictStep extends FSParam[Boolean]("frigate_push_rfph_restrict_step_enable", true) - - /** - * The number of candidates that are able to pass through the restrict step. - */ - object RestrictStepSize - extends FSBoundedParam( - name = "frigate_push_rfph_restrict_step_size", - default = 65, - min = 65, - max = 200 - ) - - /** - * Number of max crMixer candidates to send. - */ - object NumberOfMaxCrMixerCandidatesParam - extends FSBoundedParam( - name = "cr_mixer_migration_max_num_of_candidates_to_return", - default = 400, - min = 0, - max = 2000 - ) - - /** - * Duration between two MR pushes - */ - object MinDurationSincePushParam - extends FSBoundedParam[Duration]( - name = "pushcap_min_duration_since_push_hours", - default = 4.hours, - min = 0.hours, - max = 72.hours - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Each Phase duration to gradually ramp up MagicRecs for new users - */ - object GraduallyRampUpPhaseDurationDays - extends FSBoundedParam[Duration]( - name = "pushcap_gradually_ramp_up_phase_duration_days", - default = 3.days, - min = 2.days, - max = 7.days - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Param to specify interval for target pushcap fatigue - */ - object TargetPushCapFatigueIntervalHours - extends FSBoundedParam[Duration]( - name = "pushcap_fatigue_interval_hours", - default = 24.hours, - min = 1.hour, - max = 240.hours - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to specify interval for target ntabOnly fatigue - */ - object TargetNtabOnlyCapFatigueIntervalHours - extends FSBoundedParam[Duration]( - name = "pushcap_ntabonly_fatigue_interval_hours", - default = 24.hours, - min = 1.hour, - max = 240.hours - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to use completely explicit push cap instead of LTV/modeling-based - */ - object EnableExplicitPushCap - extends FSParam[Boolean]( - name = "pushcap_explicit_enable", - default = false - ) - - /** - * Param to control explicit push cap (non-LTV) - */ - object ExplicitPushCap - extends FSBoundedParam[Int]( - name = "pushcap_explicit_value", - default = 1, - min = 0, - max = 20 - ) - - /** - * Parameters for percentile thresholds of OpenOrNtabClick model in MR filtering model refreshing DDG - */ - object PercentileThresholdCohort1 - extends FSBoundedParam[Double]( - name = "frigate_push_modeling_percentile_threshold_cohort1", - default = 0.65, - min = 0.0, - max = 1.0 - ) - - object PercentileThresholdCohort2 - extends FSBoundedParam[Double]( - name = "frigate_push_modeling_percentile_threshold_cohort2", - default = 0.03, - min = 0.0, - max = 1.0 - ) - object PercentileThresholdCohort3 - extends FSBoundedParam[Double]( - name = "frigate_push_modeling_percentile_threshold_cohort3", - default = 0.03, - min = 0.0, - max = 1.0 - ) - object PercentileThresholdCohort4 - extends FSBoundedParam[Double]( - name = "frigate_push_modeling_percentile_threshold_cohort4", - default = 0.06, - min = 0.0, - max = 1.0 - ) - object PercentileThresholdCohort5 - extends FSBoundedParam[Double]( - name = "frigate_push_modeling_percentile_threshold_cohort5", - default = 0.06, - min = 0.0, - max = 1.0 - ) - object PercentileThresholdCohort6 - extends FSBoundedParam[Double]( - name = "frigate_push_modeling_percentile_threshold_cohort6", - default = 0.8, - min = 0.0, - max = 1.0 - ) - - /** - * Parameters for percentile threshold list of OpenOrNtabCLick model in MR percentile grid search experiments - */ - object MrPercentileGridSearchThresholdsCohort1 - extends FSParam[Seq[Double]]( - name = "frigate_push_modeling_percentile_grid_search_thresholds_cohort1", - default = Seq(0.8, 0.75, 0.65, 0.55, 0.45, 0.35, 0.25) - ) - object MrPercentileGridSearchThresholdsCohort2 - extends FSParam[Seq[Double]]( - name = "frigate_push_modeling_percentile_grid_search_thresholds_cohort2", - default = Seq(0.15, 0.12, 0.1, 0.08, 0.06, 0.045, 0.03) - ) - object MrPercentileGridSearchThresholdsCohort3 - extends FSParam[Seq[Double]]( - name = "frigate_push_modeling_percentile_grid_search_thresholds_cohort3", - default = Seq(0.15, 0.12, 0.1, 0.08, 0.06, 0.045, 0.03) - ) - object MrPercentileGridSearchThresholdsCohort4 - extends FSParam[Seq[Double]]( - name = "frigate_push_modeling_percentile_grid_search_thresholds_cohort4", - default = Seq(0.15, 0.12, 0.1, 0.08, 0.06, 0.045, 0.03) - ) - object MrPercentileGridSearchThresholdsCohort5 - extends FSParam[Seq[Double]]( - name = "frigate_push_modeling_percentile_grid_search_thresholds_cohort5", - default = Seq(0.3, 0.2, 0.15, 0.1, 0.08, 0.06, 0.05) - ) - object MrPercentileGridSearchThresholdsCohort6 - extends FSParam[Seq[Double]]( - name = "frigate_push_modeling_percentile_grid_search_thresholds_cohort6", - default = Seq(0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2) - ) - - /** - * Parameters for threshold list of OpenOrNtabClick model in MF grid search experiments - */ - object MfGridSearchThresholdsCohort1 - extends FSParam[Seq[Double]]( - name = "frigate_push_modeling_mf_grid_search_thresholds_cohort1", - default = Seq(0.030, 0.040, 0.050, 0.062, 0.070, 0.080, 0.090) // default: 0.062 - ) - object MfGridSearchThresholdsCohort2 - extends FSParam[Seq[Double]]( - name = "frigate_push_modeling_mf_grid_search_thresholds_cohort2", - default = Seq(0.005, 0.010, 0.015, 0.020, 0.030, 0.040, 0.050) // default: 0.020 - ) - object MfGridSearchThresholdsCohort3 - extends FSParam[Seq[Double]]( - name = "frigate_push_modeling_mf_grid_search_thresholds_cohort3", - default = Seq(0.010, 0.015, 0.020, 0.025, 0.035, 0.045, 0.055) // default: 0.025 - ) - object MfGridSearchThresholdsCohort4 - extends FSParam[Seq[Double]]( - name = "frigate_push_modeling_mf_grid_search_thresholds_cohort4", - default = Seq(0.015, 0.020, 0.025, 0.030, 0.040, 0.050, 0.060) // default: 0.030 - ) - object MfGridSearchThresholdsCohort5 - extends FSParam[Seq[Double]]( - name = "frigate_push_modeling_mf_grid_search_thresholds_cohort5", - default = Seq(0.035, 0.040, 0.045, 0.050, 0.060, 0.070, 0.080) // default: 0.050 - ) - object MfGridSearchThresholdsCohort6 - extends FSParam[Seq[Double]]( - name = "frigate_push_modeling_mf_grid_search_thresholds_cohort6", - default = Seq(0.040, 0.045, 0.050, 0.055, 0.065, 0.075, 0.085) // default: 0.055 - ) - - /** - * Param to specify which global optout models to use to first predict the global scores for users - */ - object GlobalOptoutModelParam - extends FSParam[Seq[OptoutModel.ModelNameType]]( - name = "optout_model_global_model_ids", - default = Seq.empty[OptoutModel.ModelNameType] - ) - - /** - * Param to specify which optout model to use according to the experiment bucket - */ - object BucketOptoutModelParam - extends FSParam[OptoutModel.ModelNameType]( - name = "optout_model_bucket_model_id", - default = OptoutModel.D0_has_realtime_features - ) - - /* - * Param to enable candidate generation model - * */ - object EnableCandidateGenerationModelParam - extends FSParam[Boolean]( - name = "candidate_generation_model_enable", - default = false - ) - - object EnableOverrideForSportsCandidates - extends FSParam[Boolean](name = "magicfanout_sports_event_enable_override", default = true) - - object EnableEventIdBasedOverrideForSportsCandidates - extends FSParam[Boolean]( - name = "magicfanout_sports_event_enable_event_id_based_override", - default = true) - - /** - * Param to specify the threshold to determine if a user’s optout score is high enough to enter the experiment. - */ - object GlobalOptoutThresholdParam - extends FSParam[Seq[Double]]( - name = "optout_model_global_thresholds", - default = Seq(1.0, 1.0) - ) - - /** - * Param to specify the threshold to determine if a user’s optout score is high enough to be assigned - * with a reduced pushcap based on the bucket membership. - */ - object BucketOptoutThresholdParam - extends FSBoundedParam[Double]( - name = "optout_model_bucket_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Param to specify the reduced pushcap value if the optout probability predicted by the bucket - * optout model is higher than the specified bucket optout threshold. - */ - object OptoutExptPushCapParam - extends FSBoundedParam[Int]( - name = "optout_model_expt_push_cap", - default = 10, - min = 0, - max = 10 - ) - - /** - * Param to specify the thresholds to determine which push cap slot the user should be assigned to - * according to the optout score. For example,the slot thresholds are [0.1, 0.2, ..., 1.0], the user - * is assigned to the second slot if the optout score is in (0.1, 0.2]. - */ - object BucketOptoutSlotThresholdParam - extends FSParam[Seq[Double]]( - name = "optout_model_bucket_slot_thresholds", - default = Seq.empty[Double] - ) - - /** - * Param to specify the adjusted push cap of each slot. For example, if the slot push caps are [1, 2, ..., 10] - * and the user is assigned to the 2nd slot according to the optout score, the push cap of the user - * will be adjusted to 2. - */ - object BucketOptoutSlotPushcapParam - extends FSParam[Seq[Int]]( - name = "optout_model_bucket_slot_pushcaps", - default = Seq.empty[Int] - ) - - /** - * Param to specify if the optout score based push cap adjustment is enabled - */ - object EnableOptoutAdjustedPushcap - extends FSParam[Boolean]( - "optout_model_enable_optout_adjusted_pushcap", - false - ) - - /** - * Param to specify which weighted open or ntab click model to use - */ - object WeightedOpenOrNtabClickRankingModelParam - extends FSParam[WeightedOpenOrNtabClickModel.ModelNameType]( - name = "frigate_push_modeling_oonc_ranking_model_id", - default = WeightedOpenOrNtabClickModel.Periodically_Refreshed_Prod_Model - ) - - /** - * Param to disable heavy ranker - */ - object DisableHeavyRankingModelFSParam - extends FSParam[Boolean]( - name = "frigate_push_modeling_disable_heavy_ranking", - default = false - ) - - /** - * Param to specify which weighted open or ntab click model to use for Android modelling experiment - */ - object WeightedOpenOrNtabClickRankingModelForAndroidParam - extends FSParam[WeightedOpenOrNtabClickModel.ModelNameType]( - name = "frigate_push_modeling_oonc_ranking_model_for_android_id", - default = WeightedOpenOrNtabClickModel.Periodically_Refreshed_Prod_Model - ) - - /** - * Param to specify which weighted open or ntab click model to use for FILTERING - */ - object WeightedOpenOrNtabClickFilteringModelParam - extends FSParam[WeightedOpenOrNtabClickModel.ModelNameType]( - name = "frigate_push_modeling_oonc_filtering_model_id", - default = WeightedOpenOrNtabClickModel.Periodically_Refreshed_Prod_Model - ) - - /** - * Param to specify which quality predicate to use for ML filtering - */ - object QualityPredicateIdParam - extends FSEnumParam[QualityPredicateEnum.type]( - name = "frigate_push_modeling_quality_predicate_id", - default = QualityPredicateEnum.WeightedOpenOrNtabClick, - enum = QualityPredicateEnum - ) - - /** - * Param to control threshold for any quality predicates using explicit thresholds - */ - object QualityPredicateExplicitThresholdParam - extends FSBoundedParam[Double]( - name = "frigate_push_modeling_quality_predicate_explicit_threshold", - default = 0.1, - min = 0, - max = 1) - - /** - * MagicFanout relaxed eventID fatigue interval (when we want to enable multiple updates for the same event) - */ - object MagicFanoutRelaxedEventIdFatigueIntervalInHours - extends FSBoundedParam[Int]( - name = "frigate_push_magicfanout_relaxed_event_id_fatigue_interval_in_hours", - default = 24, - min = 0, - max = 720 - ) - - /** - * MagicFanout DenyListed Countries - */ - object MagicFanoutDenyListedCountries - extends FSParam[Seq[String]]( - "frigate_push_magicfanout_denylisted_countries", - Seq.empty[String]) - - object MagicFanoutSportsEventDenyListedCountries - extends FSParam[Seq[String]]( - "magicfanout_sports_event_denylisted_countries", - Seq.empty[String]) - - /** - * MagicFanout maximum erg rank for a given push event for non heavy users - */ - object MagicFanoutRankErgThresholdNonHeavy - extends FSBoundedParam[Int]( - name = "frigate_push_magicfanout_erg_rank_threshold_non_heavy", - default = 25, - min = 1, - max = 50 - ) - - /** - * MagicFanout maximum erg rank for a given push event for heavy users - */ - object MagicFanoutRankErgThresholdHeavy - extends FSBoundedParam[Int]( - name = "frigate_push_magicfanout_erg_rank_threshold_heavy", - default = 20, - min = 1, - max = 50 - ) - - object EnablePushMixerReplacingAllSources - extends FSParam[Boolean]( - name = "push_mixer_enable_replacing_all_sources", - default = false - ) - - object EnablePushMixerReplacingAllSourcesWithControl - extends FSParam[Boolean]( - name = "push_mixer_enable_replacing_all_sources_with_control", - default = false - ) - - object EnablePushMixerReplacingAllSourcesWithExtra - extends FSParam[Boolean]( - name = "push_mixer_enable_replacing_all_sources_with_extra", - default = false - ) - - object EnablePushMixerSource - extends FSParam[Boolean]( - name = "push_mixer_enable_source", - default = false - ) - - object PushMixerMaxResults - extends FSBoundedParam[Int]( - name = "push_mixer_max_results", - default = 10, - min = 1, - max = 5000 - ) - - /** - * Enable tweets from trends that have been annotated by curators - */ - object EnableCuratedTrendTweets - extends FSParam[Boolean](name = "trend_tweet_curated_trends_enable", default = false) - - /** - * Enable tweets from trends that haven't been annotated by curators - */ - object EnableNonCuratedTrendTweets - extends FSParam[Boolean](name = "trend_tweet_non_curated_trends_enable", default = false) - - /** - * Maximum trend tweet notifications in fixed duration - */ - object MaxTrendTweetNotificationsInDuration - extends FSBoundedParam[Int]( - name = "trend_tweet_max_notifications_in_duration", - min = 0, - default = 0, - max = 20) - - /** - * Duration in days over which trend tweet notifications fatigue is applied - */ - object TrendTweetNotificationsFatigueDuration - extends FSBoundedParam[Duration]( - name = "trend_tweet_notifications_fatigue_in_days", - default = 1.day, - min = Duration.Bottom, - max = Duration.Top - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Maximum number of trends candidates to query from event-recos endpoint - */ - object MaxRecommendedTrendsToQuery - extends FSBoundedParam[Int]( - name = "trend_tweet_max_trends_to_query", - min = 0, - default = 0, - max = 100) - - /** - * Fix missing event-associated interests in MagicFanoutNoOptoutInterestsPredicate - */ - object MagicFanoutFixNoOptoutInterestsBugParam - extends FSParam[Boolean]("frigate_push_magicfanout_fix_no_optout_interests", default = true) - - object EnableSimclusterOfflineAggFeatureForExpt - extends FSParam[Boolean]("frigate_enable_simcluster_offline_agg_feature", false) - - /** - * Param to enable removal of UTT domain for - */ - object ApplyMagicFanoutBroadEntityInterestRankThresholdPredicate - extends FSParam[Boolean]( - "frigate_push_magicfanout_broad_entity_interest_rank_threshold_predicate", - false - ) - - object HydrateEventReasonsFeatures - extends FSParam[Boolean]( - name = "frigate_push_magicfanout_hydrate_event_reasons_features", - false - ) - - /** - * Param to enable online MR history features - */ - object EnableHydratingOnlineMRHistoryFeatures - extends FSParam[Boolean]( - name = "feature_hydration_online_mr_history", - default = false - ) - - /** - * Param to enable bold title on favorite and retweet push copy for Android in DDG 10220 - */ - object MRBoldTitleFavoriteAndRetweetParam - extends FSEnumParam[MRBoldTitleFavoriteAndRetweetExperimentEnum.type]( - name = "frigate_push_bold_title_favorite_and_retweet_id", - default = MRBoldTitleFavoriteAndRetweetExperimentEnum.ShortTitle, - enum = MRBoldTitleFavoriteAndRetweetExperimentEnum - ) - - /** - * Param to enable high priority push - */ - object EnableHighPriorityPush - extends FSParam[Boolean]("frigate_push_magicfanout_enable_high_priority_push", false) - - /** - * Param to redirect sports crt event to a custom url - */ - object EnableSearchURLRedirectForSportsFanout - extends FSParam[Boolean]("magicfanout_sports_event_enable_search_url_redirect", false) - - /** - * Param to enable score fanout notification for sports - */ - object EnableScoreFanoutNotification - extends FSParam[Boolean]("magicfanout_sports_event_enable_score_fanout", false) - - /** - * Param to add custom search url for sports crt event - */ - object SearchURLRedirectForSportsFanout - extends FSParam[String]( - name = "magicfanout_sports_event_search_url_redirect", - default = "https://twitter.com/explore/tabs/ipl", - ) - - /** - * Param to enable high priority sports push - */ - object EnableHighPrioritySportsPush - extends FSParam[Boolean]("magicfanout_sports_event_enable_high_priority_push", false) - - /** - * Param to control rank threshold for magicfanout user follow - */ - object MagicFanoutRealgraphRankThreshold - extends FSBoundedParam[Int]( - name = "magicfanout_realgraph_threshold", - default = 500, - max = 500, - min = 100 - ) - - /** - * Topic score threshold for topic proof tweet candidates topic annotations - * */ - object TopicProofTweetCandidatesTopicScoreThreshold - extends FSBoundedParam[Double]( - name = "topics_as_social_proof_topic_score_threshold", - default = 0.0, - min = 0.0, - max = 100.0 - ) - - /** - * Enable Topic Proof Tweet Recs - */ - object EnableTopicProofTweetRecs - extends FSParam[Boolean](name = "topics_as_social_proof_enable", default = true) - - /** - * Enable health filters for topic tweet notifications - */ - object EnableHealthFiltersForTopicProofTweet - extends FSParam[Boolean]( - name = "topics_as_social_proof_enable_health_filters", - default = false) - - /** - * Disable health filters for CrMixer candidates - */ - object DisableHealthFiltersForCrMixerCandidates - extends FSParam[Boolean]( - name = "health_and_quality_filter_disable_for_crmixer_candidates", - default = false) - - object EnableMagicFanoutNewsForYouNtabCopy - extends FSParam[Boolean](name = "send_handler_enable_nfy_ntab_copy", default = false) - - /** - * Param to enable semi-personalized high quality candidates in pushservice - * */ - object HighQualityCandidatesEnableCandidateSource - extends FSParam[Boolean]( - name = "high_quality_candidates_enable_candidate_source", - default = false - ) - - /** - * Param to decide semi-personalized high quality candidates - * */ - object HighQualityCandidatesEnableGroups - extends FSEnumSeqParam[HighQualityCandidateGroupEnum.type]( - name = "high_quality_candidates_enable_groups_ids", - default = Seq(AgeBucket, Language), - enum = HighQualityCandidateGroupEnum - ) - - /** - * Param to decide semi-personalized high quality candidates - * */ - object HighQualityCandidatesNumberOfCandidates - extends FSBoundedParam[Int]( - name = "high_quality_candidates_number_of_candidates", - default = 0, - min = 0, - max = Int.MaxValue - ) - - /** - * Param to enable small domain falling back to bigger domains for high quality candidates in pushservice - * */ - object HighQualityCandidatesEnableFallback - extends FSParam[Boolean]( - name = "high_quality_candidates_enable_fallback", - default = false - ) - - /** - * Param to decide whether to fallback to bigger domain for high quality candidates - * */ - object HighQualityCandidatesMinNumOfCandidatesToFallback - extends FSBoundedParam[Int]( - name = "high_quality_candidates_min_num_of_candidates_to_fallback", - default = 50, - min = 0, - max = Int.MaxValue - ) - - /** - * Param to specific source ids for high quality candidates - * */ - object HighQualityCandidatesFallbackSourceIds - extends FSParam[Seq[String]]( - name = "high_quality_candidates_fallback_source_ids", - default = Seq("HQ_C_COUNT_PASS_QUALITY_SCORES")) - - /** - * Param to decide groups for semi-personalized high quality candidates - * */ - object HighQualityCandidatesFallbackEnabledGroups - extends FSEnumSeqParam[HighQualityCandidateGroupEnum.type]( - name = "high_quality_candidates_fallback_enabled_groups_ids", - default = Seq(Country), - enum = HighQualityCandidateGroupEnum - ) - - /** - * Param to control what heavy ranker model to use for scribing scores - */ - object HighQualityCandidatesHeavyRankingModel - extends FSParam[String]( - name = "high_quality_candidates_heavy_ranking_model", - default = "Periodically_Refreshed_Prod_Model_V11" - ) - - /** - * Param to control what non personalized quality "Cnn" model to use for scribing scores - */ - object HighQualityCandidatesNonPersonalizedQualityCnnModel - extends FSParam[String]( - name = "high_quality_candidates_non_personalized_quality_cnn_model", - default = "Q1_2023_Mr_Tf_Quality_Model_cnn" - ) - - /** - * Param to control what nsfw health model to use for scribing scores - */ - object HighQualityCandidatesBqmlNsfwModel - extends FSParam[String]( - name = "high_quality_candidates_bqml_nsfw_model", - default = "Q2_2022_Mr_Bqml_Health_Model_NsfwV0" - ) - - /** - * Param to control what reportodel to use for scribing scores - */ - object HighQualityCandidatesBqmlReportModel - extends FSParam[String]( - name = "high_quality_candidates_bqml_report_model", - default = "Q3_2022_15266_Mr_Bqml_Non_Personalized_Report_Model_with_Media_Embeddings" - ) - - /** - * Param to specify the threshold to determine if a tweet contains nudity media - */ - object TweetMediaSensitiveCategoryThresholdParam - extends FSBoundedParam[Double]( - name = "tweet_media_sensitive_category_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Param to boost candidates from subscription creators - */ - object BoostCandidatesFromSubscriptionCreators - extends FSParam[Boolean]( - name = "subscription_enable_boost_candidates_from_active_creators", - default = false - ) - - /** - * Param to soft rank candidates from subscription creators - */ - object SoftRankCandidatesFromSubscriptionCreators - extends FSParam[Boolean]( - name = "subscription_enable_soft_rank_candidates_from_active_creators", - default = false - ) - - /** - * Param as factor to control how much we want to boost creator tweets - */ - object SoftRankFactorForSubscriptionCreators - extends FSBoundedParam[Double]( - name = "subscription_soft_rank_factor_for_boost", - default = 1.0, - min = 0.0, - max = Double.MaxValue - ) - - /** - * Param to enable new OON copy for Push Notifications - */ - object EnableNewMROONCopyForPush - extends FSParam[Boolean]( - name = "mr_copy_enable_new_mr_oon_copy_push", - default = true - ) - - /** - * Param to enable generated inline actions on OON Notifications - */ - object EnableOONGeneratedInlineActions - extends FSParam[Boolean]( - name = "mr_inline_enable_oon_generated_actions", - default = false - ) - - /** - * Param to control dynamic inline actions for Out-of-Network copies - */ - object OONTweetDynamicInlineActionsList - extends FSEnumSeqParam[InlineActionsEnum.type]( - name = "mr_inline_oon_tweet_dynamic_action_ids", - default = Seq(Follow, Retweet, Favorite), - enum = InlineActionsEnum - ) - - object HighOONCTweetFormat - extends FSEnumParam[IbisTemplateFormatEnum.type]( - name = "mr_copy_high_oonc_format_id", - default = IbisTemplateFormatEnum.template1, - enum = IbisTemplateFormatEnum - ) - - object LowOONCTweetFormat - extends FSEnumParam[IbisTemplateFormatEnum.type]( - name = "mr_copy_low_oonc_format_id", - default = IbisTemplateFormatEnum.template1, - enum = IbisTemplateFormatEnum - ) - - /** - * Param to enable dynamic inline actions based on FSParams for Tweet copies (not OON) - */ - object EnableTweetDynamicInlineActions - extends FSParam[Boolean]( - name = "mr_inline_enable_tweet_dynamic_actions", - default = false - ) - - /** - * Param to control dynamic inline actions for Tweet copies (not OON) - */ - object TweetDynamicInlineActionsList - extends FSEnumSeqParam[InlineActionsEnum.type]( - name = "mr_inline_tweet_dynamic_action_ids", - default = Seq(Reply, Retweet, Favorite), - enum = InlineActionsEnum - ) - - object UseInlineActionsV1 - extends FSParam[Boolean]( - name = "mr_inline_use_inline_action_v1", - default = true - ) - - object UseInlineActionsV2 - extends FSParam[Boolean]( - name = "mr_inline_use_inline_action_v2", - default = false - ) - - object EnableInlineFeedbackOnPush - extends FSParam[Boolean]( - name = "mr_inline_enable_inline_feedback_on_push", - default = false - ) - - object InlineFeedbackSubstitutePosition - extends FSBoundedParam[Int]( - name = "mr_inline_feedback_substitute_position", - min = 0, - max = 2, - default = 2, // default to substitute or append last inline action - ) - - /** - * Param to control dynamic inline actions for web notifications - */ - object EnableDynamicInlineActionsForDesktopWeb - extends FSParam[Boolean]( - name = "mr_inline_enable_dynamic_actions_for_desktop_web", - default = false - ) - - object EnableDynamicInlineActionsForMobileWeb - extends FSParam[Boolean]( - name = "mr_inline_enable_dynamic_actions_for_mobile_web", - default = false - ) - - /** - * Param to define dynamic inline action types for web notifications (both desktop web + mobile web) - */ - object TweetDynamicInlineActionsListForWeb - extends FSEnumSeqParam[InlineActionsEnum.type]( - name = "mr_inline_tweet_dynamic_action_for_web_ids", - default = Seq(Retweet, Favorite), - enum = InlineActionsEnum - ) - - /** - * Param to enable MR Override Notifications for Android - */ - object EnableOverrideNotificationsForAndroid - extends FSParam[Boolean]( - name = "mr_override_enable_override_notifications_for_android", - default = false - ) - - /** - * Param to enable MR Override Notifications for iOS - */ - object EnableOverrideNotificationsForIos - extends FSParam[Boolean]( - name = "mr_override_enable_override_notifications_for_ios", - default = false - ) - - /** - * Param to enable gradually ramp up notification - */ - object EnableGraduallyRampUpNotification - extends FSParam[Boolean]( - name = "pushcap_gradually_ramp_up_enable", - default = false - ) - - /** - * Param to control the minInrerval for fatigue between consecutive MFNFY pushes - */ - object MFMinIntervalFatigue - extends FSBoundedParam[Duration]( - name = "frigate_push_magicfanout_fatigue_min_interval_consecutive_pushes_minutes", - default = 240.minutes, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromMinutes - } - - /** - * Param to control the interval for MFNFY pushes - */ - object MFPushIntervalInHours - extends FSBoundedParam[Duration]( - name = "frigate_push_magicfanout_fatigue_push_interval_in_hours", - default = 24.hours, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to control the maximum number of Sports MF pushes in a period of time - */ - object SportsMaxNumberOfPushesInInterval - extends FSBoundedParam[Int]( - name = "magicfanout_sports_event_fatigue_max_pushes_in_interval", - default = 2, - min = 0, - max = 6) - - /** - * Param to control the minInterval for fatigue between consecutive sports pushes - */ - object SportsMinIntervalFatigue - extends FSBoundedParam[Duration]( - name = "magicfanout_sports_event_fatigue_min_interval_consecutive_pushes_minutes", - default = 240.minutes, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromMinutes - } - - /** - * Param to control the interval for sports pushes - */ - object SportsPushIntervalInHours - extends FSBoundedParam[Duration]( - name = "magicfanout_sports_event_fatigue_push_interval_in_hours", - default = 24.hours, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to control the maximum number of same event sports MF pushes in a period of time - */ - object SportsMaxNumberOfPushesInIntervalPerEvent - extends FSBoundedParam[Int]( - name = "magicfanout_sports_event_fatigue_max_pushes_in_per_event_interval", - default = 2, - min = 0, - max = 6) - - /** - * Param to control the minInterval for fatigue between consecutive same event sports pushes - */ - object SportsMinIntervalFatiguePerEvent - extends FSBoundedParam[Duration]( - name = "magicfanout_sports_event_fatigue_min_interval_consecutive_pushes_per_event_minutes", - default = 240.minutes, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromMinutes - } - - /** - * Param to control the interval for same event sports pushes - */ - object SportsPushIntervalInHoursPerEvent - extends FSBoundedParam[Duration]( - name = "magicfanout_sports_event_fatigue_push_interval_per_event_in_hours", - default = 24.hours, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to control the maximum number of MF pushes in a period of time - */ - object MFMaxNumberOfPushesInInterval - extends FSBoundedParam[Int]( - name = "frigate_push_magicfanout_fatigue_max_pushes_in_interval", - default = 2, - min = 0, - max = 6) - - /** - * Param to enable custom duration for fatiguing - */ - object GPEnableCustomMagicFanoutCricketFatigue - extends FSParam[Boolean]( - name = "global_participation_cricket_magicfanout_enable_custom_fatigue", - default = false - ) - - /** - * Param to enable e2e scribing for target filtering step - */ - object EnableMrRequestScribingForTargetFiltering - extends FSParam[Boolean]( - name = "mr_request_scribing_enable_for_target_filtering", - default = false - ) - - /** - * Param to enable e2e scribing for candidate filtering step - */ - object EnableMrRequestScribingForCandidateFiltering - extends FSParam[Boolean]( - name = "mr_request_scribing_enable_for_candidate_filtering", - default = false - ) - - /** - * Param to enable e2e scribing with feature hydrating - */ - object EnableMrRequestScribingWithFeatureHydrating - extends FSParam[Boolean]( - name = "mr_request_scribing_enable_with_feature_hydrating", - default = false - ) - - /* - * TargetLevel Feature list for Mr request scribing - */ - object TargetLevelFeatureListForMrRequestScribing - extends FSParam[Seq[String]]( - name = "mr_request_scribing_target_level_feature_list", - default = Seq.empty - ) - - /** - * Param to enable \eps-greedy exploration for BigFiltering/LTV-based filtering - */ - object EnableMrRequestScribingForEpsGreedyExploration - extends FSParam[Boolean]( - name = "mr_request_scribing_eps_greedy_exploration_enable", - default = false - ) - - /** - * Param to control epsilon in \eps-greedy exploration for BigFiltering/LTV-based filtering - */ - object MrRequestScribingEpsGreedyExplorationRatio - extends FSBoundedParam[Double]( - name = "mr_request_scribing_eps_greedy_exploration_ratio", - default = 0.0, - min = 0.0, - max = 1.0 - ) - - /** - * Param to enable scribing dismiss model score - */ - object EnableMrRequestScribingDismissScore - extends FSParam[Boolean]( - name = "mr_request_scribing_dismiss_score_enable", - default = false - ) - - /** - * Param to enable scribing BigFiltering supervised model(s) score(s) - */ - object EnableMrRequestScribingBigFilteringSupervisedScores - extends FSParam[Boolean]( - name = "mr_request_scribing_bigfiltering_supervised_scores_enable", - default = false - ) - - /** - * Param to enable scribing BigFiltering RL model(s) score(s) - */ - object EnableMrRequestScribingBigFilteringRLScores - extends FSParam[Boolean]( - name = "mr_request_scribing_bigfiltering_rl_scores_enable", - default = false - ) - - /** - * Param to flatten mr request scribe - */ - object EnableFlattenMrRequestScribing - extends FSParam[Boolean]( - name = "mr_request_scribing_enable_flatten", - default = false - ) - - /** - * Param to enable NSFW token based filtering - */ - object EnableNsfwTokenBasedFiltering - extends FSParam[Boolean]( - name = "health_and_quality_filter_enable_nsfw_token_based_filtering", - default = false - ) - - object NsfwTokensParam - extends FSParam[Seq[String]]( - name = "health_and_quality_filter_nsfw_tokens", - default = Seq("nsfw", "18+", "\uD83D\uDD1E")) - - object MinimumAllowedAuthorAccountAgeInHours - extends FSBoundedParam[Int]( - name = "health_and_quality_filter_minimum_allowed_author_account_age_in_hours", - default = 0, - min = 0, - max = 168 - ) - - /** - * Param to enable the profanity filter - */ - object EnableProfanityFilterParam - extends FSParam[Boolean]( - name = "health_and_quality_filter_enable_profanity_filter", - default = false - ) - - /** - * Param to enable query the author media representation store - */ - object EnableQueryAuthorMediaRepresentationStore - extends FSParam[Boolean]( - name = "health_and_quality_filter_enable_query_author_media_representation_store", - default = false - ) - - /** - * Threshold to filter a tweet based on the author sensitive media score - */ - object AuthorSensitiveMediaFilteringThreshold - extends FSBoundedParam[Double]( - name = "health_and_quality_filter_author_sensitive_media_filtering_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Threshold to filter a tweet based on the author sensitive media score - */ - object AuthorSensitiveMediaFilteringThresholdForMrTwistly - extends FSBoundedParam[Double]( - name = "health_and_quality_filter_author_sensitive_media_filtering_threshold_for_mrtwistly", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Param to enable filtering the SimCluster tweet if it has AbuseStrike_Top2Percent entitiy - */ - object EnableAbuseStrikeTop2PercentFilterSimCluster - extends FSParam[Boolean]( - name = "health_signal_store_enable_abuse_strike_top_2_percent_filter_sim_cluster", - default = false - ) - - /** - * Param to enable filtering the SimCluster tweet if it has AbuseStrike_Top1Percent entitiy - */ - object EnableAbuseStrikeTop1PercentFilterSimCluster - extends FSParam[Boolean]( - name = "health_signal_store_enable_abuse_strike_top_1_percent_filter_sim_cluster", - default = false - ) - - /** - * Param to enable filtering the SimCluster tweet if it has AbuseStrike_Top0.5Percent entitiy - */ - object EnableAbuseStrikeTop05PercentFilterSimCluster - extends FSParam[Boolean]( - name = "health_signal_store_enable_abuse_strike_top_05_percent_filter_sim_cluster", - default = false - ) - - object EnableAgathaUserHealthModelPredicate - extends FSParam[Boolean]( - name = "health_signal_store_enable_agatha_user_health_model_predicate", - default = false - ) - - /** - * Threshold to filter a tweet based on the agatha_calibrated_nsfw score of its author for MrTwistly - */ - object AgathaCalibratedNSFWThresholdForMrTwistly - extends FSBoundedParam[Double]( - name = "health_signal_store_agatha_calibrated_nsfw_threshold_for_mrtwistly", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Threshold to filter a tweet based on the agatha_calibrated_nsfw score of its author - */ - object AgathaCalibratedNSFWThreshold - extends FSBoundedParam[Double]( - name = "health_signal_store_agatha_calibrated_nsfw_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Threshold to filter a tweet based on the agatha_nsfw_text_user score of its author for MrTwistly - */ - object AgathaTextNSFWThresholdForMrTwistly - extends FSBoundedParam[Double]( - name = "health_signal_store_agatha_text_nsfw_threshold_for_mrtwistly", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Threshold to filter a tweet based on the agatha_nsfw_text_user score of its author - */ - object AgathaTextNSFWThreshold - extends FSBoundedParam[Double]( - name = "health_signal_store_agatha_text_nsfw_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Threshold to bucket a user based on the agatha_calibrated_nsfw score of the tweet author - */ - object AgathaCalibratedNSFWBucketThreshold - extends FSBoundedParam[Double]( - name = "health_signal_store_agatha_calibrated_nsfw_bucket_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Threshold to bucket a user based on the agatha_nsfw_text_user score of the tweet author - */ - object AgathaTextNSFWBucketThreshold - extends FSBoundedParam[Double]( - name = "health_signal_store_agatha_text_nsfw_bucket_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Param to enable filtering using pnsfw_text_tweet model. - */ - object EnableHealthSignalStorePnsfwTweetTextPredicate - extends FSParam[Boolean]( - name = "health_signal_store_enable_pnsfw_tweet_text_predicate", - default = false - ) - - /** - * Threshold score for filtering based on pnsfw_text_tweet Model. - */ - object PnsfwTweetTextThreshold - extends FSBoundedParam[Double]( - name = "health_signal_store_pnsfw_tweet_text_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Threshold score for bucketing based on pnsfw_text_tweet Model. - */ - object PnsfwTweetTextBucketingThreshold - extends FSBoundedParam[Double]( - name = "health_signal_store_pnsfw_tweet_text_bucketing_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Enable filtering tweets with media based on pnsfw_media_tweet Model for OON tweets only. - */ - object PnsfwTweetMediaFilterOonOnly - extends FSParam[Boolean]( - name = "health_signal_store_pnsfw_tweet_media_filter_oon_only", - default = true - ) - - /** - * Threshold score for filtering tweets with media based on pnsfw_media_tweet Model. - */ - object PnsfwTweetMediaThreshold - extends FSBoundedParam[Double]( - name = "health_signal_store_pnsfw_tweet_media_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Threshold score for filtering tweets with images based on pnsfw_media_tweet Model. - */ - object PnsfwTweetImageThreshold - extends FSBoundedParam[Double]( - name = "health_signal_store_pnsfw_tweet_image_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Threshold score for filtering quote/reply tweets based on source tweet's media - */ - object PnsfwQuoteTweetThreshold - extends FSBoundedParam[Double]( - name = "health_signal_store_pnsfw_quote_tweet_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Threshold score for bucketing based on pnsfw_media_tweet Model. - */ - object PnsfwTweetMediaBucketingThreshold - extends FSBoundedParam[Double]( - name = "health_signal_store_pnsfw_tweet_media_bucketing_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Param to enable filtering using multilingual psnfw predicate - */ - object EnableHealthSignalStoreMultilingualPnsfwTweetTextPredicate - extends FSParam[Boolean]( - name = "health_signal_store_enable_multilingual_pnsfw_tweet_text_predicate", - default = false - ) - - /** - * Language sequence we will query pnsfw scores for - */ - object MultilingualPnsfwTweetTextSupportedLanguages - extends FSParam[Seq[String]]( - name = "health_signal_store_multilingual_pnsfw_tweet_supported_languages", - default = Seq.empty[String], - ) - - /** - * Threshold score per language for bucketing based on pnsfw scores. - */ - object MultilingualPnsfwTweetTextBucketingThreshold - extends FSParam[Seq[Double]]( - name = "health_signal_store_multilingual_pnsfw_tweet_text_bucketing_thresholds", - default = Seq.empty[Double], - ) - - /** - * Threshold score per language for filtering based on pnsfw scores. - */ - object MultilingualPnsfwTweetTextFilteringThreshold - extends FSParam[Seq[Double]]( - name = "health_signal_store_multilingual_pnsfw_tweet_text_filtering_thresholds", - default = Seq.empty[Double], - ) - - /** - * List of models to threshold scores for bucketing purposes - */ - object MultilingualPnsfwTweetTextBucketingModelList - extends FSEnumSeqParam[NsfwTextDetectionModel.type]( - name = "health_signal_store_multilingual_pnsfw_tweet_text_bucketing_models_ids", - default = Seq(NsfwTextDetectionModel.ProdModel), - enum = NsfwTextDetectionModel - ) - - object MultilingualPnsfwTweetTextModel - extends FSEnumParam[NsfwTextDetectionModel.type]( - name = "health_signal_store_multilingual_pnsfw_tweet_text_model", - default = NsfwTextDetectionModel.ProdModel, - enum = NsfwTextDetectionModel - ) - - /** - * Param to determine media should be enabled for android - */ - object EnableEventSquareMediaAndroid - extends FSParam[Boolean]( - name = "mr_enable_event_media_square_android", - default = false - ) - - /** - * Param to determine expanded media should be enabled for android - */ - object EnableEventPrimaryMediaAndroid - extends FSParam[Boolean]( - name = "mr_enable_event_media_primary_android", - default = false - ) - - /** - * Param to determine media should be enabled for ios for MagicFanout - */ - object EnableEventSquareMediaIosMagicFanoutNewsEvent - extends FSParam[Boolean]( - name = "mr_enable_event_media_square_ios_mf", - default = false - ) - - /** - * Param to configure HTL Visit fatigue - */ - object HTLVisitFatigueTime - extends FSBoundedParam[Int]( - name = "frigate_push_htl_visit_fatigue_time", - default = 20, - min = 0, - max = 72) { - - // Fatigue duration for HTL visit - final val DefaultHoursToFatigueAfterHtlVisit = 20 - final val OldHoursToFatigueAfterHtlVisit = 8 - } - - object MagicFanoutNewsUserGeneratedEventsEnable - extends FSParam[Boolean]( - name = "magicfanout_news_user_generated_events_enable", - default = false) - - object MagicFanoutSkipAccountCountryPredicate - extends FSParam[Boolean]("magicfanout_news_skip_account_country_predicate", false) - - object MagicFanoutNewsEnableDescriptionCopy - extends FSParam[Boolean](name = "magicfanout_news_enable_description_copy", default = false) - - /** - * Enables Custom Targeting for MagicFnaout News events in Pushservice - */ - object MagicFanoutEnableCustomTargetingNewsEvent - extends FSParam[Boolean]("magicfanout_news_event_custom_targeting_enable", false) - - /** - * Enable Topic Copy in MF - */ - object EnableTopicCopyForMF - extends FSParam[Boolean]( - name = "magicfanout_enable_topic_copy", - default = false - ) - - /** - * Enable Topic Copy in MF for implicit topics - */ - object EnableTopicCopyForImplicitTopics - extends FSParam[Boolean]( - name = "magicfanout_enable_topic_copy_erg_interests", - default = false - ) - - /** - * Enable NewCreator push - */ - object EnableNewCreatorPush - extends FSParam[Boolean]( - name = "new_creator_enable_push", - default = false - ) - - /** - * Enable CreatorSubscription push - */ - object EnableCreatorSubscriptionPush - extends FSParam[Boolean]( - name = "creator_subscription_enable_push", - default = false - ) - - /** - * Featureswitch param to enable/disable push recommendations - */ - object EnablePushRecommendationsParam - extends FSParam[Boolean](name = "push_recommendations_enabled", default = false) - - object DisableMlInFilteringFeatureSwitchParam - extends FSParam[Boolean]( - name = "frigate_push_modeling_disable_ml_in_filtering", - default = false - ) - - object EnableMinDurationModifier - extends FSParam[Boolean]( - name = "min_duration_modifier_enable_hour_modifier", - default = false - ) - - object EnableMinDurationModifierV2 - extends FSParam[Boolean]( - name = "min_duration_modifier_enable_hour_modifier_v2", - default = false - ) - - object MinDurationModifierStartHourList - extends FSParam[Seq[Int]]( - name = "min_duration_modifier_start_time_list", - default = Seq(), - ) - - object MinDurationModifierEndHourList - extends FSParam[Seq[Int]]( - name = "min_duration_modifier_start_end_list", - default = Seq(), - ) - - object MinDurationTimeModifierConst - extends FSParam[Seq[Int]]( - name = "min_duration_modifier_const_list", - default = Seq(), - ) - - object EnableQueryUserOpenedHistory - extends FSParam[Boolean]( - name = "min_duration_modifier_enable_query_user_opened_history", - default = false - ) - - object EnableMinDurationModifierByUserHistory - extends FSParam[Boolean]( - name = "min_duration_modifier_enable_hour_modifier_by_user_history", - default = false - ) - - object EnableRandomHourForQuickSend - extends FSParam[Boolean]( - name = "min_duration_modifier_enable_random_hour_for_quick_send", - default = false - ) - - object SendTimeByUserHistoryMaxOpenedThreshold - extends FSBoundedParam[Int]( - name = "min_duration_modifier_max_opened_threshold", - default = 4, - min = 0, - max = 100) - - object SendTimeByUserHistoryNoSendsHours - extends FSBoundedParam[Int]( - name = "min_duration_modifier_no_sends_hours", - default = 1, - min = 0, - max = 24) - - object SendTimeByUserHistoryQuickSendBeforeHours - extends FSBoundedParam[Int]( - name = "min_duration_modifier_quick_send_before_hours", - default = 0, - min = 0, - max = 24) - - object SendTimeByUserHistoryQuickSendAfterHours - extends FSBoundedParam[Int]( - name = "min_duration_modifier_quick_send_after_hours", - default = 0, - min = 0, - max = 24) - - object SendTimeByUserHistoryQuickSendMinDurationInMinute - extends FSBoundedParam[Int]( - name = "min_duration_modifier_quick_send_min_duration", - default = 0, - min = 0, - max = 1440) - - object SendTimeByUserHistoryNoSendMinDuration - extends FSBoundedParam[Int]( - name = "min_duration_modifier_no_send_min_duration", - default = 24, - min = 0, - max = 24) - - object EnableMfGeoTargeting - extends FSParam[Boolean]( - name = "frigate_push_magicfanout_geo_targeting_enable", - default = false) - - /** - * Enable RUX Tweet landing page for push open. When this param is enabled, user will go to RUX - * landing page instead of Tweet details page when opening MagicRecs push. - */ - object EnableRuxLandingPage - extends FSParam[Boolean](name = "frigate_push_enable_rux_landing_page", default = false) - - /** - * Enable RUX Tweet landing page for Ntab Click. When this param is enabled, user will go to RUX - * landing page instead of Tweet details page when click MagicRecs entry on Ntab. - */ - object EnableNTabRuxLandingPage - extends FSParam[Boolean](name = "frigate_push_enable_ntab_rux_landing_page", default = false) - - /** - * Param to enable Onboarding Pushes - */ - object EnableOnboardingPushes - extends FSParam[Boolean]( - name = "onboarding_push_enable", - default = false - ) - - /** - * Param to enable Address Book Pushes - */ - object EnableAddressBookPush - extends FSParam[Boolean]( - name = "onboarding_push_enable_address_book_push", - default = false - ) - - /** - * Param to enable Complete Onboarding Pushes - */ - object EnableCompleteOnboardingPush - extends FSParam[Boolean]( - name = "onboarding_push_enable_complete_onboarding_push", - default = false - ) - - /** - * Param to enable Smart Push Config for MR Override Notifs on Android - */ - object EnableOverrideNotificationsSmartPushConfigForAndroid - extends FSParam[Boolean]( - name = "mr_override_enable_smart_push_config_for_android", - default = false) - - /** - * Param to control the min duration since last MR push for Onboarding Pushes - */ - object MrMinDurationSincePushForOnboardingPushes - extends FSBoundedParam[Duration]( - name = "onboarding_push_min_duration_since_push_days", - default = 4.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Param to control the push fatigue for Onboarding Pushes - */ - object FatigueForOnboardingPushes - extends FSBoundedParam[Duration]( - name = "onboarding_push_fatigue_days", - default = 30.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Param to specify the maximum number of Onboarding Push Notifs in a specified period of time - */ - object MaxOnboardingPushInInterval - extends FSBoundedParam[Int]( - name = "onboarding_push_max_in_interval", - default = 1, - min = 0, - max = 10 - ) - - /** - * Param to disable the Onboarding Push Notif Fatigue - */ - object DisableOnboardingPushFatigue - extends FSParam[Boolean]( - name = "onboarding_push_disable_push_fatigue", - default = false - ) - - /** - * Param to control the inverter for fatigue between consecutive TopTweetsByGeoPush - */ - object TopTweetsByGeoPushInterval - extends FSBoundedParam[Duration]( - name = "top_tweets_by_geo_interval_days", - default = 0.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Param to control the inverter for fatigue between consecutive TripTweets - */ - object HighQualityTweetsPushInterval - extends FSBoundedParam[Duration]( - name = "high_quality_candidates_push_interval_days", - default = 1.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Expiry TTL duration for Tweet Notification types written to history store - */ - object FrigateHistoryTweetNotificationWriteTtl - extends FSBoundedParam[Duration]( - name = "frigate_notification_history_tweet_write_ttl_days", - default = 60.days, - min = Duration.Bottom, - max = Duration.Top - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Expiry TTL duration for Notification written to history store - */ - object FrigateHistoryOtherNotificationWriteTtl - extends FSBoundedParam[Duration]( - name = "frigate_notification_history_other_write_ttl_days", - default = 90.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Param to control maximum number of TopTweetsByGeoPush pushes to receive in an interval - */ - object MaxTopTweetsByGeoPushGivenInterval - extends FSBoundedParam[Int]( - name = "top_tweets_by_geo_push_given_interval", - default = 1, - min = 0, - max = 10 - ) - - /** - * Param to control maximum number of HighQualityTweet pushes to receive in an interval - */ - object MaxHighQualityTweetsPushGivenInterval - extends FSBoundedParam[Int]( - name = "high_quality_candidates_max_push_given_interval", - default = 3, - min = 0, - max = 10 - ) - - /** - * Param to downrank/backfill top tweets by geo candidates - */ - object BackfillRankTopTweetsByGeoCandidates - extends FSParam[Boolean]( - name = "top_tweets_by_geo_backfill_rank", - default = false - ) - - /** - * Determine whether to use aggressive thresholds for Health filtering on SearchTweet - */ - object PopGeoTweetEnableAggressiveThresholds - extends FSParam[Boolean]( - name = "top_tweets_by_geo_enable_aggressive_health_thresholds", - default = false - ) - - /** - * Param to apply different scoring functions to select top tweets by geo candidates - */ - object ScoringFuncForTopTweetsByGeo - extends FSParam[String]( - name = "top_tweets_by_geo_scoring_function", - default = "Pop8H", - ) - - /** - * Param to query different stores in pop geo service. - */ - object TopTweetsByGeoCombinationParam - extends FSEnumParam[TopTweetsForGeoCombination.type]( - name = "top_tweets_by_geo_combination_id", - default = TopTweetsForGeoCombination.Default, - enum = TopTweetsForGeoCombination - ) - - /** - * Param for popgeo tweet version - */ - object PopGeoTweetVersionParam - extends FSEnumParam[PopGeoTweetVersion.type]( - name = "top_tweets_by_geo_version_id", - default = PopGeoTweetVersion.Prod, - enum = PopGeoTweetVersion - ) - - /** - * Param to query what length of hash for geoh store - */ - object GeoHashLengthList - extends FSParam[Seq[Int]]( - name = "top_tweets_by_geo_hash_length_list", - default = Seq(4), - ) - - /** - * Param to include country code results as back off . - */ - object EnableCountryCodeBackoffTopTweetsByGeo - extends FSParam[Boolean]( - name = "top_tweets_by_geo_enable_country_code_backoff", - default = false, - ) - - /** - * Param to decide ranking function for fetched top tweets by geo - */ - object RankingFunctionForTopTweetsByGeo - extends FSEnumParam[TopTweetsForGeoRankingFunction.type]( - name = "top_tweets_by_geo_ranking_function_id", - default = TopTweetsForGeoRankingFunction.Score, - enum = TopTweetsForGeoRankingFunction - ) - - /** - * Param to enable top tweets by geo candidates - */ - object EnableTopTweetsByGeoCandidates - extends FSParam[Boolean]( - name = "top_tweets_by_geo_enable_candidate_source", - default = false - ) - - /** - * Param to enable top tweets by geo candidates for dormant users - */ - object EnableTopTweetsByGeoCandidatesForDormantUsers - extends FSParam[Boolean]( - name = "top_tweets_by_geo_enable_candidate_source_dormant_users", - default = false - ) - - /** - * Param to specify the maximum number of Top Tweets by Geo candidates to take - */ - object MaxTopTweetsByGeoCandidatesToTake - extends FSBoundedParam[Int]( - name = "top_tweets_by_geo_candidates_to_take", - default = 10, - min = 0, - max = 100 - ) - - /** - * Param to min duration since last MR push for top tweets by geo pushes - */ - object MrMinDurationSincePushForTopTweetsByGeoPushes - extends FSBoundedParam[Duration]( - name = "top_tweets_by_geo_min_duration_since_last_mr_days", - default = 3.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Param to enable FRS candidate tweets - */ - object EnableFrsCandidates - extends FSParam[Boolean]( - name = "frs_tweet_candidate_enable_adaptor", - default = false - ) - - /** - * Param to enable FRSTweet candidates for topic setting users - * */ - object EnableFrsTweetCandidatesTopicSetting - extends FSParam[Boolean]( - name = "frs_tweet_candidate_enable_adaptor_for_topic_setting", - default = false - ) - - /** - * Param to enable topic annotations for FRSTweet candidates tweets - * */ - object EnableFrsTweetCandidatesTopicAnnotation - extends FSParam[Boolean]( - name = "frs_tweet_candidate_enable_topic_annotation", - default = false - ) - - /** - * Param to enable topic copy for FRSTweet candidates tweets - * */ - object EnableFrsTweetCandidatesTopicCopy - extends FSParam[Boolean]( - name = "frs_tweet_candidate_enable_topic_copy", - default = false - ) - - /** - * Topic score threshold for FRSTweet candidates topic annotations - * */ - object FrsTweetCandidatesTopicScoreThreshold - extends FSBoundedParam[Double]( - name = "frs_tweet_candidate_topic_score_threshold", - default = 0.0, - min = 0.0, - max = 100.0 - ) - - /** - * Param to enable mr modeling-based candidates tweets - * */ - object EnableMrModelingBasedCandidates - extends FSParam[Boolean]( - name = "candidate_generation_model_enable_adaptor", - default = false - ) - - /** - Param to enable mr modeling-based candidates tweets for topic setting users - * */ - object EnableMrModelingBasedCandidatesTopicSetting - extends FSParam[Boolean]( - name = "candidate_generation_model_enable_adaptor_for_topic_setting", - default = false - ) - - /** - * Param to enable topic annotations for mr modeling-based candidates tweets - * */ - object EnableMrModelingBasedCandidatesTopicAnnotation - extends FSParam[Boolean]( - name = "candidate_generation_model_enable_adaptor_topic_annotation", - default = false - ) - - /** - * Topic score threshold for mr modeling based candidates topic annotations - * */ - object MrModelingBasedCandidatesTopicScoreThreshold - extends FSBoundedParam[Double]( - name = "candidate_generation_model_topic_score_threshold", - default = 0.0, - min = 0.0, - max = 100.0 - ) - - /** - * Param to enable topic copy for mr modeling-based candidates tweets - * */ - object EnableMrModelingBasedCandidatesTopicCopy - extends FSParam[Boolean]( - name = "candidate_generation_model_enable_topic_copy", - default = false - ) - - /** - * Number of max mr modeling based candidates - * */ - object NumberOfMaxMrModelingBasedCandidates - extends FSBoundedParam[Int]( - name = "candidate_generation_model_max_mr_modeling_based_candidates", - default = 200, - min = 0, - max = 1000 - ) - - /** - * Enable the traffic to use fav threshold - * */ - object EnableThresholdOfFavMrModelingBasedCandidates - extends FSParam[Boolean]( - name = "candidate_generation_model_enable_fav_threshold", - default = false - ) - - /** - * Threshold of fav for mr modeling based candidates - * */ - object ThresholdOfFavMrModelingBasedCandidates - extends FSBoundedParam[Int]( - name = "candidate_generation_model_fav_threshold", - default = 0, - min = 0, - max = 500 - ) - - /** - * Filtered threshold for mr modeling based candidates - * */ - object CandidateGenerationModelCosineThreshold - extends FSBoundedParam[Double]( - name = "candidate_generation_model_cosine_threshold", - default = 0.9, - min = 0.0, - max = 1.0 - ) - - /* - * ANN hyparameters - * */ - object ANNEfQuery - extends FSBoundedParam[Int]( - name = "candidate_generation_model_ann_ef_query", - default = 300, - min = 50, - max = 1500 - ) - - /** - * Param to do real A/B impression for FRS candidates to avoid dilution - */ - object EnableResultFromFrsCandidates - extends FSParam[Boolean]( - name = "frs_tweet_candidate_enable_returned_result", - default = false - ) - - /** - * Param to enable hashspace candidate tweets - */ - object EnableHashspaceCandidates - extends FSParam[Boolean]( - name = "hashspace_candidate_enable_adaptor", - default = false - ) - - /** - * Param to enable hashspace candidates tweets for topic setting users - * */ - object EnableHashspaceCandidatesTopicSetting - extends FSParam[Boolean]( - name = "hashspace_candidate_enable_adaptor_for_topic_setting", - default = false - ) - - /** - * Param to enable topic annotations for hashspace candidates tweets - * */ - object EnableHashspaceCandidatesTopicAnnotation - extends FSParam[Boolean]( - name = "hashspace_candidate_enable_topic_annotation", - default = false - ) - - /** - * Param to enable topic copy for hashspace candidates tweets - * */ - object EnableHashspaceCandidatesTopicCopy - extends FSParam[Boolean]( - name = "hashspace_candidate_enable_topic_copy", - default = false - ) - - /** - * Topic score threshold for hashspace candidates topic annotations - * */ - object HashspaceCandidatesTopicScoreThreshold - extends FSBoundedParam[Double]( - name = "hashspace_candidate_topic_score_threshold", - default = 0.0, - min = 0.0, - max = 100.0 - ) - - /** - * Param to do real A/B impression for hashspace candidates to avoid dilution - */ - object EnableResultFromHashspaceCandidates - extends FSParam[Boolean]( - name = "hashspace_candidate_enable_returned_result", - default = false - ) - - /** - * Param to enable detopic tweet candidates in adaptor - */ - object EnableDeTopicTweetCandidates - extends FSParam[Boolean]( - name = "detopic_tweet_candidate_enable_adaptor", - default = false - ) - - /** - * Param to enable detopic tweet candidates results (to avoid dilution) - */ - object EnableDeTopicTweetCandidateResults - extends FSParam[Boolean]( - name = "detopic_tweet_candidate_enable_results", - default = false - ) - - /** - * Param to specify whether to provide a custom list of topics in request - */ - object EnableDeTopicTweetCandidatesCustomTopics - extends FSParam[Boolean]( - name = "detopic_tweet_candidate_enable_custom_topics", - default = false - ) - - /** - * Param to specify whether to provide a custom language in request - */ - object EnableDeTopicTweetCandidatesCustomLanguages - extends FSParam[Boolean]( - name = "detopic_tweet_candidate_enable_custom_languages", - default = false - ) - - /** - * Number of detopic tweet candidates in the request - * */ - object NumberOfDeTopicTweetCandidates - extends FSBoundedParam[Int]( - name = "detopic_tweet_candidate_num_candidates_in_request", - default = 600, - min = 0, - max = 3000 - ) - - /** - * Max Number of detopic tweet candidates returned in adaptor - * */ - object NumberOfMaxDeTopicTweetCandidatesReturned - extends FSBoundedParam[Int]( - name = "detopic_tweet_candidate_max_num_candidates_returned", - default = 200, - min = 0, - max = 3000 - ) - - /** - * Param to enable F1 from protected Authors - */ - object EnableF1FromProtectedTweetAuthors - extends FSParam[Boolean]( - "f1_enable_protected_tweets", - false - ) - - /** - * Param to enable safe user tweet tweetypie store - */ - object EnableSafeUserTweetTweetypieStore - extends FSParam[Boolean]( - "mr_infra_enable_use_safe_user_tweet_tweetypie", - false - ) - - /** - * Param to min duration since last MR push for top tweets by geo pushes - */ - object EnableMrMinDurationSinceMrPushFatigue - extends FSParam[Boolean]( - name = "top_tweets_by_geo_enable_min_duration_since_mr_fatigue", - default = false - ) - - /** - * Param to check time since last time user logged in for geo top tweets by geo push - */ - object TimeSinceLastLoginForGeoPopTweetPush - extends FSBoundedParam[Duration]( - name = "top_tweets_by_geo_time_since_last_login_in_days", - default = 14.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Param to check time since last time user logged in for geo top tweets by geo push - */ - object MinimumTimeSinceLastLoginForGeoPopTweetPush - extends FSBoundedParam[Duration]( - name = "top_tweets_by_geo_minimum_time_since_last_login_in_days", - default = 14.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** How long we wait after a user visited the app before sending them a space fanout rec */ - object SpaceRecsAppFatigueDuration - extends FSBoundedParam[Duration]( - name = "space_recs_app_fatigue_duration_hours", - default = 4.hours, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** The fatigue time-window for OON space fanout recs, e.g. 1 push every 3 days */ - object OONSpaceRecsFatigueDuration - extends FSBoundedParam[Duration]( - name = "space_recs_oon_fatigue_duration_days", - default = 1.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** The global fatigue time-window for space fanout recs, e.g. 1 push every 3 days */ - object SpaceRecsGlobalFatigueDuration - extends FSBoundedParam[Duration]( - name = "space_recs_global_fatigue_duration_days", - default = 1.day, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** The min-interval between space fanout recs. - * After receiving a space fanout rec, they must wait a minimum of this - * interval before eligibile for another */ - object SpaceRecsFatigueMinIntervalDuration - extends FSBoundedParam[Duration]( - name = "space_recs_fatigue_mininterval_duration_minutes", - default = 30.minutes, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromMinutes - } - - /** Space fanout user-follow rank threshold. - * Users targeted by a follow that is above this threshold will be filtered */ - object SpaceRecsRealgraphThreshold - extends FSBoundedParam[Int]( - name = "space_recs_realgraph_threshold", - default = 50, - max = 500, - min = 0 - ) - - object EnableHydratingRealGraphTargetUserFeatures - extends FSParam[Boolean]( - name = "frigate_push_modeling_enable_hydrating_real_graph_target_user_feature", - default = true - ) - - /** Param to reduce dillution when checking if a space is featured or not */ - object CheckFeaturedSpaceOON - extends FSParam[Boolean](name = "space_recs_check_if_its_featured_space", default = false) - - /** Enable Featured Spaces Rules for OON spaces */ - object EnableFeaturedSpacesOON - extends FSParam[Boolean](name = "space_recs_enable_featured_spaces_oon", default = false) - - /** Enable Geo Targeting */ - object EnableGeoTargetingForSpaces - extends FSParam[Boolean](name = "space_recs_enable_geo_targeting", default = false) - - /** Number of max pushes within the fatigue duration for OON Space Recs */ - object OONSpaceRecsPushLimit - extends FSBoundedParam[Int]( - name = "space_recs_oon_push_limit", - default = 1, - max = 3, - min = 0 - ) - - /** Space fanout recs, number of max pushes within the fatigue duration */ - object SpaceRecsGlobalPushLimit - extends FSBoundedParam[Int]( - name = "space_recs_global_push_limit", - default = 3, - max = 50, - min = 0 - ) - - /** - * Param to enable score based override. - */ - object EnableOverrideNotificationsScoreBasedOverride - extends FSParam[Boolean]( - name = "mr_override_enable_score_ranking", - default = false - ) - - /** - * Param to determine the lookback duration when searching for override info. - */ - object OverrideNotificationsLookbackDurationForOverrideInfo - extends FSBoundedParam[Duration]( - name = "mr_override_lookback_duration_override_info_in_days", - default = 30.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Param to determine the lookback duration when searching for impression ids. - */ - object OverrideNotificationsLookbackDurationForImpressionId - extends FSBoundedParam[Duration]( - name = "mr_override_lookback_duration_impression_id_in_days", - default = 30.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Param to enable sending multiple target ids in the payload. - */ - object EnableOverrideNotificationsMultipleTargetIds - extends FSParam[Boolean]( - name = "mr_override_enable_multiple_target_ids", - default = false - ) - - /** - * Param for MR Web Notifications holdback - */ - object MRWebHoldbackParam - extends FSParam[Boolean]( - name = "mr_web_notifications_holdback", - default = false - ) - - object CommonRecommendationTypeDenyListPushHoldbacks - extends FSParam[Seq[String]]( - name = "crt_to_exclude_from_holdbacks_push_holdbacks", - default = Seq.empty[String] - ) - - /** - * Param to enable sending number of slots to maintain in the payload. - */ - object EnableOverrideNotificationsNSlots - extends FSParam[Boolean]( - name = "mr_override_enable_n_slots", - default = false - ) - - /** - * Enable down ranking of NUPS and pop geo topic follow candidates for new user playbook. - */ - object EnableDownRankOfNewUserPlaybookTopicFollowPush - extends FSParam[Boolean]( - name = "topic_follow_new_user_playbook_enable_down_rank", - default = false - ) - - /** - * Enable down ranking of NUPS and pop geo topic tweet candidates for new user playbook. - */ - object EnableDownRankOfNewUserPlaybookTopicTweetPush - extends FSParam[Boolean]( - name = "topic_tweet_new_user_playbook_enable_down_rank", - default = false - ) - - /** - * Param to enable/disable employee only spaces for fanout of notifications - */ - object EnableEmployeeOnlySpaceNotifications - extends FSParam[Boolean](name = "space_recs_employee_only_enable", default = false) - - /** - * NTab spaces ttl experiments - */ - object EnableSpacesTtlForNtab - extends FSParam[Boolean]( - name = "ntab_spaces_ttl_enable", - default = false - ) - - /** - * Param to determine the ttl duration for space notifications on NTab. - */ - object SpaceNotificationsTTLDurationForNTab - extends FSBoundedParam[Duration]( - name = "ntab_spaces_ttl_hours", - default = 1.hour, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /* - * NTab override experiments - * see go/ntab-override experiment brief for more details - */ - - /** - * Override notifications for Spaces on lockscreen. - */ - object EnableOverrideForSpaces - extends FSParam[Boolean]( - name = "mr_override_spaces", - default = false - ) - - /** - * Param to enable storing the Generic Notification Key. - */ - object EnableStoringNtabGenericNotifKey - extends FSParam[Boolean]( - name = "ntab_enable_storing_generic_notif_key", - default = false - ) - - /** - * Param to enable deleting the Target's timeline. - */ - object EnableDeletingNtabTimeline - extends FSParam[Boolean]( - name = "ntab_enable_delete_timeline", - default = false - ) - - /** - * Param to enable sending the overrideId - * to NTab which enables override support in NTab-api - */ - object EnableOverrideIdNTabRequest - extends FSParam[Boolean]( - name = "ntab_enable_override_id_in_request", - default = false - ) - - /** - * [Override Workstream] Param to enable NTab override n-slot feature. - */ - object EnableNslotsForOverrideOnNtab - extends FSParam[Boolean]( - name = "ntab_enable_override_max_count", - default = false - ) - - /** - * Param to determine the lookback duration for override candidates on NTab. - */ - object OverrideNotificationsLookbackDurationForNTab - extends FSBoundedParam[Duration]( - name = "ntab_override_lookback_duration_days", - default = 30.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Param to determine the max count for candidates on NTab. - */ - object OverrideNotificationsMaxCountForNTab - extends FSBoundedParam[Int]( - name = "ntab_override_limit", - min = 0, - max = Int.MaxValue, - default = 4) - - //// end override experiments //// - /** - * Param to enable top tweet impressions notification - */ - object EnableTopTweetImpressionsNotification - extends FSParam[Boolean]( - name = "top_tweet_impressions_notification_enable", - default = false - ) - - /** - * Param to control the inverter for fatigue between consecutive TweetImpressions - */ - object TopTweetImpressionsNotificationInterval - extends FSBoundedParam[Duration]( - name = "top_tweet_impressions_notification_interval_days", - default = 7.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * The min-interval between TweetImpressions notifications. - * After receiving a TweetImpressions notif, they must wait a minimum of this - * interval before being eligible for another - */ - object TopTweetImpressionsFatigueMinIntervalDuration - extends FSBoundedParam[Duration]( - name = "top_tweet_impressions_fatigue_mininterval_duration_days", - default = 1.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Maximum number of top tweet impressions notifications to receive in an interval - */ - object MaxTopTweetImpressionsNotifications - extends FSBoundedParam( - name = "top_tweet_impressions_fatigue_max_in_interval", - default = 0, - min = 0, - max = 10 - ) - - /** - * Param for min number of impressions counts to be eligible for lonely_birds_tweet_impressions model - */ - object TopTweetImpressionsMinRequired - extends FSBoundedParam[Int]( - name = "top_tweet_impressions_min_required", - default = 25, - min = 0, - max = Int.MaxValue - ) - - /** - * Param for threshold of impressions counts to notify for lonely_birds_tweet_impressions model - */ - object TopTweetImpressionsThreshold - extends FSBoundedParam[Int]( - name = "top_tweet_impressions_threshold", - default = 25, - min = 0, - max = Int.MaxValue - ) - - /** - * Param for the number of days to search up to for a user's original tweets - */ - object TopTweetImpressionsOriginalTweetsNumDaysSearch - extends FSBoundedParam[Int]( - name = "top_tweet_impressions_original_tweets_num_days_search", - default = 3, - min = 0, - max = 21 - ) - - /** - * Param for the minimum number of original tweets a user needs to be considered an original author - */ - object TopTweetImpressionsMinNumOriginalTweets - extends FSBoundedParam[Int]( - name = "top_tweet_impressions_num_original_tweets", - default = 3, - min = 0, - max = Int.MaxValue - ) - - /** - * Param for the max number of favorites any original Tweet can have - */ - object TopTweetImpressionsMaxFavoritesPerTweet - extends FSBoundedParam[Int]( - name = "top_tweet_impressions_max_favorites_per_tweet", - default = 3, - min = 0, - max = Int.MaxValue - ) - - /** - * Param for the max number of total inbound favorites for a user's tweets - */ - object TopTweetImpressionsTotalInboundFavoritesLimit - extends FSBoundedParam[Int]( - name = "top_tweet_impressions_total_inbound_favorites_limit", - default = 60, - min = 0, - max = Int.MaxValue - ) - - /** - * Param for the number of days to search for tweets to count the total inbound favorites - */ - object TopTweetImpressionsTotalFavoritesLimitNumDaysSearch - extends FSBoundedParam[Int]( - name = "top_tweet_impressions_total_favorites_limit_num_days_search", - default = 7, - min = 0, - max = 21 - ) - - /** - * Param for the max number of recent tweets Tflock should return - */ - object TopTweetImpressionsRecentTweetsByAuthorStoreMaxResults - extends FSBoundedParam[Int]( - name = "top_tweet_impressions_recent_tweets_by_author_store_max_results", - default = 50, - min = 0, - max = 1000 - ) - - /* - * Param to represent the max number of slots to maintain for Override Notifications - */ - object OverrideNotificationsMaxNumOfSlots - extends FSBoundedParam[Int]( - name = "mr_override_max_num_slots", - default = 1, - max = 10, - min = 1 - ) - - object EnableOverrideMaxSlotFn - extends FSParam[Boolean]( - name = "mr_override_enable_max_num_slots_fn", - default = false - ) - - object OverrideMaxSlotFnPushCapKnobs - extends FSParam[Seq[Double]]("mr_override_fn_pushcap_knobs", default = Seq.empty[Double]) - - object OverrideMaxSlotFnNSlotKnobs - extends FSParam[Seq[Double]]("mr_override_fn_nslot_knobs", default = Seq.empty[Double]) - - object OverrideMaxSlotFnPowerKnobs - extends FSParam[Seq[Double]]("mr_override_fn_power_knobs", default = Seq.empty[Double]) - - object OverrideMaxSlotFnWeight - extends FSBoundedParam[Double]( - "mr_override_fn_weight", - default = 1.0, - min = 0.0, - max = Double.MaxValue) - - /** - * Use to enable sending target ids in the Smart Push Payload - */ - object EnableTargetIdsInSmartPushPayload - extends FSParam[Boolean](name = "mr_override_enable_target_ids", default = true) - - /** - * Param to enable override by target id for MagicFanoutSportsEvent candidates - */ - object EnableTargetIdInSmartPushPayloadForMagicFanoutSportsEvent - extends FSParam[Boolean]( - name = "mr_override_enable_target_id_for_magic_fanout_sports_event", - default = true) - - /** - * Param to enable secondary account predicate on MF NFY - */ - object EnableSecondaryAccountPredicateMF - extends FSParam[Boolean]( - name = "frigate_push_magicfanout_secondary_account_predicate", - default = false - ) - - /** - * Enables showing our customers videos on their notifications - */ - object EnableInlineVideo - extends FSParam[Boolean](name = "mr_inline_enable_inline_video", default = false) - - /** - * Enables autoplay for inline videos - */ - object EnableAutoplayForInlineVideo - extends FSParam[Boolean](name = "mr_inline_enable_autoplay_for_inline_video", default = false) - - /** - * Enable OON filtering based on MentionFilter. - */ - object EnableOONFilteringBasedOnUserSettings - extends FSParam[Boolean](name = "oon_filtering_enable_based_on_user_settings", false) - - /** - * Enables Custom Thread Ids which is used to ungroup notifications for N-slots on iOS - */ - object EnableCustomThreadIdForOverride - extends FSParam[Boolean](name = "mr_override_enable_custom_thread_id", default = false) - - /** - * Enables showing verified symbol in the push presentation - */ - object EnablePushPresentationVerifiedSymbol - extends FSParam[Boolean](name = "push_presentation_enable_verified_symbol", default = false) - - /** - * Decide subtext in Android push header - */ - object SubtextInAndroidPushHeaderParam - extends FSEnumParam[SubtextForAndroidPushHeader.type]( - name = "push_presentation_subtext_in_android_push_header_id", - default = SubtextForAndroidPushHeader.None, - enum = SubtextForAndroidPushHeader) - - /** - * Enable SimClusters Targeting For Spaces. If false we just drop all candidates with such targeting reason - */ - object EnableSimClusterTargetingSpaces - extends FSParam[Boolean](name = "space_recs_send_simcluster_recommendations", default = false) - - /** - * Param to control threshold for dot product of simcluster based targeting on Spaces - */ - object SpacesTargetingSimClusterDotProductThreshold - extends FSBoundedParam[Double]( - "space_recs_simclusters_dot_product_threshold", - default = 0.0, - min = 0.0, - max = 10.0) - - /** - * Param to control top-k clusters simcluster based targeting on Spaces - */ - object SpacesTopKSimClusterCount - extends FSBoundedParam[Int]( - "space_recs_simclusters_top_k_count", - default = 1, - min = 1, - max = 50) - - /** SimCluster users host/speaker must meet this follower count minimum threshold to be considered for sends */ - object SpaceRecsSimClusterUserMinimumFollowerCount - extends FSBoundedParam[Int]( - name = "space_recs_simcluster_user_min_follower_count", - default = 5000, - max = Int.MaxValue, - min = 0 - ) - - /** - * Target has been bucketed into the Inline Action App Visit Fatigue Experiment - */ - object TargetInInlineActionAppVisitFatigue - extends FSParam[Boolean](name = "inline_action_target_in_app_visit_fatigue", default = false) - - /** - * Enables Inline Action App Visit Fatigue - */ - object EnableInlineActionAppVisitFatigue - extends FSParam[Boolean](name = "inline_action_enable_app_visit_fatigue", default = false) - - /** - * Determines the fatigue that we should apply when the target user has performed an inline action - */ - object InlineActionAppVisitFatigue - extends FSBoundedParam[Duration]( - name = "inline_action_app_visit_fatigue_hours", - default = 8.hours, - min = 1.hour, - max = 48.hours) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Weight for reranking(oonc - weight * nudityRate) - */ - object AuthorSensitiveScoreWeightInReranking - extends FSBoundedParam[Double]( - name = "rerank_candidates_author_sensitive_score_weight_in_reranking", - default = 0.0, - min = -100.0, - max = 100.0 - ) - - /** - * Param to control the last active space listener threshold to filter out based on that - */ - object SpaceParticipantHistoryLastActiveThreshold - extends FSBoundedParam[Duration]( - name = "space_recs_last_active_space_listener_threshold_in_hours", - default = 0.hours, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /* - * Param to enable mr user simcluster feature set (v2020) hydration for modeling-based candidate generation - * */ - object HydrateMrUserSimclusterV2020InModelingBasedCG - extends FSParam[Boolean]( - name = "candidate_generation_model_hydrate_mr_user_simcluster_v2020", - default = false) - - /* - * Param to enable mr semantic core feature set hydration for modeling-based candidate generation - * */ - object HydrateMrUserSemanticCoreInModelingBasedCG - extends FSParam[Boolean]( - name = "candidate_generation_model_hydrate_mr_user_semantic_core", - default = false) - - /* - * Param to enable mr semantic core feature set hydration for modeling-based candidate generation - * */ - object HydrateOnboardingInModelingBasedCG - extends FSParam[Boolean]( - name = "candidate_generation_model_hydrate_onboarding", - default = false) - - /* - * Param to enable mr topic follow feature set hydration for modeling-based candidate generation - * */ - object HydrateTopicFollowInModelingBasedCG - extends FSParam[Boolean]( - name = "candidate_generation_model_hydrate_topic_follow", - default = false) - - /* - * Param to enable mr user topic feature set hydration for modeling-based candidate generation - * */ - object HydrateMrUserTopicInModelingBasedCG - extends FSParam[Boolean]( - name = "candidate_generation_model_hydrate_mr_user_topic", - default = false) - - /* - * Param to enable mr user topic feature set hydration for modeling-based candidate generation - * */ - object HydrateMrUserAuthorInModelingBasedCG - extends FSParam[Boolean]( - name = "candidate_generation_model_hydrate_mr_user_author", - default = false) - - /* - * Param to enable user penguin language feature set hydration for modeling-based candidate generation - * */ - object HydrateUserPenguinLanguageInModelingBasedCG - extends FSParam[Boolean]( - name = "candidate_generation_model_hydrate_user_penguin_language", - default = false) - /* - * Param to enable user geo feature set hydration for modeling-based candidate generation - * */ - object HydrateUseGeoInModelingBasedCG - extends FSParam[Boolean]( - name = "candidate_generation_model_hydrate_user_geo", - default = false) - - /* - * Param to enable mr user hashspace embedding feature set hydration for modeling-based candidate generation - * */ - object HydrateMrUserHashspaceEmbeddingInModelingBasedCG - extends FSParam[Boolean]( - name = "candidate_generation_model_hydrate_mr_user_hashspace_embedding", - default = false) - /* - * Param to enable user tweet text feature hydration - * */ - object EnableMrUserEngagedTweetTokensFeature - extends FSParam[Boolean]( - name = "feature_hydration_mr_user_engaged_tweet_tokens", - default = false) - - /** - * Params for CRT based see less often fatigue rules - */ - object EnableF1TriggerSeeLessOftenFatigue - extends FSParam[Boolean]( - name = "seelessoften_enable_f1_trigger_fatigue", - default = false - ) - - object EnableNonF1TriggerSeeLessOftenFatigue - extends FSParam[Boolean]( - name = "seelessoften_enable_nonf1_trigger_fatigue", - default = false - ) - - /** - * Adjust the NtabCaretClickFatigue for candidates if it is triggered by - * TripHqTweet candidates - */ - object AdjustTripHqTweetTriggeredNtabCaretClickFatigue - extends FSParam[Boolean]( - name = "seelessoften_adjust_trip_hq_tweet_triggered_fatigue", - default = false - ) - - object NumberOfDaysToFilterForSeeLessOftenForF1TriggerF1 - extends FSBoundedParam[Duration]( - name = "seelessoften_for_f1_trigger_f1_tofiltermr_days", - default = 7.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - object NumberOfDaysToReducePushCapForSeeLessOftenForF1TriggerF1 - extends FSBoundedParam[Duration]( - name = "seelessoften_for_f1_trigger_f1_toreduce_pushcap_days", - default = 30.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - object NumberOfDaysToFilterForSeeLessOftenForF1TriggerNonF1 - extends FSBoundedParam[Duration]( - name = "seelessoften_for_f1_trigger_nonf1_tofiltermr_days", - default = 7.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - object NumberOfDaysToReducePushCapForSeeLessOftenForF1TriggerNonF1 - extends FSBoundedParam[Duration]( - name = "seelessoften_for_f1_trigger_non_f1_toreduce_pushcap_days", - default = 30.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - object NumberOfDaysToFilterForSeeLessOftenForNonF1TriggerF1 - extends FSBoundedParam[Duration]( - name = "seelessoften_for_nonf1_trigger_f1_tofiltermr_days", - default = 7.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - object NumberOfDaysToReducePushCapForSeeLessOftenForNonF1TriggerF1 - extends FSBoundedParam[Duration]( - name = "seelessoften_for_nonf1_trigger_f1_toreduce_pushcap_days", - default = 30.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - object NumberOfDaysToFilterForSeeLessOftenForNonF1TriggerNonF1 - extends FSBoundedParam[Duration]( - name = "seelessoften_for_nonf1_trigger_nonf1_tofiltermr_days", - default = 7.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - object NumberOfDaysToReducePushCapForSeeLessOftenForNonF1TriggerNonF1 - extends FSBoundedParam[Duration]( - name = "seelessoften_for_nonf1_trigger_nonf1_toreduce_pushcap_days", - default = 30.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - object EnableContFnF1TriggerSeeLessOftenFatigue - extends FSParam[Boolean]( - name = "seelessoften_fn_enable_f1_trigger_fatigue", - default = false - ) - - object EnableContFnNonF1TriggerSeeLessOftenFatigue - extends FSParam[Boolean]( - name = "seelessoften_fn_enable_nonf1_trigger_fatigue", - default = false - ) - - object SeeLessOftenListOfDayKnobs - extends FSParam[Seq[Double]]("seelessoften_fn_day_knobs", default = Seq.empty[Double]) - - object SeeLessOftenListOfPushCapWeightKnobs - extends FSParam[Seq[Double]]("seelessoften_fn_pushcap_knobs", default = Seq.empty[Double]) - - object SeeLessOftenListOfPowerKnobs - extends FSParam[Seq[Double]]("seelessoften_fn_power_knobs", default = Seq.empty[Double]) - - object SeeLessOftenF1TriggerF1PushCapWeight - extends FSBoundedParam[Double]( - "seelessoften_fn_f1_trigger_f1_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - object SeeLessOftenF1TriggerNonF1PushCapWeight - extends FSBoundedParam[Double]( - "seelessoften_fn_f1_trigger_nonf1_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - object SeeLessOftenNonF1TriggerF1PushCapWeight - extends FSBoundedParam[Double]( - "seelessoften_fn_nonf1_trigger_f1_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - object SeeLessOftenNonF1TriggerNonF1PushCapWeight - extends FSBoundedParam[Double]( - "seelessoften_fn_nonf1_trigger_nonf1_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - object SeeLessOftenTripHqTweetTriggerF1PushCapWeight - extends FSBoundedParam[Double]( - "seelessoften_fn_trip_hq_tweet_trigger_f1_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - object SeeLessOftenTripHqTweetTriggerNonF1PushCapWeight - extends FSBoundedParam[Double]( - "seelessoften_fn_trip_hq_tweet_trigger_nonf1_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - object SeeLessOftenTripHqTweetTriggerTripHqTweetPushCapWeight - extends FSBoundedParam[Double]( - "seelessoften_fn_trip_hq_tweet_trigger_trip_hq_tweet_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - object SeeLessOftenTopicTriggerTopicPushCapWeight - extends FSBoundedParam[Double]( - "seelessoften_fn_topic_trigger_topic_weight", - default = 1.0, - min = 0.0, - max = Double.MaxValue) - - object SeeLessOftenTopicTriggerF1PushCapWeight - extends FSBoundedParam[Double]( - "seelessoften_fn_topic_trigger_f1_weight", - default = 100000.0, - min = 0.0, - max = Double.MaxValue) - - object SeeLessOftenTopicTriggerOONPushCapWeight - extends FSBoundedParam[Double]( - "seelessoften_fn_topic_trigger_oon_weight", - default = 100000.0, - min = 0.0, - max = Double.MaxValue) - - object SeeLessOftenF1TriggerTopicPushCapWeight - extends FSBoundedParam[Double]( - "seelessoften_fn_f1_trigger_topic_weight", - default = 100000.0, - min = 0.0, - max = Double.MaxValue) - - object SeeLessOftenOONTriggerTopicPushCapWeight - extends FSBoundedParam[Double]( - "seelessoften_fn_oon_trigger_topic_weight", - default = 1.0, - min = 0.0, - max = Double.MaxValue) - - object SeeLessOftenDefaultPushCapWeight - extends FSBoundedParam[Double]( - "seelessoften_fn_default_weight", - default = 100000.0, - min = 0.0, - max = Double.MaxValue) - - object SeeLessOftenNtabOnlyNotifUserPushCapWeight - extends FSBoundedParam[Double]( - "seelessoften_fn_ntab_only_user_weight", - default = 1.0, - min = 0.0, - max = Double.MaxValue) - - // Params for inline feedback fatigue - object EnableContFnF1TriggerInlineFeedbackFatigue - extends FSParam[Boolean]( - name = "feedback_inline_fn_enable_f1_trigger_fatigue", - default = false - ) - - object EnableContFnNonF1TriggerInlineFeedbackFatigue - extends FSParam[Boolean]( - name = "feedback_inline_fn_enable_nonf1_trigger_fatigue", - default = false - ) - - object UseInlineDislikeForFatigue - extends FSParam[Boolean]( - name = "feedback_inline_fn_use_dislike", - default = true - ) - object UseInlineDismissForFatigue - extends FSParam[Boolean]( - name = "feedback_inline_fn_use_dismiss", - default = false - ) - object UseInlineSeeLessForFatigue - extends FSParam[Boolean]( - name = "feedback_inline_fn_use_see_less", - default = false - ) - object UseInlineNotRelevantForFatigue - extends FSParam[Boolean]( - name = "feedback_inline_fn_use_not_relevant", - default = false - ) - object InlineFeedbackListOfDayKnobs - extends FSParam[Seq[Double]]("feedback_inline_fn_day_knobs", default = Seq.empty[Double]) - - object InlineFeedbackListOfPushCapWeightKnobs - extends FSParam[Seq[Double]]("feedback_inline_fn_pushcap_knobs", default = Seq.empty[Double]) - - object InlineFeedbackListOfPowerKnobs - extends FSParam[Seq[Double]]("feedback_inline_fn_power_knobs", default = Seq.empty[Double]) - - object InlineFeedbackF1TriggerF1PushCapWeight - extends FSBoundedParam[Double]( - "feedback_inline_fn_f1_trigger_f1_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - object InlineFeedbackF1TriggerNonF1PushCapWeight - extends FSBoundedParam[Double]( - "feedback_inline_fn_f1_trigger_nonf1_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - object InlineFeedbackNonF1TriggerF1PushCapWeight - extends FSBoundedParam[Double]( - "feedback_inline_fn_nonf1_trigger_f1_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - object InlineFeedbackNonF1TriggerNonF1PushCapWeight - extends FSBoundedParam[Double]( - "feedback_inline_fn_nonf1_trigger_nonf1_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - // Params for prompt feedback - object EnableContFnF1TriggerPromptFeedbackFatigue - extends FSParam[Boolean]( - name = "feedback_prompt_fn_enable_f1_trigger_fatigue", - default = false - ) - - object EnableContFnNonF1TriggerPromptFeedbackFatigue - extends FSParam[Boolean]( - name = "feedback_prompt_fn_enable_nonf1_trigger_fatigue", - default = false - ) - object PromptFeedbackListOfDayKnobs - extends FSParam[Seq[Double]]("feedback_prompt_fn_day_knobs", default = Seq.empty[Double]) - - object PromptFeedbackListOfPushCapWeightKnobs - extends FSParam[Seq[Double]]("feedback_prompt_fn_pushcap_knobs", default = Seq.empty[Double]) - - object PromptFeedbackListOfPowerKnobs - extends FSParam[Seq[Double]]("feedback_prompt_fn_power_knobs", default = Seq.empty[Double]) - - object PromptFeedbackF1TriggerF1PushCapWeight - extends FSBoundedParam[Double]( - "feedback_prompt_fn_f1_trigger_f1_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - object PromptFeedbackF1TriggerNonF1PushCapWeight - extends FSBoundedParam[Double]( - "feedback_prompt_fn_f1_trigger_nonf1_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - object PromptFeedbackNonF1TriggerF1PushCapWeight - extends FSBoundedParam[Double]( - "feedback_prompt_fn_nonf1_trigger_f1_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - object PromptFeedbackNonF1TriggerNonF1PushCapWeight - extends FSBoundedParam[Double]( - "feedback_prompt_fn_nonf1_trigger_nonf1_weight", - default = 1.0, - min = 0.0, - max = 10000000.0) - - /* - * Param to enable cohost join event notif - */ - object EnableSpaceCohostJoinEvent - extends FSParam[Boolean](name = "space_recs_cohost_join_enable", default = true) - - /* - * Param to bypass global push cap when target is device following host/speaker. - */ - object BypassGlobalSpacePushCapForSoftDeviceFollow - extends FSParam[Boolean](name = "space_recs_bypass_global_pushcap_for_soft_follow", false) - - /* - * Param to bypass active listener predicate when target is device following host/speaker. - */ - object CheckActiveListenerPredicateForSoftDeviceFollow - extends FSParam[Boolean](name = "space_recs_check_active_listener_for_soft_follow", false) - - object SpreadControlRatioParam - extends FSBoundedParam[Double]( - name = "oon_spread_control_ratio", - default = 1000.0, - min = 0.0, - max = 100000.0 - ) - - object FavOverSendThresholdParam - extends FSBoundedParam[Double]( - name = "oon_spread_control_fav_over_send_threshold", - default = 0.14, - min = 0.0, - max = 1000.0 - ) - - object AuthorReportRateThresholdParam - extends FSBoundedParam[Double]( - name = "oon_spread_control_author_report_rate_threshold", - default = 7.4e-6, - min = 0.0, - max = 1000.0 - ) - - object AuthorDislikeRateThresholdParam - extends FSBoundedParam[Double]( - name = "oon_spread_control_author_dislike_rate_threshold", - default = 1.0, - min = 0.0, - max = 1000.0 - ) - - object MinTweetSendsThresholdParam - extends FSBoundedParam[Double]( - name = "oon_spread_control_min_tweet_sends_threshold", - default = 10000000000.0, - min = 0.0, - max = 10000000000.0 - ) - - object MinAuthorSendsThresholdParam - extends FSBoundedParam[Double]( - name = "oon_spread_control_min_author_sends_threshold", - default = 10000000000.0, - min = 0.0, - max = 10000000000.0 - ) - - /* - * Tweet Ntab-dislike predicate related params - */ - object TweetNtabDislikeCountThresholdParam - extends FSBoundedParam[Double]( - name = "oon_tweet_ntab_dislike_count_threshold", - default = 10000.0, - min = 0.0, - max = 10000.0 - ) - object TweetNtabDislikeRateThresholdParam - extends FSBoundedParam[Double]( - name = "oon_tweet_ntab_dislike_rate_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Param for tweet language feature name - */ - object TweetLanguageFeatureNameParam - extends FSParam[String]( - name = "language_tweet_language_feature_name", - default = "tweet.language.tweet.identified") - - /** - * Threshold for user inferred language filtering - */ - object UserInferredLanguageThresholdParam - extends FSBoundedParam[Double]( - name = "language_user_inferred_language_threshold", - default = 0.0, - min = 0.0, - max = 1.0 - ) - - /** - * Threshold for user device language filtering - */ - object UserDeviceLanguageThresholdParam - extends FSBoundedParam[Double]( - name = "language_user_device_language_threshold", - default = 0.0, - min = 0.0, - max = 1.0 - ) - - /** - * Param to enable/disable tweet language filter - */ - object EnableTweetLanguageFilter - extends FSParam[Boolean]( - name = "language_enable_tweet_language_filter", - default = false - ) - - /** - * Param to skip language filter for media tweets - */ - object SkipLanguageFilterForMediaTweets - extends FSParam[Boolean]( - name = "language_skip_language_filter_for_media_tweets", - default = false - ) - - /* - * Tweet Ntab-dislike predicate related params for MrTwistly - */ - object TweetNtabDislikeCountThresholdForMrTwistlyParam - extends FSBoundedParam[Double]( - name = "oon_tweet_ntab_dislike_count_threshold_for_mrtwistly", - default = 10000.0, - min = 0.0, - max = 10000.0 - ) - object TweetNtabDislikeRateThresholdForMrTwistlyParam - extends FSBoundedParam[Double]( - name = "oon_tweet_ntab_dislike_rate_threshold_for_mrtwistly", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - object TweetNtabDislikeCountBucketThresholdParam - extends FSBoundedParam[Double]( - name = "oon_tweet_ntab_dislike_count_bucket_threshold", - default = 10.0, - min = 0.0, - max = 10000.0 - ) - - /* - * Tweet engagement ratio predicate related params - */ - object TweetQTtoNtabClickRatioThresholdParam - extends FSBoundedParam[Double]( - name = "oon_tweet_engagement_filter_qt_to_ntabclick_ratio_threshold", - default = 0.0, - min = 0.0, - max = 100000.0 - ) - - /** - * Lower bound threshold to filter a tweet based on its reply to like ratio - */ - object TweetReplytoLikeRatioThresholdLowerBound - extends FSBoundedParam[Double]( - name = "oon_tweet_engagement_filter_reply_to_like_ratio_threshold_lower_bound", - default = Double.MaxValue, - min = 0.0, - max = Double.MaxValue - ) - - /** - * Upper bound threshold to filter a tweet based on its reply to like ratio - */ - object TweetReplytoLikeRatioThresholdUpperBound - extends FSBoundedParam[Double]( - name = "oon_tweet_engagement_filter_reply_to_like_ratio_threshold_upper_bound", - default = 0.0, - min = 0.0, - max = Double.MaxValue - ) - - /** - * Upper bound threshold to filter a tweet based on its reply to like ratio - */ - object TweetReplytoLikeRatioReplyCountThreshold - extends FSBoundedParam[Int]( - name = "oon_tweet_engagement_filter_reply_count_threshold", - default = Int.MaxValue, - min = 0, - max = Int.MaxValue - ) - - /* - * oonTweetLengthBasedPrerankingPredicate related params - */ - object OonTweetLengthPredicateUpdatedMediaLogic - extends FSParam[Boolean]( - name = "oon_quality_filter_tweet_length_updated_media_logic", - default = false - ) - - object OonTweetLengthPredicateUpdatedQuoteTweetLogic - extends FSParam[Boolean]( - name = "oon_quality_filter_tweet_length_updated_quote_tweet_logic", - default = false - ) - - object OonTweetLengthPredicateMoreStrictForUndefinedLanguages - extends FSParam[Boolean]( - name = "oon_quality_filter_tweet_length_more_strict_for_undefined_languages", - default = false - ) - - object EnablePrerankingTweetLengthPredicate - extends FSParam[Boolean]( - name = "oon_quality_filter_enable_preranking_filter", - default = false - ) - - /* - * LengthLanguageBasedOONTweetCandidatesQualityPredicate related params - */ - object SautOonWithMediaTweetLengthThresholdParam - extends FSBoundedParam[Double]( - name = "oon_quality_filter_tweet_length_threshold_for_saut_oon_with_media", - default = 0.0, - min = 0.0, - max = 70.0 - ) - object NonSautOonWithMediaTweetLengthThresholdParam - extends FSBoundedParam[Double]( - name = "oon_quality_filter_tweet_length_threshold_for_non_saut_oon_with_media", - default = 0.0, - min = 0.0, - max = 70.0 - ) - object SautOonWithoutMediaTweetLengthThresholdParam - extends FSBoundedParam[Double]( - name = "oon_quality_filter_tweet_length_threshold_for_saut_oon_without_media", - default = 0.0, - min = 0.0, - max = 70.0 - ) - object NonSautOonWithoutMediaTweetLengthThresholdParam - extends FSBoundedParam[Double]( - name = "oon_quality_filter_tweet_length_threshold_for_non_saut_oon_without_media", - default = 0.0, - min = 0.0, - max = 70.0 - ) - - object ArgfOonWithMediaTweetWordLengthThresholdParam - extends FSBoundedParam[Double]( - name = "oon_quality_filter_tweet_word_length_threshold_for_argf_oon_with_media", - default = 0.0, - min = 0.0, - max = 18.0 - ) - object EsfthOonWithMediaTweetWordLengthThresholdParam - extends FSBoundedParam[Double]( - name = "oon_quality_filter_tweet_word_length_threshold_for_esfth_oon_with_media", - default = 0.0, - min = 0.0, - max = 10.0 - ) - - /** - * Param to enable/disable sentiment feature hydration - */ - object EnableMrTweetSentimentFeatureHydrationFS - extends FSParam[Boolean]( - name = "feature_hydration_enable_mr_tweet_sentiment_feature", - default = false - ) - - /** - * Param to enable/disable feature map scribing for staging test log - */ - object EnableMrScribingMLFeaturesAsFeatureMapForStaging - extends FSParam[Boolean]( - name = "frigate_pushservice_enable_scribing_ml_features_as_featuremap_for_staging", - default = false - ) - - /** - * Param to enable timeline health signal hydration - * */ - object EnableTimelineHealthSignalHydration - extends FSParam[Boolean]( - name = "timeline_health_signal_hydration", - default = false - ) - - /** - * Param to enable timeline health signal hydration for model training - * */ - object EnableTimelineHealthSignalHydrationForModelTraining - extends FSParam[Boolean]( - name = "timeline_health_signal_hydration_for_model_training", - default = false - ) - - /** - * Param to enable/disable mr user social context agg feature hydration - */ - object EnableMrUserSocialContextAggregateFeatureHydration - extends FSParam[Boolean]( - name = "frigate_push_modeling_hydrate_mr_user_social_context_agg_feature", - default = true - ) - - /** - * Param to enable/disable mr user semantic core agg feature hydration - */ - object EnableMrUserSemanticCoreAggregateFeatureHydration - extends FSParam[Boolean]( - name = "frigate_push_modeling_hydrate_mr_user_semantic_core_agg_feature", - default = true - ) - - /** - * Param to enable/disable mr user candidate sparse agg feature hydration - */ - object EnableMrUserCandidateSparseOfflineAggregateFeatureHydration - extends FSParam[Boolean]( - name = "frigate_push_modeling_hydrate_mr_user_candidate_sparse_agg_feature", - default = true - ) - - /** - * Param to enable/disable mr user candidate agg feature hydration - */ - object EnableMrUserCandidateOfflineAggregateFeatureHydration - extends FSParam[Boolean]( - name = "frigate_push_modeling_hydrate_mr_user_candidate_agg_feature", - default = true - ) - - /** - * Param to enable/disable mr user candidate compact agg feature hydration - */ - object EnableMrUserCandidateOfflineCompactAggregateFeatureHydration - extends FSParam[Boolean]( - name = "frigate_push_modeling_hydrate_mr_user_candidate_compact_agg_feature", - default = false - ) - - /** - * Param to enable/disable mr real graph user-author/social-context feature hydration - */ - object EnableRealGraphUserAuthorAndSocialContxtFeatureHydration - extends FSParam[Boolean]( - name = "frigate_push_modeling_hydrate_real_graph_user_social_feature", - default = true - ) - - /** - * Param to enable/disable mr user author agg feature hydration - */ - object EnableMrUserAuthorOfflineAggregateFeatureHydration - extends FSParam[Boolean]( - name = "frigate_push_modeling_hydrate_mr_user_author_agg_feature", - default = true - ) - - /** - * Param to enable/disable mr user author compact agg feature hydration - */ - object EnableMrUserAuthorOfflineCompactAggregateFeatureHydration - extends FSParam[Boolean]( - name = "frigate_push_modeling_hydrate_mr_user_author_compact_agg_feature", - default = false - ) - - /** - * Param to enable/disable mr user compact agg feature hydration - */ - object EnableMrUserOfflineCompactAggregateFeatureHydration - extends FSParam[Boolean]( - name = "frigate_push_modeling_hydrate_mr_user_compact_agg_feature", - default = false - ) - - /** - * Param to enable/disable mr user simcluster agg feature hydration - */ - object EnableMrUserSimcluster2020AggregateFeatureHydration - extends FSParam[Boolean]( - name = "frigate_push_modeling_hydrate_mr_user_simcluster_agg_feature", - default = true - ) - - /** - * Param to enable/disable mr user agg feature hydration - */ - object EnableMrUserOfflineAggregateFeatureHydration - extends FSParam[Boolean]( - name = "frigate_push_modeling_hydrate_mr_user_agg_feature", - default = true - ) - - /** - * Param to enable/disable topic engagement RTA in the ranking model - */ - object EnableTopicEngagementRealTimeAggregatesFS - extends FSParam[Boolean]( - "feature_hydration_enable_htl_topic_engagement_real_time_agg_feature", - false) - - /* - * Param to enable mr user semantic core feature hydration for heavy ranker - * */ - object EnableMrUserSemanticCoreFeatureForExpt - extends FSParam[Boolean]( - name = "frigate_push_modeling_hydrate_mr_user_semantic_core", - default = false) - - /** - * Param to enable hydrating user duration since last visit features - */ - object EnableHydratingUserDurationSinceLastVisitFeatures - extends FSParam[Boolean]( - name = "feature_hydration_user_duration_since_last_visit", - default = false) - - /** - Param to enable/disable user-topic aggregates in the ranking model - */ - object EnableUserTopicAggregatesFS - extends FSParam[Boolean]("feature_hydration_enable_htl_topic_user_agg_feature", false) - - /* - * PNegMultimodalPredicate related params - */ - object EnablePNegMultimodalPredicateParam - extends FSParam[Boolean]( - name = "pneg_multimodal_filter_enable_param", - default = false - ) - object PNegMultimodalPredicateModelThresholdParam - extends FSBoundedParam[Double]( - name = "pneg_multimodal_filter_model_threshold_param", - default = 1.0, - min = 0.0, - max = 1.0 - ) - object PNegMultimodalPredicateBucketThresholdParam - extends FSBoundedParam[Double]( - name = "pneg_multimodal_filter_bucket_threshold_param", - default = 0.4, - min = 0.0, - max = 1.0 - ) - - /* - * NegativeKeywordsPredicate related params - */ - object EnableNegativeKeywordsPredicateParam - extends FSParam[Boolean]( - name = "negative_keywords_filter_enable_param", - default = false - ) - object NegativeKeywordsPredicateDenylist - extends FSParam[Seq[String]]( - name = "negative_keywords_filter_denylist", - default = Seq.empty[String] - ) - /* - * LightRanking related params - */ - object EnableLightRankingParam - extends FSParam[Boolean]( - name = "light_ranking_enable_param", - default = false - ) - object LightRankingNumberOfCandidatesParam - extends FSBoundedParam[Int]( - name = "light_ranking_number_of_candidates_param", - default = 100, - min = 0, - max = 1000 - ) - object LightRankingModelTypeParam - extends FSParam[String]( - name = "light_ranking_model_type_param", - default = "WeightedOpenOrNtabClickProbability_Q4_2021_13172_Mr_Light_Ranker_Dbv2_Top3") - object EnableRandomBaselineLightRankingParam - extends FSParam[Boolean]( - name = "light_ranking_random_baseline_enable_param", - default = false - ) - - object LightRankingScribeCandidatesDownSamplingParam - extends FSBoundedParam[Double]( - name = "light_ranking_scribe_candidates_down_sampling_param", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /* - * Quality Upranking related params - */ - object EnableProducersQualityBoostingForHeavyRankingParam - extends FSParam[Boolean]( - name = "quality_upranking_enable_producers_quality_boosting_for_heavy_ranking_param", - default = false - ) - - object QualityUprankingBoostForHighQualityProducersParam - extends FSBoundedParam[Double]( - name = "quality_upranking_boost_for_high_quality_producers_param", - default = 1.0, - min = 0.0, - max = 10000.0 - ) - - object QualityUprankingDownboostForLowQualityProducersParam - extends FSBoundedParam[Double]( - name = "quality_upranking_downboost_for_low_quality_producers_param", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - object EnableQualityUprankingForHeavyRankingParam - extends FSParam[Boolean]( - name = "quality_upranking_enable_for_heavy_ranking_param", - default = false - ) - object QualityUprankingModelTypeParam - extends FSParam[WeightedOpenOrNtabClickModel.ModelNameType]( - name = "quality_upranking_model_id", - default = "Q4_2022_Mr_Bqml_Quality_Model_wALL" - ) - object QualityUprankingTransformTypeParam - extends FSEnumParam[MrQualityUprankingTransformTypeEnum.type]( - name = "quality_upranking_transform_id", - default = MrQualityUprankingTransformTypeEnum.Sigmoid, - enum = MrQualityUprankingTransformTypeEnum - ) - - object QualityUprankingBoostForHeavyRankingParam - extends FSBoundedParam[Double]( - name = "quality_upranking_boost_for_heavy_ranking_param", - default = 1.0, - min = -10.0, - max = 10.0 - ) - object QualityUprankingSigmoidBiasForHeavyRankingParam - extends FSBoundedParam[Double]( - name = "quality_upranking_sigmoid_bias_for_heavy_ranking_param", - default = 0.0, - min = -10.0, - max = 10.0 - ) - object QualityUprankingSigmoidWeightForHeavyRankingParam - extends FSBoundedParam[Double]( - name = "quality_upranking_sigmoid_weight_for_heavy_ranking_param", - default = 1.0, - min = -10.0, - max = 10.0 - ) - object QualityUprankingLinearBarForHeavyRankingParam - extends FSBoundedParam[Double]( - name = "quality_upranking_linear_bar_for_heavy_ranking_param", - default = 1.0, - min = 0.0, - max = 10.0 - ) - object EnableQualityUprankingCrtScoreStatsForHeavyRankingParam - extends FSParam[Boolean]( - name = "quality_upranking_enable_crt_score_stats_for_heavy_ranking_param", - default = false - ) - /* - * BQML Health Model related params - */ - object EnableBqmlHealthModelPredicateParam - extends FSParam[Boolean]( - name = "bqml_health_model_filter_enable_param", - default = false - ) - - object EnableBqmlHealthModelPredictionForInNetworkCandidatesParam - extends FSParam[Boolean]( - name = "bqml_health_model_enable_prediction_for_in_network_candidates_param", - default = false - ) - - object BqmlHealthModelTypeParam - extends FSParam[HealthNsfwModel.ModelNameType]( - name = "bqml_health_model_id", - default = HealthNsfwModel.Q2_2022_Mr_Bqml_Health_Model_NsfwV0 - ) - object BqmlHealthModelPredicateFilterThresholdParam - extends FSBoundedParam[Double]( - name = "bqml_health_model_filter_threshold_param", - default = 1.0, - min = 0.0, - max = 1.0 - ) - object BqmlHealthModelPredicateBucketThresholdParam - extends FSBoundedParam[Double]( - name = "bqml_health_model_bucket_threshold_param", - default = 0.005, - min = 0.0, - max = 1.0 - ) - - object EnableBqmlHealthModelScoreHistogramParam - extends FSParam[Boolean]( - name = "bqml_health_model_score_histogram_enable_param", - default = false - ) - - /* - * BQML Quality Model related params - */ - object EnableBqmlQualityModelPredicateParam - extends FSParam[Boolean]( - name = "bqml_quality_model_filter_enable_param", - default = false - ) - object EnableBqmlQualityModelScoreHistogramParam - extends FSParam[Boolean]( - name = "bqml_quality_model_score_histogram_enable_param", - default = false - ) - object BqmlQualityModelTypeParam - extends FSParam[WeightedOpenOrNtabClickModel.ModelNameType]( - name = "bqml_quality_model_id", - default = "Q1_2022_13562_Mr_Bqml_Quality_Model_V2" - ) - - /** - * Param to specify which quality models to use to get the scores for determining - * whether to bucket a user for the DDG - */ - object BqmlQualityModelBucketModelIdListParam - extends FSParam[Seq[WeightedOpenOrNtabClickModel.ModelNameType]]( - name = "bqml_quality_model_bucket_model_id_list", - default = Seq( - "Q1_2022_13562_Mr_Bqml_Quality_Model_V2", - "Q2_2022_DDG14146_Mr_Personalised_BQML_Quality_Model", - "Q2_2022_DDG14146_Mr_NonPersonalised_BQML_Quality_Model" - ) - ) - - object BqmlQualityModelPredicateThresholdParam - extends FSBoundedParam[Double]( - name = "bqml_quality_model_filter_threshold_param", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Param to specify the threshold to determine if a user’s quality score is high enough to enter the experiment. - */ - object BqmlQualityModelBucketThresholdListParam - extends FSParam[Seq[Double]]( - name = "bqml_quality_model_bucket_threshold_list", - default = Seq(0.7, 0.7, 0.7) - ) - - /* - * TweetAuthorAggregates related params - */ - object EnableTweetAuthorAggregatesFeatureHydrationParam - extends FSParam[Boolean]( - name = "tweet_author_aggregates_feature_hydration_enable_param", - default = false - ) - - /** - * Param to determine if we should include the relevancy score of candidates in the Ibis payload - */ - object IncludeRelevanceScoreInIbis2Payload - extends FSParam[Boolean]( - name = "relevance_score_include_in_ibis2_payload", - default = false - ) - - /** - * Param to specify supervised model to predict score by sending the notification - */ - object BigFilteringSupervisedSendingModelParam - extends FSParam[BigFilteringSupervisedModel.ModelNameType]( - name = "ltv_filtering_bigfiltering_supervised_sending_model_param", - default = BigFilteringSupervisedModel.V0_0_BigFiltering_Supervised_Sending_Model - ) - - /** - * Param to specify supervised model to predict score by not sending the notification - */ - object BigFilteringSupervisedWithoutSendingModelParam - extends FSParam[BigFilteringSupervisedModel.ModelNameType]( - name = "ltv_filtering_bigfiltering_supervised_without_sending_model_param", - default = BigFilteringSupervisedModel.V0_0_BigFiltering_Supervised_Without_Sending_Model - ) - - /** - * Param to specify RL model to predict score by sending the notification - */ - object BigFilteringRLSendingModelParam - extends FSParam[BigFilteringSupervisedModel.ModelNameType]( - name = "ltv_filtering_bigfiltering_rl_sending_model_param", - default = BigFilteringRLModel.V0_0_BigFiltering_Rl_Sending_Model - ) - - /** - * Param to specify RL model to predict score by not sending the notification - */ - object BigFilteringRLWithoutSendingModelParam - extends FSParam[BigFilteringSupervisedModel.ModelNameType]( - name = "ltv_filtering_bigfiltering_rl_without_sending_model_param", - default = BigFilteringRLModel.V0_0_BigFiltering_Rl_Without_Sending_Model - ) - - /** - * Param to specify the threshold (send notification if score >= threshold) - */ - object BigFilteringThresholdParam - extends FSBoundedParam[Double]( - name = "ltv_filtering_bigfiltering_threshold_param", - default = 0.0, - min = Double.MinValue, - max = Double.MaxValue - ) - - /** - * Param to specify normalization used for BigFiltering - */ - object BigFilteringNormalizationTypeIdParam - extends FSEnumParam[BigFilteringNormalizationEnum.type]( - name = "ltv_filtering_bigfiltering_normalization_type_id", - default = BigFilteringNormalizationEnum.NormalizationDisabled, - enum = BigFilteringNormalizationEnum - ) - - /** - * Param to specify histograms of model scores in BigFiltering - */ - object BigFilteringEnableHistogramsParam - extends FSParam[Boolean]( - name = "ltv_filtering_bigfiltering_enable_histograms_param", - default = false - ) - - /* - * Param to enable sending requests to Ins Sender - */ - object EnableInsSender extends FSParam[Boolean](name = "ins_enable_dark_traffic", default = false) - - /** - * Param to specify the range of relevance scores for MagicFanout types. - */ - object MagicFanoutRelevanceScoreRange - extends FSParam[Seq[Double]]( - name = "relevance_score_mf_range", - default = Seq(0.75, 1.0) - ) - - /** - * Param to specify the range of relevance scores for MR types. - */ - object MagicRecsRelevanceScoreRange - extends FSParam[Seq[Double]]( - name = "relevance_score_mr_range", - default = Seq(0.25, 0.5) - ) - - /** - * Param to enable backfilling OON candidates if number of F1 candidates is greater than a threshold K. - */ - object EnableOONBackfillBasedOnF1Candidates - extends FSParam[Boolean](name = "oon_enable_backfill_based_on_f1", default = false) - - /** - * Threshold for the minimum number of F1 candidates required to enable backfill of OON candidates. - */ - object NumberOfF1CandidatesThresholdForOONBackfill - extends FSBoundedParam[Int]( - name = "oon_enable_backfill_f1_threshold", - min = 0, - default = 5000, - max = 5000) - - /** - * Event ID allowlist to skip account country predicate - */ - object MagicFanoutEventAllowlistToSkipAccountCountryPredicate - extends FSParam[Seq[Long]]( - name = "magicfanout_event_allowlist_skip_account_country_predicate", - default = Seq.empty[Long] - ) - - /** - * MagicFanout Event Semantic Core Domain Ids - */ - object ListOfEventSemanticCoreDomainIds - extends FSParam[Seq[Long]]( - name = "magicfanout_automated_events_semantic_core_domain_ids", - default = Seq()) - - /** - * Adhoc id for detailed rank flow stats - */ - object ListOfAdhocIdsForStatsTracking - extends FSParam[Set[Long]]( - name = "stats_enable_detailed_stats_tracking_ids", - default = Set.empty[Long] - ) - - object EnableGenericCRTBasedFatiguePredicate - extends FSParam[Boolean]( - name = "seelessoften_enable_generic_crt_based_fatigue_predicate", - default = false) - - /** - * Param to enable copy features such as Emojis and Target Name - */ - object EnableCopyFeaturesForF1 - extends FSParam[Boolean](name = "mr_copy_enable_features_f1", default = false) - - /** - * Param to enable copy features such as Emojis and Target Name - */ - object EnableCopyFeaturesForOon - extends FSParam[Boolean](name = "mr_copy_enable_features_oon", default = false) - - /** - * Param to enable Emoji in F1 Copy - */ - object EnableEmojiInF1Copy - extends FSParam[Boolean](name = "mr_copy_enable_f1_emoji", default = false) - - /** - * Param to enable Target in F1 Copy - */ - object EnableTargetInF1Copy - extends FSParam[Boolean](name = "mr_copy_enable_f1_target", default = false) - - /** - * Param to enable Emoji in OON Copy - */ - object EnableEmojiInOonCopy - extends FSParam[Boolean](name = "mr_copy_enable_oon_emoji", default = false) - - /** - * Param to enable Target in OON Copy - */ - object EnableTargetInOonCopy - extends FSParam[Boolean](name = "mr_copy_enable_oon_target", default = false) - - /** - * Param to enable split fatigue for Target and Emoji copy for OON and F1 - */ - object EnableTargetAndEmojiSplitFatigue - extends FSParam[Boolean](name = "mr_copy_enable_target_emoji_split_fatigue", default = false) - - /** - * Param to enable experimenting string on the body - */ - object EnableF1CopyBody extends FSParam[Boolean](name = "mr_copy_f1_enable_body", default = false) - - object EnableOONCopyBody - extends FSParam[Boolean](name = "mr_copy_oon_enable_body", default = false) - - object EnableIosCopyBodyTruncate - extends FSParam[Boolean](name = "mr_copy_enable_body_truncate", default = false) - - object EnableNsfwCopy extends FSParam[Boolean](name = "mr_copy_enable_nsfw", default = false) - - /** - * Param to determine F1 candidate nsfw score threshold - */ - object NsfwScoreThresholdForF1Copy - extends FSBoundedParam[Double]( - name = "mr_copy_nsfw_threshold_f1", - default = 0.3, - min = 0.0, - max = 1.0 - ) - - /** - * Param to determine OON candidate nsfw score threshold - */ - object NsfwScoreThresholdForOONCopy - extends FSBoundedParam[Double]( - name = "mr_copy_nsfw_threshold_oon", - default = 0.2, - min = 0.0, - max = 1.0 - ) - - /** - * Param to determine the lookback duration when searching for prev copy features. - */ - object CopyFeaturesHistoryLookbackDuration - extends FSBoundedParam[Duration]( - name = "mr_copy_history_lookback_duration_in_days", - default = 30.days, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Param to determine the F1 emoji copy fatigue in # of hours. - */ - object F1EmojiCopyFatigueDuration - extends FSBoundedParam[Duration]( - name = "mr_copy_f1_emoji_copy_fatigue_in_hours", - default = 24.hours, - min = 0.hours, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to determine the F1 target copy fatigue in # of hours. - */ - object F1TargetCopyFatigueDuration - extends FSBoundedParam[Duration]( - name = "mr_copy_f1_target_copy_fatigue_in_hours", - default = 24.hours, - min = 0.hours, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to determine the OON emoji copy fatigue in # of hours. - */ - object OonEmojiCopyFatigueDuration - extends FSBoundedParam[Duration]( - name = "mr_copy_oon_emoji_copy_fatigue_in_hours", - default = 24.hours, - min = 0.hours, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to determine the OON target copy fatigue in # of hours. - */ - object OonTargetCopyFatigueDuration - extends FSBoundedParam[Duration]( - name = "mr_copy_oon_target_copy_fatigue_in_hours", - default = 24.hours, - min = 0.hours, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to turn on/off home timeline based fatigue rule, where once last home timeline visit - * is larger than the specified will evalute to not fatigue - */ - object EnableHTLBasedFatigueBasicRule - extends FSParam[Boolean]( - name = "mr_copy_enable_htl_based_fatigue_basic_rule", - default = false) - - /** - * Param to determine f1 emoji copy fatigue in # of pushes - */ - object F1EmojiCopyNumOfPushesFatigue - extends FSBoundedParam[Int]( - name = "mr_copy_f1_emoji_copy_number_of_pushes_fatigue", - default = 0, - min = 0, - max = 200 - ) - - /** - * Param to determine oon emoji copy fatigue in # of pushes - */ - object OonEmojiCopyNumOfPushesFatigue - extends FSBoundedParam[Int]( - name = "mr_copy_oon_emoji_copy_number_of_pushes_fatigue", - default = 0, - min = 0, - max = 200 - ) - - /** - * If user haven't visited home timeline for certain duration, we will - * exempt user from feature copy fatigue. This param is used to control - * how long it is before we enter exemption. - */ - object MinFatigueDurationSinceLastHTLVisit - extends FSBoundedParam[Duration]( - name = "mr_copy_min_duration_since_last_htl_visit_hours", - default = Duration.Top, - min = 0.hour, - max = Duration.Top, - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * If a user haven't visit home timeline very long, the user will return - * to fatigue state under the home timeline based fatigue rule. There will - * only be a window, where the user is out of fatigue state under the rule. - * This param control the length of the non fatigue period. - */ - object LastHTLVisitBasedNonFatigueWindow - extends FSBoundedParam[Duration]( - name = "mr_copy_last_htl_visit_based_non_fatigue_window_hours", - default = 48.hours, - min = 0.hour, - max = Duration.Top, - ) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - object EnableOONCBasedCopy - extends FSParam[Boolean]( - name = "mr_copy_enable_oonc_based_copy", - default = false - ) - - object HighOONCThresholdForCopy - extends FSBoundedParam[Double]( - name = "mr_copy_high_oonc_threshold_for_copy", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - object LowOONCThresholdForCopy - extends FSBoundedParam[Double]( - name = "mr_copy_low_oonc_threshold_for_copy", - default = 0.0, - min = 0.0, - max = 1.0 - ) - - object EnableTweetTranslation - extends FSParam[Boolean](name = "tweet_translation_enable", default = false) - - object TripTweetCandidateReturnEnable - extends FSParam[Boolean](name = "trip_tweet_candidate_enable", default = false) - - object TripTweetCandidateSourceIds - extends FSParam[Seq[String]]( - name = "trip_tweet_candidate_source_ids", - default = Seq("TOP_GEO_V3")) - - object TripTweetMaxTotalCandidates - extends FSBoundedParam[Int]( - name = "trip_tweet_max_total_candidates", - default = 500, - min = 10, - max = 1000) - - object EnableEmptyBody - extends FSParam[Boolean](name = "push_presentation_enable_empty_body", default = false) - - object EnableSocialContextForRetweet - extends FSParam[Boolean](name = "push_presentation_social_context_retweet", default = false) - - /** - * Param to enable/disable simcluster feature hydration - */ - object EnableMrTweetSimClusterFeatureHydrationFS - extends FSParam[Boolean]( - name = "feature_hydration_enable_mr_tweet_simcluster_feature", - default = false - ) - - /** - * Param to disable OON candidates based on tweetAuthor - */ - object DisableOutNetworkTweetCandidatesFS - extends FSParam[Boolean](name = "oon_filtering_disable_oon_candidates", default = false) - - /** - * Param to enable Local Viral Tweets - */ - object EnableLocalViralTweets - extends FSParam[Boolean](name = "local_viral_tweets_enable", default = true) - - /** - * Param to enable Explore Video Tweets - */ - object EnableExploreVideoTweets - extends FSParam[Boolean](name = "explore_video_tweets_enable", default = false) - - /** - * Param to enable List Recommendations - */ - object EnableListRecommendations - extends FSParam[Boolean](name = "list_recommendations_enable", default = false) - - /** - * Param to enable IDS List Recommendations - */ - object EnableIDSListRecommendations - extends FSParam[Boolean](name = "list_recommendations_ids_enable", default = false) - - /** - * Param to enable PopGeo List Recommendations - */ - object EnablePopGeoListRecommendations - extends FSParam[Boolean](name = "list_recommendations_pop_geo_enable", default = false) - - /** - * Param to control the inverter for fatigue between consecutive ListRecommendations - */ - object ListRecommendationsPushInterval - extends FSBoundedParam[Duration]( - name = "list_recommendations_interval_days", - default = 24.hours, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromDays - } - - /** - * Param to control the granularity of GeoHash for ListRecommendations - */ - object ListRecommendationsGeoHashLength - extends FSBoundedParam[Int]( - name = "list_recommendations_geo_hash_length", - default = 5, - min = 3, - max = 5) - - /** - * Param to control maximum number of ListRecommendation pushes to receive in an interval - */ - object MaxListRecommendationsPushGivenInterval - extends FSBoundedParam[Int]( - name = "list_recommendations_push_given_interval", - default = 1, - min = 0, - max = 10 - ) - - /** - * Param to control the subscriber count for list recommendation - */ - object ListRecommendationsSubscriberCount - extends FSBoundedParam[Int]( - name = "list_recommendations_subscriber_count", - default = 0, - min = 0, - max = Integer.MAX_VALUE) - - /** - * Param to define dynamic inline action types for web notifications (both desktop web + mobile web) - */ - object LocalViralTweetsBucket - extends FSParam[String]( - name = "local_viral_tweets_bucket", - default = "high", - ) - - /** - * List of CrTags to disable - */ - object OONCandidatesDisabledCrTagParam - extends FSParam[Seq[String]]( - name = "oon_enable_oon_candidates_disabled_crtag", - default = Seq.empty[String] - ) - - /** - * List of Crt groups to disable - */ - object OONCandidatesDisabledCrtGroupParam - extends FSEnumSeqParam[CrtGroupEnum.type]( - name = "oon_enable_oon_candidates_disabled_crt_group_ids", - default = Seq.empty[CrtGroupEnum.Value], - enum = CrtGroupEnum - ) - - /** - * Param to enable launching video tweets in the Immersive Explore timeline - */ - object EnableLaunchVideosInImmersiveExplore - extends FSParam[Boolean](name = "launch_videos_in_immersive_explore", default = false) - - /** - * Param to enable Ntab Entries for Sports Event Notifications - */ - object EnableNTabEntriesForSportsEventNotifications - extends FSParam[Boolean]( - name = "magicfanout_sports_event_enable_ntab_entries", - default = false) - - /** - * Param to enable Ntab Facepiles for teams in Sport Notifs - */ - object EnableNTabFacePileForSportsEventNotifications - extends FSParam[Boolean]( - name = "magicfanout_sports_event_enable_ntab_facepiles", - default = false) - - /** - * Param to enable Ntab Override for Sports Event Notifications - */ - object EnableNTabOverrideForSportsEventNotifications - extends FSParam[Boolean]( - name = "magicfanout_sports_event_enable_ntab_override", - default = false) - - /** - * Param to control the interval for MF Product Launch Notifs - */ - object ProductLaunchPushIntervalInHours - extends FSBoundedParam[Duration]( - name = "product_launch_fatigue_push_interval_in_hours", - default = 24.hours, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to control the maximum number of MF Product Launch Notifs in a period of time - */ - object ProductLaunchMaxNumberOfPushesInInterval - extends FSBoundedParam[Int]( - name = "product_launch_fatigue_max_pushes_in_interval", - default = 1, - min = 0, - max = 10) - - /** - * Param to control the minInterval for fatigue between consecutive MF Product Launch Notifs - */ - object ProductLaunchMinIntervalFatigue - extends FSBoundedParam[Duration]( - name = "product_launch_fatigue_min_interval_consecutive_pushes_in_hours", - default = 24.hours, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to control the interval for MF New Creator Notifs - */ - object NewCreatorPushIntervalInHours - extends FSBoundedParam[Duration]( - name = "new_creator_fatigue_push_interval_in_hours", - default = 24.hours, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to control the maximum number of MF New Creator Notifs in a period of time - */ - object NewCreatorPushMaxNumberOfPushesInInterval - extends FSBoundedParam[Int]( - name = "new_creator_fatigue_max_pushes_in_interval", - default = 1, - min = 0, - max = 10) - - /** - * Param to control the minInterval for fatigue between consecutive MF New Creator Notifs - */ - object NewCreatorPushMinIntervalFatigue - extends FSBoundedParam[Duration]( - name = "new_creator_fatigue_min_interval_consecutive_pushes_in_hours", - default = 24.hours, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to control the interval for MF New Creator Notifs - */ - object CreatorSubscriptionPushIntervalInHours - extends FSBoundedParam[Duration]( - name = "creator_subscription_fatigue_push_interval_in_hours", - default = 24.hours, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to control the maximum number of MF New Creator Notifs in a period of time - */ - object CreatorSubscriptionPushMaxNumberOfPushesInInterval - extends FSBoundedParam[Int]( - name = "creator_subscription_fatigue_max_pushes_in_interval", - default = 1, - min = 0, - max = 10) - - /** - * Param to control the minInterval for fatigue between consecutive MF New Creator Notifs - */ - object CreatorSubscriptionPushhMinIntervalFatigue - extends FSBoundedParam[Duration]( - name = "creator_subscription_fatigue_min_interval_consecutive_pushes_in_hours", - default = 24.hours, - min = Duration.Bottom, - max = Duration.Top) - with HasDurationConversion { - override val durationConversion = DurationConversion.FromHours - } - - /** - * Param to define the landing page deeplink of product launch notifications - */ - object ProductLaunchLandingPageDeepLink - extends FSParam[String]( - name = "product_launch_landing_page_deeplink", - default = "" - ) - - /** - * Param to define the tap through of product launch notifications - */ - object ProductLaunchTapThrough - extends FSParam[String]( - name = "product_launch_tap_through", - default = "" - ) - - /** - * Param to skip checking isTargetBlueVerified - */ - object DisableIsTargetBlueVerifiedPredicate - extends FSParam[Boolean]( - name = "product_launch_disable_is_target_blue_verified_predicate", - default = false - ) - - /** - * Param to enable Ntab Entries for Sports Event Notifications - */ - object EnableNTabEntriesForProductLaunchNotifications - extends FSParam[Boolean](name = "product_launch_enable_ntab_entry", default = true) - - /** - * Param to skip checking isTargetLegacyVerified - */ - object DisableIsTargetLegacyVerifiedPredicate - extends FSParam[Boolean]( - name = "product_launch_disable_is_target_legacy_verified_predicate", - default = false - ) - - /** - * Param to enable checking isTargetSuperFollowCreator - */ - object EnableIsTargetSuperFollowCreatorPredicate - extends FSParam[Boolean]( - name = "product_launch_is_target_super_follow_creator_predicate_enabled", - default = false - ) - - /** - * Param to enable Spammy Tweet filter - */ - object EnableSpammyTweetFilter - extends FSParam[Boolean]( - name = "health_signal_store_enable_spammy_tweet_filter", - default = false) - - /** - * Param to enable Push to Home Android - */ - object EnableTweetPushToHomeAndroid - extends FSParam[Boolean](name = "push_to_home_tweet_recs_android", default = false) - - /** - * Param to enable Push to Home iOS - */ - object EnableTweetPushToHomeiOS - extends FSParam[Boolean](name = "push_to_home_tweet_recs_iOS", default = false) - - /** - * Param to set Spammy Tweet score threshold for OON candidates - */ - object SpammyTweetOonThreshold - extends FSBoundedParam[Double]( - name = "health_signal_store_spammy_tweet_oon_threshold", - default = 1.1, - min = 0.0, - max = 1.1 - ) - - object NumFollowerThresholdForHealthAndQualityFilters - extends FSBoundedParam[Double]( - name = "health_signal_store_num_follower_threshold_for_health_and_quality_filters", - default = 10000000000.0, - min = 0.0, - max = 10000000000.0 - ) - - object NumFollowerThresholdForHealthAndQualityFiltersPreranking - extends FSBoundedParam[Double]( - name = - "health_signal_store_num_follower_threshold_for_health_and_quality_filters_preranking", - default = 10000000.0, - min = 0.0, - max = 10000000000.0 - ) - - /** - * Param to set Spammy Tweet score threshold for IN candidates - */ - object SpammyTweetInThreshold - extends FSBoundedParam[Double]( - name = "health_signal_store_spammy_tweet_in_threshold", - default = 1.1, - min = 0.0, - max = 1.1 - ) - - /** - * Param to control bucketing for the Spammy Tweet score - */ - object SpammyTweetBucketingThreshold - extends FSBoundedParam[Double]( - name = "health_signal_store_spammy_tweet_bucketing_threshold", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - /** - * Param to specify the maximum number of Explore Video Tweets to request - */ - object MaxExploreVideoTweets - extends FSBoundedParam[Int]( - name = "explore_video_tweets_max_candidates", - default = 100, - min = 0, - max = 500 - ) - - /** - * Param to enable social context feature set - */ - object EnableBoundedFeatureSetForSocialContext - extends FSParam[Boolean]( - name = "feature_hydration_user_social_context_bounded_feature_set_enable", - default = true) - - /** - * Param to enable stp user social context feature set - */ - object EnableStpBoundedFeatureSetForUserSocialContext - extends FSParam[Boolean]( - name = "feature_hydration_stp_social_context_bounded_feature_set_enable", - default = true) - - /** - * Param to enable core user history social context feature set - */ - object EnableCoreUserHistoryBoundedFeatureSetForSocialContext - extends FSParam[Boolean]( - name = "feature_hydration_core_user_history_social_context_bounded_feature_set_enable", - default = true) - - /** - * Param to enable skipping post-ranking filters - */ - object SkipPostRankingFilters - extends FSParam[Boolean]( - name = "frigate_push_modeling_skip_post_ranking_filters", - default = false) - - object MagicFanoutSimClusterDotProductNonHeavyUserThreshold - extends FSBoundedParam[Double]( - name = "frigate_push_magicfanout_simcluster_non_heavy_user_dot_product_threshold", - default = 0.0, - min = 0.0, - max = 100.0 - ) - - object MagicFanoutSimClusterDotProductHeavyUserThreshold - extends FSBoundedParam[Double]( - name = "frigate_push_magicfanout_simcluster_heavy_user_dot_product_threshold", - default = 10.0, - min = 0.0, - max = 100.0 - ) - - object EnableReducedFatigueRulesForSeeLessOften - extends FSParam[Boolean]( - name = "seelessoften_enable_reduced_fatigue", - default = false - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushFeatureSwitches.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushFeatureSwitches.docx new file mode 100644 index 000000000..e4833200d Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushFeatureSwitches.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushFeatureSwitches.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushFeatureSwitches.scala deleted file mode 100644 index 96167c134..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushFeatureSwitches.scala +++ /dev/null @@ -1,751 +0,0 @@ -package com.twitter.frigate.pushservice.params - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.{FeatureSwitchParams => Common} -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => Pushservice} -import com.twitter.logging.Logger -import com.twitter.servo.decider.DeciderGateBuilder -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.decider.DeciderUtils - -case class PushFeatureSwitches( - deciderGateBuilder: DeciderGateBuilder, - statsReceiver: StatsReceiver) { - - private[this] val logger = Logger(classOf[PushFeatureSwitches]) - private[this] val stat = statsReceiver.scope("PushFeatureSwitches") - - private val booleanDeciderOverrides = DeciderUtils.getBooleanDeciderOverrides( - deciderGateBuilder, - PushParams.DisableAllRelevanceParam, - PushParams.DisableHeavyRankingParam, - PushParams.RestrictLightRankingParam, - PushParams.UTEGTweetCandidateSourceParam, - PushParams.EnableWritesToNotificationServiceParam, - PushParams.EnableWritesToNotificationServiceForAllEmployeesParam, - PushParams.EnableWritesToNotificationServiceForEveryoneParam, - PushParams.EnablePromptFeedbackFatigueResponseNoPredicate, - PushParams.EarlyBirdSCBasedCandidatesParam, - PushParams.MRTweetFavRecsParam, - PushParams.MRTweetRetweetRecsParam, - PushParams.EnablePushSendEventBus, - PushParams.DisableMlInFilteringParam, - PushParams.DownSampleLightRankingScribeCandidatesParam, - PushParams.EnableMrRequestScribing, - PushParams.EnableHighQualityCandidateScoresScribing, - PushParams.EnablePnegMultimodalPredictionForF1Tweets, - PushParams.EnableScribeOonFavScoreForF1Tweets, - PushParams.EnableMrUserSemanticCoreFeaturesHydration, - PushParams.EnableMrUserSemanticCoreNoZeroFeaturesHydration, - PushParams.EnableHtlOfflineUserAggregatesExtendedHydration, - PushParams.EnableNerErgFeatureHydration, - PushParams.EnableDaysSinceRecentResurrectionFeatureHydration, - PushParams.EnableUserPastAggregatesFeatureHydration, - PushParams.EnableMrUserSimclusterV2020FeaturesHydration, - PushParams.EnableMrUserSimclusterV2020NoZeroFeaturesHydration, - PushParams.EnableTopicEngagementRealTimeAggregatesFeatureHydration, - PushParams.EnableUserTopicAggregatesFeatureHydration, - PushParams.EnableHtlUserAuthorRTAFeaturesFromFeatureStoreHydration, - PushParams.EnableDurationSinceLastVisitFeatures, - PushParams.EnableTweetAnnotationFeaturesHydration, - PushParams.EnableSpaceVisibilityLibraryFiltering, - PushParams.EnableUserTopicFollowFeatureSetHydration, - PushParams.EnableOnboardingNewUserFeatureSetHydration, - PushParams.EnableMrUserAuthorSparseContFeatureSetHydration, - PushParams.EnableMrUserTopicSparseContFeatureSetHydration, - PushParams.EnableUserPenguinLanguageFeatureSetHydration, - PushParams.EnableMrUserHashspaceEmbeddingFeatureHydration, - PushParams.EnableMrUserEngagedTweetTokensFeatureHydration, - PushParams.EnableMrCandidateTweetTokensFeatureHydration, - PushParams.EnableMrTweetSentimentFeatureHydration, - PushParams.EnableMrTweetAuthorAggregatesFeatureHydration, - PushParams.EnableUserGeoFeatureSetHydration, - PushParams.EnableAuthorGeoFeatureSetHydration, - PushParams.EnableTwHINUserEngagementFeaturesHydration, - PushParams.EnableTwHINUserFollowFeaturesHydration, - PushParams.EnableTwHINAuthorFollowFeaturesHydration, - PushParams.EnableAuthorFollowTwhinEmbeddingFeatureHydration, - PushParams.RampupUserGeoFeatureSetHydration, - PushParams.RampupAuthorGeoFeatureSetHydration, - PushParams.EnablePredicateDetailedInfoScribing, - PushParams.EnablePushCapInfoScribing, - PushParams.EnableUserSignalLanguageFeatureHydration, - PushParams.EnableUserPreferredLanguageFeatureHydration, - PushParams.PopGeoCandidatesDecider, - PushParams.TrendsCandidateDecider, - PushParams.EnableInsTrafficDecider, - PushParams.EnableModelBasedPushcapAssignments, - PushParams.TripGeoTweetCandidatesDecider, - PushParams.ContentRecommenderMixerAdaptorDecider, - PushParams.GenericCandidateAdaptorDecider, - PushParams.TripGeoTweetContentMixerDarkTrafficDecider, - PushParams.EnableIsTweetTranslatableCheck, - PushParams.EnableMrTweetSimClusterFeatureHydration, - PushParams.EnableTwistlyAggregatesFeatureHydration, - PushParams.EnableTweetTwHINFavFeatureHydration, - PushParams.EnableRealGraphV2FeatureHydration, - PushParams.EnableTweetBeTFeatureHydration, - PushParams.EnableMrOfflineUserTweetTopicAggregateHydration, - PushParams.EnableMrOfflineUserTweetSimClusterAggregateHydration, - PushParams.EnableUserSendTimeFeatureHydration, - PushParams.EnableMrUserUtcSendTimeAggregateFeaturesHydration, - PushParams.EnableMrUserLocalSendTimeAggregateFeaturesHydration, - PushParams.EnableBqmlReportModelPredictionForF1Tweets, - PushParams.EnableUserTwhinEmbeddingFeatureHydration, - PushParams.EnableScribingMLFeaturesAsDataRecord, - PushParams.EnableAuthorVerifiedFeatureHydration, - PushParams.EnableAuthorCreatorSubscriptionFeatureHydration, - PushParams.EnableDirectHydrationForUserFeatures - ) - - private val intFeatureSwitchOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( - Pushservice.SportsMaxNumberOfPushesInIntervalPerEvent, - Pushservice.SportsMaxNumberOfPushesInInterval, - Pushservice.PushMixerMaxResults, - Pushservice.MaxTrendTweetNotificationsInDuration, - Pushservice.MaxRecommendedTrendsToQuery, - Pushservice.NumberOfMaxEarlybirdInNetworkCandidatesParam, - Pushservice.NumberOfMaxCandidatesToBatchInRFPHTakeStep, - Pushservice.MaxMrPushSends24HoursParam, - Pushservice.MaxMrPushSends24HoursNtabOnlyUsersParam, - Pushservice.NumberOfMaxCrMixerCandidatesParam, - Pushservice.RestrictStepSize, - Pushservice.MagicFanoutRankErgThresholdHeavy, - Pushservice.MagicFanoutRankErgThresholdNonHeavy, - Pushservice.MagicFanoutRelaxedEventIdFatigueIntervalInHours, - Pushservice.NumberOfMaxUTEGCandidatesQueriedParam, - Pushservice.HTLVisitFatigueTime, - Pushservice.MaxOnboardingPushInInterval, - Pushservice.MaxTopTweetsByGeoPushGivenInterval, - Pushservice.MaxHighQualityTweetsPushGivenInterval, - Pushservice.MaxTopTweetsByGeoCandidatesToTake, - Pushservice.SpaceRecsRealgraphThreshold, - Pushservice.SpaceRecsGlobalPushLimit, - Pushservice.OptoutExptPushCapParam, - Pushservice.MaxTopTweetImpressionsNotifications, - Pushservice.TopTweetImpressionsMinRequired, - Pushservice.TopTweetImpressionsThreshold, - Pushservice.TopTweetImpressionsOriginalTweetsNumDaysSearch, - Pushservice.TopTweetImpressionsMinNumOriginalTweets, - Pushservice.TopTweetImpressionsMaxFavoritesPerTweet, - Pushservice.TopTweetImpressionsTotalInboundFavoritesLimit, - Pushservice.TopTweetImpressionsTotalFavoritesLimitNumDaysSearch, - Pushservice.TopTweetImpressionsRecentTweetsByAuthorStoreMaxResults, - Pushservice.ANNEfQuery, - Pushservice.NumberOfMaxMrModelingBasedCandidates, - Pushservice.ThresholdOfFavMrModelingBasedCandidates, - Pushservice.LightRankingNumberOfCandidatesParam, - Pushservice.NumberOfDeTopicTweetCandidates, - Pushservice.NumberOfMaxDeTopicTweetCandidatesReturned, - Pushservice.OverrideNotificationsMaxNumOfSlots, - Pushservice.OverrideNotificationsMaxCountForNTab, - Pushservice.MFMaxNumberOfPushesInInterval, - Pushservice.SpacesTopKSimClusterCount, - Pushservice.SpaceRecsSimClusterUserMinimumFollowerCount, - Pushservice.OONSpaceRecsPushLimit, - Pushservice.MagicFanoutRealgraphRankThreshold, - Pushservice.CustomizedPushCapOffset, - Pushservice.NumberOfF1CandidatesThresholdForOONBackfill, - Pushservice.MinimumAllowedAuthorAccountAgeInHours, - Pushservice.RestrictedMinModelPushcap, - Pushservice.ListRecommendationsGeoHashLength, - Pushservice.ListRecommendationsSubscriberCount, - Pushservice.MaxListRecommendationsPushGivenInterval, - Pushservice.SendTimeByUserHistoryMaxOpenedThreshold, - Pushservice.SendTimeByUserHistoryNoSendsHours, - Pushservice.SendTimeByUserHistoryQuickSendBeforeHours, - Pushservice.SendTimeByUserHistoryQuickSendAfterHours, - Pushservice.SendTimeByUserHistoryQuickSendMinDurationInMinute, - Pushservice.SendTimeByUserHistoryNoSendMinDuration, - Pushservice.F1EmojiCopyNumOfPushesFatigue, - Pushservice.OonEmojiCopyNumOfPushesFatigue, - Pushservice.TripTweetMaxTotalCandidates, - Pushservice.InlineFeedbackSubstitutePosition, - Pushservice.HighQualityCandidatesNumberOfCandidates, - Pushservice.HighQualityCandidatesMinNumOfCandidatesToFallback, - Pushservice.ProductLaunchMaxNumberOfPushesInInterval, - Pushservice.CreatorSubscriptionPushMaxNumberOfPushesInInterval, - Pushservice.NewCreatorPushMaxNumberOfPushesInInterval, - Pushservice.TweetReplytoLikeRatioReplyCountThreshold, - Pushservice.MaxExploreVideoTweets, - ) - - private val doubleFeatureSwitchOverrides = - FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides( - Pushservice.PercentileThresholdCohort1, - Pushservice.PercentileThresholdCohort2, - Pushservice.PercentileThresholdCohort3, - Pushservice.PercentileThresholdCohort4, - Pushservice.PercentileThresholdCohort5, - Pushservice.PercentileThresholdCohort6, - Pushservice.PnsfwTweetTextThreshold, - Pushservice.PnsfwTweetTextBucketingThreshold, - Pushservice.PnsfwTweetMediaThreshold, - Pushservice.PnsfwTweetImageThreshold, - Pushservice.PnsfwQuoteTweetThreshold, - Pushservice.PnsfwTweetMediaBucketingThreshold, - Pushservice.AgathaCalibratedNSFWThreshold, - Pushservice.AgathaCalibratedNSFWThresholdForMrTwistly, - Pushservice.AgathaTextNSFWThreshold, - Pushservice.AgathaTextNSFWThresholdForMrTwistly, - Pushservice.AgathaCalibratedNSFWBucketThreshold, - Pushservice.AgathaTextNSFWBucketThreshold, - Pushservice.BucketOptoutThresholdParam, - Pushservice.TweetMediaSensitiveCategoryThresholdParam, - Pushservice.CandidateGenerationModelCosineThreshold, - Pushservice.MrModelingBasedCandidatesTopicScoreThreshold, - Pushservice.HashspaceCandidatesTopicScoreThreshold, - Pushservice.FrsTweetCandidatesTopicScoreThreshold, - Pushservice.TopicProofTweetCandidatesTopicScoreThreshold, - Pushservice.SpacesTargetingSimClusterDotProductThreshold, - Pushservice.SautOonWithMediaTweetLengthThresholdParam, - Pushservice.NonSautOonWithMediaTweetLengthThresholdParam, - Pushservice.SautOonWithoutMediaTweetLengthThresholdParam, - Pushservice.NonSautOonWithoutMediaTweetLengthThresholdParam, - Pushservice.ArgfOonWithMediaTweetWordLengthThresholdParam, - Pushservice.EsfthOonWithMediaTweetWordLengthThresholdParam, - Pushservice.BqmlQualityModelPredicateThresholdParam, - Pushservice.LightRankingScribeCandidatesDownSamplingParam, - Pushservice.QualityUprankingBoostForHeavyRankingParam, - Pushservice.QualityUprankingSigmoidBiasForHeavyRankingParam, - Pushservice.QualityUprankingSigmoidWeightForHeavyRankingParam, - Pushservice.QualityUprankingLinearBarForHeavyRankingParam, - Pushservice.QualityUprankingBoostForHighQualityProducersParam, - Pushservice.QualityUprankingDownboostForLowQualityProducersParam, - Pushservice.BqmlHealthModelPredicateFilterThresholdParam, - Pushservice.BqmlHealthModelPredicateBucketThresholdParam, - Pushservice.PNegMultimodalPredicateModelThresholdParam, - Pushservice.PNegMultimodalPredicateBucketThresholdParam, - Pushservice.SeeLessOftenF1TriggerF1PushCapWeight, - Pushservice.SeeLessOftenF1TriggerNonF1PushCapWeight, - Pushservice.SeeLessOftenNonF1TriggerF1PushCapWeight, - Pushservice.SeeLessOftenNonF1TriggerNonF1PushCapWeight, - Pushservice.SeeLessOftenTripHqTweetTriggerF1PushCapWeight, - Pushservice.SeeLessOftenTripHqTweetTriggerNonF1PushCapWeight, - Pushservice.SeeLessOftenTripHqTweetTriggerTripHqTweetPushCapWeight, - Pushservice.SeeLessOftenNtabOnlyNotifUserPushCapWeight, - Pushservice.PromptFeedbackF1TriggerF1PushCapWeight, - Pushservice.PromptFeedbackF1TriggerNonF1PushCapWeight, - Pushservice.PromptFeedbackNonF1TriggerF1PushCapWeight, - Pushservice.PromptFeedbackNonF1TriggerNonF1PushCapWeight, - Pushservice.InlineFeedbackF1TriggerF1PushCapWeight, - Pushservice.InlineFeedbackF1TriggerNonF1PushCapWeight, - Pushservice.InlineFeedbackNonF1TriggerF1PushCapWeight, - Pushservice.InlineFeedbackNonF1TriggerNonF1PushCapWeight, - Pushservice.TweetNtabDislikeCountThresholdParam, - Pushservice.TweetNtabDislikeRateThresholdParam, - Pushservice.TweetNtabDislikeCountThresholdForMrTwistlyParam, - Pushservice.TweetNtabDislikeRateThresholdForMrTwistlyParam, - Pushservice.TweetNtabDislikeCountBucketThresholdParam, - Pushservice.MinAuthorSendsThresholdParam, - Pushservice.MinTweetSendsThresholdParam, - Pushservice.AuthorDislikeRateThresholdParam, - Pushservice.AuthorReportRateThresholdParam, - Pushservice.FavOverSendThresholdParam, - Pushservice.SpreadControlRatioParam, - Pushservice.TweetQTtoNtabClickRatioThresholdParam, - Pushservice.TweetReplytoLikeRatioThresholdLowerBound, - Pushservice.TweetReplytoLikeRatioThresholdUpperBound, - Pushservice.AuthorSensitiveMediaFilteringThreshold, - Pushservice.AuthorSensitiveMediaFilteringThresholdForMrTwistly, - Pushservice.MrRequestScribingEpsGreedyExplorationRatio, - Pushservice.SeeLessOftenTopicTriggerTopicPushCapWeight, - Pushservice.SeeLessOftenTopicTriggerF1PushCapWeight, - Pushservice.SeeLessOftenTopicTriggerOONPushCapWeight, - Pushservice.SeeLessOftenF1TriggerTopicPushCapWeight, - Pushservice.SeeLessOftenOONTriggerTopicPushCapWeight, - Pushservice.SeeLessOftenDefaultPushCapWeight, - Pushservice.OverrideMaxSlotFnWeight, - Pushservice.QualityPredicateExplicitThresholdParam, - Pushservice.AuthorSensitiveScoreWeightInReranking, - Pushservice.BigFilteringThresholdParam, - Pushservice.NsfwScoreThresholdForF1Copy, - Pushservice.NsfwScoreThresholdForOONCopy, - Pushservice.HighOONCThresholdForCopy, - Pushservice.LowOONCThresholdForCopy, - Pushservice.UserDeviceLanguageThresholdParam, - Pushservice.UserInferredLanguageThresholdParam, - Pushservice.SpammyTweetOonThreshold, - Pushservice.SpammyTweetInThreshold, - Pushservice.SpammyTweetBucketingThreshold, - Pushservice.NumFollowerThresholdForHealthAndQualityFilters, - Pushservice.NumFollowerThresholdForHealthAndQualityFiltersPreranking, - Pushservice.SoftRankFactorForSubscriptionCreators, - Pushservice.MagicFanoutSimClusterDotProductHeavyUserThreshold, - Pushservice.MagicFanoutSimClusterDotProductNonHeavyUserThreshold - ) - - private val doubleSeqFeatureSwitchOverrides = - FeatureSwitchOverrideUtil.getDoubleSeqFSOverrides( - Pushservice.MfGridSearchThresholdsCohort1, - Pushservice.MfGridSearchThresholdsCohort2, - Pushservice.MfGridSearchThresholdsCohort3, - Pushservice.MfGridSearchThresholdsCohort4, - Pushservice.MfGridSearchThresholdsCohort5, - Pushservice.MfGridSearchThresholdsCohort6, - Pushservice.MrPercentileGridSearchThresholdsCohort1, - Pushservice.MrPercentileGridSearchThresholdsCohort2, - Pushservice.MrPercentileGridSearchThresholdsCohort3, - Pushservice.MrPercentileGridSearchThresholdsCohort4, - Pushservice.MrPercentileGridSearchThresholdsCohort5, - Pushservice.MrPercentileGridSearchThresholdsCohort6, - Pushservice.GlobalOptoutThresholdParam, - Pushservice.BucketOptoutSlotThresholdParam, - Pushservice.BqmlQualityModelBucketThresholdListParam, - Pushservice.SeeLessOftenListOfDayKnobs, - Pushservice.SeeLessOftenListOfPushCapWeightKnobs, - Pushservice.SeeLessOftenListOfPowerKnobs, - Pushservice.PromptFeedbackListOfDayKnobs, - Pushservice.PromptFeedbackListOfPushCapWeightKnobs, - Pushservice.PromptFeedbackListOfPowerKnobs, - Pushservice.InlineFeedbackListOfDayKnobs, - Pushservice.InlineFeedbackListOfPushCapWeightKnobs, - Pushservice.InlineFeedbackListOfPowerKnobs, - Pushservice.OverrideMaxSlotFnPushCapKnobs, - Pushservice.OverrideMaxSlotFnPowerKnobs, - Pushservice.OverrideMaxSlotFnPushCapKnobs, - Pushservice.MagicRecsRelevanceScoreRange, - Pushservice.MagicFanoutRelevanceScoreRange, - Pushservice.MultilingualPnsfwTweetTextBucketingThreshold, - Pushservice.MultilingualPnsfwTweetTextFilteringThreshold, - ) - - private val booleanFeatureSwitchOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - Pushservice.EnablePushRecommendationsParam, - Pushservice.DisableHeavyRankingModelFSParam, - Pushservice.EnablePushMixerReplacingAllSources, - Pushservice.EnablePushMixerReplacingAllSourcesWithControl, - Pushservice.EnablePushMixerReplacingAllSourcesWithExtra, - Pushservice.EnablePushMixerSource, - Common.EnableScheduledSpaceSpeakers, - Common.EnableScheduledSpaceSubscribers, - Pushservice.MagicFanoutNewsUserGeneratedEventsEnable, - Pushservice.MagicFanoutSkipAccountCountryPredicate, - Pushservice.MagicFanoutNewsEnableDescriptionCopy, - Pushservice.EnableF1TriggerSeeLessOftenFatigue, - Pushservice.EnableNonF1TriggerSeeLessOftenFatigue, - Pushservice.AdjustTripHqTweetTriggeredNtabCaretClickFatigue, - Pushservice.EnableCuratedTrendTweets, - Pushservice.EnableNonCuratedTrendTweets, - Pushservice.DisableMlInFilteringFeatureSwitchParam, - Pushservice.EnableTopicCopyForMF, - Pushservice.EnableTopicCopyForImplicitTopics, - Pushservice.EnableRestrictStep, - Pushservice.EnableHighPriorityPush, - Pushservice.BoostCandidatesFromSubscriptionCreators, - Pushservice.SoftRankCandidatesFromSubscriptionCreators, - Pushservice.EnableNewMROONCopyForPush, - Pushservice.EnableQueryAuthorMediaRepresentationStore, - Pushservice.EnableProfanityFilterParam, - Pushservice.EnableAbuseStrikeTop2PercentFilterSimCluster, - Pushservice.EnableAbuseStrikeTop1PercentFilterSimCluster, - Pushservice.EnableAbuseStrikeTop05PercentFilterSimCluster, - Pushservice.EnableAgathaUserHealthModelPredicate, - Pushservice.PnsfwTweetMediaFilterOonOnly, - Pushservice.EnableHealthSignalStorePnsfwTweetTextPredicate, - Pushservice.EnableHealthSignalStoreMultilingualPnsfwTweetTextPredicate, - Pushservice.DisableHealthFiltersForCrMixerCandidates, - Pushservice.EnableOverrideNotificationsForAndroid, - Pushservice.EnableOverrideNotificationsForIos, - Pushservice.EnableMrRequestScribingForTargetFiltering, - Pushservice.EnableMrRequestScribingForCandidateFiltering, - Pushservice.EnableMrRequestScribingWithFeatureHydrating, - Pushservice.EnableFlattenMrRequestScribing, - Pushservice.EnableMrRequestScribingForEpsGreedyExploration, - Pushservice.EnableMrRequestScribingDismissScore, - Pushservice.EnableMrRequestScribingBigFilteringSupervisedScores, - Pushservice.EnableMrRequestScribingBigFilteringRLScores, - Pushservice.EnableEventPrimaryMediaAndroid, - Pushservice.EnableEventSquareMediaIosMagicFanoutNewsEvent, - Pushservice.EnableEventSquareMediaAndroid, - Pushservice.EnableMagicFanoutNewsForYouNtabCopy, - Pushservice.EnableMfGeoTargeting, - Pushservice.EnableRuxLandingPage, - Pushservice.EnableNTabRuxLandingPage, - Pushservice.EnableGraduallyRampUpNotification, - Pushservice.EnableOnboardingPushes, - Pushservice.EnableAddressBookPush, - Pushservice.EnableCompleteOnboardingPush, - Pushservice.EnableOverrideNotificationsSmartPushConfigForAndroid, - Pushservice.DisableOnboardingPushFatigue, - Pushservice.EnableTopTweetsByGeoCandidates, - Pushservice.BackfillRankTopTweetsByGeoCandidates, - Pushservice.PopGeoTweetEnableAggressiveThresholds, - Pushservice.EnableMrMinDurationSinceMrPushFatigue, - Pushservice.EnableF1FromProtectedTweetAuthors, - Pushservice.MagicFanoutEnableCustomTargetingNewsEvent, - Pushservice.EnableSafeUserTweetTweetypieStore, - Pushservice.EnableMrMinDurationSinceMrPushFatigue, - Pushservice.EnableHydratingOnlineMRHistoryFeatures, - Common.SpaceRecsEnableHostNotifs, - Common.SpaceRecsEnableSpeakerNotifs, - Common.SpaceRecsEnableListenerNotifs, - Common.EnableMagicFanoutProductLaunch, - Pushservice.EnableTopTweetsByGeoCandidatesForDormantUsers, - Pushservice.EnableOverrideNotificationsScoreBasedOverride, - Pushservice.EnableOverrideNotificationsMultipleTargetIds, - Pushservice.EnableMinDurationModifier, - Pushservice.EnableMinDurationModifierV2, - Pushservice.EnableMinDurationModifierByUserHistory, - Pushservice.EnableQueryUserOpenedHistory, - Pushservice.EnableRandomHourForQuickSend, - Pushservice.EnableFrsCandidates, - Pushservice.EnableFrsTweetCandidatesTopicSetting, - Pushservice.EnableFrsTweetCandidatesTopicAnnotation, - Pushservice.EnableFrsTweetCandidatesTopicCopy, - Pushservice.EnableCandidateGenerationModelParam, - Pushservice.EnableOverrideForSportsCandidates, - Pushservice.EnableEventIdBasedOverrideForSportsCandidates, - Pushservice.EnableMrModelingBasedCandidates, - Pushservice.EnableMrModelingBasedCandidatesTopicSetting, - Pushservice.EnableMrModelingBasedCandidatesTopicAnnotation, - Pushservice.EnableMrModelingBasedCandidatesTopicCopy, - Pushservice.EnableResultFromFrsCandidates, - Pushservice.EnableHashspaceCandidates, - Pushservice.EnableHashspaceCandidatesTopicSetting, - Pushservice.EnableHashspaceCandidatesTopicAnnotation, - Pushservice.EnableHashspaceCandidatesTopicCopy, - Pushservice.EnableResultFromHashspaceCandidates, - Pushservice.EnableDownRankOfNewUserPlaybookTopicFollowPush, - Pushservice.EnableDownRankOfNewUserPlaybookTopicTweetPush, - Pushservice.EnableTopTweetImpressionsNotification, - Pushservice.EnableLightRankingParam, - Pushservice.EnableRandomBaselineLightRankingParam, - Pushservice.EnableQualityUprankingForHeavyRankingParam, - Pushservice.EnableQualityUprankingCrtScoreStatsForHeavyRankingParam, - Pushservice.EnableProducersQualityBoostingForHeavyRankingParam, - Pushservice.EnableMrScribingMLFeaturesAsFeatureMapForStaging, - Pushservice.EnableMrTweetSentimentFeatureHydrationFS, - Pushservice.EnableTimelineHealthSignalHydration, - Pushservice.EnableTopicEngagementRealTimeAggregatesFS, - Pushservice.EnableMrUserSemanticCoreFeatureForExpt, - Pushservice.EnableHydratingRealGraphTargetUserFeatures, - Pushservice.EnableHydratingUserDurationSinceLastVisitFeatures, - Pushservice.EnableRealGraphUserAuthorAndSocialContxtFeatureHydration, - Pushservice.EnableUserTopicAggregatesFS, - Pushservice.EnableTimelineHealthSignalHydrationForModelTraining, - Pushservice.EnableMrUserSocialContextAggregateFeatureHydration, - Pushservice.EnableMrUserSemanticCoreAggregateFeatureHydration, - Pushservice.EnableMrUserCandidateSparseOfflineAggregateFeatureHydration, - Pushservice.EnableMrUserCandidateOfflineAggregateFeatureHydration, - Pushservice.EnableMrUserCandidateOfflineCompactAggregateFeatureHydration, - Pushservice.EnableMrUserAuthorOfflineAggregateFeatureHydration, - Pushservice.EnableMrUserAuthorOfflineCompactAggregateFeatureHydration, - Pushservice.EnableMrUserOfflineCompactAggregateFeatureHydration, - Pushservice.EnableMrUserSimcluster2020AggregateFeatureHydration, - Pushservice.EnableMrUserOfflineAggregateFeatureHydration, - Pushservice.EnableBqmlQualityModelPredicateParam, - Pushservice.EnableBqmlQualityModelScoreHistogramParam, - Pushservice.EnableBqmlHealthModelPredicateParam, - Pushservice.EnableBqmlHealthModelPredictionForInNetworkCandidatesParam, - Pushservice.EnableBqmlHealthModelScoreHistogramParam, - Pushservice.EnablePNegMultimodalPredicateParam, - Pushservice.EnableNegativeKeywordsPredicateParam, - Pushservice.EnableTweetAuthorAggregatesFeatureHydrationParam, - Pushservice.OonTweetLengthPredicateUpdatedMediaLogic, - Pushservice.OonTweetLengthPredicateUpdatedQuoteTweetLogic, - Pushservice.OonTweetLengthPredicateMoreStrictForUndefinedLanguages, - Pushservice.EnablePrerankingTweetLengthPredicate, - Pushservice.EnableDeTopicTweetCandidates, - Pushservice.EnableDeTopicTweetCandidateResults, - Pushservice.EnableDeTopicTweetCandidatesCustomTopics, - Pushservice.EnableDeTopicTweetCandidatesCustomLanguages, - Pushservice.EnableMrTweetSimClusterFeatureHydrationFS, - Pushservice.DisableOutNetworkTweetCandidatesFS, - Pushservice.EnableLaunchVideosInImmersiveExplore, - Pushservice.EnableStoringNtabGenericNotifKey, - Pushservice.EnableDeletingNtabTimeline, - Pushservice.EnableOverrideNotificationsNSlots, - Pushservice.EnableNslotsForOverrideOnNtab, - Pushservice.EnableOverrideMaxSlotFn, - Pushservice.EnableTargetIdInSmartPushPayloadForMagicFanoutSportsEvent, - Pushservice.EnableOverrideIdNTabRequest, - Pushservice.EnableOverrideForSpaces, - Pushservice.EnableTopicProofTweetRecs, - Pushservice.EnableHealthFiltersForTopicProofTweet, - Pushservice.EnableTargetIdsInSmartPushPayload, - Pushservice.EnableSecondaryAccountPredicateMF, - Pushservice.EnableInlineVideo, - Pushservice.EnableAutoplayForInlineVideo, - Pushservice.EnableOONGeneratedInlineActions, - Pushservice.EnableInlineFeedbackOnPush, - Pushservice.UseInlineActionsV1, - Pushservice.UseInlineActionsV2, - Pushservice.EnableFeaturedSpacesOON, - Pushservice.CheckFeaturedSpaceOON, - Pushservice.EnableGeoTargetingForSpaces, - Pushservice.EnableEmployeeOnlySpaceNotifications, - Pushservice.EnableSpacesTtlForNtab, - Pushservice.EnableCustomThreadIdForOverride, - Pushservice.EnableSimClusterTargetingSpaces, - Pushservice.TargetInInlineActionAppVisitFatigue, - Pushservice.EnableInlineActionAppVisitFatigue, - Pushservice.EnableThresholdOfFavMrModelingBasedCandidates, - Pushservice.HydrateMrUserSimclusterV2020InModelingBasedCG, - Pushservice.HydrateMrUserSemanticCoreInModelingBasedCG, - Pushservice.HydrateOnboardingInModelingBasedCG, - Pushservice.HydrateTopicFollowInModelingBasedCG, - Pushservice.HydrateMrUserTopicInModelingBasedCG, - Pushservice.HydrateMrUserAuthorInModelingBasedCG, - Pushservice.HydrateUserPenguinLanguageInModelingBasedCG, - Pushservice.EnableMrUserEngagedTweetTokensFeature, - Pushservice.HydrateMrUserHashspaceEmbeddingInModelingBasedCG, - Pushservice.HydrateUseGeoInModelingBasedCG, - Pushservice.EnableSpaceCohostJoinEvent, - Pushservice.EnableOONFilteringBasedOnUserSettings, - Pushservice.EnableContFnF1TriggerSeeLessOftenFatigue, - Pushservice.EnableContFnNonF1TriggerSeeLessOftenFatigue, - Pushservice.EnableContFnF1TriggerPromptFeedbackFatigue, - Pushservice.EnableContFnNonF1TriggerPromptFeedbackFatigue, - Pushservice.EnableContFnF1TriggerInlineFeedbackFatigue, - Pushservice.EnableContFnNonF1TriggerInlineFeedbackFatigue, - Pushservice.UseInlineDislikeForFatigue, - Pushservice.UseInlineDismissForFatigue, - Pushservice.UseInlineSeeLessForFatigue, - Pushservice.UseInlineNotRelevantForFatigue, - Pushservice.GPEnableCustomMagicFanoutCricketFatigue, - Pushservice.IncludeRelevanceScoreInIbis2Payload, - Pushservice.BypassGlobalSpacePushCapForSoftDeviceFollow, - Pushservice.EnableCountryCodeBackoffTopTweetsByGeo, - Pushservice.EnableNewCreatorPush, - Pushservice.EnableCreatorSubscriptionPush, - Pushservice.EnableInsSender, - Pushservice.EnableOptoutAdjustedPushcap, - Pushservice.EnableOONBackfillBasedOnF1Candidates, - Pushservice.EnableVFInTweetypie, - Pushservice.EnablePushPresentationVerifiedSymbol, - Pushservice.EnableHighPrioritySportsPush, - Pushservice.EnableSearchURLRedirectForSportsFanout, - Pushservice.EnableScoreFanoutNotification, - Pushservice.EnableExplicitPushCap, - Pushservice.EnableNsfwTokenBasedFiltering, - Pushservice.EnableRestrictedMinModelPushcap, - Pushservice.EnableGenericCRTBasedFatiguePredicate, - Pushservice.EnableCopyFeaturesForF1, - Pushservice.EnableEmojiInF1Copy, - Pushservice.EnableTargetInF1Copy, - Pushservice.EnableCopyFeaturesForOon, - Pushservice.EnableEmojiInOonCopy, - Pushservice.EnableTargetInOonCopy, - Pushservice.EnableF1CopyBody, - Pushservice.EnableOONCopyBody, - Pushservice.EnableIosCopyBodyTruncate, - Pushservice.EnableHTLBasedFatigueBasicRule, - Pushservice.EnableTargetAndEmojiSplitFatigue, - Pushservice.EnableNsfwCopy, - Pushservice.EnableOONCopyBody, - Pushservice.EnableTweetDynamicInlineActions, - Pushservice.EnablePushcapRefactor, - Pushservice.BigFilteringEnableHistogramsParam, - Pushservice.EnableTweetTranslation, - Pushservice.TripTweetCandidateReturnEnable, - Pushservice.EnableSocialContextForRetweet, - Pushservice.EnableEmptyBody, - Pushservice.EnableLocalViralTweets, - Pushservice.EnableExploreVideoTweets, - Pushservice.EnableDynamicInlineActionsForDesktopWeb, - Pushservice.EnableDynamicInlineActionsForMobileWeb, - Pushservice.EnableNTabEntriesForSportsEventNotifications, - Pushservice.EnableNTabFacePileForSportsEventNotifications, - Pushservice.DisableIsTargetBlueVerifiedPredicate, - Pushservice.EnableNTabEntriesForProductLaunchNotifications, - Pushservice.DisableIsTargetLegacyVerifiedPredicate, - Pushservice.EnableNTabOverrideForSportsEventNotifications, - Pushservice.EnableOONCBasedCopy, - Pushservice.HighQualityCandidatesEnableCandidateSource, - Pushservice.HighQualityCandidatesEnableFallback, - Pushservice.EnableTweetLanguageFilter, - Pushservice.EnableListRecommendations, - Pushservice.EnableIDSListRecommendations, - Pushservice.EnablePopGeoListRecommendations, - Pushservice.SkipLanguageFilterForMediaTweets, - Pushservice.EnableSpammyTweetFilter, - Pushservice.EnableTweetPushToHomeAndroid, - Pushservice.EnableTweetPushToHomeiOS, - Pushservice.EnableBoundedFeatureSetForSocialContext, - Pushservice.EnableStpBoundedFeatureSetForUserSocialContext, - Pushservice.EnableCoreUserHistoryBoundedFeatureSetForSocialContext, - Pushservice.SkipPostRankingFilters, - Pushservice.MRWebHoldbackParam, - Pushservice.EnableIsTargetSuperFollowCreatorPredicate - ) - - private val longSeqFeatureSwitchOverrides = - FeatureSwitchOverrideUtil.getLongSeqFSOverrides( - Pushservice.MagicFanoutEventAllowlistToSkipAccountCountryPredicate - ) - - private val longSetFeatureSwitchOverrides = - FeatureSwitchOverrideUtil.getLongSetFSOverrides( - Pushservice.ListOfAdhocIdsForStatsTracking - ) - - private val stringSeqFeatureSwitchOverrides = - FeatureSwitchOverrideUtil.getStringSeqFSOverrides( - Pushservice.ListOfCrtsForOpenApp, - Pushservice.ListOfCrtsToUpRank, - Pushservice.OONCandidatesDisabledCrTagParam, - Pushservice.ListOfCrtsToDownRank, - Pushservice.MagicFanoutDenyListedCountries, - Pushservice.GlobalOptoutModelParam, - Pushservice.BqmlQualityModelBucketModelIdListParam, - Pushservice.CommonRecommendationTypeDenyListPushHoldbacks, - Pushservice.TargetLevelFeatureListForMrRequestScribing, - Pushservice.MagicFanoutSportsEventDenyListedCountries, - Pushservice.MultilingualPnsfwTweetTextSupportedLanguages, - Pushservice.NegativeKeywordsPredicateDenylist, - Pushservice.TripTweetCandidateSourceIds, - Pushservice.NsfwTokensParam, - Pushservice.HighQualityCandidatesFallbackSourceIds - ) - - private val intSeqFeatureSwitchOverrides = - FeatureSwitchOverrideUtil.getIntSeqFSOverrides( - Pushservice.BucketOptoutSlotPushcapParam, - Pushservice.GeoHashLengthList, - Pushservice.MinDurationModifierStartHourList, - Pushservice.MinDurationModifierEndHourList, - Pushservice.MinDurationTimeModifierConst - ) - - private val enumFeatureSwitchOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( - stat, - logger, - Pushservice.MRBoldTitleFavoriteAndRetweetParam, - Pushservice.QualityUprankingTransformTypeParam, - Pushservice.QualityPredicateIdParam, - Pushservice.BigFilteringNormalizationTypeIdParam, - Common.PushcapModelType, - Common.MFCricketTargetingPredicate, - Pushservice.RankingFunctionForTopTweetsByGeo, - Pushservice.TopTweetsByGeoCombinationParam, - Pushservice.PopGeoTweetVersionParam, - Pushservice.SubtextInAndroidPushHeaderParam, - Pushservice.HighOONCTweetFormat, - Pushservice.LowOONCTweetFormat, - ) - - private val enumSeqFeatureSwitchOverrides = FeatureSwitchOverrideUtil.getEnumSeqFSOverrides( - stat, - logger, - Pushservice.OONTweetDynamicInlineActionsList, - Pushservice.TweetDynamicInlineActionsList, - Pushservice.TweetDynamicInlineActionsListForWeb, - Pushservice.HighQualityCandidatesEnableGroups, - Pushservice.HighQualityCandidatesFallbackEnabledGroups, - Pushservice.OONCandidatesDisabledCrtGroupParam, - Pushservice.MultilingualPnsfwTweetTextBucketingModelList, - ) - - private val stringFeatureSwitchOverrides = FeatureSwitchOverrideUtil.getStringFSOverrides( - Common.PushcapModelPredictionVersion, - Pushservice.WeightedOpenOrNtabClickRankingModelParam, - Pushservice.WeightedOpenOrNtabClickFilteringModelParam, - Pushservice.BucketOptoutModelParam, - Pushservice.ScoringFuncForTopTweetsByGeo, - Pushservice.LightRankingModelTypeParam, - Pushservice.BigFilteringSupervisedSendingModelParam, - Pushservice.BigFilteringSupervisedWithoutSendingModelParam, - Pushservice.BigFilteringRLSendingModelParam, - Pushservice.BigFilteringRLWithoutSendingModelParam, - Pushservice.BqmlQualityModelTypeParam, - Pushservice.BqmlHealthModelTypeParam, - Pushservice.QualityUprankingModelTypeParam, - Pushservice.SearchURLRedirectForSportsFanout, - Pushservice.LocalViralTweetsBucket, - Pushservice.HighQualityCandidatesHeavyRankingModel, - Pushservice.HighQualityCandidatesNonPersonalizedQualityCnnModel, - Pushservice.HighQualityCandidatesBqmlNsfwModel, - Pushservice.HighQualityCandidatesBqmlReportModel, - Pushservice.ProductLaunchLandingPageDeepLink, - Pushservice.ProductLaunchTapThrough, - Pushservice.TweetLanguageFeatureNameParam - ) - - private val durationFeatureSwitchOverrides = - FeatureSwitchOverrideUtil.getBoundedDurationFSOverrides( - Common.NumberOfDaysToFilterMRForSeeLessOften, - Common.NumberOfDaysToReducePushCapForSeeLessOften, - Pushservice.NumberOfDaysToFilterForSeeLessOftenForF1TriggerF1, - Pushservice.NumberOfDaysToReducePushCapForSeeLessOftenForF1TriggerF1, - Pushservice.NumberOfDaysToFilterForSeeLessOftenForF1TriggerNonF1, - Pushservice.NumberOfDaysToReducePushCapForSeeLessOftenForF1TriggerNonF1, - Pushservice.NumberOfDaysToFilterForSeeLessOftenForNonF1TriggerF1, - Pushservice.NumberOfDaysToReducePushCapForSeeLessOftenForNonF1TriggerF1, - Pushservice.NumberOfDaysToFilterForSeeLessOftenForNonF1TriggerNonF1, - Pushservice.NumberOfDaysToReducePushCapForSeeLessOftenForNonF1TriggerNonF1, - Pushservice.TrendTweetNotificationsFatigueDuration, - Pushservice.MinDurationSincePushParam, - Pushservice.MFMinIntervalFatigue, - Pushservice.SimclusterBasedCandidateMaxTweetAgeParam, - Pushservice.DetopicBasedCandidateMaxTweetAgeParam, - Pushservice.F1CandidateMaxTweetAgeParam, - Pushservice.MaxTweetAgeParam, - Pushservice.ModelingBasedCandidateMaxTweetAgeParam, - Pushservice.GeoPopTweetMaxAgeInHours, - Pushservice.MinDurationSincePushParam, - Pushservice.GraduallyRampUpPhaseDurationDays, - Pushservice.MrMinDurationSincePushForOnboardingPushes, - Pushservice.FatigueForOnboardingPushes, - Pushservice.FrigateHistoryOtherNotificationWriteTtl, - Pushservice.FrigateHistoryTweetNotificationWriteTtl, - Pushservice.TopTweetsByGeoPushInterval, - Pushservice.HighQualityTweetsPushInterval, - Pushservice.MrMinDurationSincePushForTopTweetsByGeoPushes, - Pushservice.TimeSinceLastLoginForGeoPopTweetPush, - Pushservice.NewUserPlaybookAllowedLastLoginHours, - Pushservice.SpaceRecsAppFatigueDuration, - Pushservice.OONSpaceRecsFatigueDuration, - Pushservice.SpaceRecsFatigueMinIntervalDuration, - Pushservice.SpaceRecsGlobalFatigueDuration, - Pushservice.MinimumTimeSinceLastLoginForGeoPopTweetPush, - Pushservice.MinFatigueDurationSinceLastHTLVisit, - Pushservice.LastHTLVisitBasedNonFatigueWindow, - Pushservice.SpaceNotificationsTTLDurationForNTab, - Pushservice.OverrideNotificationsLookbackDurationForOverrideInfo, - Pushservice.OverrideNotificationsLookbackDurationForImpressionId, - Pushservice.OverrideNotificationsLookbackDurationForNTab, - Pushservice.TopTweetImpressionsNotificationInterval, - Pushservice.TopTweetImpressionsFatigueMinIntervalDuration, - Pushservice.MFPushIntervalInHours, - Pushservice.InlineActionAppVisitFatigue, - Pushservice.SpaceParticipantHistoryLastActiveThreshold, - Pushservice.SportsMinIntervalFatigue, - Pushservice.SportsPushIntervalInHours, - Pushservice.SportsMinIntervalFatiguePerEvent, - Pushservice.SportsPushIntervalInHoursPerEvent, - Pushservice.TargetNtabOnlyCapFatigueIntervalHours, - Pushservice.TargetPushCapFatigueIntervalHours, - Pushservice.CopyFeaturesHistoryLookbackDuration, - Pushservice.F1EmojiCopyFatigueDuration, - Pushservice.F1TargetCopyFatigueDuration, - Pushservice.OonEmojiCopyFatigueDuration, - Pushservice.OonTargetCopyFatigueDuration, - Pushservice.ProductLaunchPushIntervalInHours, - Pushservice.ExploreVideoTweetAgeParam, - Pushservice.ListRecommendationsPushInterval, - Pushservice.ProductLaunchMinIntervalFatigue, - Pushservice.NewCreatorPushIntervalInHours, - Pushservice.NewCreatorPushMinIntervalFatigue, - Pushservice.CreatorSubscriptionPushIntervalInHours, - Pushservice.CreatorSubscriptionPushhMinIntervalFatigue - ) - - private[params] val allFeatureSwitchOverrides = - booleanDeciderOverrides ++ - booleanFeatureSwitchOverrides ++ - intFeatureSwitchOverrides ++ - doubleFeatureSwitchOverrides ++ - doubleSeqFeatureSwitchOverrides ++ - enumFeatureSwitchOverrides ++ - stringSeqFeatureSwitchOverrides ++ - stringFeatureSwitchOverrides ++ - durationFeatureSwitchOverrides ++ - intSeqFeatureSwitchOverrides ++ - longSeqFeatureSwitchOverrides ++ - enumSeqFeatureSwitchOverrides ++ - longSetFeatureSwitchOverrides - - val config = BaseConfigBuilder(allFeatureSwitchOverrides).build() -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushMLModelParams.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushMLModelParams.docx new file mode 100644 index 000000000..4aa7a4dbf Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushMLModelParams.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushMLModelParams.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushMLModelParams.scala deleted file mode 100644 index c451a61bc..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushMLModelParams.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.frigate.pushservice.params - -/** - * This enum defines ML models for push - */ -object PushMLModel extends Enumeration { - type PushMLModel = Value - - val WeightedOpenOrNtabClickProbability = Value - val DauProbability = Value - val OptoutProbability = Value - val FilteringProbability = Value - val BigFilteringSupervisedSendingModel = Value - val BigFilteringSupervisedWithoutSendingModel = Value - val BigFilteringRLSendingModel = Value - val BigFilteringRLWithoutSendingModel = Value - val HealthNsfwProbability = Value -} - -object WeightedOpenOrNtabClickModel { - type ModelNameType = String - - // MR models - val Periodically_Refreshed_Prod_Model = - "Periodically_Refreshed_Prod_Model" // used in DBv2 service, needed for gradually migrate via feature switch -} - - -object OptoutModel { - type ModelNameType = String - val D0_has_realtime_features = "D0_has_realtime_features" - val D0_no_realtime_features = "D0_no_realtime_features" -} - -object HealthNsfwModel { - type ModelNameType = String - val Q2_2022_Mr_Bqml_Health_Model_NsfwV0 = "Q2_2022_Mr_Bqml_Health_Model_NsfwV0" -} - -object BigFilteringSupervisedModel { - type ModelNameType = String - val V0_0_BigFiltering_Supervised_Sending_Model = "Q3_2022_bigfiltering_supervised_send_model_v0" - val V0_0_BigFiltering_Supervised_Without_Sending_Model = - "Q3_2022_bigfiltering_supervised_not_send_model_v0" -} - -object BigFilteringRLModel { - type ModelNameType = String - val V0_0_BigFiltering_Rl_Sending_Model = "Q3_2022_bigfiltering_rl_send_model_dqn_dau_15_open" - val V0_0_BigFiltering_Rl_Without_Sending_Model = - "Q3_2022_bigfiltering_rl_not_send_model_dqn_dau_15_open" -} - -case class PushModelName( - modelType: PushMLModel.Value, - version: WeightedOpenOrNtabClickModel.ModelNameType) { - override def toString: String = { - modelType.toString + "_" + version - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushParams.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushParams.docx new file mode 100644 index 000000000..957bfa90d Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushParams.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushParams.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushParams.scala deleted file mode 100644 index 5e5f6af6a..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushParams.scala +++ /dev/null @@ -1,534 +0,0 @@ -package com.twitter.frigate.pushservice.params - -import com.twitter.rux.common.context.thriftscala.ExperimentKey -import com.twitter.timelines.configapi.Param -import com.twitter.timelines.configapi.decider.BooleanDeciderParam - -object PushParams { - - /** - * Disable ML models in filtering - */ - object DisableMlInFilteringParam extends BooleanDeciderParam(DeciderKey.disableMLInFiltering) - - /** - * Disable ML models in ranking, use random ranking instead - * This param is used for ML holdback and training data collection - */ - object UseRandomRankingParam extends Param(false) - - /** - * Disable feature hydration, ML ranking, and ML filtering - * Use default order from candidate source - * This param is for service continuity - */ - object DisableAllRelevanceParam extends BooleanDeciderParam(DeciderKey.disableAllRelevance) - - /** - * Disable ML heavy ranking - * Use default order from candidate source - * This param is for service continuity - */ - object DisableHeavyRankingParam extends BooleanDeciderParam(DeciderKey.disableHeavyRanking) - - /** - * Restrict ML light ranking by selecting top3 candidates - * Use default order from candidate source - * This param is for service continuity - */ - object RestrictLightRankingParam extends BooleanDeciderParam(DeciderKey.restrictLightRanking) - - /** - * Downsample ML light ranking scribed candidates - */ - object DownSampleLightRankingScribeCandidatesParam - extends BooleanDeciderParam(DeciderKey.downSampleLightRankingScribeCandidates) - - /** - * Set it to true only for Android only ranking experiments - */ - object AndroidOnlyRankingExperimentParam extends Param(false) - - /** - * Enable the user_tweet_entity_graph tweet candidate source. - */ - object UTEGTweetCandidateSourceParam - extends BooleanDeciderParam(DeciderKey.entityGraphTweetRecsDeciderKey) - - /** - * Enable writes to Notification Service - */ - object EnableWritesToNotificationServiceParam - extends BooleanDeciderParam(DeciderKey.enablePushserviceWritesToNotificationServiceDeciderKey) - - /** - * Enable writes to Notification Service for all employees - */ - object EnableWritesToNotificationServiceForAllEmployeesParam - extends BooleanDeciderParam( - DeciderKey.enablePushserviceWritesToNotificationServiceForAllEmployeesDeciderKey) - - /** - * Enable writes to Notification Service for everyone - */ - object EnableWritesToNotificationServiceForEveryoneParam - extends BooleanDeciderParam( - DeciderKey.enablePushserviceWritesToNotificationServiceForEveryoneDeciderKey) - - /** - * Enable fatiguing MR for Ntab caret click - */ - object EnableFatigueNtabCaretClickingParam extends Param(true) - - /** - * Param for disabling in-network Tweet candidates - */ - object DisableInNetworkTweetCandidatesParam extends Param(false) - - /** - * Decider controlled param to enable prompt feedback response NO predicate - */ - object EnablePromptFeedbackFatigueResponseNoPredicate - extends BooleanDeciderParam( - DeciderKey.enablePromptFeedbackFatigueResponseNoPredicateDeciderKey) - - /** - * Enable hydration and generation of Social context (TF, TR) based candidates for Earlybird Tweets - */ - object EarlyBirdSCBasedCandidatesParam - extends BooleanDeciderParam(DeciderKey.enableUTEGSCForEarlybirdTweetsDecider) - - /** - * Param to allow reduce to one social proof for tweet param in UTEG - */ - object AllowOneSocialProofForTweetInUTEGParam extends Param(true) - - /** - * Param to query UTEG for out network tweets only - */ - object OutNetworkTweetsOnlyForUTEGParam extends Param(false) - - object EnablePushSendEventBus extends BooleanDeciderParam(DeciderKey.enablePushSendEventBus) - - /** - * Enable RUX Tweet landing page for push open on iOS - */ - object EnableRuxLandingPageIOSParam extends Param[Boolean](true) - - /** - * Enable RUX Tweet landing page for push open on Android - */ - object EnableRuxLandingPageAndroidParam extends Param[Boolean](true) - - /** - * Param to decide which ExperimentKey to be encoded into Rux landing page context object. - * The context object is sent to rux-api and rux-api applies logic (e.g. show reply module on - * rux landing page or not) accordingly based on the experiment key. - */ - object RuxLandingPageExperimentKeyIOSParam extends Param[Option[ExperimentKey]](None) - object RuxLandingPageExperimentKeyAndroidParam extends Param[Option[ExperimentKey]](None) - - /** - * Param to enable MR Tweet Fav Recs - */ - object MRTweetFavRecsParam extends BooleanDeciderParam(DeciderKey.enableTweetFavRecs) - - /** - * Param to enable MR Tweet Retweet Recs - */ - object MRTweetRetweetRecsParam extends BooleanDeciderParam(DeciderKey.enableTweetRetweetRecs) - - /** - * Param to disable writing to NTAB - * */ - object DisableWritingToNTAB extends Param[Boolean](default = false) - - /** - * Param to show RUX landing page as a modal on iOS - */ - object ShowRuxLandingPageAsModalOnIOS extends Param[Boolean](default = false) - - /** - * Param to enable mr end to end scribing - */ - object EnableMrRequestScribing extends BooleanDeciderParam(DeciderKey.enableMrRequestScribing) - - /** - * Param to enable scribing of high quality candidate scores - */ - object EnableHighQualityCandidateScoresScribing - extends BooleanDeciderParam(DeciderKey.enableHighQualityCandidateScoresScribing) - - /** - * Decider controlled param to pNeg multimodal predictions for F1 tweets - */ - object EnablePnegMultimodalPredictionForF1Tweets - extends BooleanDeciderParam(DeciderKey.enablePnegMultimodalPredictionForF1Tweets) - - /** - * Decider controlled param to scribe oonFav score for F1 tweets - */ - object EnableScribeOonFavScoreForF1Tweets - extends BooleanDeciderParam(DeciderKey.enableScribingOonFavScoreForF1Tweets) - - /** - * Param to enable htl user aggregates extended hydration - */ - object EnableHtlOfflineUserAggregatesExtendedHydration - extends BooleanDeciderParam(DeciderKey.enableHtlOfflineUserAggregateExtendedFeaturesHydration) - - /** - * Param to enable predicate detailed info scribing - */ - object EnablePredicateDetailedInfoScribing - extends BooleanDeciderParam(DeciderKey.enablePredicateDetailedInfoScribing) - - /** - * Param to enable predicate detailed info scribing - */ - object EnablePushCapInfoScribing - extends BooleanDeciderParam(DeciderKey.enablePredicateDetailedInfoScribing) - - /** - * Param to enable user signal language feature hydration - */ - object EnableUserSignalLanguageFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableUserSignalLanguageFeatureHydration) - - /** - * Param to enable user preferred language feature hydration - */ - object EnableUserPreferredLanguageFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableUserPreferredLanguageFeatureHydration) - - /** - * Param to enable ner erg feature hydration - */ - object EnableNerErgFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableNerErgFeaturesHydration) - - /** - * Param to enable inline action on push copy for Android - */ - object MRAndroidInlineActionOnPushCopyParam extends Param[Boolean](default = true) - - /** - * Param to enable hydrating mr user semantic core embedding features - * */ - object EnableMrUserSemanticCoreFeaturesHydration - extends BooleanDeciderParam(DeciderKey.enableMrUserSemanticCoreFeaturesHydration) - - /** - * Param to enable hydrating mr user semantic core embedding features filtered by 0.0000001 - * */ - object EnableMrUserSemanticCoreNoZeroFeaturesHydration - extends BooleanDeciderParam(DeciderKey.enableMrUserSemanticCoreNoZeroFeaturesHydration) - - /* - * Param to enable days since user's recent resurrection features hydration - */ - object EnableDaysSinceRecentResurrectionFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableDaysSinceRecentResurrectionFeatureHydration) - - /* - * Param to enable days since user past aggregates features hydration - */ - object EnableUserPastAggregatesFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableUserPastAggregatesFeatureHydration) - - /* - * Param to enable mr user simcluster features (v2020) hydration - * */ - object EnableMrUserSimclusterV2020FeaturesHydration - extends BooleanDeciderParam(DeciderKey.enableMrUserSimclusterV2020FeaturesHydration) - - /* - * Param to enable mr user simcluster features (v2020) hydration - * */ - object EnableMrUserSimclusterV2020NoZeroFeaturesHydration - extends BooleanDeciderParam(DeciderKey.enableMrUserSimclusterV2020NoZeroFeaturesHydration) - - /* - * Param to enable HTL topic engagement realtime aggregate features - * */ - object EnableTopicEngagementRealTimeAggregatesFeatureHydration - extends BooleanDeciderParam( - DeciderKey.enableTopicEngagementRealTimeAggregatesFeatureHydration) - - object EnableUserTopicAggregatesFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableUserTopicAggregatesFeatureHydration) - - /** - * Param to enable user author RTA feature hydration - */ - object EnableHtlUserAuthorRTAFeaturesFromFeatureStoreHydration - extends BooleanDeciderParam(DeciderKey.enableHtlUserAuthorRealTimeAggregateFeatureHydration) - - /** - * Param to enable duration since last visit features - */ - object EnableDurationSinceLastVisitFeatures - extends BooleanDeciderParam(DeciderKey.enableDurationSinceLastVisitFeatureHydration) - - object EnableTweetAnnotationFeaturesHydration - extends BooleanDeciderParam(DeciderKey.enableTweetAnnotationFeatureHydration) - - /** - * Param to Enable visibility filtering through SpaceVisibilityLibrary from SpacePredicate - */ - object EnableSpaceVisibilityLibraryFiltering - extends BooleanDeciderParam(DeciderKey.enableSpaceVisibilityLibraryFiltering) - - /* - * Param to enable user topic follow feature set hydration - * */ - object EnableUserTopicFollowFeatureSetHydration - extends BooleanDeciderParam(DeciderKey.enableUserTopicFollowFeatureSet) - - /* - * Param to enable onboarding new user feature set hydration - * */ - object EnableOnboardingNewUserFeatureSetHydration - extends BooleanDeciderParam(DeciderKey.enableOnboardingNewUserFeatureSet) - - /* - * Param to enable mr user author sparse continuous feature set hydration - * */ - object EnableMrUserAuthorSparseContFeatureSetHydration - extends BooleanDeciderParam(DeciderKey.enableMrUserAuthorSparseContFeatureSet) - - /* - * Param to enable mr user topic sparse continuous feature set hydration - * */ - object EnableMrUserTopicSparseContFeatureSetHydration - extends BooleanDeciderParam(DeciderKey.enableMrUserTopicSparseContFeatureSet) - - /* - * Param to enable penguin language feature set hydration - * */ - object EnableUserPenguinLanguageFeatureSetHydration - extends BooleanDeciderParam(DeciderKey.enableUserPenguinLanguageFeatureSet) - - /* - * Param to enable user engaged tweet tokens feature hydration - * */ - object EnableMrUserEngagedTweetTokensFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableMrUserEngagedTweetTokensFeaturesHydration) - - /* - * Param to enable candidate tweet tokens feature hydration - * */ - object EnableMrCandidateTweetTokensFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableMrCandidateTweetTokensFeaturesHydration) - - /* - * Param to enable mr user hashspace embedding feature set hydration - * */ - object EnableMrUserHashspaceEmbeddingFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableMrUserHashspaceEmbeddingFeatureSet) - - /* - * Param to enable mr tweet sentiment feature set hydration - * */ - object EnableMrTweetSentimentFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableMrTweetSentimentFeatureSet) - - /* - * Param to enable mr tweet_author aggregates feature set hydration - * */ - object EnableMrTweetAuthorAggregatesFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableMrTweetAuthorAggregatesFeatureSet) - - /** - * Param to enable twistly aggregated features - */ - object EnableTwistlyAggregatesFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableTwistlyAggregatesFeatureHydration) - - /** - * Param to enable tweet twhin favoriate features - */ - object EnableTweetTwHINFavFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableTweetTwHINFavFeaturesHydration) - - /* - * Param to enable mr user geo feature set hydration - * */ - object EnableUserGeoFeatureSetHydration - extends BooleanDeciderParam(DeciderKey.enableUserGeoFeatureSet) - - /* - * Param to enable mr author geo feature set hydration - * */ - object EnableAuthorGeoFeatureSetHydration - extends BooleanDeciderParam(DeciderKey.enableAuthorGeoFeatureSet) - - /* - * Param to ramp up mr user geo feature set hydration - * */ - object RampupUserGeoFeatureSetHydration - extends BooleanDeciderParam(DeciderKey.rampupUserGeoFeatureSet) - - /* - * Param to ramp up mr author geo feature set hydration - * */ - object RampupAuthorGeoFeatureSetHydration - extends BooleanDeciderParam(DeciderKey.rampupAuthorGeoFeatureSet) - - /* - * Decider controlled param to enable Pop Geo Tweets - * */ - object PopGeoCandidatesDecider extends BooleanDeciderParam(DeciderKey.enablePopGeoTweets) - - /** - * Decider controlled param to enable Trip Geo Tweets - */ - object TripGeoTweetCandidatesDecider - extends BooleanDeciderParam(DeciderKey.enableTripGeoTweetCandidates) - - /** - * Decider controlled param to enable ContentRecommenderMixerAdaptor - */ - object ContentRecommenderMixerAdaptorDecider - extends BooleanDeciderParam(DeciderKey.enableContentRecommenderMixerAdaptor) - - /** - * Decider controlled param to enable GenericCandidateAdaptor - */ - object GenericCandidateAdaptorDecider - extends BooleanDeciderParam(DeciderKey.enableGenericCandidateAdaptor) - - /** - * Decider controlled param to enable dark traffic to ContentMixer for Trip Geo Tweets - */ - object TripGeoTweetContentMixerDarkTrafficDecider - extends BooleanDeciderParam(DeciderKey.enableTripGeoTweetContentMixerDarkTraffic) - - /* - * Decider controlled param to enable Pop Geo Tweets - * */ - object TrendsCandidateDecider extends BooleanDeciderParam(DeciderKey.enableTrendsTweets) - - /* - * Decider controlled param to enable INS Traffic - **/ - object EnableInsTrafficDecider extends BooleanDeciderParam(DeciderKey.enableInsTraffic) - - /** - * Param to enable assigning pushcap with ML predictions (read from MH table). - * Disabling will fallback to only use heuristics and default values. - */ - object EnableModelBasedPushcapAssignments - extends BooleanDeciderParam(DeciderKey.enableModelBasedPushcapAssignments) - - /** - * Param to enable twhin user engagement feature hydration - */ - object EnableTwHINUserEngagementFeaturesHydration - extends BooleanDeciderParam(DeciderKey.enableTwHINUserEngagementFeaturesHydration) - - /** - * Param to enable twhin user follow feature hydration - */ - object EnableTwHINUserFollowFeaturesHydration - extends BooleanDeciderParam(DeciderKey.enableTwHINUserFollowFeaturesHydration) - - /** - * Param to enable twhin author follow feature hydration - */ - object EnableTwHINAuthorFollowFeaturesHydration - extends BooleanDeciderParam(DeciderKey.enableTwHINAuthorFollowFeaturesHydration) - - /** - * Param to enable calls to the IsTweetTranslatable strato column - */ - object EnableIsTweetTranslatableCheck - extends BooleanDeciderParam(DeciderKey.enableIsTweetTranslatable) - - /** - * Decider controlled param to enable mr tweet simcluster feature set hydration - */ - object EnableMrTweetSimClusterFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableMrTweetSimClusterFeatureSet) - - /** - * Decider controlled param to enable real graph v2 feature set hydration - */ - object EnableRealGraphV2FeatureHydration - extends BooleanDeciderParam(DeciderKey.enableRealGraphV2FeatureHydration) - - /** - * Decider controlled param to enable Tweet BeT feature set hydration - */ - object EnableTweetBeTFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableTweetBeTFeatureHydration) - - /** - * Decider controlled param to enable mr user tweet topic feature set hydration - */ - object EnableMrOfflineUserTweetTopicAggregateHydration - extends BooleanDeciderParam(DeciderKey.enableMrOfflineUserTweetTopicAggregate) - - /** - * Decider controlled param to enable mr tweet simcluster feature set hydration - */ - object EnableMrOfflineUserTweetSimClusterAggregateHydration - extends BooleanDeciderParam(DeciderKey.enableMrOfflineUserTweetSimClusterAggregate) - - /** - * Decider controlled param to enable user send time features - */ - object EnableUserSendTimeFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableUserSendTimeFeatureHydration) - - /** - * Decider controlled param to enable mr user utc send time aggregate features - */ - object EnableMrUserUtcSendTimeAggregateFeaturesHydration - extends BooleanDeciderParam(DeciderKey.enableMrUserUtcSendTimeAggregateFeaturesHydration) - - /** - * Decider controlled param to enable mr user local send time aggregate features - */ - object EnableMrUserLocalSendTimeAggregateFeaturesHydration - extends BooleanDeciderParam(DeciderKey.enableMrUserLocalSendTimeAggregateFeaturesHydration) - - /** - * Decider controlled param to enable BQML report model predictions for F1 tweets - */ - object EnableBqmlReportModelPredictionForF1Tweets - extends BooleanDeciderParam(DeciderKey.enableBqmlReportModelPredictionForF1Tweets) - - /** - * Decider controlled param to enable user Twhin embedding feature hydration - */ - object EnableUserTwhinEmbeddingFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableUserTwhinEmbeddingFeatureHydration) - - /** - * Decider controlled param to enable author follow Twhin embedding feature hydration - */ - object EnableAuthorFollowTwhinEmbeddingFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableAuthorFollowTwhinEmbeddingFeatureHydration) - - object EnableScribingMLFeaturesAsDataRecord - extends BooleanDeciderParam(DeciderKey.enableScribingMLFeaturesAsDataRecord) - - /** - * Decider controlled param to enable feature hydration for Verified related feature - */ - object EnableAuthorVerifiedFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableAuthorVerifiedFeatureHydration) - - /** - * Decider controlled param to enable feature hydration for creator subscription related feature - */ - object EnableAuthorCreatorSubscriptionFeatureHydration - extends BooleanDeciderParam(DeciderKey.enableAuthorCreatorSubscriptionFeatureHydration) - - /** - * Decider controlled param to direct MH+Memcache hydration for the UserFeaturesDataset - */ - object EnableDirectHydrationForUserFeatures - extends BooleanDeciderParam(DeciderKey.enableDirectHydrationForUserFeatures) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushServiceTunableKeys.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushServiceTunableKeys.docx new file mode 100644 index 000000000..dc0be8b98 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushServiceTunableKeys.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushServiceTunableKeys.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushServiceTunableKeys.scala deleted file mode 100644 index 7920bb6cd..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushServiceTunableKeys.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.frigate.pushservice.params - -import com.twitter.util.tunable.TunableMap - -object PushServiceTunableKeys { - final val IbisQpsLimitTunableKey = TunableMap.Key[Int]("ibis2.qps.limit") - final val NtabQpsLimitTunableKey = TunableMap.Key[Int]("ntab.qps.limit") - final val TweetPerspectiveStoreQpsLimit = TunableMap.Key[Int]("tweetperspective.qps.limit") -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/ShardParams.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/ShardParams.docx new file mode 100644 index 000000000..0d4f391eb Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/ShardParams.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/ShardParams.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/ShardParams.scala deleted file mode 100644 index c0a68c939..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/ShardParams.scala +++ /dev/null @@ -1,3 +0,0 @@ -package com.twitter.frigate.pushservice.params - -case class ShardParams(numShards: Int, shardId: Int) diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BigFilteringEpsilonGreedyExplorationPredicate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BigFilteringEpsilonGreedyExplorationPredicate.docx new file mode 100644 index 000000000..0ec00ab2e Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BigFilteringEpsilonGreedyExplorationPredicate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BigFilteringEpsilonGreedyExplorationPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BigFilteringEpsilonGreedyExplorationPredicate.scala deleted file mode 100644 index 67a117cc5..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BigFilteringEpsilonGreedyExplorationPredicate.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.tracing.Trace -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.hashing.KeyHasher -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.util.Future - -/* - * A predicate for epsilon-greedy exploration; - * We defined it as a candidate level predicate to avoid changing the predicate and scribing pipeline, - * but it is actually a post-ranking target level predicate: - * if a target user IS ENABLED for \epsilon-greedy exploration, - * then with probability epsilon, the user (and thus all candidates) will be blocked - */ -object BigFilteringEpsilonGreedyExplorationPredicate { - - val name = "BigFilteringEpsilonGreedyExplorationPredicate" - - private def shouldFilterBasedOnEpsilonGreedyExploration( - target: Target - ): Boolean = { - val seed = KeyHasher.FNV1A_64.hashKey(s"${target.targetId}".getBytes("UTF8")) - val hashKey = KeyHasher.FNV1A_64 - .hashKey( - s"${Trace.id.traceId.toString}:${seed.toString}".getBytes("UTF8") - ) - - math.abs(hashKey).toDouble / Long.MaxValue < - target.params(PushFeatureSwitchParams.MrRequestScribingEpsGreedyExplorationRatio) - } - - def apply()(implicit statsReceiver: StatsReceiver): NamedPredicate[PushCandidate] = { - val stats = statsReceiver.scope(s"predicate_$name") - - val enabledForEpsilonGreedyCounter = stats.counter("enabled_for_eps_greedy") - - new Predicate[PushCandidate] { - def apply(candidates: Seq[PushCandidate]): Future[Seq[Boolean]] = { - val results = candidates.map { candidate => - if (!candidate.target.skipFilters && candidate.target.params( - PushFeatureSwitchParams.EnableMrRequestScribingForEpsGreedyExploration)) { - enabledForEpsilonGreedyCounter.incr() - !shouldFilterBasedOnEpsilonGreedyExploration(candidate.target) - } else { - true - } - } - Future.value(results) - } - }.withStats(stats) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BqmlHealthModelPredicates.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BqmlHealthModelPredicates.docx new file mode 100644 index 000000000..b8573ece3 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BqmlHealthModelPredicates.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BqmlHealthModelPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BqmlHealthModelPredicates.scala deleted file mode 100644 index f7ff95c9b..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BqmlHealthModelPredicates.scala +++ /dev/null @@ -1,129 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringRequest -import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringResponse -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.ml.HealthFeatureGetter -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.params.PushMLModel -import com.twitter.util.Future -import com.twitter.frigate.pushservice.util.CandidateUtil -import com.twitter.frigate.thriftscala.UserMediaRepresentation -import com.twitter.hss.api.thriftscala.UserHealthSignalResponse -import com.twitter.storehaus.ReadableStore - -object BqmlHealthModelPredicates { - - def healthModelOonPredicate( - bqmlHealthModelScorer: PushMLModelScorer, - producerMediaRepresentationStore: ReadableStore[Long, UserMediaRepresentation], - userHealthScoreStore: ReadableStore[Long, UserHealthSignalResponse], - tweetHealthScoreStore: ReadableStore[TweetScoringRequest, TweetScoringResponse] - )( - implicit stats: StatsReceiver - ): NamedPredicate[ - PushCandidate with TweetCandidate with RecommendationType with TweetAuthor - ] = { - val name = "bqml_health_model_based_predicate" - val scopedStatsReceiver = stats.scope(name) - - val allCandidatesCounter = scopedStatsReceiver.counter("all_candidates") - val oonCandidatesCounter = scopedStatsReceiver.counter("oon_candidates") - val filteredOonCandidatesCounter = - scopedStatsReceiver.counter("filtered_oon_candidates") - val emptyScoreCandidatesCounter = scopedStatsReceiver.counter("empty_score_candidates") - val healthScoreStat = scopedStatsReceiver.stat("health_model_dist") - - Predicate - .fromAsync { candidate: PushCandidate with TweetCandidate with RecommendationType => - val target = candidate.target - val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(candidate.commonRecType) || - RecTypes.outOfNetworkTopicTweetTypes.contains(candidate.commonRecType) - - lazy val enableBqmlHealthModelPredicateParam = - target.params(PushFeatureSwitchParams.EnableBqmlHealthModelPredicateParam) - lazy val enableBqmlHealthModelPredictionForInNetworkCandidates = - target.params( - PushFeatureSwitchParams.EnableBqmlHealthModelPredictionForInNetworkCandidatesParam) - lazy val bqmlHealthModelPredicateFilterThresholdParam = - target.params(PushFeatureSwitchParams.BqmlHealthModelPredicateFilterThresholdParam) - lazy val healthModelId = target.params(PushFeatureSwitchParams.BqmlHealthModelTypeParam) - lazy val enableBqmlHealthModelScoreHistogramParam = - target.params(PushFeatureSwitchParams.EnableBqmlHealthModelScoreHistogramParam) - val healthModelScoreFeature = "bqml_health_model_score" - - val histogramBinSize = 0.05 - lazy val healthCandidateScoreHistogramCounters = - bqmlHealthModelScorer.getScoreHistogramCounters( - scopedStatsReceiver, - "health_score_histogram", - histogramBinSize) - - candidate match { - case candidate: PushCandidate with TweetAuthor with TweetAuthorDetails - if enableBqmlHealthModelPredicateParam && (isOonCandidate || enableBqmlHealthModelPredictionForInNetworkCandidates) => - HealthFeatureGetter - .getFeatures( - candidate, - producerMediaRepresentationStore, - userHealthScoreStore, - Some(tweetHealthScoreStore)) - .flatMap { healthFeatures => - allCandidatesCounter.incr() - candidate.mergeFeatures(healthFeatures) - - val healthModelScoreFutOpt = - if (candidate.numericFeatures.contains(healthModelScoreFeature)) { - Future.value(candidate.numericFeatures.get(healthModelScoreFeature)) - } else - bqmlHealthModelScorer.singlePredicationForModelVersion( - healthModelId, - candidate - ) - - candidate.populateQualityModelScore( - PushMLModel.HealthNsfwProbability, - healthModelId, - healthModelScoreFutOpt - ) - - healthModelScoreFutOpt.map { - case Some(healthModelScore) => - healthScoreStat.add((healthModelScore * 10000).toFloat) - if (enableBqmlHealthModelScoreHistogramParam) { - healthCandidateScoreHistogramCounters( - math.ceil(healthModelScore / histogramBinSize).toInt).incr() - } - - if (CandidateUtil.shouldApplyHealthQualityFilters( - candidate) && isOonCandidate) { - oonCandidatesCounter.incr() - val threshold = bqmlHealthModelPredicateFilterThresholdParam - candidate.cachePredicateInfo( - name, - healthModelScore, - threshold, - healthModelScore > threshold) - if (healthModelScore > threshold) { - filteredOonCandidatesCounter.incr() - false - } else true - } else true - case _ => - emptyScoreCandidatesCounter.incr() - true - } - } - case _ => Future.True - } - } - .withStats(stats.scope(name)) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BqmlQualityModelPredicates.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BqmlQualityModelPredicates.docx new file mode 100644 index 000000000..1596a9ab1 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BqmlQualityModelPredicates.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BqmlQualityModelPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BqmlQualityModelPredicates.scala deleted file mode 100644 index 76d52992b..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BqmlQualityModelPredicates.scala +++ /dev/null @@ -1,141 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.params.PushConstants.TweetMediaEmbeddingBQKeyIds -import com.twitter.frigate.pushservice.params.PushMLModel -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.util.CandidateUtil -import com.twitter.util.Future -import com.twitter.frigate.pushservice.util.CandidateUtil._ - -object BqmlQualityModelPredicates { - - def ingestExtraFeatures(cand: PushCandidate): Unit = { - val tagsCRCountFeature = "tagsCR_count" - val hasPushOpenOrNtabClickFeature = "has_PushOpenOrNtabClick" - val onlyPushOpenOrNtabClickFeature = "only_PushOpenOrNtabClick" - val firstTweetMediaEmbeddingFeature = "media_embedding_0" - val tweetMediaEmbeddingFeature = - "media.mediaunderstanding.media_embeddings.twitter_clip_as_sparse_continuous_feature" - - if (!cand.numericFeatures.contains(tagsCRCountFeature)) { - cand.numericFeatures(tagsCRCountFeature) = getTagsCRCount(cand) - } - if (!cand.booleanFeatures.contains(hasPushOpenOrNtabClickFeature)) { - cand.booleanFeatures(hasPushOpenOrNtabClickFeature) = isRelatedToMrTwistlyCandidate(cand) - } - if (!cand.booleanFeatures.contains(onlyPushOpenOrNtabClickFeature)) { - cand.booleanFeatures(onlyPushOpenOrNtabClickFeature) = isMrTwistlyCandidate(cand) - } - if (!cand.numericFeatures.contains(firstTweetMediaEmbeddingFeature)) { - val tweetMediaEmbedding = cand.sparseContinuousFeatures - .getOrElse(tweetMediaEmbeddingFeature, Map.empty[String, Double]) - Seq.range(0, TweetMediaEmbeddingBQKeyIds.size).foreach { i => - cand.numericFeatures(s"media_embedding_$i") = - tweetMediaEmbedding.getOrElse(TweetMediaEmbeddingBQKeyIds(i).toString, 0.0) - } - } - } - - def BqmlQualityModelOonPredicate( - bqmlQualityModelScorer: PushMLModelScorer - )( - implicit stats: StatsReceiver - ): NamedPredicate[ - PushCandidate with TweetCandidate with RecommendationType - ] = { - - val name = "bqml_quality_model_based_predicate" - val scopedStatsReceiver = stats.scope(name) - val oonCandidatesCounter = scopedStatsReceiver.counter("oon_candidates") - val inCandidatesCounter = scopedStatsReceiver.counter("in_candidates") - val filteredOonCandidatesCounter = - scopedStatsReceiver.counter("filtered_oon_candidates") - val bucketedCandidatesCounter = scopedStatsReceiver.counter("bucketed_oon_candidates") - val emptyScoreCandidatesCounter = scopedStatsReceiver.counter("empty_score_candidates") - val histogramBinSize = 0.05 - - Predicate - .fromAsync { candidate: PushCandidate with TweetCandidate with RecommendationType => - val target = candidate.target - val crt = candidate.commonRecType - val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(crt) || - RecTypes.outOfNetworkTopicTweetTypes.contains(crt) - - lazy val enableBqmlQualityModelScoreHistogramParam = - target.params(PushFeatureSwitchParams.EnableBqmlQualityModelScoreHistogramParam) - - lazy val qualityCandidateScoreHistogramCounters = - bqmlQualityModelScorer.getScoreHistogramCounters( - scopedStatsReceiver, - "quality_score_histogram", - histogramBinSize) - - if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && (isOonCandidate || target - .params(PushParams.EnableBqmlReportModelPredictionForF1Tweets)) - && target.params(PushFeatureSwitchParams.EnableBqmlQualityModelPredicateParam)) { - ingestExtraFeatures(candidate) - - lazy val shouldFilterFutSeq = - target - .params(PushFeatureSwitchParams.BqmlQualityModelBucketModelIdListParam) - .zip(target.params(PushFeatureSwitchParams.BqmlQualityModelBucketThresholdListParam)) - .map { - case (modelId, bucketThreshold) => - val scoreFutOpt = - bqmlQualityModelScorer.singlePredicationForModelVersion(modelId, candidate) - - candidate.populateQualityModelScore( - PushMLModel.FilteringProbability, - modelId, - scoreFutOpt - ) - - if (isOonCandidate) { - oonCandidatesCounter.incr() - scoreFutOpt.map { - case Some(score) => - if (score >= bucketThreshold) { - bucketedCandidatesCounter.incr() - if (modelId == target.params( - PushFeatureSwitchParams.BqmlQualityModelTypeParam)) { - if (enableBqmlQualityModelScoreHistogramParam) { - val scoreHistogramBinId = - math.ceil(score / histogramBinSize).toInt - qualityCandidateScoreHistogramCounters(scoreHistogramBinId).incr() - } - if (score >= target.params( - PushFeatureSwitchParams.BqmlQualityModelPredicateThresholdParam)) { - filteredOonCandidatesCounter.incr() - true - } else false - } else false - } else false - case _ => - emptyScoreCandidatesCounter.incr() - false - } - } else { - inCandidatesCounter.incr() - Future.False - } - } - - Future.collect(shouldFilterFutSeq).flatMap { shouldFilterSeq => - if (shouldFilterSeq.contains(true)) { - Future.False - } else Future.True - } - } else Future.True - } - .withStats(stats.scope(name)) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CaretFeedbackHistoryFilter.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CaretFeedbackHistoryFilter.docx new file mode 100644 index 000000000..f38ac7fa0 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CaretFeedbackHistoryFilter.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CaretFeedbackHistoryFilter.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CaretFeedbackHistoryFilter.scala deleted file mode 100644 index 8ccccd14d..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CaretFeedbackHistoryFilter.scala +++ /dev/null @@ -1,99 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.frigate.common.base.TargetUser -import com.twitter.frigate.common.candidate.CaretFeedbackHistory -import com.twitter.frigate.common.candidate.TargetABDecider -import com.twitter.frigate.common.util.MrNtabCopyObjects -import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails -import com.twitter.notificationservice.thriftscala.GenericNotificationMetadata -import com.twitter.notificationservice.thriftscala.GenericType - -object CaretFeedbackHistoryFilter { - - def caretFeedbackHistoryFilter( - categories: Seq[String] - ): TargetUser with TargetABDecider with CaretFeedbackHistory => Seq[CaretFeedbackDetails] => Seq[ - CaretFeedbackDetails - ] = { target => caretFeedbackDetailsSeq => - caretFeedbackDetailsSeq.filter { caretFeedbackDetails => - caretFeedbackDetails.genericNotificationMetadata match { - case Some(genericNotificationMetadata) => - isFeedbackSupportedGenericType(genericNotificationMetadata) - case None => false - } - } - } - - private def filterCriteria( - caretFeedbackDetails: CaretFeedbackDetails, - genericTypes: Seq[GenericType] - ): Boolean = { - caretFeedbackDetails.genericNotificationMetadata match { - case Some(genericNotificationMetadata) => - genericTypes.contains(genericNotificationMetadata.genericType) - case None => false - } - } - - def caretFeedbackHistoryFilterByGenericType( - genericTypes: Seq[GenericType] - ): TargetUser with TargetABDecider with CaretFeedbackHistory => Seq[CaretFeedbackDetails] => Seq[ - CaretFeedbackDetails - ] = { target => caretFeedbackDetailsSeq => - caretFeedbackDetailsSeq.filter { caretFeedbackDetails => - filterCriteria(caretFeedbackDetails, genericTypes) - } - } - - def caretFeedbackHistoryFilterByGenericTypeDenyList( - genericTypes: Seq[GenericType] - ): TargetUser with TargetABDecider with CaretFeedbackHistory => Seq[CaretFeedbackDetails] => Seq[ - CaretFeedbackDetails - ] = { target => caretFeedbackDetailsSeq => - caretFeedbackDetailsSeq.filterNot { caretFeedbackDetails => - filterCriteria(caretFeedbackDetails, genericTypes) - } - } - - def caretFeedbackHistoryFilterByRefreshableType( - refreshableTypes: Set[Option[String]] - ): TargetUser with TargetABDecider with CaretFeedbackHistory => Seq[CaretFeedbackDetails] => Seq[ - CaretFeedbackDetails - ] = { target => caretFeedbackDetailsSeq => - caretFeedbackDetailsSeq.filter { caretFeedbackDetails => - caretFeedbackDetails.genericNotificationMetadata match { - case Some(genericNotificationMetadata) => - refreshableTypes.contains(genericNotificationMetadata.refreshableType) - case None => false - } - } - } - - def caretFeedbackHistoryFilterByRefreshableTypeDenyList( - refreshableTypes: Set[Option[String]] - ): TargetUser with TargetABDecider with CaretFeedbackHistory => Seq[CaretFeedbackDetails] => Seq[ - CaretFeedbackDetails - ] = { target => caretFeedbackDetailsSeq => - caretFeedbackDetailsSeq.filter { caretFeedbackDetails => - caretFeedbackDetails.genericNotificationMetadata match { - case Some(genericNotificationMetadata) => - !refreshableTypes.contains(genericNotificationMetadata.refreshableType) - case None => true - } - } - } - - private def isFeedbackSupportedGenericType( - notificationMetadata: GenericNotificationMetadata - ): Boolean = { - val genericNotificationTypeName = - (notificationMetadata.genericType, notificationMetadata.refreshableType) match { - case (GenericType.RefreshableNotification, Some(refreshableType)) => refreshableType - case _ => notificationMetadata.genericType.name - } - - MrNtabCopyObjects.AllNtabCopyTypes - .flatMap(_.refreshableType) - .contains(genericNotificationTypeName) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CasLockPredicate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CasLockPredicate.docx new file mode 100644 index 000000000..ffc74d3d5 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CasLockPredicate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CasLockPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CasLockPredicate.scala deleted file mode 100644 index 22067405a..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CasLockPredicate.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.CasLock -import com.twitter.frigate.common.util.CasSuccess -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.util.Duration -import com.twitter.util.Future - -object CasLockPredicate { - def apply( - casLock: CasLock, - expiryDuration: Duration - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val stats = statsReceiver.scope("predicate_addcaslock_for_candidate") - Predicate - .fromAsync { candidate: PushCandidate => - if (candidate.target.pushContext.exists(_.darkWrite.exists(_ == true))) { - Future.True - } else if (candidate.commonRecType == CommonRecommendationType.MagicFanoutSportsEvent) { - Future.True - } else { - candidate.target.history flatMap { h => - val now = candidate.createdAt - val expiry = now + expiryDuration - val oldTimestamp = h.lastNotificationTime map { - _.inSeconds - } getOrElse 0 - casLock.cas(candidate.target.targetId, oldTimestamp, now.inSeconds, expiry) map { - casResult => - stats.counter(s"cas_$casResult").incr() - casResult == CasSuccess - } - } - } - } - .withStats(stats) - .withName("add_cas_lock") - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CrtDeciderPredicate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CrtDeciderPredicate.docx new file mode 100644 index 000000000..f72eab8fb Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CrtDeciderPredicate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CrtDeciderPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CrtDeciderPredicate.scala deleted file mode 100644 index 4b1abf221..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CrtDeciderPredicate.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.decider.Decider -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate - -object CrtDeciderPredicate { - val name = "crt_decider" - def apply( - decider: Decider - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate] = { - Predicate - .from { (candidate: PushCandidate) => - val prefix = "frigate_pushservice_" - val deciderKey = prefix + candidate.commonRecType - decider.feature(deciderKey).isAvailable - } - .withStats(statsReceiver.scope(s"predicate_$name")) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/DiscoverTwitterPredicate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/DiscoverTwitterPredicate.docx new file mode 100644 index 000000000..da5c67d98 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/DiscoverTwitterPredicate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/DiscoverTwitterPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/DiscoverTwitterPredicate.scala deleted file mode 100644 index cb55be356..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/DiscoverTwitterPredicate.scala +++ /dev/null @@ -1,47 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.TargetUser -import com.twitter.frigate.common.candidate.FrigateHistory -import com.twitter.frigate.common.history.History -import com.twitter.frigate.common.predicate.FrigateHistoryFatiguePredicate -import com.twitter.frigate.common.predicate.{FatiguePredicate => TargetFatiguePredicate} -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.hermit.predicate.Predicate -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration - -object DiscoverTwitterPredicate { - - /** - * Predicate used to determine if a minimum duration has elapsed since the last MR push - * for a CRT to be valid. - * @param name Identifier of the caller (used for stats) - * @param intervalParam The minimum duration interval - * @param stats StatsReceiver - * @return Target Predicate - */ - def minDurationElapsedSinceLastMrPushPredicate( - name: String, - intervalParam: Param[Duration], - stats: StatsReceiver - ): Predicate[Target] = - Predicate - .fromAsync { target: Target => - val interval = - target.params(intervalParam) - FrigateHistoryFatiguePredicate( - minInterval = interval, - getSortedHistory = { h: History => - val magicRecsOnlyHistory = - TargetFatiguePredicate.magicRecsPushOnlyFilter(h.sortedPushDmHistory) - TargetFatiguePredicate.magicRecsNewUserPlaybookPushFilter(magicRecsOnlyHistory) - } - ).flatContraMap { target: TargetUser with FrigateHistory => - target.history - }.apply(Seq(target)).map { - _.head - } - }.withStats(stats.scope(s"${name}_predicate_mr_push_min_interval")) - .withName(s"${name}_predicate_mr_push_min_interval") -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/FatiguePredicate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/FatiguePredicate.docx new file mode 100644 index 000000000..ba322f102 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/FatiguePredicate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/FatiguePredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/FatiguePredicate.scala deleted file mode 100644 index 457dc879c..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/FatiguePredicate.scala +++ /dev/null @@ -1,74 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.predicate.FatiguePredicate._ -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.{NotificationDisplayLocation => DisplayLocation} -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.util.Duration - -object FatiguePredicate { - - /** - * Predicate that operates on a candidate, and applies custom fatigue rules for the slice of history only - * corresponding to a given rec type. - * - * @param interval - * @param maxInInterval - * @param minInterval - * @param recommendationType - * @param statsReceiver - * @return - */ - def recTypeOnly( - interval: Duration, - maxInInterval: Int, - minInterval: Duration, - recommendationType: CommonRecommendationType, - notificationDisplayLocation: DisplayLocation = DisplayLocation.PushToMobileDevice - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate] = { - build( - interval = interval, - maxInInterval = maxInInterval, - minInterval = minInterval, - filterHistory = recOnlyFilter(recommendationType), - notificationDisplayLocation = notificationDisplayLocation - ).flatContraMap { candidate: PushCandidate => candidate.target.history } - .withStats(statsReceiver.scope(s"predicate_${recTypeOnlyFatigue}")) - .withName(recTypeOnlyFatigue) - } - - /** - * Predicate that operates on a candidate, and applies custom fatigue rules for the slice of history only - * corresponding to specified rec types - * - * @param interval - * @param maxInInterval - * @param minInterval - * @param statsReceiver - * @return - */ - def recTypeSetOnly( - interval: Duration, - maxInInterval: Int, - minInterval: Duration, - recTypes: Set[CommonRecommendationType], - notificationDisplayLocation: DisplayLocation = DisplayLocation.PushToMobileDevice - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "rec_type_set_fatigue" - build( - interval = interval, - maxInInterval = maxInInterval, - minInterval = minInterval, - filterHistory = recTypesOnlyFilter(recTypes), - notificationDisplayLocation = notificationDisplayLocation - ).flatContraMap { candidate: PushCandidate => candidate.target.history } - .withStats(statsReceiver.scope(s"${name}_predicate")) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/HealthPredicates.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/HealthPredicates.docx new file mode 100644 index 000000000..d878d5992 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/HealthPredicates.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/HealthPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/HealthPredicates.scala deleted file mode 100644 index f11ed1400..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/HealthPredicates.scala +++ /dev/null @@ -1,740 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringRequest -import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringResponse -import com.twitter.abuse.detection.scoring.thriftscala.{Model => TweetHealthModel} -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.NsfwTextDetectionModel -import com.twitter.frigate.pushservice.params.PushConstants -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.util.CandidateHydrationUtil -import com.twitter.frigate.pushservice.util.CandidateUtil -import com.twitter.frigate.pushservice.util.MediaAnnotationsUtil -import com.twitter.frigate.thriftscala.UserMediaRepresentation -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.hss.api.thriftscala.UserHealthSignal._ -import com.twitter.hss.api.thriftscala.SignalValue -import com.twitter.hss.api.thriftscala.UserHealthSignalResponse -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future -import com.twitter.util.Time - -object HealthPredicates { - - private val NsfwTextDetectionModelMap: Map[NsfwTextDetectionModel.Value, TweetHealthModel] = - Map( - NsfwTextDetectionModel.ProdModel -> TweetHealthModel.PnsfwTweetText, - NsfwTextDetectionModel.RetrainedModel -> TweetHealthModel.ExperimentalHealthModelScore1, - ) - - private def tweetIsSupportedLanguage( - candidate: PushCandidate, - supportedLanguages: Set[String] - ): Boolean = { - val tweetLanguage = - candidate.categoricalFeatures.getOrElse("RecTweet.TweetyPieResult.Language", "") - supportedLanguages.contains(tweetLanguage) - } - - def tweetHealthSignalScorePredicate( - tweetHealthScoreStore: ReadableStore[TweetScoringRequest, TweetScoringResponse], - applyToQuoteTweet: Boolean = false - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with TweetCandidate with TweetDetails] = { - val name = "tweet_health_signal_store_applyToQuoteTweet_" + applyToQuoteTweet.toString - val scopedStatsReceiver = stats.scope(name) - val numCandidatesStats = scopedStatsReceiver.scope("num_candidates") - val numCandidatesMediaNsfwScoreStats = numCandidatesStats.scope("media_nsfw_score") - - Predicate - .fromAsync { candidate: PushCandidate with TweetCandidate with TweetDetails => - numCandidatesStats.counter("all").incr() - val target = candidate.target - val tweetIdOpt = if (!applyToQuoteTweet) { - Some(candidate.tweetId) - } else candidate.tweetyPieResult.flatMap(_.quotedTweet.map(_.id)) - - tweetIdOpt match { - case Some(tweetId) => - val pMediaNsfwRequest = - TweetScoringRequest(tweetId, TweetHealthModel.ExperimentalHealthModelScore4) - tweetHealthScoreStore.get(pMediaNsfwRequest).map { - case Some(tweetScoringResponse) => - numCandidatesMediaNsfwScoreStats.counter("non_empty").incr() - val pMediaNsfwScore = tweetScoringResponse.score - - if (!applyToQuoteTweet) { - candidate - .cacheExternalScore("NsfwMediaProbability", Future.value(Some(pMediaNsfwScore))) - } - - val pMediaNsfwShouldBucket = - pMediaNsfwScore > target.params( - PushFeatureSwitchParams.PnsfwTweetMediaBucketingThreshold) - if (CandidateUtil.shouldApplyHealthQualityFilters( - candidate) && pMediaNsfwShouldBucket) { - numCandidatesMediaNsfwScoreStats.counter("bucketed").incr() - if (target.params(PushFeatureSwitchParams.PnsfwTweetMediaFilterOonOnly) - && !RecTypes.isOutOfNetworkTweetRecType(candidate.commonRecType)) { - true - } else { - val pMediaNsfwScoreThreshold = - if (applyToQuoteTweet) - target.params(PushFeatureSwitchParams.PnsfwQuoteTweetThreshold) - else if (candidate.hasPhoto) - target.params(PushFeatureSwitchParams.PnsfwTweetImageThreshold) - else target.params(PushFeatureSwitchParams.PnsfwTweetMediaThreshold) - candidate.cachePredicateInfo( - name + "_nsfwMedia", - pMediaNsfwScore, - pMediaNsfwScoreThreshold, - pMediaNsfwScore > pMediaNsfwScoreThreshold) - if (pMediaNsfwScore > pMediaNsfwScoreThreshold) { - numCandidatesMediaNsfwScoreStats.counter("filtered").incr() - false - } else true - } - } else true - case _ => - numCandidatesMediaNsfwScoreStats.counter("empty").incr() - if (candidate.hasPhoto || candidate.hasVideo) { - numCandidatesMediaNsfwScoreStats.counter("media_tweet_with_empty_score").incr() - } - true - } - case _ => Future.True - } - } - .withStats(stats.scope(s"predicate_$name")) - .withName(name) - } - - def healthSignalScoreSpammyTweetPredicate( - tweetHealthScoreStore: ReadableStore[TweetScoringRequest, TweetScoringResponse] - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with TweetCandidate with TweetDetails] = { - val name = "health_signal_store_spammy_tweet" - val statsScope = stats.scope(name) - val allCandidatesCounter = statsScope.counter("all_candidates") - val eligibleCandidatesCounter = statsScope.counter("eligible_candidates") - val oonCandidatesCounter = statsScope.counter("oon_candidates") - val inCandidatesCounter = statsScope.counter("in_candidates") - val bucketedCandidatesCounter = statsScope.counter("num_bucketed") - val nonEmptySpamScoreCounter = statsScope.counter("non_empty_spam_score") - val filteredOonCandidatesCounter = statsScope.counter("num_filtered_oon") - val filteredInCandidatesCounter = statsScope.counter("num_filtered_in") - - Predicate - .fromAsync { candidate: PushCandidate with TweetCandidate with TweetDetails => - allCandidatesCounter.incr() - val crt = candidate.commonRecType - val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(crt) || - RecTypes.outOfNetworkTopicTweetTypes.contains(crt) - if (isOonCandidate) { - oonCandidatesCounter.incr() - } - val target = candidate.target - if (target.params(PushFeatureSwitchParams.EnableSpammyTweetFilter)) { - eligibleCandidatesCounter.incr() - val tweetSpamScore = - TweetScoringRequest(candidate.tweetId, TweetHealthModel.SpammyTweetContent) - tweetHealthScoreStore.get(tweetSpamScore).map { - case (Some(tweetScoringResponse)) => - nonEmptySpamScoreCounter.incr() - val candidateSpamScore = tweetScoringResponse.score - - candidate - .cacheExternalScore("SpammyTweetScore", Future.value(Some(candidateSpamScore))) - - val tweetSpamShouldBucket = - candidateSpamScore > target.params( - PushFeatureSwitchParams.SpammyTweetBucketingThreshold) - if (CandidateUtil.shouldApplyHealthQualityFilters( - candidate) && tweetSpamShouldBucket) { - bucketedCandidatesCounter.incr() - if (isOonCandidate) { - val spamScoreThreshold = - target.params(PushFeatureSwitchParams.SpammyTweetOonThreshold) - if (candidateSpamScore > spamScoreThreshold) { - filteredOonCandidatesCounter.incr() - false - } else true - } else { - inCandidatesCounter.incr() - val spamScoreThreshold = - target.params(PushFeatureSwitchParams.SpammyTweetInThreshold) - if (candidateSpamScore > spamScoreThreshold) { - filteredInCandidatesCounter.incr() - false - } else true - } - } else true - case _ => true - } - } else Future.True - } - .withStats(stats.scope(s"predicate_$name")) - .withName(name) - } - - def healthSignalScorePnsfwTweetTextPredicate( - tweetHealthScoreStore: ReadableStore[TweetScoringRequest, TweetScoringResponse] - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with TweetCandidate] = { - val name = "health_signal_store_pnsfw_tweet_text" - val statsScope = stats.scope(name) - val allCandidatesCounter = statsScope.counter("all_candidates") - val nonEmptyNsfwTextScoreNum = statsScope.counter("non_empty_nsfw_text_score") - val filteredCounter = statsScope.counter("num_filtered") - val lowScoreCounter = statsScope.counter("low_score_count") - - Predicate - .fromAsync { candidate: PushCandidate with TweetCandidate => - val target = candidate.target - val predEnabled = - target.params(PushFeatureSwitchParams.EnableHealthSignalStorePnsfwTweetTextPredicate) - if (CandidateUtil.shouldApplyHealthQualityFilters( - candidate) && predEnabled && tweetIsSupportedLanguage(candidate, Set(""))) { - allCandidatesCounter.incr() - val pnsfwTextRequest = - TweetScoringRequest(candidate.tweetId, TweetHealthModel.PnsfwTweetText) - tweetHealthScoreStore.get(pnsfwTextRequest).flatMap { - case Some(tweetScoringResponse) => { - nonEmptyNsfwTextScoreNum.incr() - if (tweetScoringResponse.score < 1e-8) { - lowScoreCounter.incr() - } - - candidate - .cacheExternalScore( - "NsfwTextProbability-en", - Future.value(Some(tweetScoringResponse.score))) - val threshold = target.params(PushFeatureSwitchParams.PnsfwTweetTextThreshold) - candidate.cachePredicateInfo( - name, - tweetScoringResponse.score, - threshold, - tweetScoringResponse.score > threshold) - if (tweetScoringResponse.score > threshold) { - filteredCounter.incr() - Future.False - } else Future.True - } - case _ => Future.True - } - } else Future.True - } - .withStats(stats.scope(s"predicate_$name")) - .withName(name) - } - - def healthSignalScoreMultilingualPnsfwTweetTextPredicate( - tweetHealthScoreStore: ReadableStore[TweetScoringRequest, TweetScoringResponse] - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with TweetCandidate] = { - val name = "health_signal_store_multilingual_pnsfw_tweet_text" - val statsScope = stats.scope(name) - - val allLanguagesIdentifier = "all" - val languagesSelectedForStats = - Set("") + allLanguagesIdentifier - - val candidatesCounterMap: Map[String, Counter] = languagesSelectedForStats.map { lang => - lang -> statsScope.counter(f"candidates_$lang") - }.toMap - val nonEmptyHealthScoreMap: Map[String, Counter] = languagesSelectedForStats.map { lang => - lang -> statsScope.counter(f"non_empty_health_score_$lang") - }.toMap - val emptyHealthScoreMap: Map[String, Counter] = languagesSelectedForStats.map { lang => - lang -> statsScope.counter(f"empty_health_score_$lang") - }.toMap - val bucketedCounterMap: Map[String, Counter] = languagesSelectedForStats.map { lang => - lang -> statsScope.counter(f"num_candidates_bucketed_$lang") - }.toMap - val filteredCounterMap: Map[String, Counter] = languagesSelectedForStats.map { lang => - lang -> statsScope.counter(f"num_filtered_$lang") - }.toMap - val lowScoreCounterMap: Map[String, Counter] = languagesSelectedForStats.map { lang => - lang -> statsScope.counter(f"low_score_count_$lang") - }.toMap - - val wrongBucketingModelCounter = statsScope.counter("wrong_bucketing_model_count") - val wrongDetectionModelCounter = statsScope.counter("wrong_detection_model_count") - - def increaseCounterForLanguage(counterMap: Map[String, Counter], language: String): Unit = { - counterMap.get(allLanguagesIdentifier) match { - case Some(counter) => counter.incr() - case _ => - } - counterMap.get(language) match { - case Some(counter) => counter.incr() - case _ => - } - } - - Predicate - .fromAsync { candidate: PushCandidate with TweetCandidate => - val target = candidate.target - - val languageFeatureName = "RecTweet.TweetyPieResult.Language" - - lazy val isPredicateEnabledForTarget = target.params( - PushFeatureSwitchParams.EnableHealthSignalStoreMultilingualPnsfwTweetTextPredicate) - - lazy val targetNsfwTextDetectionModel: NsfwTextDetectionModel.Value = - target.params(PushFeatureSwitchParams.MultilingualPnsfwTweetTextModel) - - lazy val targetPredicateSupportedLanguageSeq: Seq[String] = - target.params(PushFeatureSwitchParams.MultilingualPnsfwTweetTextSupportedLanguages) - - lazy val bucketingModelSeq: Seq[NsfwTextDetectionModel.Value] = - target.params(PushFeatureSwitchParams.MultilingualPnsfwTweetTextBucketingModelList) - - lazy val bucketingThresholdPerLanguageSeq: Seq[Double] = - target.params(PushFeatureSwitchParams.MultilingualPnsfwTweetTextBucketingThreshold) - - lazy val filteringThresholdPerLanguageSeq: Seq[Double] = - target.params(PushFeatureSwitchParams.MultilingualPnsfwTweetTextFilteringThreshold) - - if (CandidateUtil.shouldApplyHealthQualityFilters( - candidate) && isPredicateEnabledForTarget) { - val candidateLanguage = - candidate.categoricalFeatures.getOrElse(languageFeatureName, "") - - val indexOfCandidateLanguage = - targetPredicateSupportedLanguageSeq.indexOf(candidateLanguage) - - val isCandidateLanguageSupported = indexOfCandidateLanguage >= 0 - - if (isCandidateLanguageSupported) { - increaseCounterForLanguage(candidatesCounterMap, candidateLanguage) - - val bucketingModelScoreMap: Map[NsfwTextDetectionModel.Value, Future[Option[Double]]] = - bucketingModelSeq.map { modelName => - NsfwTextDetectionModelMap.get(modelName) match { - case Some(targetNsfwTextDetectionModel) => - val pnsfwTweetTextRequest: TweetScoringRequest = - TweetScoringRequest(candidate.tweetId, targetNsfwTextDetectionModel) - - val scoreOptFut: Future[Option[Double]] = - tweetHealthScoreStore.get(pnsfwTweetTextRequest).map(_.map(_.score)) - - candidate - .cacheExternalScore("NsfwTextProbability", scoreOptFut) - - modelName -> scoreOptFut - case _ => - wrongBucketingModelCounter.incr() - modelName -> Future.None - } - }.toMap - - val candidateLanguageBucketingThreshold = - bucketingThresholdPerLanguageSeq(indexOfCandidateLanguage) - - val userShouldBeBucketedFut: Future[Boolean] = - Future - .collect(bucketingModelScoreMap.map { - case (_, modelScoreOptFut) => - modelScoreOptFut.map { - case Some(score) => - increaseCounterForLanguage(nonEmptyHealthScoreMap, candidateLanguage) - score > candidateLanguageBucketingThreshold - case _ => - increaseCounterForLanguage(emptyHealthScoreMap, candidateLanguage) - false - } - }.toSeq).map(_.contains(true)) - - val candidateShouldBeFilteredFut: Future[Boolean] = userShouldBeBucketedFut.flatMap { - userShouldBeBucketed => - if (userShouldBeBucketed) { - increaseCounterForLanguage(bucketedCounterMap, candidateLanguage) - - val candidateLanguageFilteringThreshold = - filteringThresholdPerLanguageSeq(indexOfCandidateLanguage) - - bucketingModelScoreMap.get(targetNsfwTextDetectionModel) match { - case Some(scoreOptFut) => - scoreOptFut.map { - case Some(score) => - val candidateShouldBeFiltered = - score > candidateLanguageFilteringThreshold - if (candidateShouldBeFiltered) { - increaseCounterForLanguage(filteredCounterMap, candidateLanguage) - } - candidateShouldBeFiltered - case _ => false - } - case _ => - wrongDetectionModelCounter.incr() - Future.False - } - } else { - increaseCounterForLanguage(lowScoreCounterMap, candidateLanguage) - Future.False - } - } - candidateShouldBeFilteredFut.map(result => !result) - } else Future.True - } else Future.True - } - .withStats(stats.scope(s"predicate_$name")) - .withName(name) - } - - def authorProfileBasedPredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with TweetCandidate] = { - val name = "author_profile" - val statsScope = stats.scope(name) - val filterByNsfwToken = statsScope.counter("filter_by_nsfw_token") - val filterByAccountAge = statsScope.counter("filter_by_account_age") - - Predicate - .fromAsync { candidate: PushCandidate with TweetCandidate => - val target = candidate.target - candidate match { - case cand: PushCandidate with TweetAuthorDetails => - cand.tweetAuthor.map { - case Some(author) => - val nsfwTokens = target.params(PushFeatureSwitchParams.NsfwTokensParam) - val accountAgeInHours = - (Time.now - Time.fromMilliseconds(author.createdAtMsec)).inHours - val isNsfwAccount = CandidateHydrationUtil.isNsfwAccount(author, nsfwTokens) - val isVerified = author.safety.map(_.verified).getOrElse(false) - - if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && !isVerified) { - val enableNsfwTokenCheck = - target.params(PushFeatureSwitchParams.EnableNsfwTokenBasedFiltering) - val minimumAllowedAge = - target.params(PushFeatureSwitchParams.MinimumAllowedAuthorAccountAgeInHours) - cand.cachePredicateInfo( - name + "_nsfwToken", - if (isNsfwAccount) 1.0 else 0.0, - 0.0, - enableNsfwTokenCheck && isNsfwAccount) - cand.cachePredicateInfo( - name + "_authorAge", - accountAgeInHours, - minimumAllowedAge, - accountAgeInHours < minimumAllowedAge) - - if (enableNsfwTokenCheck && isNsfwAccount) { - filterByNsfwToken.incr() - false - } else if (accountAgeInHours < minimumAllowedAge) { - filterByAccountAge.incr() - false - } else true - } else true - case _ => true - } - case _ => Future.value(true) - } - } - .withStats(stats.scope(s"predicate_$name")) - .withName(name) - } - - def authorSensitiveMediaPredicate( - producerMediaRepresentationStore: ReadableStore[Long, UserMediaRepresentation] - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with TweetAuthor] = { - val name = "author_sensitive_media_mrtwistly" - val statsScope = stats.scope(name) - val enableQueryNum = statsScope.counter("enable_query") - val nonEmptyMediaRepresentationNum = statsScope.counter("non_empty_media_representation") - val filteredOON = statsScope.counter("filtered_oon") - - Predicate - .fromAsync { candidate: PushCandidate with TweetAuthor => - val target = candidate.target - val useAggressiveThresholds = CandidateUtil.useAggressiveHealthThresholds(candidate) - - if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && - RecTypes.isOutOfNetworkTweetRecType(candidate.commonRecType) && - target.params(PushFeatureSwitchParams.EnableQueryAuthorMediaRepresentationStore)) { - enableQueryNum.incr() - - candidate.authorId match { - case Some(authorId) => - producerMediaRepresentationStore.get(authorId).map { - case Some(mediaRepresentation) => - nonEmptyMediaRepresentationNum.incr() - val sumScore: Double = mediaRepresentation.mediaRepresentation.values.sum - val nudityScore: Double = mediaRepresentation.mediaRepresentation - .getOrElse(MediaAnnotationsUtil.nudityCategoryId, 0.0) - val nudityRate = if (sumScore > 0) nudityScore / sumScore else 0.0 - - candidate - .cacheExternalScore("AuthorNudityScore", Future.value(Some(nudityScore))) - candidate.cacheExternalScore("AuthorNudityRate", Future.value(Some(nudityRate))) - - val threshold = if (useAggressiveThresholds) { - target.params( - PushFeatureSwitchParams.AuthorSensitiveMediaFilteringThresholdForMrTwistly) - } else { - target.params(PushFeatureSwitchParams.AuthorSensitiveMediaFilteringThreshold) - } - candidate.cachePredicateInfo( - name, - nudityRate, - threshold, - nudityRate > threshold, - Some(Map[String, Double]("sumScore" -> sumScore, "nudityScore" -> nudityScore))) - - if (nudityRate > threshold) { - filteredOON.incr() - false - } else true - case _ => true - } - case _ => Future.True - } - } else { - Future.True - } - } - .withStats(stats.scope(s"predicate_$name")) - .withName(name) - } - - def sensitiveMediaCategoryPredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with TweetCandidate] = { - val name = "sensitive_media_category" - val tweetMediaAnnotationFeature = - "tweet.mediaunderstanding.tweet_annotations.sensitive_category_probabilities" - val scopedStatsReceiver = stats.scope(name) - val allCandidatesCounter = scopedStatsReceiver.counter("all_candidates") - val nonZeroNudityCandidatesCounter = scopedStatsReceiver.counter("non_zero_nudity_candidates") - val nudityScoreStats = scopedStatsReceiver.stat("nudity_scores") - - Predicate - .fromAsync { candidate: PushCandidate => - allCandidatesCounter.incr() - val target = candidate.target - val nudityScore = candidate.sparseContinuousFeatures - .getOrElse(tweetMediaAnnotationFeature, Map.empty[String, Double]).getOrElse( - MediaAnnotationsUtil.nudityCategoryId, - 0.0) - if (nudityScore > 0) nonZeroNudityCandidatesCounter.incr() - nudityScoreStats.add(nudityScore.toFloat) - val threshold = - target.params(PushFeatureSwitchParams.TweetMediaSensitiveCategoryThresholdParam) - candidate.cachePredicateInfo(name, nudityScore, threshold, nudityScore > threshold) - if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && nudityScore > threshold) { - Future.False - } else { - Future.True - } - } - .withStats(stats.scope(s"predicate_$name")) - .withName(name) - } - - def profanityPredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with TweetCandidate] = { - val name = "profanity_filter" - val scopedStatsReceiver = stats.scope(name) - val allCandidatesCounter = scopedStatsReceiver.counter("all_candidates") - - Predicate - .fromAsync { candidate: PushCandidate => - allCandidatesCounter.incr() - val target = candidate.target - - lazy val enableFilter = - target.params(PushFeatureSwitchParams.EnableProfanityFilterParam) - val tweetSemanticCoreIds = candidate.sparseBinaryFeatures - .getOrElse(PushConstants.TweetSemanticCoreIdFeature, Set.empty[String]) - - if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && - tweetSemanticCoreIds.contains(PushConstants.ProfanityFilter_Id) && enableFilter) { - Future.False - } else { - Future.True - } - } - .withStats(stats.scope(s"predicate_$name")) - .withName(name) - } - - def agathaAbusiveTweetAuthorPredicateMrTwistly( - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with OutOfNetworkTweetCandidate] = { - val name = "agatha_abusive_tweet_author_mr_twistly" - val scopedStatsReceiver = stats.scope(name) - val allCandidatesCounter = scopedStatsReceiver.counter("all_candidates") - val isMrBackfillCRCandidateCounter = scopedStatsReceiver.counter("isMrBackfillCR_candidates") - Predicate - .fromAsync { cand: PushCandidate with OutOfNetworkTweetCandidate => - allCandidatesCounter.incr() - val target = cand.target - val tweetSemanticCoreIds = cand.sparseBinaryFeatures - .getOrElse(PushConstants.TweetSemanticCoreIdFeature, Set.empty[String]) - - val hasAbuseStrikeTop2Percent = - tweetSemanticCoreIds.contains(PushConstants.AbuseStrike_Top2Percent_Id) - val hasAbuseStrikeTop1Percent = - tweetSemanticCoreIds.contains(PushConstants.AbuseStrike_Top1Percent_Id) - val hasAbuseStrikeTop05Percent = - tweetSemanticCoreIds.contains(PushConstants.AbuseStrike_Top05Percent_Id) - - if (hasAbuseStrikeTop2Percent) { - scopedStatsReceiver.counter("abuse_strike_top_2_percent_candidates").incr() - } - if (hasAbuseStrikeTop1Percent) { - scopedStatsReceiver.counter("abuse_strike_top_1_percent_candidates").incr() - } - if (hasAbuseStrikeTop05Percent) { - scopedStatsReceiver.counter("abuse_strike_top_05_percent_candidates").incr() - } - - if (CandidateUtil.shouldApplyHealthQualityFilters(cand) && cand.isMrBackfillCR.getOrElse( - false)) { - isMrBackfillCRCandidateCounter.incr() - if (hasAbuseStrikeTop2Percent) { - if (target.params( - PushFeatureSwitchParams.EnableAbuseStrikeTop2PercentFilterSimCluster) && hasAbuseStrikeTop2Percent || - target.params( - PushFeatureSwitchParams.EnableAbuseStrikeTop1PercentFilterSimCluster) && hasAbuseStrikeTop1Percent || - target.params( - PushFeatureSwitchParams.EnableAbuseStrikeTop05PercentFilterSimCluster) && hasAbuseStrikeTop05Percent) { - Future.False - } else { - Future.True - } - } else { - Future.True - } - } else Future.True - } - .withStats(stats.scope(s"predicate_$name")) - .withName(name) - } - - def userHealthSignalsPredicate( - userHealthSignalStore: ReadableStore[Long, UserHealthSignalResponse] - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with TweetDetails] = { - val name = "agatha_user_health_model_score" - val scopedStatsReceiver = stats.scope(name) - val allCandidatesCounter = scopedStatsReceiver.counter("all_candidates") - val bucketedUserCandidatesCounter = - scopedStatsReceiver.counter("bucketed_user_candidates") - val filteredOON = scopedStatsReceiver.counter("filtered_oon") - - Predicate - .fromAsync { candidate: PushCandidate with TweetDetails => - allCandidatesCounter.incr() - val target = candidate.target - val useAggressiveThresholds = CandidateUtil.useAggressiveHealthThresholds(candidate) - - if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && target.params( - PushFeatureSwitchParams.EnableAgathaUserHealthModelPredicate)) { - val healthSignalsResponseFutOpt: Future[Option[UserHealthSignalResponse]] = - candidate.authorId match { - case Some(authorId) => userHealthSignalStore.get(authorId) - case _ => Future.None - } - healthSignalsResponseFutOpt.map { - case Some(response) => - val agathaRecentAbuseStrikeScore: Double = userHealthSignalValueToDouble( - response.signalValues - .getOrElse(AgathaRecentAbuseStrikeDouble, SignalValue.DoubleValue(0.0))) - val agathaCalibratedNSFWScore: Double = userHealthSignalValueToDouble( - response.signalValues - .getOrElse(AgathaCalibratedNsfwDouble, SignalValue.DoubleValue(0.0))) - val agathaTextNSFWScore: Double = userHealthSignalValueToDouble(response.signalValues - .getOrElse(NsfwTextUserScoreDouble, SignalValue.DoubleValue(0.0))) - - candidate - .cacheExternalScore( - "agathaRecentAbuseStrikeScore", - Future.value(Some(agathaRecentAbuseStrikeScore))) - candidate - .cacheExternalScore( - "agathaCalibratedNSFWScore", - Future.value(Some(agathaCalibratedNSFWScore))) - candidate - .cacheExternalScore("agathaTextNSFWScore", Future.value(Some(agathaTextNSFWScore))) - - val NSFWShouldBucket = agathaCalibratedNSFWScore > target.params( - PushFeatureSwitchParams.AgathaCalibratedNSFWBucketThreshold) - val textNSFWShouldBucket = agathaTextNSFWScore > target.params( - PushFeatureSwitchParams.AgathaTextNSFWBucketThreshold) - - if (NSFWShouldBucket || textNSFWShouldBucket) { - bucketedUserCandidatesCounter.incr() - if (NSFWShouldBucket) { - scopedStatsReceiver.counter("calibrated_nsfw_bucketed_user_candidates").incr() - } - if (textNSFWShouldBucket) { - scopedStatsReceiver.counter("text_nsfw_bucketed_user_candidates").incr() - } - - val (thresholdAgathaNsfw, thresholdTextNsfw) = if (useAggressiveThresholds) { - ( - target.params( - PushFeatureSwitchParams.AgathaCalibratedNSFWThresholdForMrTwistly), - target - .params(PushFeatureSwitchParams.AgathaTextNSFWThresholdForMrTwistly)) - } else { - ( - target.params(PushFeatureSwitchParams.AgathaCalibratedNSFWThreshold), - target.params(PushFeatureSwitchParams.AgathaTextNSFWThreshold)) - } - candidate.cachePredicateInfo( - name + "_agathaNsfw", - agathaCalibratedNSFWScore, - thresholdAgathaNsfw, - agathaCalibratedNSFWScore > thresholdAgathaNsfw) - candidate.cachePredicateInfo( - name + "_authorTextNsfw", - agathaTextNSFWScore, - thresholdTextNsfw, - agathaTextNSFWScore > thresholdTextNsfw) - - if ((agathaCalibratedNSFWScore > thresholdAgathaNsfw) || - (agathaTextNSFWScore > thresholdTextNsfw)) { - filteredOON.incr() - false - } else true - } else { - true - } - case _ => true - } - } else { - Future.True - } - } - .withStats(stats.scope(s"predicate_$name")) - .withName(name) - } - - def userHealthSignalValueToDouble(signalValue: SignalValue): Double = { - signalValue match { - case SignalValue.DoubleValue(value) => value - case _ => throw new Exception(f"Could not convert signal value to double") - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/JointDauAndQualityModelPredicate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/JointDauAndQualityModelPredicate.docx new file mode 100644 index 000000000..133f68fa0 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/JointDauAndQualityModelPredicate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/JointDauAndQualityModelPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/JointDauAndQualityModelPredicate.scala deleted file mode 100644 index 63095f4db..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/JointDauAndQualityModelPredicate.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams.QualityPredicateIdParam -import com.twitter.frigate.pushservice.predicate.quality_model_predicate._ -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.util.Future - -object JointDauAndQualityModelPredicate { - - val name = "JointDauAndQualityModelPredicate" - - def apply()(implicit statsReceiver: StatsReceiver): NamedPredicate[PushCandidate] = { - val stats = statsReceiver.scope(s"predicate_$name") - - val defaultPred = WeightedOpenOrNtabClickQualityPredicate() - val qualityPredicateMap = QualityPredicateMap() - - Predicate - .fromAsync { candidate: PushCandidate => - if (!candidate.target.skipModelPredicate) { - - val modelPredicate = - qualityPredicateMap.getOrElse( - candidate.target.params(QualityPredicateIdParam), - defaultPred) - - val modelPredicateResultFut = - modelPredicate.apply(Seq(candidate)).map(_.headOption.getOrElse(false)) - - modelPredicateResultFut - } else Future.True - } - .withStats(stats) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ListPredicates.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ListPredicates.docx new file mode 100644 index 000000000..67b2a85a2 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ListPredicates.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ListPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ListPredicates.scala deleted file mode 100644 index cbfb670d8..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ListPredicates.scala +++ /dev/null @@ -1,110 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.model.ListRecommendationPushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.hermit.predicate.socialgraph.Edge -import com.twitter.hermit.predicate.socialgraph.RelationEdge -import com.twitter.hermit.predicate.socialgraph.SocialGraphPredicate -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.socialgraph.thriftscala.RelationshipType -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -object ListPredicates { - - def listNameExistsPredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[ListRecommendationPushCandidate] = { - Predicate - .fromAsync { candidate: ListRecommendationPushCandidate => - candidate.listName.map(_.isDefined) - } - .withStats(stats) - .withName("list_name_exists") - } - - def listAuthorExistsPredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[ListRecommendationPushCandidate] = { - Predicate - .fromAsync { candidate: ListRecommendationPushCandidate => - candidate.listOwnerId.map(_.isDefined) - } - .withStats(stats) - .withName("list_owner_exists") - } - - def listAuthorAcceptableToTargetUser( - edgeStore: ReadableStore[RelationEdge, Boolean] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[ListRecommendationPushCandidate] = { - val name = "list_author_acceptable_to_target_user" - val sgsPredicate = SocialGraphPredicate - .anyRelationExists( - edgeStore, - Set( - RelationshipType.Blocking, - RelationshipType.BlockedBy, - RelationshipType.Muting - ) - ) - .withStats(statsReceiver.scope("list_sgs_any_relation_exists")) - .withName("list_sgs_any_relation_exists") - - Predicate - .fromAsync { candidate: ListRecommendationPushCandidate => - candidate.listOwnerId.flatMap { - case Some(ownerId) => - sgsPredicate.apply(Seq(Edge(candidate.target.targetId, ownerId))).map(_.head) - case _ => Future.True - } - } - .withStats(statsReceiver.scope(s"predicate_$name")) - .withName(name) - } - - /** - * Checks if the list is acceptable to Target user => - * - Is Target not following the list - * - Is Target not muted the list - */ - def listAcceptablePredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[ListRecommendationPushCandidate] = { - val name = "list_acceptable_to_target_user" - Predicate - .fromAsync { candidate: ListRecommendationPushCandidate => - candidate.apiList.map { - case Some(apiList) => - !(apiList.following.contains(true) || apiList.muting.contains(true)) - case _ => false - } - } - .withStats(stats.scope(name)) - .withName(name) - } - - def listSubscriberCountPredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[ListRecommendationPushCandidate] = { - val name = "list_subscribe_count" - Predicate - .fromAsync { candidate: ListRecommendationPushCandidate => - candidate.apiList.map { apiListOpt => - apiListOpt.exists { apiList => - apiList.subscriberCount >= candidate.target.params( - PushFeatureSwitchParams.ListRecommendationsSubscriberCount) - } - } - } - .withStats(stats.scope(name)) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/LoggedOutPreRankingPredicates.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/LoggedOutPreRankingPredicates.docx new file mode 100644 index 000000000..9fd556985 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/LoggedOutPreRankingPredicates.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/LoggedOutPreRankingPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/LoggedOutPreRankingPredicates.scala deleted file mode 100644 index 9ba1c9f6f..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/LoggedOutPreRankingPredicates.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.common.base.TweetDetails -import com.twitter.frigate.common.predicate.tweet._ -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.hermit.predicate.NamedPredicate - -class LoggedOutPreRankingPredicatesBuilder(implicit statsReceiver: StatsReceiver) { - - private val TweetPredicates = List[NamedPredicate[PushCandidate]]( - TweetObjectExistsPredicate[ - TweetCandidate with TweetDetails - ].applyOnlyToTweetCandidatesWithTweetDetails - .withName("tweet_object_exists"), - PredicatesForCandidate.oldTweetRecsPredicate.applyOnlyToTweetCandidateWithTargetAndABDeciderAndMaxTweetAge - .withName("old_tweet"), - PredicatesForCandidate.tweetIsNotAreply.applyOnlyToTweetCandidateWithoutSocialContextWithTweetDetails - .withName("tweet_candidate_not_a_reply"), - TweetAuthorPredicates - .recTweetAuthorUnsuitable[TweetCandidate with TweetAuthorDetails] - .applyOnlyToTweetCandidateWithTweetAuthorDetails - .withName("tweet_author_unsuitable") - ) - - final def build(): List[NamedPredicate[PushCandidate]] = { - TweetPredicates - } - -} - -object LoggedOutPreRankingPredicates { - def apply(statsReceiver: StatsReceiver): List[NamedPredicate[PushCandidate]] = - new LoggedOutPreRankingPredicatesBuilder()(statsReceiver).build() -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/LoggedOutTargetPredicates.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/LoggedOutTargetPredicates.docx new file mode 100644 index 000000000..6f06803b5 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/LoggedOutTargetPredicates.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/LoggedOutTargetPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/LoggedOutTargetPredicates.scala deleted file mode 100644 index 085ad73e9..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/LoggedOutTargetPredicates.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.abdecider.GuestRecipient -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.common.predicate.{FatiguePredicate => CommonFatiguePredicate} -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.conversions.DurationOps._ -import com.twitter.frigate.common.util.Experiments.LoggedOutRecsHoldback -import com.twitter.hermit.predicate.Predicate - -object LoggedOutTargetPredicates { - - def targetFatiguePredicate[T <: Target]( - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[T] = { - val name = "logged_out_target_min_duration_since_push" - CommonFatiguePredicate - .magicRecsPushTargetFatiguePredicate( - minInterval = 24.hours, - maxInInterval = 1 - ).withStats(statsReceiver.scope(name)) - .withName(name) - } - - def loggedOutRecsHoldbackPredicate[T <: Target]( - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[T] = { - val name = "logged_out_recs_holdback" - val guestIdNotFoundCounter = statsReceiver.scope("logged_out").counter("guest_id_not_found") - val controlBucketCounter = statsReceiver.scope("logged_out").counter("holdback_control") - val allowTrafficCounter = statsReceiver.scope("logged_out").counter("allow_traffic") - Predicate.from { target: T => - val guestId = target.targetGuestId match { - case Some(guest) => guest - case _ => - guestIdNotFoundCounter.incr() - throw new IllegalStateException("guest_id_not_found") - } - target.abDecider - .bucket(LoggedOutRecsHoldback.exptName, GuestRecipient(guestId)).map(_.name) match { - case Some(LoggedOutRecsHoldback.control) => - controlBucketCounter.incr() - false - case _ => - allowTrafficCounter.incr() - true - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/MlModelsHoldbackExperimentPredicate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/MlModelsHoldbackExperimentPredicate.docx new file mode 100644 index 000000000..207b661e2 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/MlModelsHoldbackExperimentPredicate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/MlModelsHoldbackExperimentPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/MlModelsHoldbackExperimentPredicate.scala deleted file mode 100644 index 014393870..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/MlModelsHoldbackExperimentPredicate.scala +++ /dev/null @@ -1,71 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.util.Future - -object MlModelsHoldbackExperimentPredicate { - - val name = "MlModelsHoldbackExperimentPredicate" - - private val alwaysTruePred = PredicatesForCandidate.alwaysTruePushCandidatePredicate - - def getPredicateBasedOnCandidate( - pc: PushCandidate, - treatmentPred: Predicate[PushCandidate] - )( - implicit statsReceiver: StatsReceiver - ): Future[Predicate[PushCandidate]] = { - - Future - .join(Future.value(pc.target.skipFilters), pc.target.isInModelExclusionList) - .map { - case (skipFilters, isInModelExclusionList) => - if (skipFilters || - isInModelExclusionList || - pc.target.params(PushParams.DisableMlInFilteringParam) || - pc.target.params(PushFeatureSwitchParams.DisableMlInFilteringFeatureSwitchParam) || - pc.target.params(PushParams.DisableAllRelevanceParam) || - pc.target.params(PushParams.DisableHeavyRankingParam)) { - alwaysTruePred - } else { - treatmentPred - } - } - } - - def apply()(implicit statsReceiver: StatsReceiver): NamedPredicate[PushCandidate] = { - val stats = statsReceiver.scope(s"predicate_$name") - val statsProd = stats.scope("prod") - val counterAcceptedByModel = statsProd.counter("accepted") - val counterRejectedByModel = statsProd.counter("rejected") - val counterHoldback = stats.scope("holdback").counter("all") - val jointDauQualityPredicate = JointDauAndQualityModelPredicate() - - new Predicate[PushCandidate] { - def apply(items: Seq[PushCandidate]): Future[Seq[Boolean]] = { - val boolFuts = items.map { item => - getPredicateBasedOnCandidate(item, jointDauQualityPredicate)(statsReceiver) - .flatMap { predicate => - val predictionFut = predicate.apply(Seq(item)).map(_.headOption.getOrElse(false)) - predictionFut.foreach { prediction => - if (item.target.params(PushParams.DisableMlInFilteringParam) || item.target.params( - PushFeatureSwitchParams.DisableMlInFilteringFeatureSwitchParam)) { - counterHoldback.incr() - } else { - if (prediction) counterAcceptedByModel.incr() else counterRejectedByModel.incr() - } - } - predictionFut - } - } - Future.collect(boolFuts) - } - }.withStats(stats) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OONSpreadControlPredicate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OONSpreadControlPredicate.docx new file mode 100644 index 000000000..1d4b17741 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OONSpreadControlPredicate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OONSpreadControlPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OONSpreadControlPredicate.scala deleted file mode 100644 index bcd9e30d0..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OONSpreadControlPredicate.scala +++ /dev/null @@ -1,116 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushConstants._ -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.util.CandidateUtil -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.util.Future - -object OONSpreadControlPredicate { - - def oonTweetSpreadControlPredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[ - PushCandidate with TweetCandidate with RecommendationType - ] = { - val name = "oon_tweet_spread_control_predicate" - val scopedStatsReceiver = stats.scope(name) - val allOonCandidatesCounter = scopedStatsReceiver.counter("all_oon_candidates") - val filteredCandidatesCounter = - scopedStatsReceiver.counter("filtered_oon_candidates") - - Predicate - .fromAsync { candidate: PushCandidate with TweetCandidate with RecommendationType => - val target = candidate.target - val crt = candidate.commonRecType - val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(crt) || - RecTypes.outOfNetworkTopicTweetTypes.contains(crt) - - lazy val minTweetSendsThreshold = - target.params(PushFeatureSwitchParams.MinTweetSendsThresholdParam) - lazy val spreadControlRatio = - target.params(PushFeatureSwitchParams.SpreadControlRatioParam) - lazy val favOverSendThreshold = - target.params(PushFeatureSwitchParams.FavOverSendThresholdParam) - - lazy val sentCount = candidate.numericFeatures.getOrElse(sentFeatureName, 0.0) - lazy val followerCount = - candidate.numericFeatures.getOrElse(authorActiveFollowerFeatureName, 0.0) - lazy val favCount = candidate.numericFeatures.getOrElse(favFeatureName, 0.0) - lazy val favOverSends = favCount / (sentCount + 1.0) - - if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && isOonCandidate) { - allOonCandidatesCounter.incr() - if (sentCount > minTweetSendsThreshold && - sentCount > spreadControlRatio * followerCount && - favOverSends < favOverSendThreshold) { - filteredCandidatesCounter.incr() - Future.False - } else Future.True - } else Future.True - } - .withStats(stats.scope(name)) - .withName(name) - } - - def oonAuthorSpreadControlPredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[ - PushCandidate with TweetCandidate with RecommendationType - ] = { - val name = "oon_author_spread_control_predicate" - val scopedStatsReceiver = stats.scope(name) - val allOonCandidatesCounter = scopedStatsReceiver.counter("all_oon_candidates") - val filteredCandidatesCounter = - scopedStatsReceiver.counter("filtered_oon_candidates") - - Predicate - .fromAsync { candidate: PushCandidate with TweetCandidate with RecommendationType => - val target = candidate.target - val crt = candidate.commonRecType - val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(crt) || - RecTypes.outOfNetworkTopicTweetTypes.contains(crt) - - lazy val minAuthorSendsThreshold = - target.params(PushFeatureSwitchParams.MinAuthorSendsThresholdParam) - lazy val spreadControlRatio = - target.params(PushFeatureSwitchParams.SpreadControlRatioParam) - lazy val reportRateThreshold = - target.params(PushFeatureSwitchParams.AuthorReportRateThresholdParam) - lazy val dislikeRateThreshold = - target.params(PushFeatureSwitchParams.AuthorDislikeRateThresholdParam) - - lazy val authorSentCount = - candidate.numericFeatures.getOrElse(authorSendCountFeatureName, 0.0) - lazy val authorReportCount = - candidate.numericFeatures.getOrElse(authorReportCountFeatureName, 0.0) - lazy val authorDislikeCount = - candidate.numericFeatures.getOrElse(authorDislikeCountFeatureName, 0.0) - lazy val followerCount = candidate.numericFeatures - .getOrElse(authorActiveFollowerFeatureName, 0.0) - lazy val reportRate = - authorReportCount / (authorSentCount + 1.0) - lazy val dislikeRate = - authorDislikeCount / (authorSentCount + 1.0) - - if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && isOonCandidate) { - allOonCandidatesCounter.incr() - if (authorSentCount > minAuthorSendsThreshold && - authorSentCount > spreadControlRatio * followerCount && - (reportRate > reportRateThreshold || dislikeRate > dislikeRateThreshold)) { - filteredCandidatesCounter.incr() - Future.False - } else Future.True - } else Future.True - } - .withStats(stats.scope(name)) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OONTweetNegativeFeedbackBasedPredicate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OONTweetNegativeFeedbackBasedPredicate.docx new file mode 100644 index 000000000..3318da1a6 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OONTweetNegativeFeedbackBasedPredicate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OONTweetNegativeFeedbackBasedPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OONTweetNegativeFeedbackBasedPredicate.scala deleted file mode 100644 index 3efb23d88..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OONTweetNegativeFeedbackBasedPredicate.scala +++ /dev/null @@ -1,82 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.util.CandidateUtil -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.util.Future - -object OONTweetNegativeFeedbackBasedPredicate { - - def ntabDislikeBasedPredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[ - PushCandidate with TweetCandidate with RecommendationType - ] = { - val name = "oon_tweet_dislike_based_predicate" - val scopedStatsReceiver = stats.scope(name) - val allOonCandidatesCounter = scopedStatsReceiver.counter("all_oon_candidates") - val oonCandidatesImpressedCounter = - scopedStatsReceiver.counter("oon_candidates_impressed") - val filteredCandidatesCounter = - scopedStatsReceiver.counter("filtered_oon_candidates") - - val ntabDislikeCountFeature = - "tweet.magic_recs_tweet_real_time_aggregates_v2.pair.v2.magicrecs.realtime.is_ntab_disliked.any_feature.Duration.Top.count" - val sentFeature = - "tweet.magic_recs_tweet_real_time_aggregates_v2.pair.v2.magicrecs.realtime.is_sent.any_feature.Duration.Top.count" - - Predicate - .fromAsync { candidate: PushCandidate with TweetCandidate with RecommendationType => - val target = candidate.target - val crt = candidate.commonRecType - val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(crt) || - RecTypes.outOfNetworkTopicTweetTypes.contains(crt) - - lazy val ntabDislikeCountThreshold = - target.params(PushFeatureSwitchParams.TweetNtabDislikeCountThresholdParam) - lazy val ntabDislikeRateThreshold = - target.params(PushFeatureSwitchParams.TweetNtabDislikeRateThresholdParam) - lazy val ntabDislikeCountThresholdForMrTwistly = - target.params(PushFeatureSwitchParams.TweetNtabDislikeCountThresholdForMrTwistlyParam) - lazy val ntabDislikeRateThresholdForMrTwistly = - target.params(PushFeatureSwitchParams.TweetNtabDislikeRateThresholdForMrTwistlyParam) - - val isMrTwistly = CandidateUtil.isMrTwistlyCandidate(candidate) - - lazy val dislikeCount = candidate.numericFeatures.getOrElse(ntabDislikeCountFeature, 0.0) - lazy val sentCount = candidate.numericFeatures.getOrElse(sentFeature, 0.0) - lazy val dislikeRate = if (sentCount > 0) dislikeCount / sentCount else 0.0 - - if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && isOonCandidate) { - allOonCandidatesCounter.incr() - val (countThreshold, rateThreshold) = if (isMrTwistly) { - (ntabDislikeCountThresholdForMrTwistly, ntabDislikeRateThresholdForMrTwistly) - } else { - (ntabDislikeCountThreshold, ntabDislikeRateThreshold) - } - candidate.cachePredicateInfo( - name + "_count", - dislikeCount, - countThreshold, - dislikeCount > countThreshold) - candidate.cachePredicateInfo( - name + "_rate", - dislikeRate, - rateThreshold, - dislikeRate > rateThreshold) - if (dislikeCount > countThreshold && dislikeRate > rateThreshold) { - filteredCandidatesCounter.incr() - Future.False - } else Future.True - } else Future.True - } - .withStats(stats.scope(name)) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OutOfNetworkCandidatesQualityPredicates.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OutOfNetworkCandidatesQualityPredicates.docx new file mode 100644 index 000000000..46c43f0cd Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OutOfNetworkCandidatesQualityPredicates.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OutOfNetworkCandidatesQualityPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OutOfNetworkCandidatesQualityPredicates.scala deleted file mode 100644 index 6f09df0c7..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OutOfNetworkCandidatesQualityPredicates.scala +++ /dev/null @@ -1,221 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.candidate.TargetABDecider -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.data_pipeline.features_common.MrRequestContextForFeatureStore -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.ml.featurestore.lib.dynamic.DynamicFeatureStoreClient -import com.twitter.util.Future -import com.twitter.frigate.pushservice.predicate.PostRankingPredicateHelper._ -import com.twitter.frigate.pushservice.util.CandidateUtil - -object OutOfNetworkCandidatesQualityPredicates { - - def getTweetCharLengthThreshold( - target: TargetUser with TargetABDecider, - language: String, - useMediaThresholds: Boolean - ): Double = { - lazy val sautOonWithMediaTweetLengthThreshold = - target.params(PushFeatureSwitchParams.SautOonWithMediaTweetLengthThresholdParam) - lazy val nonSautOonWithMediaTweetLengthThreshold = - target.params(PushFeatureSwitchParams.NonSautOonWithMediaTweetLengthThresholdParam) - lazy val sautOonWithoutMediaTweetLengthThreshold = - target.params(PushFeatureSwitchParams.SautOonWithoutMediaTweetLengthThresholdParam) - lazy val nonSautOonWithoutMediaTweetLengthThreshold = - target.params(PushFeatureSwitchParams.NonSautOonWithoutMediaTweetLengthThresholdParam) - val moreStrictForUndefinedLanguages = - target.params(PushFeatureSwitchParams.OonTweetLengthPredicateMoreStrictForUndefinedLanguages) - val isSautLanguage = if (moreStrictForUndefinedLanguages) { - isTweetLanguageInSautOrUndefined(language) - } else isTweetLanguageInSaut(language) - - (useMediaThresholds, isSautLanguage) match { - case (true, true) => - sautOonWithMediaTweetLengthThreshold - case (true, false) => - nonSautOonWithMediaTweetLengthThreshold - case (false, true) => - sautOonWithoutMediaTweetLengthThreshold - case (false, false) => - nonSautOonWithoutMediaTweetLengthThreshold - case _ => -1 - } - } - - def getTweetWordLengthThreshold( - target: TargetUser with TargetABDecider, - language: String, - useMediaThresholds: Boolean - ): Double = { - lazy val argfOonWithMediaTweetWordLengthThresholdParam = - target.params(PushFeatureSwitchParams.ArgfOonWithMediaTweetWordLengthThresholdParam) - lazy val esfthOonWithMediaTweetWordLengthThresholdParam = - target.params(PushFeatureSwitchParams.EsfthOonWithMediaTweetWordLengthThresholdParam) - - lazy val argfOonCandidatesWithMediaCondition = - isTweetLanguageInArgf(language) && useMediaThresholds - lazy val esfthOonCandidatesWithMediaCondition = - isTweetLanguageInEsfth(language) && useMediaThresholds - lazy val afirfOonCandidatesWithoutMediaCondition = - isTweetLanguageInAfirf(language) && !useMediaThresholds - - val afirfOonCandidatesWithoutMediaTweetWordLengthThreshold = 5 - if (argfOonCandidatesWithMediaCondition) { - argfOonWithMediaTweetWordLengthThresholdParam - } else if (esfthOonCandidatesWithMediaCondition) { - esfthOonWithMediaTweetWordLengthThresholdParam - } else if (afirfOonCandidatesWithoutMediaCondition) { - afirfOonCandidatesWithoutMediaTweetWordLengthThreshold - } else -1 - } - - def oonTweetLengthBasedPrerankingPredicate( - characterBased: Boolean - )( - implicit stats: StatsReceiver - ): NamedPredicate[OutOfNetworkTweetCandidate with TargetInfo[ - TargetUser with TargetABDecider - ]] = { - val name = "oon_tweet_length_based_preranking_predicate" - val scopedStats = stats.scope(s"${name}_charBased_$characterBased") - - Predicate - .fromAsync { - cand: OutOfNetworkTweetCandidate with TargetInfo[TargetUser with TargetABDecider] => - cand match { - case candidate: TweetAuthorDetails => - val target = candidate.target - val crt = candidate.commonRecType - - val updatedMediaLogic = - target.params(PushFeatureSwitchParams.OonTweetLengthPredicateUpdatedMediaLogic) - val updatedQuoteTweetLogic = - target.params(PushFeatureSwitchParams.OonTweetLengthPredicateUpdatedQuoteTweetLogic) - val useMediaThresholds = if (updatedMediaLogic || updatedQuoteTweetLogic) { - val hasMedia = updatedMediaLogic && (candidate.hasPhoto || candidate.hasVideo) - val hasQuoteTweet = updatedQuoteTweetLogic && candidate.quotedTweet.nonEmpty - hasMedia || hasQuoteTweet - } else RecTypes.isMediaType(crt) - val enableFilter = - target.params(PushFeatureSwitchParams.EnablePrerankingTweetLengthPredicate) - - val language = candidate.tweet.flatMap(_.language.map(_.language)).getOrElse("") - val tweetTextOpt = candidate.tweet.flatMap(_.coreData.map(_.text)) - - val (length: Double, threshold: Double) = if (characterBased) { - ( - tweetTextOpt.map(_.size.toDouble).getOrElse(9999.0), - getTweetCharLengthThreshold(target, language, useMediaThresholds)) - } else { - ( - tweetTextOpt.map(getTweetWordLength).getOrElse(999.0), - getTweetWordLengthThreshold(target, language, useMediaThresholds)) - } - scopedStats.counter("threshold_" + threshold.toString).incr() - - CandidateUtil.shouldApplyHealthQualityFiltersForPrerankingPredicates(candidate).map { - case true if enableFilter => - length > threshold - case _ => true - } - case _ => - scopedStats.counter("author_is_not_hydrated").incr() - Future.True - } - }.withStats(scopedStats) - .withName(name) - } - - private def isTweetLanguageInAfirf(candidateLanguage: String): Boolean = { - val setAFIRF: Set[String] = Set("") - setAFIRF.contains(candidateLanguage) - } - private def isTweetLanguageInEsfth(candidateLanguage: String): Boolean = { - val setESFTH: Set[String] = Set("") - setESFTH.contains(candidateLanguage) - } - private def isTweetLanguageInArgf(candidateLanguage: String): Boolean = { - val setARGF: Set[String] = Set("") - setARGF.contains(candidateLanguage) - } - - private def isTweetLanguageInSaut(candidateLanguage: String): Boolean = { - val setSAUT = Set("") - setSAUT.contains(candidateLanguage) - } - - private def isTweetLanguageInSautOrUndefined(candidateLanguage: String): Boolean = { - val setSautOrUndefined = Set("") - setSautOrUndefined.contains(candidateLanguage) - } - - def containTargetNegativeKeywords(text: String, denylist: Seq[String]): Boolean = { - if (denylist.isEmpty) - false - else { - denylist - .map { negativeKeyword => - text.toLowerCase().contains(negativeKeyword) - }.reduce(_ || _) - } - } - - def NegativeKeywordsPredicate( - postRankingFeatureStoreClient: DynamicFeatureStoreClient[MrRequestContextForFeatureStore] - )( - implicit stats: StatsReceiver - ): NamedPredicate[ - PushCandidate with TweetCandidate with RecommendationType - ] = { - - val name = "negative_keywords_predicate" - val scopedStatsReceiver = stats.scope(name) - val allOonCandidatesCounter = scopedStatsReceiver.counter("all_oon_candidates") - val filteredOonCandidatesCounter = scopedStatsReceiver.counter("filtered_oon_candidates") - val tweetLanguageFeature = "RecTweet.TweetyPieResult.Language" - - Predicate - .fromAsync { candidate: PushCandidate with TweetCandidate with RecommendationType => - val target = candidate.target - val crt = candidate.commonRecType - val isTwistlyCandidate = RecTypes.twistlyTweets.contains(crt) - - lazy val enableNegativeKeywordsPredicateParam = - target.params(PushFeatureSwitchParams.EnableNegativeKeywordsPredicateParam) - lazy val negativeKeywordsPredicateDenylist = - target.params(PushFeatureSwitchParams.NegativeKeywordsPredicateDenylist) - lazy val candidateLanguage = - candidate.categoricalFeatures.getOrElse(tweetLanguageFeature, "") - - if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && candidateLanguage.equals( - "en") && isTwistlyCandidate && enableNegativeKeywordsPredicateParam) { - allOonCandidatesCounter.incr() - - val tweetTextFuture: Future[String] = - getTweetText(candidate, postRankingFeatureStoreClient) - - tweetTextFuture.map { tweetText => - val containsNegativeWords = - containTargetNegativeKeywords(tweetText, negativeKeywordsPredicateDenylist) - candidate.cachePredicateInfo( - name, - if (containsNegativeWords) 1.0 else 0.0, - 0.0, - containsNegativeWords) - if (containsNegativeWords) { - filteredOonCandidatesCounter.incr() - false - } else true - } - } else Future.True - } - .withStats(stats.scope(name)) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PNegMultimodalPredicates.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PNegMultimodalPredicates.docx new file mode 100644 index 000000000..4e5740c8a Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PNegMultimodalPredicates.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PNegMultimodalPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PNegMultimodalPredicates.scala deleted file mode 100644 index f838d7ae6..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PNegMultimodalPredicates.scala +++ /dev/null @@ -1,83 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.abuse.detection.scoring.thriftscala.Model -import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringRequest -import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringResponse -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.util.CandidateUtil -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -object PNegMultimodalPredicates { - - def healthSignalScorePNegMultimodalPredicate( - tweetHealthScoreStore: ReadableStore[TweetScoringRequest, TweetScoringResponse] - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with TweetCandidate] = { - val name = "pneg_multimodal_predicate" - val statsScope = stats.scope(name) - val oonCandidatesCounter = statsScope.counter("oon_candidates") - val nonEmptyModelScoreCounter = statsScope.counter("non_empty_model_score") - val bucketedCounter = statsScope.counter("bucketed_oon_candidates") - val filteredCounter = statsScope.counter("filtered_oon_candidates") - - Predicate - .fromAsync { candidate: PushCandidate with TweetCandidate => - val target = candidate.target - val crt = candidate.commonRecType - val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(crt) || - RecTypes.outOfNetworkTopicTweetTypes.contains(crt) - - lazy val enablePNegMultimodalPredicateParam = - target.params(PushFeatureSwitchParams.EnablePNegMultimodalPredicateParam) - lazy val pNegMultimodalPredicateModelThresholdParam = - target.params(PushFeatureSwitchParams.PNegMultimodalPredicateModelThresholdParam) - lazy val pNegMultimodalPredicateBucketThresholdParam = - target.params(PushFeatureSwitchParams.PNegMultimodalPredicateBucketThresholdParam) - val pNegMultimodalEnabledForF1Tweets = - target.params(PushParams.EnablePnegMultimodalPredictionForF1Tweets) - - if (CandidateUtil.shouldApplyHealthQualityFilters( - candidate) && (isOonCandidate || pNegMultimodalEnabledForF1Tweets) && enablePNegMultimodalPredicateParam) { - - val pNegMultimodalRequest = TweetScoringRequest(candidate.tweetId, Model.PNegMultimodal) - tweetHealthScoreStore.get(pNegMultimodalRequest).map { - case Some(tweetScoringResponse) => - nonEmptyModelScoreCounter.incr() - - val pNegMultimodalScore = 1.0 - tweetScoringResponse.score - - candidate - .cacheExternalScore("PNegMultimodalScore", Future.value(Some(pNegMultimodalScore))) - - if (isOonCandidate) { - oonCandidatesCounter.incr() - - if (pNegMultimodalScore > pNegMultimodalPredicateBucketThresholdParam) { - bucketedCounter.incr() - if (pNegMultimodalScore > pNegMultimodalPredicateModelThresholdParam) { - filteredCounter.incr() - false - } else true - } else true - } else { - true - } - case _ => true - } - } else { - Future.True - } - } - .withStats(stats.scope(name)) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PostRankingPredicateHelper.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PostRankingPredicateHelper.docx new file mode 100644 index 000000000..f9e66ac3b Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PostRankingPredicateHelper.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PostRankingPredicateHelper.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PostRankingPredicateHelper.scala deleted file mode 100644 index 604f7b07c..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PostRankingPredicateHelper.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.frigate.common.base._ -import com.twitter.frigate.data_pipeline.features_common.MrRequestContextForFeatureStore -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.ml.featurestore.catalog.entities.core.Tweet -import com.twitter.ml.featurestore.catalog.features.core.Tweet.Text -import com.twitter.ml.featurestore.lib.TweetId -import com.twitter.ml.featurestore.lib.dynamic.DynamicFeatureStoreClient -import com.twitter.ml.featurestore.lib.online.FeatureStoreRequest -import com.twitter.util.Future - -object PostRankingPredicateHelper { - - val tweetTextFeature = "tweet.core.tweet.text" - - def getTweetText( - candidate: PushCandidate with TweetCandidate, - dynamicClient: DynamicFeatureStoreClient[MrRequestContextForFeatureStore] - ): Future[String] = { - if (candidate.categoricalFeatures.contains(tweetTextFeature)) { - Future.value(candidate.categoricalFeatures.getOrElse(tweetTextFeature, "")) - } else { - val candidateTweetEntity = Tweet.withId(TweetId(candidate.tweetId)) - val featureStoreRequests = Seq( - FeatureStoreRequest( - entityIds = Seq(candidateTweetEntity) - )) - val predictionRecords = dynamicClient( - featureStoreRequests, - requestContext = candidate.target.mrRequestContextForFeatureStore) - - predictionRecords.map { records => - val tweetText = records.head - .getFeatureValue(candidateTweetEntity, Text).getOrElse( - "" - ) - candidate.categoricalFeatures(tweetTextFeature) = tweetText - tweetText - } - } - } - - def getTweetWordLength(tweetText: String): Double = { - val tweetTextWithoutUrl: String = - tweetText.replaceAll("https?://\\S+\\s?", "").replaceAll("[\\s]+", " ") - tweetTextWithoutUrl.trim().split(" ").length.toDouble - } - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PreRankingPredicates.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PreRankingPredicates.docx new file mode 100644 index 000000000..741684b47 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PreRankingPredicates.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PreRankingPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PreRankingPredicates.scala deleted file mode 100644 index 4b61b23e3..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PreRankingPredicates.scala +++ /dev/null @@ -1,158 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.SocialContextActions -import com.twitter.frigate.common.base.SocialContextUserDetails -import com.twitter.frigate.common.base.TargetInfo -import com.twitter.frigate.common.base.TargetUser -import com.twitter.frigate.common.base.TweetAuthor -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.common.base.TweetDetails -import com.twitter.frigate.common.candidate.FrigateHistory -import com.twitter.frigate.common.candidate.TargetABDecider -import com.twitter.frigate.common.candidate.TweetImpressionHistory -import com.twitter.frigate.common.predicate.socialcontext.{Predicates => SocialContextPredicates, _} -import com.twitter.frigate.common.predicate.tweet._ -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue.NtabCaretClickContFnFatiguePredicate -import com.twitter.hermit.predicate.NamedPredicate - -class PreRankingPredicatesBuilder( -)( - implicit statsReceiver: StatsReceiver) { - - private val SocialProofPredicates = List[NamedPredicate[PushCandidate]]( - SocialContextPredicates - .authorInSocialContext() - .applyOnlyToTweetAuthorWithSocialContextActions - .withName("author_social_context"), - SocialContextPredicates - .selfInSocialContext[TargetUser, SocialContextActions with TargetInfo[TargetUser]]() - .applyOnlyToSocialContextActionsWithTargetUser - .withName("self_social_context"), - SocialContextPredicates - .duplicateSocialContext[SocialContextActions]() - .applyOnlyToSocialContextActions - .withName("duplicate_social_context"), - SocialContextPredicates - .socialContextProtected[SocialContextUserDetails]() - .applyOnlyToSocialContextUserDetails - .withName("social_context_protected"), - SocialContextPredicates - .socialContextUnsuitable[SocialContextUserDetails]() - .applyOnlyToSocialContextUserDetails - .withName("social_context_unsuitable"), - SocialContextPredicates - .socialContextBlink[SocialContextUserDetails]() - .applyOnlyToSocialContextUserDetails - .withName("social_context_blink") - ) - - private val CommonPredicates = List[NamedPredicate[PushCandidate]]( - PredicatesForCandidate.candidateEnabledForEmailPredicate(), - PredicatesForCandidate.openAppExperimentUserCandidateAllowList(statsReceiver) - ) - - private val TweetPredicates = List[NamedPredicate[PushCandidate]]( - PredicatesForCandidate.tweetCandidateWithLessThan2SocialContextsIsAReply.applyOnlyToTweetCandidatesWithSocialContextActions - .withName("tweet_candidate_with_less_than_2_social_contexts_is_not_a_reply"), - PredicatesForCandidate.filterOONCandidatePredicate(), - PredicatesForCandidate.oldTweetRecsPredicate.applyOnlyToTweetCandidateWithTargetAndABDeciderAndMaxTweetAge - .withName("old_tweet"), - DuplicatePushTweetPredicate - .apply[ - TargetUser with FrigateHistory, - TweetCandidate with TargetInfo[TargetUser with FrigateHistory] - ] - .applyOnlyToTweetCandidateWithTargetAndFrigateHistory - .withName("duplicate_push_tweet"), - DuplicateEmailTweetPredicate - .apply[ - TargetUser with FrigateHistory, - TweetCandidate with TargetInfo[TargetUser with FrigateHistory] - ] - .applyOnlyToTweetCandidateWithTargetAndFrigateHistory - .withName("duplicate_email_tweet"), - TweetAuthorPredicates - .recTweetAuthorUnsuitable[TweetCandidate with TweetAuthorDetails] - .applyOnlyToTweetCandidateWithTweetAuthorDetails - .withName("tweet_author_unsuitable"), - TweetObjectExistsPredicate[ - TweetCandidate with TweetDetails - ].applyOnlyToTweetCandidatesWithTweetDetails - .withName("tweet_object_exists"), - TweetImpressionPredicate[ - TargetUser with TweetImpressionHistory, - TweetCandidate with TargetInfo[TargetUser with TweetImpressionHistory] - ].applyOnlyToTweetCandidateWithTargetAndTweetImpressionHistory - .withStats(statsReceiver.scope("tweet_impression")) - .withName("tweet_impression"), - SelfTweetPredicate[ - TargetUser, - TweetAuthor with TargetInfo[TargetUser]]().applyOnlyToTweetAuthorWithTargetInfo - .withName("self_author"), - PredicatesForCandidate.tweetIsNotAreply.applyOnlyToTweetCandidateWithoutSocialContextWithTweetDetails - .withName("tweet_candidate_not_a_reply"), - PredicatesForCandidate.f1CandidateIsNotAReply.applyOnlyToF1CandidateWithTargetAndABDecider - .withName("f1_candidate_is_not_a_reply"), - PredicatesForCandidate.outOfNetworkTweetCandidateIsNotAReply.applyOnlyToOutOfNetworkTweetCandidateWithTargetAndABDecider - .withName("out_of_network_tweet_candidate_is_not_a_reply"), - PredicatesForCandidate.outOfNetworkTweetCandidateEnabledCrTag.applyOnlyToOutOfNetworkTweetCandidateWithTargetAndABDecider - .withName("out_of_network_tweet_candidate_enabled_crtag"), - PredicatesForCandidate.outOfNetworkTweetCandidateEnabledCrtGroup.applyOnlyToOutOfNetworkTweetCandidateWithTargetAndABDecider - .withName("out_of_network_tweet_candidate_enabled_crt_group"), - OutOfNetworkCandidatesQualityPredicates - .oonTweetLengthBasedPrerankingPredicate(characterBased = true) - .applyOnlyToOutOfNetworkTweetCandidateWithTargetAndABDecider - .withName("oon_tweet_char_length_too_short"), - OutOfNetworkCandidatesQualityPredicates - .oonTweetLengthBasedPrerankingPredicate(characterBased = false) - .applyOnlyToOutOfNetworkTweetCandidateWithTargetAndABDecider - .withName("oon_tweet_word_length_too_short"), - PredicatesForCandidate - .protectedTweetF1ExemptPredicate[ - TargetUser with TargetABDecider, - TweetCandidate with TweetAuthorDetails with TargetInfo[ - TargetUser with TargetABDecider - ] - ] - .applyOnlyToTweetCandidateWithAuthorDetailsWithTargetABDecider - .withName("f1_exempt_tweet_author_protected"), - ) - - private val SgsPreRankingPredicates = List[NamedPredicate[PushCandidate]]( - SGSPredicatesForCandidate.authorBeingFollowed.applyOnlyToAuthorBeingFollowPredicates - .withName("author_not_being_followed"), - SGSPredicatesForCandidate.authorNotBeingDeviceFollowed.applyOnlyToBasicTweetPredicates - .withName("author_being_device_followed"), - SGSPredicatesForCandidate.recommendedTweetAuthorAcceptableToTargetUser.applyOnlyToBasicTweetPredicates - .withName("recommended_tweet_author_not_acceptable_to_target_user"), - SGSPredicatesForCandidate.disableInNetworkTweetPredicate.applyOnlyToBasicTweetPredicates - .withName("enable_in_network_tweet"), - SGSPredicatesForCandidate.disableOutNetworkTweetPredicate.applyOnlyToBasicTweetPredicates - .withName("enable_out_network_tweet") - ) - - private val SeeLessOftenPredicates = List[NamedPredicate[PushCandidate]]( - NtabCaretClickContFnFatiguePredicate - .ntabCaretClickContFnFatiguePredicates( - ) - .withName("seelessoften_cont_fn_fatigue") - ) - - final def build(): List[NamedPredicate[PushCandidate]] = { - TweetPredicates ++ - CommonPredicates ++ - SocialProofPredicates ++ - SgsPreRankingPredicates ++ - SeeLessOftenPredicates - } -} - -object PreRankingPredicates { - def apply( - statsReceiver: StatsReceiver - ): List[NamedPredicate[PushCandidate]] = - new PreRankingPredicatesBuilder()(statsReceiver).build() -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PredicatesForCandidate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PredicatesForCandidate.docx new file mode 100644 index 000000000..d1e639b29 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PredicatesForCandidate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PredicatesForCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PredicatesForCandidate.scala deleted file mode 100644 index e18667b51..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PredicatesForCandidate.scala +++ /dev/null @@ -1,874 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.candidate.MaxTweetAge -import com.twitter.frigate.common.candidate.TargetABDecider -import com.twitter.frigate.common.predicate.tweet.TweetAuthorPredicates -import com.twitter.frigate.common.predicate._ -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.common.util.SnowflakeUtils -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.util.CandidateUtil -import com.twitter.frigate.thriftscala.ChannelName -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.gizmoduck.thriftscala.UserType -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.hermit.predicate.gizmoduck._ -import com.twitter.hermit.predicate.socialgraph.Edge -import com.twitter.hermit.predicate.socialgraph.MultiEdge -import com.twitter.hermit.predicate.socialgraph.RelationEdge -import com.twitter.hermit.predicate.socialgraph.SocialGraphPredicate -import com.twitter.service.metastore.gen.thriftscala.Location -import com.twitter.socialgraph.thriftscala.RelationshipType -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration -import com.twitter.util.Future - -object PredicatesForCandidate { - - def oldTweetRecsPredicate(implicit stats: StatsReceiver): Predicate[ - TweetCandidate with RecommendationType with TargetInfo[ - TargetUser with TargetABDecider with MaxTweetAge - ] - ] = { - val name = "old_tweet" - Predicate - .from[TweetCandidate with RecommendationType with TargetInfo[ - TargetUser with TargetABDecider with MaxTweetAge - ]] { candidate => - { - val crt = candidate.commonRecType - val defaultAge = if (RecTypes.mrModelingBasedTypes.contains(crt)) { - candidate.target.params(PushFeatureSwitchParams.ModelingBasedCandidateMaxTweetAgeParam) - } else if (RecTypes.GeoPopTweetTypes.contains(crt)) { - candidate.target.params(PushFeatureSwitchParams.GeoPopTweetMaxAgeInHours) - } else if (RecTypes.simclusterBasedTweets.contains(crt)) { - candidate.target.params( - PushFeatureSwitchParams.SimclusterBasedCandidateMaxTweetAgeParam) - } else if (RecTypes.detopicTypes.contains(crt)) { - candidate.target.params(PushFeatureSwitchParams.DetopicBasedCandidateMaxTweetAgeParam) - } else if (RecTypes.f1FirstDegreeTypes.contains(crt)) { - candidate.target.params(PushFeatureSwitchParams.F1CandidateMaxTweetAgeParam) - } else if (crt == CommonRecommendationType.ExploreVideoTweet) { - candidate.target.params(PushFeatureSwitchParams.ExploreVideoTweetAgeParam) - } else - candidate.target.params(PushFeatureSwitchParams.MaxTweetAgeParam) - SnowflakeUtils.isRecent(candidate.tweetId, defaultAge) - } - } - .withStats(stats.scope(name)) - .withName(name) - } - - def tweetIsNotAreply( - implicit stats: StatsReceiver - ): NamedPredicate[TweetCandidate with TweetDetails] = { - val name = "tweet_candidate_not_a_reply" - Predicate - .from[TweetCandidate with TweetDetails] { c => - c.isReply match { - case Some(true) => false - case _ => true - } - } - .withStats(stats.scope(name)) - .withName(name) - } - - /** - * Check if tweet contains any optouted free form interests. - * Currently, we use it for media categories and semantic core - * @param stats - * @return - */ - def noOptoutFreeFormInterestPredicate( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "free_form_interest_opt_out" - val tweetMediaAnnotationFeature = - "tweet.mediaunderstanding.tweet_annotations.safe_category_probabilities" - val tweetSemanticCoreFeature = - "tweet.core.tweet.semantic_core_annotations" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - val withOptOutFreeFormInterestsCounter = stats.counter("with_optout_interests") - val withoutOptOutInterestsCounter = stats.counter("without_optout_interests") - val withOptOutFreeFormInterestsFromMediaAnnotationCounter = - stats.counter("with_optout_interests_from_media_annotation") - val withOptOutFreeFormInterestsFromSemanticCoreCounter = - stats.counter("with_optout_interests_from_semantic_core") - Predicate - .fromAsync { candidate: PushCandidate => - val tweetSemanticCoreEntityIds = candidate.sparseBinaryFeatures - .getOrElse(tweetSemanticCoreFeature, Set.empty[String]).map { id => - id.split('.')(2) - }.toSet - val tweetMediaAnnotationIds = candidate.sparseContinuousFeatures - .getOrElse(tweetMediaAnnotationFeature, Map.empty[String, Double]).keys.toSet - - candidate.target.optOutFreeFormUserInterests.map { - case optOutUserInterests: Seq[String] => - withOptOutFreeFormInterestsCounter.incr() - val optOutUserInterestsSet = optOutUserInterests.toSet - val mediaAnnoIntersect = optOutUserInterestsSet.intersect(tweetMediaAnnotationIds) - val semanticCoreIntersect = optOutUserInterestsSet.intersect(tweetSemanticCoreEntityIds) - if (!mediaAnnoIntersect.isEmpty) { - withOptOutFreeFormInterestsFromMediaAnnotationCounter.incr() - } - if (!semanticCoreIntersect.isEmpty) { - withOptOutFreeFormInterestsFromSemanticCoreCounter.incr() - } - semanticCoreIntersect.isEmpty && mediaAnnoIntersect.isEmpty - case _ => - withoutOptOutInterestsCounter.incr() - true - } - } - .withStats(scopedStatsReceiver) - .withName(name) - } - - def tweetCandidateWithLessThan2SocialContextsIsAReply( - implicit stats: StatsReceiver - ): NamedPredicate[TweetCandidate with TweetDetails with SocialContextActions] = { - val name = "tweet_candidate_with_less_than_2_social_contexts_is_not_a_reply" - Predicate - .from[TweetCandidate with TweetDetails with SocialContextActions] { cand => - cand.isReply match { - case Some(true) if cand.socialContextTweetIds.size < 2 => false - case _ => true - } - } - .withStats(stats.scope(name)) - .withName(name) - } - - def f1CandidateIsNotAReply(implicit stats: StatsReceiver): NamedPredicate[F1Candidate] = { - val name = "f1_candidate_is_not_a_reply" - Predicate - .from[F1Candidate] { candidate => - candidate.isReply match { - case Some(true) => false - case _ => true - } - } - .withStats(stats.scope(name)) - .withName(name) - } - - def outOfNetworkTweetCandidateEnabledCrTag( - implicit stats: StatsReceiver - ): NamedPredicate[OutOfNetworkTweetCandidate with TargetInfo[TargetUser with TargetABDecider]] = { - val name = "out_of_network_tweet_candidate_enabled_crtag" - val scopedStats = stats.scope(name) - Predicate - .from[OutOfNetworkTweetCandidate with TargetInfo[TargetUser with TargetABDecider]] { cand => - val disabledCrTag = cand.target - .params(PushFeatureSwitchParams.OONCandidatesDisabledCrTagParam) - val candGeneratedByDisabledSignal = cand.tagsCR.exists { tagsCR => - val tagsCRSet = tagsCR.map(_.toString).toSet - tagsCRSet.nonEmpty && tagsCRSet.subsetOf(disabledCrTag.toSet) - } - if (candGeneratedByDisabledSignal) { - cand.tagsCR.getOrElse(Nil).foreach(tag => scopedStats.counter(tag.toString).incr()) - false - } else true - } - .withStats(scopedStats) - .withName(name) - } - - def outOfNetworkTweetCandidateEnabledCrtGroup( - implicit stats: StatsReceiver - ): NamedPredicate[OutOfNetworkTweetCandidate with TargetInfo[TargetUser with TargetABDecider]] = { - val name = "out_of_network_tweet_candidate_enabled_crt_group" - val scopedStats = stats.scope(name) - Predicate - .from[OutOfNetworkTweetCandidate with TargetInfo[TargetUser with TargetABDecider]] { cand => - val disabledCrtGroup = cand.target - .params(PushFeatureSwitchParams.OONCandidatesDisabledCrtGroupParam) - val crtGroup = CandidateUtil.getCrtGroup(cand.commonRecType) - val candGeneratedByDisabledCrt = disabledCrtGroup.contains(crtGroup) - if (candGeneratedByDisabledCrt) { - scopedStats.counter("filter_" + crtGroup.toString).incr() - false - } else true - } - .withStats(scopedStats) - .withName(name) - } - - def outOfNetworkTweetCandidateIsNotAReply( - implicit stats: StatsReceiver - ): NamedPredicate[OutOfNetworkTweetCandidate] = { - val name = "out_of_network_tweet_candidate_is_not_a_reply" - Predicate - .from[OutOfNetworkTweetCandidate] { cand => - cand.isReply match { - case Some(true) => false - case _ => true - } - } - .withStats(stats.scope(name)) - .withName(name) - } - - def recommendedTweetIsAuthoredBySelf( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate] = - Predicate - .from[PushCandidate] { - case tweetCandidate: PushCandidate with TweetDetails => - tweetCandidate.authorId match { - case Some(authorId) => authorId != tweetCandidate.target.targetId - case None => true - } - case _ => - true - } - .withStats(statsReceiver.scope("predicate_self_author")) - .withName("self_author") - - def authorInSocialContext(implicit statsReceiver: StatsReceiver): NamedPredicate[PushCandidate] = - Predicate - .from[PushCandidate] { - case tweetCandidate: PushCandidate with TweetDetails with SocialContextActions => - tweetCandidate.authorId match { - case Some(authorId) => - !tweetCandidate.socialContextUserIds.contains(authorId) - case None => true - } - case _ => true - } - .withStats(statsReceiver.scope("predicate_author_social_context")) - .withName("author_social_context") - - def selfInSocialContext(implicit statsReceiver: StatsReceiver): NamedPredicate[PushCandidate] = { - val name = "self_social_context" - Predicate - .from[PushCandidate] { - case candidate: PushCandidate with SocialContextActions => - !candidate.socialContextUserIds.contains(candidate.target.targetId) - case _ => - true - } - .withStats(statsReceiver.scope(s"${name}_predicate")) - .withName(name) - } - - def minSocialContext( - threshold: Int - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with SocialContextActions] = { - Predicate - .from { candidate: PushCandidate with SocialContextActions => - candidate.socialContextUserIds.size >= threshold - } - .withStats(statsReceiver.scope("predicate_min_social_context")) - .withName("min_social_context") - } - - private def anyWithheldContent( - userStore: ReadableStore[Long, User], - userCountryStore: ReadableStore[Long, Location] - )( - implicit statsReceiver: StatsReceiver - ): Predicate[TargetRecUser] = - GizmoduckUserPredicate.withheldContentPredicate( - userStore = userStore, - userCountryStore = userCountryStore, - statsReceiver = statsReceiver, - checkAllCountries = true - ) - - def targetUserExists(implicit statsReceiver: StatsReceiver): NamedPredicate[PushCandidate] = { - TargetUserPredicates - .targetUserExists()(statsReceiver) - .flatContraMap { candidate: PushCandidate => Future.value(candidate.target) } - .withName("target_user_exists") - } - - def secondaryDormantAccountPredicate( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "secondary_dormant_account" - TargetUserPredicates - .secondaryDormantAccountPredicate()(statsReceiver) - .on { candidate: PushCandidate => candidate.target } - .withStats(statsReceiver.scope(s"predicate_$name")) - .withName(name) - } - - def socialContextBeingFollowed( - edgeStore: ReadableStore[RelationEdge, Boolean] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with SocialContextActions] = - SocialGraphPredicate - .allRelationEdgesExist(edgeStore, RelationshipType.Following) - .on { candidate: PushCandidate with SocialContextActions => - candidate.socialContextUserIds.map { u => Edge(candidate.target.targetId, u) } - } - .withStats(statsReceiver.scope("predicate_social_context_being_followed")) - .withName("social_context_being_followed") - - private def edgeFromCandidate(candidate: PushCandidate with TweetAuthor): Option[Edge] = { - candidate.authorId map { authorId => Edge(candidate.target.targetId, authorId) } - } - - def authorNotBeingDeviceFollowed( - edgeStore: ReadableStore[RelationEdge, Boolean] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetAuthor] = { - SocialGraphPredicate - .relationExists(edgeStore, RelationshipType.DeviceFollowing) - .optionalOn( - edgeFromCandidate, - missingResult = false - ) - .flip - .withStats(statsReceiver.scope("predicate_author_not_device_followed")) - .withName("author_not_device_followed") - } - - def authorBeingFollowed( - edgeStore: ReadableStore[RelationEdge, Boolean] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetAuthor] = { - SocialGraphPredicate - .relationExists(edgeStore, RelationshipType.Following) - .optionalOn( - edgeFromCandidate, - missingResult = false - ) - .withStats(statsReceiver.scope("predicate_author_being_followed")) - .withName("author_being_followed") - } - - def authorNotBeingFollowed( - edgeStore: ReadableStore[RelationEdge, Boolean] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetAuthor] = { - SocialGraphPredicate - .relationExists(edgeStore, RelationshipType.Following) - .optionalOn( - edgeFromCandidate, - missingResult = false - ) - .flip - .withStats(statsReceiver.scope("predicate_author_not_being_followed")) - .withName("author_not_being_followed") - } - - def recommendedTweetAuthorAcceptableToTargetUser( - edgeStore: ReadableStore[RelationEdge, Boolean] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetAuthor] = { - val name = "recommended_tweet_author_acceptable_to_target_user" - SocialGraphPredicate - .anyRelationExists( - edgeStore, - Set( - RelationshipType.Blocking, - RelationshipType.BlockedBy, - RelationshipType.HideRecommendations, - RelationshipType.Muting - ) - ) - .flip - .optionalOn( - edgeFromCandidate, - missingResult = false - ) - .withStats(statsReceiver.scope(s"predicate_$name")) - .withName(name) - } - - def relationNotExistsPredicate( - edgeStore: ReadableStore[RelationEdge, Boolean], - relations: Set[RelationshipType] - ): Predicate[(Long, Iterable[Long])] = - SocialGraphPredicate - .anyRelationExistsForMultiEdge( - edgeStore, - relations - ) - .flip - .on { - case (targetUserId, userIds) => - MultiEdge(targetUserId, userIds.toSet) - } - - def blocking(edgeStore: ReadableStore[RelationEdge, Boolean]): Predicate[(Long, Iterable[Long])] = - relationNotExistsPredicate( - edgeStore, - Set(RelationshipType.BlockedBy, RelationshipType.Blocking) - ) - - def blockingOrMuting( - edgeStore: ReadableStore[RelationEdge, Boolean] - ): Predicate[(Long, Iterable[Long])] = - relationNotExistsPredicate( - edgeStore, - Set(RelationshipType.BlockedBy, RelationshipType.Blocking, RelationshipType.Muting) - ) - - def socialContextNotRetweetFollowing( - edgeStore: ReadableStore[RelationEdge, Boolean] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with SocialContextActions] = { - val name = "social_context_not_retweet_following" - relationNotExistsPredicate(edgeStore, Set(RelationshipType.NotRetweetFollowing)) - .optionalOn[PushCandidate with SocialContextActions]( - { - case candidate: PushCandidate with SocialContextActions - if RecTypes.isTweetRetweetType(candidate.commonRecType) => - Some((candidate.target.targetId, candidate.socialContextUserIds)) - case _ => - None - }, - missingResult = true - ) - .withStats(statsReceiver.scope(s"predicate_$name")) - .withName(name) - } - - def socialContextBlockingOrMuting( - edgeStore: ReadableStore[RelationEdge, Boolean] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with SocialContextActions] = - blockingOrMuting(edgeStore) - .on { candidate: PushCandidate with SocialContextActions => - (candidate.target.targetId, candidate.socialContextUserIds) - } - .withStats(statsReceiver.scope("predicate_social_context_blocking_or_muting")) - .withName("social_context_blocking_or_muting") - - /** - * Use hyrated Tweet object for F1 Protected experiment for checking null cast as Tweetypie hydration - * fails for protected Authors without passing in Target id. We do this specifically for - * F1 Protected Tweet Experiment in Earlybird Adaptor. - * For rest of the traffic refer to existing Nullcast Predicate - */ - def nullCastF1ProtectedExperientPredicate( - tweetypieStore: ReadableStore[Long, TweetyPieResult] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetCandidate with TweetDetails] = { - val name = "f1_exempted_null_cast_tweet" - val f1NullCastCheckCounter = statsReceiver.scope(name).counter("f1_null_cast_check") - Predicate - .fromAsync { tweetCandidate: PushCandidate with TweetCandidate with TweetDetails => - if (RecTypes.f1FirstDegreeTypes(tweetCandidate.commonRecType) && tweetCandidate.target - .params(PushFeatureSwitchParams.EnableF1FromProtectedTweetAuthors)) { - f1NullCastCheckCounter.incr() - tweetCandidate.tweet match { - case Some(tweetObj) => - baseNullCastTweet().apply(Seq(TweetyPieResult(tweetObj, None, None))).map(_.head) - case _ => Future.False - } - } else { - nullCastTweet(tweetypieStore).apply(Seq(tweetCandidate)).map(_.head) - } - } - .withStats(statsReceiver.scope(s"predicate_$name")) - .withName(name) - } - - private def baseNullCastTweet(): Predicate[TweetyPieResult] = - Predicate.from { t: TweetyPieResult => !t.tweet.coreData.exists { cd => cd.nullcast } } - - def nullCastTweet( - tweetyPieStore: ReadableStore[Long, TweetyPieResult] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetCandidate] = { - val name = "null_cast_tweet" - baseNullCastTweet() - .flatOptionContraMap[PushCandidate with TweetCandidate]( - f = (tweetCandidate: PushCandidate - with TweetCandidate) => tweetyPieStore.get(tweetCandidate.tweetId), - missingResult = false - ) - .withStats(statsReceiver.scope(s"predicate_$name")) - .withName(name) - } - - /** - * Use the predicate except fn is true. - */ - def exceptedPredicate[T <: PushCandidate]( - name: String, - fn: T => Future[Boolean], - predicate: Predicate[T] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[T] = { - Predicate - .fromAsync { e: T => fn(e) } - .or(predicate) - .withStats(statsReceiver.scope(name)) - .withName(name) - } - - /** - * - * @param edgeStore [[ReadableStore[RelationEdge, Boolean]]] - * @return - allow only out-network tweets if in-network tweets are disabled - */ - def disableInNetworkTweetPredicate( - edgeStore: ReadableStore[RelationEdge, Boolean] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetAuthor] = { - val name = "disable_in_network_tweet" - Predicate - .fromAsync { candidate: PushCandidate with TweetAuthor => - if (candidate.target.params(PushParams.DisableInNetworkTweetCandidatesParam)) { - authorNotBeingFollowed(edgeStore) - .apply(Seq(candidate)) - .map(_.head) - } else Future.True - }.withStats(statsReceiver.scope(name)) - .withName(name) - } - - /** - * - * @param edgeStore [[ReadableStore[RelationEdge, Boolean]]] - * @return - allow only in-network tweets if out-network tweets are disabled - */ - def disableOutNetworkTweetPredicate( - edgeStore: ReadableStore[RelationEdge, Boolean] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetAuthor] = { - val name = "disable_out_network_tweet" - Predicate - .fromAsync { candidate: PushCandidate with TweetAuthor => - if (candidate.target.params(PushFeatureSwitchParams.DisableOutNetworkTweetCandidatesFS)) { - authorBeingFollowed(edgeStore) - .apply(Seq(candidate)) - .map(_.head) - } else Future.True - }.withStats(statsReceiver.scope(name)) - .withName(name) - } - - def alwaysTruePredicate: NamedPredicate[PushCandidate] = { - Predicate - .all[PushCandidate] - .withName("predicate_AlwaysTrue") - } - - def alwaysTruePushCandidatePredicate: NamedPredicate[PushCandidate] = { - Predicate - .all[PushCandidate] - .withName("predicate_AlwaysTrue") - } - - def alwaysFalsePredicate(implicit statsReceiver: StatsReceiver): NamedPredicate[PushCandidate] = { - val name = "predicate_AlwaysFalse" - val scopedStatsReceiver = statsReceiver.scope(name) - Predicate - .from { candidate: PushCandidate => false } - .withStats(scopedStatsReceiver) - .withName(name) - } - - def accountCountryPredicate( - allowedCountries: Set[String] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "AccountCountryPredicate" - val stats = statsReceiver.scope(name) - AccountCountryPredicate(allowedCountries) - .on { candidate: PushCandidate => candidate.target } - .withStats(stats) - .withName(name) - } - - def paramPredicate[T <: PushCandidate]( - param: Param[Boolean] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[T] = { - val name = param.getClass.getSimpleName.stripSuffix("$") - TargetPredicates - .paramPredicate(param) - .on { candidate: PushCandidate => candidate.target } - .withStats(statsReceiver.scope(s"param_${name}_controlled_predicate")) - .withName(s"param_${name}_controlled_predicate") - } - - def isDeviceEligibleForNewsOrSports( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "is_device_eligible_for_news_or_sports" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - Predicate - .fromAsync { candidate: PushCandidate => - candidate.target.deviceInfo.map(_.exists(_.isNewsEligible)) - } - .withStats(scopedStatsReceiver) - .withName(name) - } - - def isDeviceEligibleForCreatorPush( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "is_device_eligible_for_creator_push" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - Predicate - .fromAsync { candidate: PushCandidate => - candidate.target.deviceInfo.map(_.exists(settings => - settings.isNewsEligible || settings.isRecommendationsEligible)) - } - .withStats(scopedStatsReceiver) - .withName(name) - } - - /** - * Like [[TargetUserPredicates.homeTimelineFatigue()]] but for candidate. - */ - def htlFatiguePredicate( - fatigueDuration: Param[Duration] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "htl_fatigue" - Predicate - .fromAsync { candidate: PushCandidate => - val _fatigueDuration = candidate.target.params(fatigueDuration) - TargetUserPredicates - .homeTimelineFatigue( - fatigueDuration = _fatigueDuration - ).apply(Seq(candidate.target)).map(_.head) - } - .withStats(statsReceiver.scope(name)) - .withName(name) - } - - def mrWebHoldbackPredicate( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "mr_web_holdback_for_candidate" - val scopedStats = stats.scope(name) - PredicatesForCandidate.exludeCrtFromPushHoldback - .or( - TargetPredicates - .webNotifsHoldback() - .on { candidate: PushCandidate => candidate.target } - ) - .withStats(scopedStats) - .withName(name) - } - - def candidateEnabledForEmailPredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "candidates_enabled_for_email" - Predicate - .from { candidate: PushCandidate => - if (candidate.target.isEmailUser) - candidate.isInstanceOf[TweetCandidate with TweetAuthor with RecommendationType] - else true - } - .withStats(stats.scope(name)) - .withName(name) - } - - def protectedTweetF1ExemptPredicate[ - T <: TargetUser with TargetABDecider, - Cand <: TweetCandidate with TweetAuthorDetails with TargetInfo[T] - ]( - implicit stats: StatsReceiver - ): NamedPredicate[ - TweetCandidate with TweetAuthorDetails with TargetInfo[ - TargetUser with TargetABDecider - ] - ] = { - val name = "f1_exempt_tweet_author_protected" - val skipForProtectedAuthorScope = stats.scope(name).scope("skip_protected_author_for_f1") - val authorIsProtectedCounter = skipForProtectedAuthorScope.counter("author_protected_true") - val authorIsNotProtectedCounter = skipForProtectedAuthorScope.counter("author_protected_false") - val authorNotFoundCounter = stats.scope(name).counter("author_not_found") - Predicate - .fromAsync[TweetCandidate with TweetAuthorDetails with TargetInfo[ - TargetUser with TargetABDecider - ]] { - case candidate: F1Candidate - if candidate.target.params(PushFeatureSwitchParams.EnableF1FromProtectedTweetAuthors) => - candidate.tweetAuthor.foreach { - case Some(author) => - if (GizmoduckUserPredicate.isProtected(author)) { - authorIsProtectedCounter.incr() - } else authorIsNotProtectedCounter.incr() - case _ => authorNotFoundCounter.incr() - } - Future.True - case cand => - TweetAuthorPredicates.recTweetAuthorProtected.apply(Seq(cand)).map(_.head) - } - .withStats(stats.scope(name)) - .withName(name) - } - - /** - * filter a notification if user has already received ANY prior notification about the space id - * @param stats - * @return - */ - def duplicateSpacesPredicate( - implicit stats: StatsReceiver - ): NamedPredicate[Space with PushCandidate] = { - val name = "duplicate_spaces_predicate" - Predicate - .fromAsync { c: Space with PushCandidate => - c.target.pushRecItems.map { pushRecItems => - !pushRecItems.spaceIds.contains(c.spaceId) - } - } - .withStats(stats.scope(name)) - .withName(name) - } - - def filterOONCandidatePredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "filter_oon_candidate" - - Predicate - .fromAsync[PushCandidate] { cand => - val crt = cand.commonRecType - val isOONCandidate = - RecTypes.isOutOfNetworkTweetRecType(crt) || RecTypes.outOfNetworkTopicTweetTypes - .contains(crt) || RecTypes.isOutOfNetworkSpaceType(crt) || RecTypes.userTypes.contains( - crt) - if (isOONCandidate) { - cand.target.notificationsFromOnlyPeopleIFollow.map { inNetworkOnly => - if (inNetworkOnly) { - stats.scope(name, crt.toString).counter("inNetworkOnlyOn").incr() - } else { - stats.scope(name, crt.toString).counter("inNetworkOnlyOff").incr() - } - !(inNetworkOnly && cand.target.params( - PushFeatureSwitchParams.EnableOONFilteringBasedOnUserSettings)) - } - } else Future.True - } - .withStats(stats.scope(name)) - .withName(name) - } - - def exludeCrtFromPushHoldback( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate] = Predicate - .from { candidate: PushCandidate => - val crtName = candidate.commonRecType.name - val target = candidate.target - target - .params(PushFeatureSwitchParams.CommonRecommendationTypeDenyListPushHoldbacks) - .exists(crtName.equalsIgnoreCase) - } - .withStats(stats.scope("exclude_crt_from_push_holdbacks")) - - def enableSendHandlerCandidates(implicit stats: StatsReceiver): NamedPredicate[PushCandidate] = { - val name = "sendhandler_enable_push_recommendations" - PredicatesForCandidate.exludeCrtFromPushHoldback - .or(PredicatesForCandidate.paramPredicate( - PushFeatureSwitchParams.EnablePushRecommendationsParam)) - .withStats(stats.scope(name)) - .withName(name) - } - - def openAppExperimentUserCandidateAllowList( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "open_app_experiment_user_candidate_allow_list" - Predicate - .fromAsync { candidate: PushCandidate => - val target = candidate.target - Future.join(target.isOpenAppExperimentUser, target.targetUser).map { - case (isOpenAppUser, targetUser) => - val shouldLimitOpenAppCrts = - isOpenAppUser || targetUser.exists(_.userType == UserType.Soft) - - if (shouldLimitOpenAppCrts) { - val listOfAllowedCrt = target - .params(PushFeatureSwitchParams.ListOfCrtsForOpenApp) - .flatMap(CommonRecommendationType.valueOf) - listOfAllowedCrt.contains(candidate.commonRecType) - } else true - } - }.withStats(stats.scope(name)) - .withName(name) - } - - def isTargetBlueVerified( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "is_target_already_blue_verified" - Predicate - .fromAsync { candidate: PushCandidate => - val target = candidate.target - target.isBlueVerified.map(_.getOrElse(false)) - }.withStats(stats.scope(name)) - .withName(name) - } - - def isTargetLegacyVerified( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "is_target_already_legacy_verified" - Predicate - .fromAsync { candidate: PushCandidate => - val target = candidate.target - target.isVerified.map(_.getOrElse(false)) - }.withStats(stats.scope(name)) - .withName(name) - } - - def isTargetSuperFollowCreator(implicit stats: StatsReceiver): NamedPredicate[PushCandidate] = { - val name = "is_target_already_super_follow_creator" - Predicate - .fromAsync { candidate: PushCandidate => - val target = candidate.target - target.isSuperFollowCreator.map( - _.getOrElse(false) - ) - }.withStats(stats.scope(name)) - .withName(name) - } - - def isChannelValidPredicate( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "is_channel_valid" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - Predicate - .fromAsync { candidate: PushCandidate => - candidate - .getChannels().map(channels => - !(channels.toSet.size == 1 && channels.head == ChannelName.None)) - } - .withStats(scopedStatsReceiver) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/SGSPredicatesForCandidate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/SGSPredicatesForCandidate.docx new file mode 100644 index 000000000..d9caaf871 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/SGSPredicatesForCandidate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/SGSPredicatesForCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/SGSPredicatesForCandidate.scala deleted file mode 100644 index e335c8d9c..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/SGSPredicatesForCandidate.scala +++ /dev/null @@ -1,174 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.SocialGraphServiceRelationshipMap -import com.twitter.frigate.common.base.TweetAuthor -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.gizmoduck.thriftscala.UserType -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.hermit.predicate.socialgraph.Edge -import com.twitter.hermit.predicate.socialgraph.RelationEdge -import com.twitter.socialgraph.thriftscala.RelationshipType -import com.twitter.util.Future - -/** - * Refactor SGS predicates so that predicates can use relationshipMap we generate in hydrate step - */ -object SGSPredicatesForCandidate { - - case class RelationshipMapEdge(edge: Edge, relationshipMap: Map[RelationEdge, Boolean]) - - private def relationshipMapEdgeFromCandidate( - candidate: PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap - ): Option[RelationshipMapEdge] = { - candidate.authorId map { authorId => - RelationshipMapEdge(Edge(candidate.target.targetId, authorId), candidate.relationshipMap) - } - } - - def authorBeingFollowed( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap] = { - val name = "author_not_being_followed" - val stats = statsReceiver.scope(name) - val softUserCounter = stats.counter("soft_user") - - val sgsAuthorBeingFollowedPredicate = Predicate - .from { relationshipMapEdge: RelationshipMapEdge => - anyRelationExist(relationshipMapEdge, Set(RelationshipType.Following)) - } - - Predicate - .fromAsync { - candidate: PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap => - val target = candidate.target - target.targetUser.flatMap { - case Some(gizmoduckUser) if gizmoduckUser.userType == UserType.Soft => - softUserCounter.incr() - target.seedsWithWeight.map { followedUsersWithWeightOpt => - candidate.authorId match { - case Some(authorId) => - val followedUsers = followedUsersWithWeightOpt.getOrElse(Map.empty).keys - followedUsers.toSet.contains(authorId) - - case None => false - } - } - - case _ => - sgsAuthorBeingFollowedPredicate - .optionalOn(relationshipMapEdgeFromCandidate, missingResult = false) - .apply(Seq(candidate)) - .map(_.head) - } - }.withStats(stats) - .withName(name) - } - - def authorNotBeingDeviceFollowed( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap] = { - val name = "author_being_device_followed" - Predicate - .from { relationshipMapEdge: RelationshipMapEdge => - { - anyRelationExist(relationshipMapEdge, Set(RelationshipType.DeviceFollowing)) - } - } - .optionalOn(relationshipMapEdgeFromCandidate, missingResult = false) - .flip - .withStats(statsReceiver.scope(name)) - .withName(name) - } - - def recommendedTweetAuthorAcceptableToTargetUser( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap] = { - val name = "recommended_tweet_author_not_acceptable_to_target_user" - Predicate - .from { relationshipMapEdge: RelationshipMapEdge => - { - anyRelationExist( - relationshipMapEdge, - Set( - RelationshipType.Blocking, - RelationshipType.BlockedBy, - RelationshipType.HideRecommendations, - RelationshipType.Muting - )) - } - } - .flip - .optionalOn(relationshipMapEdgeFromCandidate, missingResult = false) - .withStats(statsReceiver.scope(name)) - .withName(name) - } - - def authorNotBeingFollowed( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap] = { - Predicate - .from { relationshipMapEdge: RelationshipMapEdge => - { - anyRelationExist(relationshipMapEdge, Set(RelationshipType.Following)) - } - } - .optionalOn(relationshipMapEdgeFromCandidate, missingResult = false) - .flip - .withStats(statsReceiver.scope("predicate_author_not_being_followed_pre_ranking")) - .withName("author_not_being_followed") - } - - def disableInNetworkTweetPredicate( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap] = { - val name = "enable_in_network_tweet" - Predicate - .fromAsync { - candidate: PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap => - if (candidate.target.params(PushParams.DisableInNetworkTweetCandidatesParam)) { - authorNotBeingFollowed - .apply(Seq(candidate)) - .map(_.head) - } else Future.True - }.withStats(statsReceiver.scope(name)) - .withName(name) - } - - def disableOutNetworkTweetPredicate( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap] = { - val name = "enable_out_network_tweet" - Predicate - .fromAsync { - candidate: PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap => - if (candidate.target.params(PushFeatureSwitchParams.DisableOutNetworkTweetCandidatesFS)) { - authorBeingFollowed - .apply(Seq(candidate)) - .map(_.head) - } else Future.True - }.withStats(statsReceiver.scope(name)) - .withName(name) - } - - /** - * Returns true if the provided relationshipEdge exists among - * @param candidate candidate - * @param relationships relaionships - * @return Boolean result - */ - private def anyRelationExist( - relationshipMapEdge: RelationshipMapEdge, - relationships: Set[RelationshipType] - ): Boolean = { - val resultSeq = relationships.map { relationship => - relationshipMapEdge.relationshipMap.getOrElse( - RelationEdge(relationshipMapEdge.edge, relationship), - false) - }.toSeq - resultSeq.contains(true) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ScarecrowPredicate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ScarecrowPredicate.docx new file mode 100644 index 000000000..a44df9443 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ScarecrowPredicate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ScarecrowPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ScarecrowPredicate.scala deleted file mode 100644 index a4728eba2..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ScarecrowPredicate.scala +++ /dev/null @@ -1,138 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.thriftscala._ -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.scarecrow.{ScarecrowPredicate => HermitScarecrowPredicate} -import com.twitter.relevance.feature_store.thriftscala.FeatureData -import com.twitter.relevance.feature_store.thriftscala.FeatureValue -import com.twitter.service.gen.scarecrow.thriftscala.Event -import com.twitter.service.gen.scarecrow.thriftscala.TieredActionResult -import com.twitter.storehaus.ReadableStore - -object ScarecrowPredicate { - val name = "" - - def candidateToEvent(candidate: PushCandidate): Event = { - val recommendedUserIdOpt = candidate match { - case tweetCandidate: TweetCandidate with TweetAuthor => - tweetCandidate.authorId - case userCandidate: UserCandidate => - Some(userCandidate.userId) - case _ => None - } - val hashtagsInTweet = candidate match { - case tweetCandidate: TweetCandidate with TweetDetails => - tweetCandidate.tweetyPieResult - .flatMap { tweetPieResult => - tweetPieResult.tweet.hashtags.map(_.map(_.text)) - }.getOrElse(Nil) - case _ => - Nil - } - val urlsInTweet = candidate match { - case tweetCandidate: TweetCandidate with TweetDetails => - tweetCandidate.tweetyPieResult - .flatMap { tweetPieResult => - tweetPieResult.tweet.urls.map(_.flatMap(_.expanded)) - } - case _ => None - } - val tweetIdOpt = candidate match { - case tweetCandidate: TweetCandidate => - Some(tweetCandidate.tweetId) - case _ => - None - } - val urlOpt = candidate match { - case candidate: UrlCandidate => - Some(candidate.url) - case _ => - None - } - val scUserIds = candidate match { - case hasSocialContext: SocialContextActions => Some(hasSocialContext.socialContextUserIds) - case _ => None - } - - val eventTitleOpt = candidate match { - case eventCandidate: EventCandidate with EventDetails => - Some(eventCandidate.eventTitle) - case _ => - None - } - - val urlTitleOpt = candidate match { - case candidate: UrlCandidate => - candidate.title - case _ => - None - } - - val urlDescriptionOpt = candidate match { - case candidate: UrlCandidate with UrlCandidateWithDetails => - candidate.description - case _ => - None - } - - Event( - "magicrecs_recommendation_write", - Map( - "targetUserId" -> FeatureData(Some(FeatureValue.LongValue(candidate.target.targetId))), - "type" -> FeatureData( - Some( - FeatureValue.StrValue(candidate.commonRecType.name) - ) - ), - "recommendedUserId" -> FeatureData(recommendedUserIdOpt map { id => - FeatureValue.LongValue(id) - }), - "tweetId" -> FeatureData(tweetIdOpt map { id => - FeatureValue.LongValue(id) - }), - "url" -> FeatureData(urlOpt map { url => - FeatureValue.StrValue(url) - }), - "hashtagsInTweet" -> FeatureData(Some(FeatureValue.StrListValue(hashtagsInTweet))), - "urlsInTweet" -> FeatureData(urlsInTweet.map(FeatureValue.StrListValue)), - "socialContexts" -> FeatureData(scUserIds.map { sc => - FeatureValue.LongListValue(sc) - }), - "eventTitle" -> FeatureData(eventTitleOpt.map { eventTitle => - FeatureValue.StrValue(eventTitle) - }), - "urlTitle" -> FeatureData(urlTitleOpt map { title => - FeatureValue.StrValue(title) - }), - "urlDescription" -> FeatureData(urlDescriptionOpt map { des => - FeatureValue.StrValue(des) - }) - ) - ) - } - - def candidateToPossibleEvent(c: PushCandidate): Option[Event] = { - if (c.frigateNotification.notificationDisplayLocation == NotificationDisplayLocation.PushToMobileDevice) { - Some(candidateToEvent(c)) - } else { - None - } - } - - def apply( - scarecrowCheckEventStore: ReadableStore[Event, TieredActionResult] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate] = { - HermitScarecrowPredicate(scarecrowCheckEventStore) - .optionalOn( - candidateToPossibleEvent, - missingResult = true - ) - .withStats(statsReceiver.scope(s"predicate_$name")) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/SpacePredicate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/SpacePredicate.docx new file mode 100644 index 000000000..73b0e7997 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/SpacePredicate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/SpacePredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/SpacePredicate.scala deleted file mode 100644 index 044c0afdb..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/SpacePredicate.scala +++ /dev/null @@ -1,153 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.SpaceCandidate -import com.twitter.frigate.common.base.SpaceCandidateDetails -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.hermit.predicate.socialgraph.Edge -import com.twitter.hermit.predicate.socialgraph.RelationEdge -import com.twitter.hermit.predicate.socialgraph.SocialGraphPredicate -import com.twitter.socialgraph.thriftscala.RelationshipType -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.response.Err -import com.twitter.ubs.thriftscala.AudioSpace -import com.twitter.ubs.thriftscala.BroadcastState -import com.twitter.ubs.thriftscala.ParticipantUser -import com.twitter.ubs.thriftscala.Participants -import com.twitter.util.Future - -object SpacePredicate { - - /** Filters the request if the target is present in the space as a listener, speakeTestConfigr, or admin */ - def targetInSpace( - audioSpaceParticipantsStore: ReadableStore[String, Participants] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[SpaceCandidateDetails with RawCandidate] = { - val name = "target_in_space" - Predicate - .fromAsync[SpaceCandidateDetails with RawCandidate] { spaceCandidate => - audioSpaceParticipantsStore.get(spaceCandidate.spaceId).map { - case Some(participants) => - val allParticipants: Seq[ParticipantUser] = - (participants.admins ++ participants.speakers ++ participants.listeners).flatten.toSeq - val isInSpace = allParticipants.exists { participant => - participant.twitterUserId.contains(spaceCandidate.target.targetId) - } - !isInSpace - case None => false - } - }.withStats(statsReceiver.scope(name)) - .withName(name) - } - - /** - * - * @param audioSpaceStore: space metadata store - * @param statsReceiver: record stats - * @return: true if the space not started ELSE false to filter out notification - */ - def scheduledSpaceStarted( - audioSpaceStore: ReadableStore[String, AudioSpace] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[SpaceCandidate with RawCandidate] = { - val name = "scheduled_space_started" - Predicate - .fromAsync[SpaceCandidate with RawCandidate] { spaceCandidate => - audioSpaceStore - .get(spaceCandidate.spaceId) - .map(_.exists(_.state.contains(BroadcastState.NotStarted))) - .rescue { - case Err(Err.Authorization, _, _) => - Future.False - } - } - .withStats(statsReceiver.scope(name)) - .withName(name) - } - - private def relationshipMapEdgeFromSpaceCandidate( - candidate: RawCandidate with SpaceCandidate - ): Option[(Long, Seq[Long])] = { - candidate.hostId.map { spaceHostId => - (candidate.target.targetId, Seq(spaceHostId)) - } - } - - /** - * Check only host block for scheduled space reminders - * @return: True if no blocking relation between host and target user, else False - */ - def spaceHostTargetUserBlocking( - edgeStore: ReadableStore[RelationEdge, Boolean] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[SpaceCandidate with RawCandidate] = { - val name = "space_host_target_user_blocking" - PredicatesForCandidate - .blocking(edgeStore) - .optionalOn(relationshipMapEdgeFromSpaceCandidate, false) - .withStats(statsReceiver.scope(name)) - .withName(name) - } - - private def edgeFromCandidate( - candidate: PushCandidate with TweetAuthorDetails - ): Future[Option[Edge]] = { - candidate.tweetAuthor.map(_.map { author => Edge(candidate.target.targetId, author.id) }) - } - - def recommendedTweetAuthorAcceptableToTargetUser( - edgeStore: ReadableStore[RelationEdge, Boolean] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetAuthorDetails] = { - val name = "recommended_tweet_author_acceptable_to_target_user" - SocialGraphPredicate - .anyRelationExists( - edgeStore, - Set( - RelationshipType.Blocking, - RelationshipType.BlockedBy, - RelationshipType.HideRecommendations, - RelationshipType.Muting - ) - ) - .flip - .flatOptionContraMap( - edgeFromCandidate, - missingResult = false - ) - .withStats(statsReceiver.scope(s"predicate_$name")) - .withName(name) - } - - def narrowCastSpace( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[SpaceCandidateDetails with RawCandidate] = { - val name = "narrow_cast_space" - val narrowCastSpaceScope = statsReceiver.scope(name) - val employeeSpaceCounter = narrowCastSpaceScope.counter("employees") - val superFollowerSpaceCounter = narrowCastSpaceScope.counter("super_followers") - - Predicate - .fromAsync[SpaceCandidateDetails with RawCandidate] { candidate => - candidate.audioSpaceFut.map { - case Some(audioSpace) if audioSpace.narrowCastSpaceType.contains(1L) => - employeeSpaceCounter.incr() - candidate.target.params(PushFeatureSwitchParams.EnableEmployeeOnlySpaceNotifications) - case Some(audioSpace) if audioSpace.narrowCastSpaceType.contains(2L) => - superFollowerSpaceCounter.incr() - false - case _ => true - } - }.withStats(narrowCastSpaceScope) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetEngagementPredicate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetEngagementPredicate.docx new file mode 100644 index 000000000..30e2d93ef Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetEngagementPredicate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetEngagementPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetEngagementPredicate.scala deleted file mode 100644 index d02e5b89a..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetEngagementPredicate.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.tweetypie.EngagementsPredicate -import com.twitter.hermit.predicate.tweetypie.Perspective -import com.twitter.hermit.predicate.tweetypie.UserTweet -import com.twitter.storehaus.ReadableStore - -object TargetEngagementPredicate { - val name = "target_engagement" - def apply( - perspectiveStore: ReadableStore[UserTweet, Perspective], - defaultForMissing: Boolean - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetCandidate] = { - EngagementsPredicate(perspectiveStore, defaultForMissing) - .on { candidate: PushCandidate with TweetCandidate => - UserTweet(candidate.target.targetId, candidate.tweetId) - } - .withStats(statsReceiver.scope(s"predicate_$name")) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetNtabCaretClickFatiguePredicate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetNtabCaretClickFatiguePredicate.docx new file mode 100644 index 000000000..57317bc2a Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetNtabCaretClickFatiguePredicate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetNtabCaretClickFatiguePredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetNtabCaretClickFatiguePredicate.scala deleted file mode 100644 index c5042bbc8..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetNtabCaretClickFatiguePredicate.scala +++ /dev/null @@ -1,91 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.TargetUser -import com.twitter.frigate.common.candidate.CaretFeedbackHistory -import com.twitter.frigate.common.candidate.FrigateHistory -import com.twitter.frigate.common.candidate.HTLVisitHistory -import com.twitter.frigate.common.candidate.TargetABDecider -import com.twitter.frigate.common.history.History -import com.twitter.frigate.common.predicate.FrigateHistoryFatiguePredicate.TimeSeries -import com.twitter.frigate.common.predicate.ntab_caret_fatigue.NtabCaretClickFatiguePredicateHelper -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.common.util.FeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.frigate.common.predicate.{FatiguePredicate => CommonFatiguePredicate} - -object TargetNtabCaretClickFatiguePredicate { - import NtabCaretClickFatiguePredicateHelper._ - - private val MagicRecsCategory = "MagicRecs" - - def apply[ - T <: TargetUser with TargetABDecider with CaretFeedbackHistory with FrigateHistory with HTLVisitHistory - ]( - filterHistory: TimeSeries => TimeSeries = - CommonFatiguePredicate.recTypesOnlyFilter(RecTypes.sharedNTabCaretFatigueTypes), - filterCaretFeedbackHistory: TargetUser with TargetABDecider with CaretFeedbackHistory => Seq[ - CaretFeedbackDetails - ] => Seq[CaretFeedbackDetails] = - CaretFeedbackHistoryFilter.caretFeedbackHistoryFilter(Seq(MagicRecsCategory)), - calculateFatiguePeriod: Seq[CaretFeedbackDetails] => Duration = calculateFatiguePeriodMagicRecs, - useMostRecentDislikeTime: Boolean = false, - name: String = "NtabCaretClickFatiguePredicate" - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[T] = { - - val scopedStats = statsReceiver.scope(name) - val crtStats = scopedStats.scope("crt") - Predicate - .fromAsync { target: T => - Future.join(target.history, target.caretFeedbacks).map { - case (history, Some(feedbackDetails)) => { - val feedbackDetailsDeduped = dedupFeedbackDetails( - filterCaretFeedbackHistory(target)(feedbackDetails), - scopedStats - ) - - val fatiguePeriod = - if (hasUserDislikeInLast30Days(feedbackDetailsDeduped) && target.params( - PushFeatureSwitchParams.EnableReducedFatigueRulesForSeeLessOften)) { - durationToFilterMRForSeeLessOftenExpt( - feedbackDetailsDeduped, - target.params(FeatureSwitchParams.NumberOfDaysToFilterMRForSeeLessOften), - target.params(FeatureSwitchParams.NumberOfDaysToReducePushCapForSeeLessOften), - scopedStats - ) - } else { - calculateFatiguePeriod(feedbackDetailsDeduped) - } - - val crtlist = feedbackDetailsDeduped - .flatMap { fd => - fd.genericNotificationMetadata.map { gm => - gm.genericType.name - } - }.distinct.sorted.mkString("-") - - if (fatiguePeriod > 0.days) { - crtStats.scope(crtlist).counter("fatigued").incr() - } else { - crtStats.scope(crtlist).counter("non_fatigued").incr() - } - - val hasRecentSent = - hasRecentSend(History(filterHistory(history.history.toSeq).toMap), fatiguePeriod) - !hasRecentSent - } - case _ => true - } - } - .withStats(scopedStats) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetPredicates.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetPredicates.docx new file mode 100644 index 000000000..fa6704467 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetPredicates.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetPredicates.scala deleted file mode 100644 index 45d0b7578..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetPredicates.scala +++ /dev/null @@ -1,292 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.TargetUser -import com.twitter.frigate.common.candidate.FrigateHistory -import com.twitter.frigate.common.candidate.HTLVisitHistory -import com.twitter.frigate.common.candidate.TargetABDecider -import com.twitter.frigate.common.candidate.UserDetails -import com.twitter.frigate.common.predicate.TargetUserPredicates -import com.twitter.frigate.common.predicate.{FatiguePredicate => CommonFatiguePredicate} -import com.twitter.frigate.common.store.deviceinfo.MobileClientType -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.target.TargetScoringDetails -import com.twitter.frigate.pushservice.util.PushCapUtil -import com.twitter.frigate.thriftscala.NotificationDisplayLocation -import com.twitter.frigate.thriftscala.{CommonRecommendationType => CRT} -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration -import com.twitter.util.Future - -object TargetPredicates { - - def paramPredicate[T <: Target]( - param: Param[Boolean] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[T] = { - val name = param.getClass.getSimpleName.stripSuffix("$") - Predicate - .from { target: T => target.params(param) } - .withStats(statsReceiver.scope(s"param_${name}_controlled_predicate")) - .withName(s"param_${name}_controlled_predicate") - } - - /** - * Use the predicate except fn is true., Same as the candidate version but for Target - */ - def exceptedPredicate[T <: TargetUser]( - name: String, - fn: T => Future[Boolean], - predicate: Predicate[T] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[T] = { - Predicate - .fromAsync { e: T => fn(e) } - .or(predicate) - .withStats(statsReceiver.scope(name)) - .withName(name) - } - - /** - * Refresh For push handler target user predicate to fatigue on visiting Home timeline - */ - def targetHTLVisitPredicate[ - T <: TargetUser with UserDetails with TargetABDecider with HTLVisitHistory - ]( - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[T] = { - val name = "target_htl_visit_predicate" - Predicate - .fromAsync { target: T => - val hoursToFatigue = target.params(PushFeatureSwitchParams.HTLVisitFatigueTime) - TargetUserPredicates - .homeTimelineFatigue(hoursToFatigue.hours) - .apply(Seq(target)) - .map(_.head) - } - .withStats(statsReceiver.scope(name)) - .withName(name) - } - - def targetPushBitEnabledPredicate[T <: Target]( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[T] = { - val name = "push_bit_enabled" - val scopedStats = statsReceiver.scope(s"targetpredicate_$name") - - Predicate - .fromAsync { target: T => - target.deviceInfo - .map { info => - info.exists { deviceInfo => - deviceInfo.isRecommendationsEligible || - deviceInfo.isNewsEligible || - deviceInfo.isTopicsEligible || - deviceInfo.isSpacesEligible - } - } - }.withStats(scopedStats) - .withName(name) - } - - def targetFatiguePredicate[T <: Target]( - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[T] = { - val name = "target_fatigue_predicate" - val predicateStatScope = statsReceiver.scope(name) - Predicate - .fromAsync { target: T => - PushCapUtil - .getPushCapFatigue(target, predicateStatScope) - .flatMap { pushCapInfo => - CommonFatiguePredicate - .magicRecsPushTargetFatiguePredicate( - interval = pushCapInfo.fatigueInterval, - maxInInterval = pushCapInfo.pushcap - ) - .apply(Seq(target)) - .map(_.headOption.getOrElse(false)) - } - } - .withStats(predicateStatScope) - .withName(name) - } - - def teamExceptedPredicate[T <: TargetUser]( - predicate: NamedPredicate[T] - )( - implicit stats: StatsReceiver - ): NamedPredicate[T] = { - Predicate - .fromAsync { t: T => t.isTeamMember } - .or(predicate) - .withStats(stats.scope(predicate.name)) - .withName(predicate.name) - } - - def targetValidMobileSDKPredicate[T <: Target]( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[T] = { - val name = "valid_mobile_sdk" - val scopedStats = statsReceiver.scope(s"targetpredicate_$name") - - Predicate - .fromAsync { target: T => - TargetUserPredicates.validMobileSDKPredicate - .apply(Seq(target)).map(_.headOption.getOrElse(false)) - }.withStats(scopedStats) - .withName(name) - } - - def magicRecsMinDurationSinceSent[T <: Target]( - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[T] = { - val name = "target_min_duration_since_push" - Predicate - .fromAsync { target: T => - PushCapUtil.getMinDurationSincePush(target, statsReceiver).flatMap { minDurationSincePush => - CommonFatiguePredicate - .magicRecsMinDurationSincePush(interval = minDurationSincePush) - .apply(Seq(target)).map(_.head) - } - } - .withStats(statsReceiver.scope(name)) - .withName(name) - } - - def optoutProbPredicate[ - T <: TargetUser with TargetABDecider with TargetScoringDetails with FrigateHistory - ]( - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[T] = { - val name = "target_has_high_optout_probability" - Predicate - .fromAsync { target: T => - val isNewUser = target.is30DayNewUserFromSnowflakeIdTime - if (isNewUser) { - statsReceiver.scope(name).counter("all_new_users").incr() - } - target.bucketOptoutProbability - .flatMap { - case Some(optoutProb) => - if (optoutProb >= target.params(PushFeatureSwitchParams.BucketOptoutThresholdParam)) { - CommonFatiguePredicate - .magicRecsPushTargetFatiguePredicate( - interval = 24.hours, - maxInInterval = target.params(PushFeatureSwitchParams.OptoutExptPushCapParam) - ) - .apply(Seq(target)) - .map { values => - val isValid = values.headOption.getOrElse(false) - if (!isValid && isNewUser) { - statsReceiver.scope(name).counter("filtered_new_users").incr() - } - isValid - } - } else Future.True - case _ => Future.True - } - } - .withStats(statsReceiver.scope(name)) - .withName(name) - } - - /** - * Predicate used to specify CRT fatigue given interval and max number of candidates within interval. - * @param crt The specific CRT that this predicate is being applied to - * @param intervalParam The fatigue interval - * @param maxInIntervalParam The max number of the given CRT's candidates that are acceptable - * in the interval - * @param stats StatsReceiver - * @return Target Predicate - */ - def pushRecTypeFatiguePredicate( - crt: CRT, - intervalParam: Param[Duration], - maxInIntervalParam: FSBoundedParam[Int], - stats: StatsReceiver - ): Predicate[Target] = - Predicate.fromAsync { target: Target => - val interval = target.params(intervalParam) - val maxIninterval = target.params(maxInIntervalParam) - CommonFatiguePredicate - .recTypeTargetFatiguePredicate( - interval = interval, - maxInInterval = maxIninterval, - recommendationType = crt, - notificationDisplayLocation = NotificationDisplayLocation.PushToMobileDevice, - minInterval = 30.minutes - )(stats.scope(s"${crt}_push_candidate_fatigue")).apply(Seq(target)).map(_.head) - } - - def inlineActionFatiguePredicate( - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[Target] = { - val name = "inline_action_fatigue" - val predicateRequests = statsReceiver.scope(name).counter("requests") - val targetIsInExpt = statsReceiver.scope(name).counter("target_in_expt") - val predicateEnabled = statsReceiver.scope(name).counter("enabled") - val predicateDisabled = statsReceiver.scope(name).counter("disabled") - val inlineFatigueDisabled = statsReceiver.scope(name).counter("inline_fatigue_disabled") - - Predicate - .fromAsync { target: Target => - predicateRequests.incr() - if (target.params(PushFeatureSwitchParams.TargetInInlineActionAppVisitFatigue)) { - targetIsInExpt.incr() - target.inlineActionHistory.map { inlineHistory => - if (inlineHistory.nonEmpty && target.params( - PushFeatureSwitchParams.EnableInlineActionAppVisitFatigue)) { - predicateEnabled.incr() - val inlineFatigue = target.params(PushFeatureSwitchParams.InlineActionAppVisitFatigue) - val lookbackInMs = inlineFatigue.ago.inMilliseconds - val filteredHistory = inlineHistory.filter { - case (time, _) => time > lookbackInMs - } - filteredHistory.isEmpty - } else { - inlineFatigueDisabled.incr() - true - } - } - } else { - predicateDisabled.incr() - Future.True - } - } - .withStats(statsReceiver.scope(name)) - .withName(name) - } - - def webNotifsHoldback[T <: TargetUser with UserDetails with TargetABDecider]( - )( - implicit stats: StatsReceiver - ): NamedPredicate[T] = { - val name = "mr_web_notifs_holdback" - Predicate - .fromAsync { targetUserContext: T => - targetUserContext.deviceInfo.map { deviceInfoOpt => - val isPrimaryWeb = deviceInfoOpt.exists { - _.guessedPrimaryClient.exists { clientType => - clientType == MobileClientType.Web - } - } - !(isPrimaryWeb && targetUserContext.params(PushFeatureSwitchParams.MRWebHoldbackParam)) - } - } - .withStats(stats.scope(s"predicate_$name")) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TopTweetImpressionsPredicates.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TopTweetImpressionsPredicates.docx new file mode 100644 index 000000000..6b85b2e5e Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TopTweetImpressionsPredicates.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TopTweetImpressionsPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TopTweetImpressionsPredicates.scala deleted file mode 100644 index be5993b71..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TopTweetImpressionsPredicates.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.model.TopTweetImpressionsPushCandidate -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate - -object TopTweetImpressionsPredicates { - - def topTweetImpressionsFatiguePredicate( - implicit stats: StatsReceiver - ): NamedPredicate[TopTweetImpressionsPushCandidate] = { - val name = "top_tweet_impressions_fatigue" - val scopedStats = stats.scope(name) - val bucketImpressionCounter = scopedStats.counter("bucket_impression_count") - Predicate - .fromAsync { candidate: TopTweetImpressionsPushCandidate => - val interval = candidate.target.params(FS.TopTweetImpressionsNotificationInterval) - val maxInInterval = candidate.target.params(FS.MaxTopTweetImpressionsNotifications) - val minInterval = candidate.target.params(FS.TopTweetImpressionsFatigueMinIntervalDuration) - bucketImpressionCounter.incr() - - val fatiguePredicate = FatiguePredicate.recTypeOnly( - interval = interval, - maxInInterval = maxInInterval, - minInterval = minInterval, - recommendationType = CommonRecommendationType.TweetImpressions - ) - fatiguePredicate.apply(Seq(candidate)).map(_.head) - } - .withStats(stats.scope(s"predicate_${name}")) - .withName(name) - } - - def topTweetImpressionsThreshold( - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[TopTweetImpressionsPushCandidate] = { - val name = "top_tweet_impressions_threshold" - val scopedStats = statsReceiver.scope(name) - val meetsImpressionsCounter = scopedStats.counter("meets_impressions_count") - val bucketImpressionCounter = scopedStats.counter("bucket_impression_count") - Predicate - .from[TopTweetImpressionsPushCandidate] { candidate => - val meetsImpressionsThreshold = - candidate.impressionsCount >= candidate.target.params(FS.TopTweetImpressionsThreshold) - if (meetsImpressionsThreshold) meetsImpressionsCounter.incr() - bucketImpressionCounter.incr() - meetsImpressionsThreshold - } - .withStats(statsReceiver.scope(s"predicate_${name}")) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetEngagementRatioPredicate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetEngagementRatioPredicate.docx new file mode 100644 index 000000000..4ca7f5954 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetEngagementRatioPredicate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetEngagementRatioPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetEngagementRatioPredicate.scala deleted file mode 100644 index d1a3a1c64..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetEngagementRatioPredicate.scala +++ /dev/null @@ -1,112 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushConstants -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.util.CandidateUtil -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.util.Future - -object TweetEngagementRatioPredicate { - - def QTtoNtabClickBasedPredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[ - PushCandidate with TweetCandidate with RecommendationType - ] = { - val name = "oon_tweet_engagement_filter_qt_to_ntabclick_ratio_based_predicate" - val scopedStatsReceiver = stats.scope(name) - val allOonCandidatesCounter = scopedStatsReceiver.counter("all_oon_candidates") - val filteredCandidatesCounter = - scopedStatsReceiver.counter("filtered_oon_candidates") - - val quoteCountFeature = - "tweet.core.tweet_counts.quote_count" - val ntabClickCountFeature = - "tweet.magic_recs_tweet_real_time_aggregates_v2.pair.v2.magicrecs.realtime.is_ntab_clicked.any_feature.Duration.Top.count" - - Predicate - .fromAsync { candidate: PushCandidate with TweetCandidate with RecommendationType => - val target = candidate.target - val crt = candidate.commonRecType - val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(crt) || - RecTypes.outOfNetworkTopicTweetTypes.contains(crt) - - lazy val QTtoNtabClickRatioThreshold = - target.params(PushFeatureSwitchParams.TweetQTtoNtabClickRatioThresholdParam) - lazy val quoteCount = candidate.numericFeatures.getOrElse(quoteCountFeature, 0.0) - lazy val ntabClickCount = candidate.numericFeatures.getOrElse(ntabClickCountFeature, 0.0) - lazy val quoteRate = if (ntabClickCount > 0) quoteCount / ntabClickCount else 1.0 - - if (isOonCandidate) allOonCandidatesCounter.incr() - if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && isOonCandidate) { - val ntabClickThreshold = 1000 - candidate.cachePredicateInfo( - name + "_count", - ntabClickCount, - ntabClickThreshold, - ntabClickCount >= ntabClickThreshold) - candidate.cachePredicateInfo( - name + "_ratio", - quoteRate, - QTtoNtabClickRatioThreshold, - quoteRate < QTtoNtabClickRatioThreshold) - if (ntabClickCount >= ntabClickThreshold && quoteRate < QTtoNtabClickRatioThreshold) { - filteredCandidatesCounter.incr() - Future.False - } else Future.True - } else Future.True - } - .withStats(stats.scope(name)) - .withName(name) - } - - def TweetReplyLikeRatioPredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with TweetCandidate] = { - val name = "tweet_reply_like_ratio" - val scopedStatsReceiver = stats.scope(name) - val allCandidatesCounter = scopedStatsReceiver.counter("all_candidates") - val filteredCandidatesCounter = scopedStatsReceiver.counter("filtered_candidates") - val bucketedCandidatesCounter = scopedStatsReceiver.counter("bucketed_candidates") - - Predicate - .fromAsync { candidate: PushCandidate => - allCandidatesCounter.incr() - val target = candidate.target - val likeCount = candidate.numericFeatures - .getOrElse(PushConstants.TweetLikesFeatureName, 0.0) - val replyCount = candidate.numericFeatures - .getOrElse(PushConstants.TweetRepliesFeatureName, 0.0) - val ratio = replyCount / likeCount.max(1) - val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(candidate.commonRecType) || - RecTypes.outOfNetworkTopicTweetTypes.contains(candidate.commonRecType) - - if (isOonCandidate - && CandidateUtil.shouldApplyHealthQualityFilters(candidate) - && replyCount > target.params( - PushFeatureSwitchParams.TweetReplytoLikeRatioReplyCountThreshold)) { - bucketedCandidatesCounter.incr() - if (ratio > target.params( - PushFeatureSwitchParams.TweetReplytoLikeRatioThresholdLowerBound) - && ratio < target.params( - PushFeatureSwitchParams.TweetReplytoLikeRatioThresholdUpperBound)) { - filteredCandidatesCounter.incr() - Future.False - } else { - Future.True - } - } else { - Future.True - } - } - .withStats(stats.scope(s"predicate_$name")) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetLanguagePredicate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetLanguagePredicate.docx new file mode 100644 index 000000000..c2474f1e9 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetLanguagePredicate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetLanguagePredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetLanguagePredicate.scala deleted file mode 100644 index 4ff24ae77..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetLanguagePredicate.scala +++ /dev/null @@ -1,109 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.util.CandidateUtil -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.language.normalization.UserDisplayLanguage -import com.twitter.util.Future - -object TweetLanguagePredicate { - - def oonTweeetLanguageMatch( - )( - implicit stats: StatsReceiver - ): NamedPredicate[ - PushCandidate with RecommendationType with TweetDetails - ] = { - val name = "oon_tweet_language_predicate" - val scopedStatsReceiver = stats.scope(name) - val oonCandidatesCounter = - scopedStatsReceiver.counter("oon_candidates") - val enableFilterCounter = - scopedStatsReceiver.counter("enabled_filter") - val skipMediaTweetsCounter = - scopedStatsReceiver.counter("skip_media_tweets") - - Predicate - .fromAsync { candidate: PushCandidate with RecommendationType with TweetDetails => - val target = candidate.target - val crt = candidate.commonRecType - val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(crt) || - RecTypes.outOfNetworkTopicTweetTypes.contains(crt) - - if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && isOonCandidate) { - oonCandidatesCounter.incr() - - target.featureMap.map { featureMap => - val userPreferredLanguages = featureMap.sparseBinaryFeatures - .getOrElse("user.language.user.preferred_contents", Set.empty[String]) - val userEngagementLanguages = featureMap.sparseContinuousFeatures.getOrElse( - "user.language.user.engagements", - Map.empty[String, Double]) - val userFollowLanguages = featureMap.sparseContinuousFeatures.getOrElse( - "user.language.user.following_accounts", - Map.empty[String, Double]) - val userProducedTweetLanguages = featureMap.sparseContinuousFeatures - .getOrElse("user.language.user.produced_tweets", Map.empty) - val userDeviceLanguages = featureMap.sparseContinuousFeatures.getOrElse( - "user.language.user.recent_devices", - Map.empty[String, Double]) - val tweetLanguageOpt = candidate.categoricalFeatures - .get(target.params(PushFeatureSwitchParams.TweetLanguageFeatureNameParam)) - - if (userPreferredLanguages.isEmpty) - scopedStatsReceiver.counter("userPreferredLanguages_empty").incr() - if (userEngagementLanguages.isEmpty) - scopedStatsReceiver.counter("userEngagementLanguages_empty").incr() - if (userFollowLanguages.isEmpty) - scopedStatsReceiver.counter("userFollowLanguages_empty").incr() - if (userProducedTweetLanguages.isEmpty) - scopedStatsReceiver - .counter("userProducedTweetLanguages_empty") - .incr() - if (userDeviceLanguages.isEmpty) - scopedStatsReceiver.counter("userDeviceLanguages_empty").incr() - if (tweetLanguageOpt.isEmpty) scopedStatsReceiver.counter("tweetLanguage_empty").incr() - - val tweetLanguage = tweetLanguageOpt.getOrElse("und") - val undefinedTweetLanguages = Set("") - - if (!undefinedTweetLanguages.contains(tweetLanguage)) { - lazy val userInferredLanguageThreshold = - target.params(PushFeatureSwitchParams.UserInferredLanguageThresholdParam) - lazy val userDeviceLanguageThreshold = - target.params(PushFeatureSwitchParams.UserDeviceLanguageThresholdParam) - lazy val enableTweetLanguageFilter = - target.params(PushFeatureSwitchParams.EnableTweetLanguageFilter) - lazy val skipLanguageFilterForMediaTweets = - target.params(PushFeatureSwitchParams.SkipLanguageFilterForMediaTweets) - - lazy val allLanguages = userPreferredLanguages ++ - userEngagementLanguages.filter(_._2 > userInferredLanguageThreshold).keySet ++ - userFollowLanguages.filter(_._2 > userInferredLanguageThreshold).keySet ++ - userProducedTweetLanguages.filter(_._2 > userInferredLanguageThreshold).keySet ++ - userDeviceLanguages.filter(_._2 > userDeviceLanguageThreshold).keySet - - if (enableTweetLanguageFilter && allLanguages.nonEmpty) { - enableFilterCounter.incr() - val hasMedia = candidate.hasPhoto || candidate.hasVideo - - if (hasMedia && skipLanguageFilterForMediaTweets) { - skipMediaTweetsCounter.incr() - true - } else { - allLanguages.map(UserDisplayLanguage.toTweetLanguage).contains(tweetLanguage) - } - } else true - } else true - } - } else Future.True - } - .withStats(stats.scope(name)) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetWithheldContentPredicate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetWithheldContentPredicate.docx new file mode 100644 index 000000000..684c04b17 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetWithheldContentPredicate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetWithheldContentPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetWithheldContentPredicate.scala deleted file mode 100644 index c05536909..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetWithheldContentPredicate.scala +++ /dev/null @@ -1,35 +0,0 @@ -package com.twitter.frigate.pushservice.predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.TweetDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.hermit.predicate.tweetypie.UserLocationAndTweet -import com.twitter.hermit.predicate.tweetypie.WithheldTweetPredicate -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.service.metastore.gen.thriftscala.Location -import com.twitter.util.Future - -object TweetWithheldContentPredicate { - val name = "withheld_content" - val defaultLocation = Location(city = "", region = "", countryCode = "", confidence = 0.0) - - def apply( - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with TweetDetails] = { - Predicate - .fromAsync { candidate: PushCandidate with TweetDetails => - candidate.tweet match { - case Some(tweet) => - WithheldTweetPredicate(checkAllCountries = true) - .apply(Seq(UserLocationAndTweet(defaultLocation, tweet))) - .map(_.head) - case None => - Future.value(false) - } - } - .withStats(statsReceiver.scope(s"predicate_$name")) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/event/EventPredicatesForCandidate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/event/EventPredicatesForCandidate.docx new file mode 100644 index 000000000..71d35a4ff Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/event/EventPredicatesForCandidate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/event/EventPredicatesForCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/event/EventPredicatesForCandidate.scala deleted file mode 100644 index 86c1f8abd..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/event/EventPredicatesForCandidate.scala +++ /dev/null @@ -1,155 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.event - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.EventCandidate -import com.twitter.frigate.common.base.TargetInfo -import com.twitter.frigate.common.base.TargetUser -import com.twitter.frigate.common.candidate.FrigateHistory -import com.twitter.frigate.common.history.RecItems -import com.twitter.frigate.magic_events.thriftscala.Locale -import com.twitter.frigate.pushservice.model.MagicFanoutEventHydratedCandidate -import com.twitter.frigate.pushservice.model.MagicFanoutEventPushCandidate -import com.twitter.frigate.pushservice.model.MagicFanoutNewsEventPushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutPredicatesUtil._ -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.util.Future - -object EventPredicatesForCandidate { - def hasTitle( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[MagicFanoutEventHydratedCandidate] = { - val name = "event_title_available" - val scopedStatsReceiver = statsReceiver.scope(s"predicate_$name") - Predicate - .fromAsync { candidate: MagicFanoutEventHydratedCandidate => - candidate.eventTitleFut.map(_.nonEmpty) - } - .withStats(scopedStatsReceiver) - .withName(name) - } - - def isNotDuplicateWithEventId( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[MagicFanoutEventHydratedCandidate] = { - val name = "duplicate_event_id" - Predicate - .fromAsync { candidate: MagicFanoutEventHydratedCandidate => - val useRelaxedFatigueLengthFut: Future[Boolean] = - candidate match { - case mfNewsEvent: MagicFanoutNewsEventPushCandidate => - mfNewsEvent.isHighPriorityEvent - case _ => Future.value(false) - } - Future.join(candidate.target.history, useRelaxedFatigueLengthFut).map { - case (history, useRelaxedFatigueLength) => - val filteredNotifications = if (useRelaxedFatigueLength) { - val relaxedFatigueInterval = - candidate.target - .params( - PushFeatureSwitchParams.MagicFanoutRelaxedEventIdFatigueIntervalInHours).hours - history.notificationMap.filterKeys { time => - time.untilNow <= relaxedFatigueInterval - }.values - } else history.notificationMap.values - !RecItems(filteredNotifications.toSeq).events.exists(_.eventId == candidate.eventId) - } - } - .withStats(statsReceiver.scope(s"predicate_$name")) - .withName(name) - } - - def isNotDuplicateWithEventIdForCandidate[ - T <: TargetUser with FrigateHistory, - Cand <: EventCandidate with TargetInfo[T] - ]( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[Cand] = { - val name = "is_not_duplicate_event" - Predicate - .fromAsync { candidate: Cand => - candidate.target.pushRecItems.map { - !_.events.map(_.eventId).contains(candidate.eventId) - } - } - .withStats(statsReceiver.scope(name)) - .withName(name) - } - - def accountCountryPredicateWithAllowlist( - implicit stats: StatsReceiver - ): NamedPredicate[MagicFanoutEventPushCandidate] = { - val name = "account_country_predicate_with_allowlist" - val scopedStats = stats.scope(name) - - val skipPredicate = Predicate - .from { candidate: MagicFanoutEventPushCandidate => - candidate.target.params(PushFeatureSwitchParams.MagicFanoutSkipAccountCountryPredicate) - } - .withStats(stats.scope("skip_account_country_predicate_mf")) - .withName("skip_account_country_predicate_mf") - - val excludeEventFromAccountCountryPredicateFiltering = Predicate - .from { candidate: MagicFanoutEventPushCandidate => - val eventId = candidate.eventId - val target = candidate.target - target - .params(PushFeatureSwitchParams.MagicFanoutEventAllowlistToSkipAccountCountryPredicate) - .exists(eventId.equals) - } - .withStats(stats.scope("exclude_event_from_account_country_predicate_filtering")) - .withName("exclude_event_from_account_country_predicate_filtering") - - skipPredicate - .or(excludeEventFromAccountCountryPredicateFiltering) - .or(accountCountryPredicate) - .withStats(scopedStats) - .withName(name) - } - - /** - * Check if user's country is targeted - * @param stats - */ - def accountCountryPredicate( - implicit stats: StatsReceiver - ): NamedPredicate[MagicFanoutEventPushCandidate] = { - val name = "account_country_predicate" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - val internationalLocalePassedCounter = - scopedStatsReceiver.counter("international_locale_passed") - val internationalLocaleFilteredCounter = - scopedStatsReceiver.counter("international_locale_filtered") - Predicate - .fromAsync { candidate: MagicFanoutEventPushCandidate => - candidate.target.countryCode.map { - case Some(countryCode) => - val denyListedCountryCodes: Seq[String] = - if (candidate.commonRecType == CommonRecommendationType.MagicFanoutNewsEvent) { - candidate.target - .params(PushFeatureSwitchParams.MagicFanoutDenyListedCountries) - } else if (candidate.commonRecType == CommonRecommendationType.MagicFanoutSportsEvent) { - candidate.target - .params(PushFeatureSwitchParams.MagicFanoutSportsEventDenyListedCountries) - } else Seq() - val eventCountries = - candidate.newsForYouMetadata - .flatMap(_.locales).getOrElse(Seq.empty[Locale]).flatMap(_.country) - if (isInCountryList(countryCode, eventCountries) - && !isInCountryList(countryCode, denyListedCountryCodes)) { - internationalLocalePassedCounter.incr() - true - } else { - internationalLocaleFilteredCounter.incr() - false - } - case _ => false - } - } - .withStats(scopedStatsReceiver) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutPredicatesForCandidate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutPredicatesForCandidate.docx new file mode 100644 index 000000000..1be3f50d1 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutPredicatesForCandidate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutPredicatesForCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutPredicatesForCandidate.scala deleted file mode 100644 index 52371b488..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutPredicatesForCandidate.scala +++ /dev/null @@ -1,525 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.magic_fanout - -import com.twitter.audience_rewards.thriftscala.HasSuperFollowingRelationshipRequest -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.MagicFanoutCandidate -import com.twitter.frigate.common.base.MagicFanoutCreatorEventCandidate -import com.twitter.frigate.common.base.MagicFanoutProductLaunchCandidate -import com.twitter.frigate.common.history.RecItems -import com.twitter.frigate.common.predicate.FatiguePredicate.build -import com.twitter.frigate.common.predicate.FatiguePredicate.productLaunchTypeRecTypesOnlyFilter -import com.twitter.frigate.common.predicate.FatiguePredicate.recOnlyFilter -import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext -import com.twitter.frigate.common.store.interests.SemanticCoreEntityId -import com.twitter.frigate.common.util.IbisAppPushDeviceSettingsUtil -import com.twitter.frigate.magic_events.thriftscala.CreatorFanoutType -import com.twitter.frigate.magic_events.thriftscala.ProductType -import com.twitter.frigate.magic_events.thriftscala.TargetID -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.MagicFanoutEventHydratedCandidate -import com.twitter.frigate.pushservice.model.MagicFanoutEventPushCandidate -import com.twitter.frigate.pushservice.model.MagicFanoutNewsEventPushCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.predicate.FatiguePredicate -import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.NotificationDisplayLocation -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.interests.thriftscala.UserInterests -import com.twitter.simclusters_v2.thriftscala.EmbeddingType -import com.twitter.simclusters_v2.thriftscala.ModelVersion -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration -import com.twitter.util.Future - -object MagicFanoutPredicatesForCandidate { - - /** - * Check if Semantic Core reasons satisfy rank threshold ( for heavy users a non broad entity should satisfy the threshold) - */ - def magicFanoutErgInterestRankThresholdPredicate( - implicit stats: StatsReceiver - ): NamedPredicate[MagicFanoutEventHydratedCandidate] = { - val name = "magicfanout_interest_erg_rank_threshold" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - Predicate - .fromAsync { candidate: MagicFanoutEventHydratedCandidate => - candidate.target.isHeavyUserState.map { isHeavyUser => - lazy val rankThreshold = - if (isHeavyUser) { - candidate.target.params(PushFeatureSwitchParams.MagicFanoutRankErgThresholdHeavy) - } else { - candidate.target.params(PushFeatureSwitchParams.MagicFanoutRankErgThresholdNonHeavy) - } - MagicFanoutPredicatesUtil - .checkIfValidErgScEntityReasonExists( - candidate.effectiveMagicEventsReasons, - rankThreshold - ) - } - } - .withStats(scopedStatsReceiver) - .withName(name) - } - - def newsNotificationFatigue( - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val name = "news_notification_fatigue" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - Predicate - .fromAsync { candidate: PushCandidate => - FatiguePredicate - .recTypeSetOnly( - notificationDisplayLocation = NotificationDisplayLocation.PushToMobileDevice, - recTypes = Set(CommonRecommendationType.MagicFanoutNewsEvent), - maxInInterval = - candidate.target.params(PushFeatureSwitchParams.MFMaxNumberOfPushesInInterval), - interval = candidate.target.params(PushFeatureSwitchParams.MFPushIntervalInHours), - minInterval = candidate.target.params(PushFeatureSwitchParams.MFMinIntervalFatigue) - ) - .apply(Seq(candidate)) - .map(_.headOption.getOrElse(false)) - - } - .withStats(scopedStatsReceiver) - .withName(name) - } - - /** - * Check if reason contains any optouted semantic core entity interests. - * - * @param stats - * - * @return - */ - def magicFanoutNoOptoutInterestPredicate( - implicit stats: StatsReceiver - ): NamedPredicate[MagicFanoutEventPushCandidate] = { - val name = "magicfanout_optout_interest_predicate" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - val withOptOutInterestsCounter = stats.counter("with_optout_interests") - val withoutOptOutInterestsCounter = stats.counter("without_optout_interests") - Predicate - .fromAsync { candidate: MagicFanoutEventPushCandidate => - candidate.target.optOutSemanticCoreInterests.map { - case ( - optOutUserInterests: Seq[SemanticCoreEntityId] - ) => - withOptOutInterestsCounter.incr() - optOutUserInterests - .intersect(candidate.annotatedAndInferredSemanticCoreEntities).isEmpty - case _ => - withoutOptOutInterestsCounter.incr() - true - } - } - .withStats(scopedStatsReceiver) - .withName(name) - } - - /** - * Checks if the target has only one device language language, - * and that language is targeted for that event - * - * @param statsReceiver - * - * @return - */ - def inferredUserDeviceLanguagePredicate( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[MagicFanoutEventPushCandidate] = { - val name = "inferred_device_language" - val scopedStats = statsReceiver.scope(s"predicate_$name") - Predicate - .fromAsync { candidate: MagicFanoutEventPushCandidate => - val target = candidate.target - target.deviceInfo.map { - _.flatMap { deviceInfo => - val languages = deviceInfo.deviceLanguages.getOrElse(Seq.empty[String]) - val distinctDeviceLanguages = - IbisAppPushDeviceSettingsUtil.distinctDeviceLanguages(languages) - - candidate.newsForYouMetadata.map { newsForYouMetadata => - val eventLocales = newsForYouMetadata.locales.getOrElse(Seq.empty) - val eventLanguages = eventLocales.flatMap(_.language).map(_.toLowerCase).distinct - - eventLanguages.intersect(distinctDeviceLanguages).nonEmpty - } - }.getOrElse(false) - } - } - .withStats(scopedStats) - .withName(name) - } - - /** - * Bypass predicate if high priority push - */ - def highPriorityNewsEventExceptedPredicate( - predicate: NamedPredicate[MagicFanoutNewsEventPushCandidate] - )( - implicit config: Config - ): NamedPredicate[MagicFanoutNewsEventPushCandidate] = { - PredicatesForCandidate.exceptedPredicate( - name = "high_priority_excepted_" + predicate.name, - fn = MagicFanoutPredicatesUtil.checkIfHighPriorityNewsEventForCandidate, - predicate - )(config.statsReceiver) - } - - /** - * Bypass predicate if high priority push - */ - def highPriorityEventExceptedPredicate( - predicate: NamedPredicate[MagicFanoutEventPushCandidate] - )( - implicit config: Config - ): NamedPredicate[MagicFanoutEventPushCandidate] = { - PredicatesForCandidate.exceptedPredicate( - name = "high_priority_excepted_" + predicate.name, - fn = MagicFanoutPredicatesUtil.checkIfHighPriorityEventForCandidate, - predicate - )(config.statsReceiver) - } - - def magicFanoutSimClusterTargetingPredicate( - implicit stats: StatsReceiver - ): NamedPredicate[MagicFanoutEventPushCandidate] = { - val name = "simcluster_targeting" - val scopedStats = stats.scope(s"predicate_$name") - val userStateCounters = scopedStats.scope("user_state") - Predicate - .fromAsync { candidate: MagicFanoutEventPushCandidate => - candidate.target.isHeavyUserState.map { isHeavyUser => - val simClusterEmbeddings = candidate.newsForYouMetadata.flatMap( - _.eventContextScribe.flatMap(_.simClustersEmbeddings)) - val TopKSimClustersCount = 50 - val eventSimClusterVectorOpt: Option[MagicFanoutPredicatesUtil.SimClusterScores] = - MagicFanoutPredicatesUtil.getEventSimClusterVector( - simClusterEmbeddings.map(_.toMap), - (ModelVersion.Model20m145kUpdated, EmbeddingType.FollowBasedTweet), - TopKSimClustersCount - ) - val userSimClusterVectorOpt: Option[MagicFanoutPredicatesUtil.SimClusterScores] = - MagicFanoutPredicatesUtil.getUserSimClusterVector(candidate.effectiveMagicEventsReasons) - (eventSimClusterVectorOpt, userSimClusterVectorOpt) match { - case ( - Some(eventSimClusterVector: MagicFanoutPredicatesUtil.SimClusterScores), - Some(userSimClusterVector)) => - val score = eventSimClusterVector - .normedDotProduct(userSimClusterVector, eventSimClusterVector) - val threshold = if (isHeavyUser) { - candidate.target.params( - PushFeatureSwitchParams.MagicFanoutSimClusterDotProductHeavyUserThreshold) - } else { - candidate.target.params( - PushFeatureSwitchParams.MagicFanoutSimClusterDotProductNonHeavyUserThreshold) - } - val isPassed = score >= threshold - userStateCounters.scope(isHeavyUser.toString).counter(s"$isPassed").incr() - isPassed - - case (None, Some(userSimClusterVector)) => - candidate.commonRecType == CommonRecommendationType.MagicFanoutSportsEvent - - case _ => false - } - } - } - .withStats(scopedStats) - .withName(name) - } - - def geoTargetingHoldback( - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with MagicFanoutCandidate] = { - Predicate - .from[PushCandidate with MagicFanoutCandidate] { candidate => - if (MagicFanoutPredicatesUtil.reasonsContainGeoTarget( - candidate.candidateMagicEventsReasons)) { - candidate.target.params(PushFeatureSwitchParams.EnableMfGeoTargeting) - } else true - } - .withStats(stats.scope("geo_targeting_holdback")) - .withName("geo_targeting_holdback") - } - - def geoOptOutPredicate( - userStore: ReadableStore[Long, User] - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with MagicFanoutCandidate] = { - Predicate - .fromAsync[PushCandidate with MagicFanoutCandidate] { candidate => - if (MagicFanoutPredicatesUtil.reasonsContainGeoTarget( - candidate.candidateMagicEventsReasons)) { - userStore.get(candidate.target.targetId).map { userOpt => - val isGeoAllowed = userOpt - .flatMap(_.account) - .exists(_.allowLocationHistoryPersonalization) - isGeoAllowed - } - } else { - Future.True - } - } - .withStats(stats.scope("geo_opt_out_predicate")) - .withName("geo_opt_out_predicate") - } - - /** - * Check if Semantic Core reasons contains valid utt reason & reason is within top k topics followed by user - */ - def magicFanoutTopicFollowsTargetingPredicate( - implicit stats: StatsReceiver, - interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests] - ): NamedPredicate[MagicFanoutEventHydratedCandidate] = { - val name = "magicfanout_topic_follows_targeting" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - Predicate - .fromAsync[PushCandidate with MagicFanoutEventHydratedCandidate] { candidate => - candidate.followedTopicLocalizedEntities.map(_.nonEmpty) - } - .withStats(scopedStatsReceiver) - .withName(name) - } - - /** Requires the magicfanout candidate to have a UserID reason which ranks below the follow - * rank threshold. If no UserID target exists the candidate is dropped. */ - def followRankThreshold( - threshold: Param[Int] - )( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with MagicFanoutCandidate] = { - val name = "follow_rank_threshold" - Predicate - .from[PushCandidate with MagicFanoutCandidate] { c => - c.candidateMagicEventsReasons.exists { fanoutReason => - fanoutReason.reason match { - case TargetID.UserID(_) => - fanoutReason.rank.exists { rank => - rank <= c.target.params(threshold) - } - case _ => false - } - } - } - .withStats(statsReceiver.scope(name)) - .withName(name) - } - - def userGeneratedEventsPredicate( - implicit statsReceiver: StatsReceiver - ): NamedPredicate[PushCandidate with MagicFanoutEventHydratedCandidate] = { - val name = "user_generated_moments" - val stats = statsReceiver.scope(name) - - Predicate - .from { candidate: PushCandidate with MagicFanoutEventHydratedCandidate => - val isUgmMoment = candidate.semanticCoreEntityTags.values.flatten.toSet - .contains(MagicFanoutPredicatesUtil.UgmMomentTag) - if (isUgmMoment) { - candidate.target.params(PushFeatureSwitchParams.MagicFanoutNewsUserGeneratedEventsEnable) - } else true - }.withStats(stats) - .withName(name) - } - def escherbirdMagicfanoutEventParam( - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with MagicFanoutEventPushCandidate] = { - val name = "magicfanout_escherbird_fs" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - - Predicate - .fromAsync[PushCandidate with MagicFanoutEventPushCandidate] { candidate => - val candidateFrigateNotif = candidate.frigateNotification.magicFanoutEventNotification - val isEscherbirdEvent = candidateFrigateNotif.exists(_.isEscherbirdEvent.contains(true)) - scopedStatsReceiver.counter(s"with_escherbird_flag_$isEscherbirdEvent").incr() - - if (isEscherbirdEvent) { - - val listOfEventsSemanticCoreDomainIds = - candidate.target.params(PushFeatureSwitchParams.ListOfEventSemanticCoreDomainIds) - - val candScDomainEvent = - if (listOfEventsSemanticCoreDomainIds.nonEmpty) { - candidate.eventSemanticCoreDomainIds - .intersect(listOfEventsSemanticCoreDomainIds).nonEmpty - } else { - false - } - scopedStatsReceiver - .counter( - s"with_escherbird_fs_in_list_of_event_semantic_core_domains_$candScDomainEvent").incr() - Future.value(candScDomainEvent) - } else { - Future.True - } - } - .withStats(scopedStatsReceiver) - .withName(name) - } - - /** - * Checks if the user has custom targeting enabled.If so, bucket the user in experiment. This custom targeting refers to adding - * tweet authors as targets in the eventfanout service. - * @param stats [StatsReceiver] - * @return NamedPredicate[PushCandidate with MagicFanoutEventPushCandidate] - */ - def hasCustomTargetingForNewsEventsParam( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with MagicFanoutEventPushCandidate] = { - val name = "magicfanout_hascustomtargeting" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - - Predicate - .from[PushCandidate with MagicFanoutEventPushCandidate] { candidate => - candidate.candidateMagicEventsReasons.exists { fanoutReason => - fanoutReason.reason match { - case userIdReason: TargetID.UserID => - if (userIdReason.userID.hasCustomTargeting.contains(true)) { - candidate.target.params( - PushFeatureSwitchParams.MagicFanoutEnableCustomTargetingNewsEvent) - } else true - case _ => true - } - } - } - .withStats(scopedStatsReceiver) - .withName(name) - - } - - def magicFanoutProductLaunchFatigue( - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with MagicFanoutProductLaunchCandidate] = { - val name = "magic_fanout_product_launch_fatigue" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - Predicate - .fromAsync { candidate: PushCandidate with MagicFanoutProductLaunchCandidate => - val target = candidate.target - val (interval, maxInInterval, minInterval) = { - candidate.productLaunchType match { - case ProductType.BlueVerified => - ( - target.params(PushFeatureSwitchParams.ProductLaunchPushIntervalInHours), - target.params(PushFeatureSwitchParams.ProductLaunchMaxNumberOfPushesInInterval), - target.params(PushFeatureSwitchParams.ProductLaunchMinIntervalFatigue)) - case _ => - (Duration.fromDays(1), 0, Duration.Zero) - } - } - build( - interval = interval, - maxInInterval = maxInInterval, - minInterval = minInterval, - filterHistory = productLaunchTypeRecTypesOnlyFilter( - Set(CommonRecommendationType.MagicFanoutProductLaunch), - candidate.productLaunchType.toString), - notificationDisplayLocation = NotificationDisplayLocation.PushToMobileDevice - ).flatContraMap { candidate: PushCandidate => candidate.target.history } - .apply(Seq(candidate)) - .map(_.headOption.getOrElse(false)) - } - .withStats(scopedStatsReceiver) - .withName(name) - } - - def creatorPushTargetIsNotCreator( - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with MagicFanoutCreatorEventCandidate] = { - val name = "magic_fanout_creator_is_self" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - Predicate - .from { candidate: PushCandidate with MagicFanoutCreatorEventCandidate => - candidate.target.targetId != candidate.creatorId - } - .withStats(scopedStatsReceiver) - .withName(name) - } - - def duplicateCreatorPredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with MagicFanoutCreatorEventCandidate] = { - val name = "magic_fanout_creator_duplicate_creator_id" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - Predicate - .fromAsync { cand: PushCandidate with MagicFanoutCreatorEventCandidate => - cand.target.pushRecItems.map { recItems: RecItems => - !recItems.creatorIds.contains(cand.creatorId) - } - } - .withStats(scopedStatsReceiver) - .withName(name) - } - - def isSuperFollowingCreator( - )( - implicit config: Config, - stats: StatsReceiver - ): NamedPredicate[PushCandidate with MagicFanoutCreatorEventCandidate] = { - val name = "magic_fanout_is_already_superfollowing_creator" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - Predicate - .fromAsync { cand: PushCandidate with MagicFanoutCreatorEventCandidate => - config.hasSuperFollowingRelationshipStore - .get( - HasSuperFollowingRelationshipRequest( - sourceUserId = cand.target.targetId, - targetUserId = cand.creatorId)).map(_.getOrElse(false)) - } - .withStats(scopedStatsReceiver) - .withName(name) - } - - def magicFanoutCreatorPushFatiguePredicate( - )( - implicit stats: StatsReceiver - ): NamedPredicate[PushCandidate with MagicFanoutCreatorEventCandidate] = { - val name = "magic_fanout_creator_fatigue" - val scopedStatsReceiver = stats.scope(s"predicate_$name") - Predicate - .fromAsync { candidate: PushCandidate with MagicFanoutCreatorEventCandidate => - val target = candidate.target - val (interval, maxInInterval, minInterval) = { - candidate.creatorFanoutType match { - case CreatorFanoutType.UserSubscription => - ( - target.params(PushFeatureSwitchParams.CreatorSubscriptionPushIntervalInHours), - target.params( - PushFeatureSwitchParams.CreatorSubscriptionPushMaxNumberOfPushesInInterval), - target.params(PushFeatureSwitchParams.CreatorSubscriptionPushhMinIntervalFatigue)) - case CreatorFanoutType.NewCreator => - ( - target.params(PushFeatureSwitchParams.NewCreatorPushIntervalInHours), - target.params(PushFeatureSwitchParams.NewCreatorPushMaxNumberOfPushesInInterval), - target.params(PushFeatureSwitchParams.NewCreatorPushMinIntervalFatigue)) - case _ => - (Duration.fromDays(1), 0, Duration.Zero) - } - } - build( - interval = interval, - maxInInterval = maxInInterval, - minInterval = minInterval, - filterHistory = recOnlyFilter(candidate.commonRecType), - notificationDisplayLocation = NotificationDisplayLocation.PushToMobileDevice - ).flatContraMap { candidate: PushCandidate => candidate.target.history } - .apply(Seq(candidate)) - .map(_.headOption.getOrElse(false)) - } - .withStats(scopedStatsReceiver) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutPredicatesUtil.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutPredicatesUtil.docx new file mode 100644 index 000000000..855fdb4f3 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutPredicatesUtil.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutPredicatesUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutPredicatesUtil.scala deleted file mode 100644 index 306f2b3b6..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutPredicatesUtil.scala +++ /dev/null @@ -1,218 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.magic_fanout - -import com.twitter.eventdetection.event_context.util.SimClustersUtil -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.magic_events.thriftscala._ -import com.twitter.frigate.pushservice.model.MagicFanoutEventPushCandidate -import com.twitter.frigate.pushservice.model.MagicFanoutNewsEventPushCandidate -import com.twitter.frigate.pushservice.model.MagicFanoutProductLaunchPushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.simclusters_v2.common.SimClustersEmbedding -import com.twitter.simclusters_v2.thriftscala.EmbeddingType -import com.twitter.simclusters_v2.thriftscala.ModelVersion -import com.twitter.simclusters_v2.thriftscala.SimClustersEmbeddingId -import com.twitter.simclusters_v2.thriftscala.{SimClustersEmbedding => ThriftSimClustersEmbedding} -import com.twitter.util.Future - -object MagicFanoutPredicatesUtil { - - val UttDomain: Long = 0L - type DomainId = Long - type EntityId = Long - val BroadCategoryTag = "utt:broad_category" - val UgmMomentTag = "MMTS.isUGMMoment" - val TopKSimClustersCount = 50 - - case class SimClusterScores(simClusterScoreVector: Map[Int, Double]) { - def dotProduct(other: SimClusterScores): Double = { - simClusterScoreVector - .map { - case (clusterId, score) => other.simClusterScoreVector.getOrElse(clusterId, 0.0) * score - }.foldLeft(0.0) { _ + _ } - } - - def norm(): Double = { - val sumOfSquares: Double = simClusterScoreVector - .map { - case (clusterId, score) => score * score - }.foldLeft(0.0)(_ + _) - scala.math.sqrt(sumOfSquares) - } - - def normedDotProduct(other: SimClusterScores, normalizer: SimClusterScores): Double = { - val denominator = normalizer.norm() - val score = dotProduct(other) - if (denominator != 0.0) { - score / denominator - } else { - score - } - } - } - - private def isSemanticCoreEntityBroad( - semanticCoreEntityTags: Map[(DomainId, EntityId), Set[String]], - scEntityId: SemanticCoreID - ): Boolean = { - semanticCoreEntityTags - .getOrElse((scEntityId.domainId, scEntityId.entityId), Set.empty).contains(BroadCategoryTag) - } - - def isInCountryList(accountCountryCode: String, locales: Seq[String]): Boolean = { - locales.map(_.toLowerCase).contains(accountCountryCode.toLowerCase) - } - - /** - * Boolean check of if a MagicFanout is high priority push - */ - def checkIfHighPriorityNewsEventForCandidate( - candidate: MagicFanoutNewsEventPushCandidate - ): Future[Boolean] = { - candidate.isHighPriorityEvent.map { isHighPriority => - isHighPriority && (candidate.target.params(PushFeatureSwitchParams.EnableHighPriorityPush)) - } - } - - /** - * Boolean check of if a MagicFanout event is high priority push - */ - def checkIfHighPriorityEventForCandidate( - candidate: MagicFanoutEventPushCandidate - ): Future[Boolean] = { - candidate.isHighPriorityEvent.map { isHighPriority => - candidate.commonRecType match { - case CommonRecommendationType.MagicFanoutSportsEvent => - isHighPriority && (candidate.target.params( - PushFeatureSwitchParams.EnableHighPrioritySportsPush)) - case _ => false - } - } - } - - /** - * Boolean check if to skip target blue verified - */ - def shouldSkipBlueVerifiedCheckForCandidate( - candidate: MagicFanoutProductLaunchPushCandidate - ): Future[Boolean] = - Future.value( - candidate.target.params(PushFeatureSwitchParams.DisableIsTargetBlueVerifiedPredicate)) - - /** - * Boolean check if to skip target is legacy verified - */ - def shouldSkipLegacyVerifiedCheckForCandidate( - candidate: MagicFanoutProductLaunchPushCandidate - ): Future[Boolean] = - Future.value( - candidate.target.params(PushFeatureSwitchParams.DisableIsTargetLegacyVerifiedPredicate)) - - def shouldSkipSuperFollowCreatorCheckForCandidate( - candidate: MagicFanoutProductLaunchPushCandidate - ): Future[Boolean] = - Future.value( - !candidate.target.params(PushFeatureSwitchParams.EnableIsTargetSuperFollowCreatorPredicate)) - - /** - * Boolean check of if a reason of a MagicFanout is higher than the rank threshold of an event - */ - def checkIfErgScEntityReasonMeetsThreshold( - rankThreshold: Int, - reason: MagicEventsReason, - ): Boolean = { - reason.reason match { - case TargetID.SemanticCoreID(scEntityId: SemanticCoreID) => - reason.rank match { - case Some(rank) => rank < rankThreshold - case _ => false - } - case _ => false - } - } - - /** - * Check if MagicEventsReasons contains a reason that matches the thresholdw - */ - def checkIfValidErgScEntityReasonExists( - magicEventsReasons: Option[Seq[MagicEventsReason]], - rankThreshold: Int - )( - implicit stats: StatsReceiver - ): Boolean = { - magicEventsReasons match { - case Some(reasons) if reasons.exists(_.isNewUser.contains(true)) => true - case Some(reasons) => - reasons.exists { reason => - reason.source.contains(ReasonSource.ErgShortTermInterestSemanticCore) && - checkIfErgScEntityReasonMeetsThreshold( - rankThreshold, - reason - ) - } - - case _ => false - } - } - - /** - * Get event simcluster vector from event context - */ - def getEventSimClusterVector( - simClustersEmbeddingOption: Option[Map[SimClustersEmbeddingId, ThriftSimClustersEmbedding]], - embeddingMapKey: (ModelVersion, EmbeddingType), - topKSimClustersCount: Int - ): Option[SimClusterScores] = { - simClustersEmbeddingOption.map { thriftSimClustersEmbeddings => - val simClustersEmbeddings: Map[SimClustersEmbeddingId, SimClustersEmbedding] = - thriftSimClustersEmbeddings.map { - case (simClustersEmbeddingId, simClustersEmbeddingValue) => - (simClustersEmbeddingId, SimClustersEmbedding(simClustersEmbeddingValue)) - }.toMap - val emptySeq = Seq[(Int, Double)]() - val simClusterScoreTuple: Map[(ModelVersion, EmbeddingType), Seq[(Int, Double)]] = - SimClustersUtil - .getMaxTopKTweetSimClusters(simClustersEmbeddings, topKSimClustersCount) - SimClusterScores(simClusterScoreTuple.getOrElse(embeddingMapKey, emptySeq).toMap) - } - } - - /** - * Get user simcluster vector magic events reasons - */ - def getUserSimClusterVector( - magicEventsReasonsOpt: Option[Seq[MagicEventsReason]] - ): Option[SimClusterScores] = { - magicEventsReasonsOpt.map { magicEventsReasons: Seq[MagicEventsReason] => - val reasons: Seq[(Int, Double)] = magicEventsReasons.flatMap { reason => - reason.reason match { - case TargetID.SimClusterID(simClusterId: SimClusterID) => - Some((simClusterId.clusterId, reason.score.getOrElse(0.0))) - case _ => - None - } - } - SimClusterScores(reasons.toMap) - } - } - - def reasonsContainGeoTarget(reasons: Seq[MagicEventsReason]): Boolean = { - reasons.exists { reason => - val isGeoGraphSource = reason.source.contains(ReasonSource.GeoGraph) - reason.reason match { - case TargetID.PlaceID(_) if isGeoGraphSource => true - case _ => false - } - } - } - - def geoPlaceIdsFromReasons(reasons: Seq[MagicEventsReason]): Set[Long] = { - reasons.flatMap { reason => - val isGeoGraphSource = reason.source.contains(ReasonSource.GeoGraph) - reason.reason match { - case TargetID.PlaceID(PlaceID(id)) if isGeoGraphSource => Some(id) - case _ => None - } - }.toSet - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutSportsUtil.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutSportsUtil.docx new file mode 100644 index 000000000..fe9c69f64 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutSportsUtil.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutSportsUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutSportsUtil.scala deleted file mode 100644 index 224be3ad5..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutSportsUtil.scala +++ /dev/null @@ -1,231 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.magic_fanout - -import com.twitter.datatools.entityservice.entities.sports.thriftscala.NflFootballGameLiveUpdate -import com.twitter.datatools.entityservice.entities.sports.thriftscala.SoccerMatchLiveUpdate -import com.twitter.datatools.entityservice.entities.sports.thriftscala.SoccerPeriod -import com.twitter.datatools.entityservice.entities.sports.thriftscala.SportsEventHomeAwayTeamScore -import com.twitter.datatools.entityservice.entities.sports.thriftscala.SportsEventStatus -import com.twitter.datatools.entityservice.entities.sports.thriftscala.SportsEventTeamAlignment.Away -import com.twitter.datatools.entityservice.entities.sports.thriftscala.SportsEventTeamAlignment.Home -import com.twitter.escherbird.metadata.thriftscala.EntityMegadata -import com.twitter.frigate.pushservice.params.SportGameEnum -import com.twitter.frigate.common.base.GenericGameScore -import com.twitter.frigate.common.base.NflGameScore -import com.twitter.frigate.common.base.SoccerGameScore -import com.twitter.frigate.common.base.TeamInfo -import com.twitter.frigate.common.base.TeamScore -import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -object MagicFanoutSportsUtil { - - def transformSoccerGameScore(game: SoccerMatchLiveUpdate): Option[SoccerGameScore] = { - require(game.status.isDefined) - val gameScore = transformToGameScore(game.score, game.status.get) - val _penaltyKicks = transformToGameScore(game.penaltyScore, game.status.get) - gameScore.map { score => - val _isGameEnd = game.status.get match { - case SportsEventStatus.Completed(_) => true - case _ => false - } - - val _isHalfTime = game.period.exists { period => - period match { - case SoccerPeriod.Halftime(_) => true - case _ => false - } - } - - val _isOvertime = game.period.exists { period => - period match { - case SoccerPeriod.PreOvertime(_) => true - case _ => false - } - } - - val _isPenaltyKicks = game.period.exists { period => - period match { - case SoccerPeriod.PrePenalty(_) => true - case SoccerPeriod.Penalty(_) => true - case _ => false - } - } - - val _gameMinute = game.gameMinute.map { soccerGameMinute => - game.minutesInInjuryTime match { - case Some(injuryTime) => s"($soccerGameMinute+$injuryTime′)" - case None => s"($soccerGameMinute′)" - } - } - - SoccerGameScore( - score.home, - score.away, - isGameOngoing = score.isGameOngoing, - penaltyKicks = _penaltyKicks, - gameMinute = _gameMinute, - isHalfTime = _isHalfTime, - isOvertime = _isOvertime, - isPenaltyKicks = _isPenaltyKicks, - isGameEnd = _isGameEnd - ) - } - } - - def transformNFLGameScore(game: NflFootballGameLiveUpdate): Option[NflGameScore] = { - require(game.status.isDefined) - - val gameScore = transformToGameScore(game.score, game.status.get) - gameScore.map { score => - val _isGameEnd = game.status.get match { - case SportsEventStatus.Completed(_) => true - case _ => false - } - - val _matchTime = (game.quarter, game.remainingSecondsInQuarter) match { - case (Some(quarter), Some(remainingSeconds)) if remainingSeconds != 0L => - val m = (remainingSeconds / 60) % 60 - val s = remainingSeconds % 60 - val formattedSeconds = "%02d:%02d".format(m, s) - s"(Q$quarter - $formattedSeconds)" - case (Some(quarter), None) => s"(Q$quarter)" - case _ => "" - } - - NflGameScore( - score.home, - score.away, - isGameOngoing = score.isGameOngoing, - isGameEnd = _isGameEnd, - matchTime = _matchTime - ) - } - } - - /** - Takes a score from Strato columns and turns it into an easier to handle structure (GameScore class) - We do this to easily access the home/away scenario for copy setting - */ - def transformToGameScore( - scoreOpt: Option[SportsEventHomeAwayTeamScore], - status: SportsEventStatus - ): Option[GenericGameScore] = { - val isGameOngoing = status match { - case SportsEventStatus.InProgress(_) => true - case SportsEventStatus.Completed(_) => false - case _ => false - } - - val scoresWithTeam = scoreOpt - .map { score => - score.scores.map { score => (score.score, score.participantAlignment, score.participantId) } - }.getOrElse(Seq()) - - val tuple = scoresWithTeam match { - case Seq(teamOne, teamTwo, _*) => Some((teamOne, teamTwo)) - case _ => None - } - tuple.flatMap { - case ((Some(teamOneScore), teamOneAlignment, teamOne), (Some(teamTwoScore), _, teamTwo)) => - teamOneAlignment.flatMap { - case Home(_) => - val home = TeamScore(teamOneScore, teamOne.entityId, teamOne.domainId) - val away = TeamScore(teamTwoScore, teamTwo.entityId, teamTwo.domainId) - Some(GenericGameScore(home, away, isGameOngoing)) - case Away(_) => - val away = TeamScore(teamOneScore, teamOne.entityId, teamOne.domainId) - val home = TeamScore(teamTwoScore, teamTwo.entityId, teamTwo.domainId) - Some(GenericGameScore(home, away, isGameOngoing)) - case _ => None - } - case _ => None - } - } - - def getTeamInfo( - team: TeamScore, - semanticCoreMegadataStore: ReadableStore[SemanticEntityForQuery, EntityMegadata] - ): Future[Option[TeamInfo]] = { - semanticCoreMegadataStore - .get(SemanticEntityForQuery(team.teamDomainId, team.teamEntityId)).map { - _.flatMap { - _.basicMetadata.map { metadata => - TeamInfo( - name = metadata.name, - twitterUserId = metadata.twitter.flatMap(_.preferredTwitterUserId)) - } - } - } - } - - def getNFLReadableName(name: String): String = { - val teamNames = - Seq("") - teamNames.find(teamName => name.contains(teamName)).getOrElse(name) - } - - def getSoccerIbisMap(game: SoccerGameScore): Map[String, String] = { - val gameMinuteMap = game.gameMinute - .map { gameMinute => Map("match_time" -> gameMinute) } - .getOrElse(Map.empty) - - val updateTypeMap = { - if (game.isGameEnd) Map("is_game_end" -> "true") - else if (game.isHalfTime) Map("is_half_time" -> "true") - else if (game.isOvertime) Map("is_overtime" -> "true") - else if (game.isPenaltyKicks) Map("is_penalty_kicks" -> "true") - else Map("is_score_update" -> "true") - } - - val awayScore = game match { - case SoccerGameScore(_, away, _, None, _, _, _, _, _) => - away.score.toString - case SoccerGameScore(_, away, _, Some(penaltyKick), _, _, _, _, _) => - s"${away.score} (${penaltyKick.away.score}) " - case _ => "" - } - - val homeScore = game match { - case SoccerGameScore(home, _, _, None, _, _, _, _, _) => - home.score.toString - case SoccerGameScore(home, _, _, Some(penaltyKick), _, _, _, _, _) => - s"${home.score} (${penaltyKick.home.score}) " - case _ => "" - } - - val scoresMap = Map( - "away_score" -> awayScore, - "home_score" -> homeScore, - ) - - gameType(SportGameEnum.Soccer) ++ updateTypeMap ++ gameMinuteMap ++ scoresMap - } - - def getNflIbisMap(game: NflGameScore): Map[String, String] = { - val gameMinuteMap = Map("match_time" -> game.matchTime) - - val updateTypeMap = { - if (game.isGameEnd) Map("is_game_end" -> "true") - else Map("is_score_update" -> "true") - } - - val awayScore = game.away.score - val homeScore = game.home.score - - val scoresMap = Map( - "away_score" -> awayScore.toString, - "home_score" -> homeScore.toString, - ) - - gameType(SportGameEnum.Nfl) ++ updateTypeMap ++ gameMinuteMap ++ scoresMap - } - - private def gameType(game: SportGameEnum.Value): Map[String, String] = { - game match { - case SportGameEnum.Soccer => Map("is_soccer_game" -> "true") - case SportGameEnum.Nfl => Map("is_nfl_game" -> "true") - case _ => Map.empty - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutTargetingPredicateWrappersForCandidate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutTargetingPredicateWrappersForCandidate.docx new file mode 100644 index 000000000..7578a379e Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutTargetingPredicateWrappersForCandidate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutTargetingPredicateWrappersForCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutTargetingPredicateWrappersForCandidate.scala deleted file mode 100644 index 758c9ef34..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutTargetingPredicateWrappersForCandidate.scala +++ /dev/null @@ -1,133 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.magic_fanout - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext -import com.twitter.frigate.common.util.FeatureSwitchParams -import com.twitter.frigate.common.util.MagicFanoutTargetingPredicatesEnum -import com.twitter.frigate.common.util.MagicFanoutTargetingPredicatesEnum.MagicFanoutTargetingPredicatesEnum -import com.twitter.frigate.pushservice.model.MagicFanoutEventPushCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.interests.thriftscala.UserInterests -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi.FSEnumParam - -object MagicFanoutTargetingPredicateWrappersForCandidate { - - /** - * Combine Prod and Experimental Targeting predicate logic - * @return: NamedPredicate[MagicFanoutNewsEventPushCandidate] - */ - def magicFanoutTargetingPredicate( - stats: StatsReceiver, - config: Config - ): NamedPredicate[MagicFanoutEventPushCandidate] = { - val name = "magic_fanout_targeting_predicate" - Predicate - .fromAsync { candidate: MagicFanoutEventPushCandidate => - val mfTargetingPredicateParam = getTargetingPredicateParams(candidate) - val mfTargetingPredicate = MagicFanoutTargetingPredicateMapForCandidate - .apply(config) - .get(candidate.target.params(mfTargetingPredicateParam)) - mfTargetingPredicate match { - case Some(predicate) => - predicate.apply(Seq(candidate)).map(_.head) - case None => - throw new Exception( - s"MFTargetingPredicateMap doesnt contain value for TargetingParam: ${FeatureSwitchParams.MFTargetingPredicate}") - } - } - .withStats(stats.scope(name)) - .withName(name) - } - - private def getTargetingPredicateParams( - candidate: MagicFanoutEventPushCandidate - ): FSEnumParam[MagicFanoutTargetingPredicatesEnum.type] = { - if (candidate.commonRecType == CommonRecommendationType.MagicFanoutSportsEvent) { - FeatureSwitchParams.MFCricketTargetingPredicate - } else FeatureSwitchParams.MFTargetingPredicate - } - - /** - * SimCluster and ERG and Topic Follows Targeting Predicate - */ - def simClusterErgTopicFollowsTargetingPredicate( - implicit stats: StatsReceiver, - interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests] - ): NamedPredicate[MagicFanoutEventPushCandidate] = { - simClusterErgTargetingPredicate - .or(MagicFanoutPredicatesForCandidate.magicFanoutTopicFollowsTargetingPredicate) - .withName("sim_cluster_erg_topic_follows_targeting") - } - - /** - * SimCluster and ERG and Topic Follows Targeting Predicate - */ - def simClusterErgTopicFollowsUserFollowsTargetingPredicate( - implicit stats: StatsReceiver, - interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests] - ): NamedPredicate[MagicFanoutEventPushCandidate] = { - simClusterErgTopicFollowsTargetingPredicate - .or( - MagicFanoutPredicatesForCandidate.followRankThreshold( - PushFeatureSwitchParams.MagicFanoutRealgraphRankThreshold)) - .withName("sim_cluster_erg_topic_follows_user_follows_targeting") - } - - /** - * SimCluster and ERG Targeting Predicate - */ - def simClusterErgTargetingPredicate( - implicit stats: StatsReceiver - ): NamedPredicate[MagicFanoutEventPushCandidate] = { - MagicFanoutPredicatesForCandidate.magicFanoutSimClusterTargetingPredicate - .or(MagicFanoutPredicatesForCandidate.magicFanoutErgInterestRankThresholdPredicate) - .withName("sim_cluster_erg_targeting") - } -} - -/** - * Object to initalze and get predicate map - */ -object MagicFanoutTargetingPredicateMapForCandidate { - - /** - * Called from the Config.scala at the time of server initialization - * @param statsReceiver: implict stats receiver - * @return Map[MagicFanoutTargetingPredicatesEnum, NamedPredicate[MagicFanoutNewsEventPushCandidate]] - */ - def apply( - config: Config - ): Map[MagicFanoutTargetingPredicatesEnum, NamedPredicate[MagicFanoutEventPushCandidate]] = { - Map( - MagicFanoutTargetingPredicatesEnum.SimClusterAndERGAndTopicFollows -> MagicFanoutTargetingPredicateWrappersForCandidate - .simClusterErgTopicFollowsTargetingPredicate( - config.statsReceiver, - config.interestsWithLookupContextStore), - MagicFanoutTargetingPredicatesEnum.SimClusterAndERG -> MagicFanoutTargetingPredicateWrappersForCandidate - .simClusterErgTargetingPredicate(config.statsReceiver), - MagicFanoutTargetingPredicatesEnum.SimCluster -> MagicFanoutPredicatesForCandidate - .magicFanoutSimClusterTargetingPredicate(config.statsReceiver), - MagicFanoutTargetingPredicatesEnum.ERG -> MagicFanoutPredicatesForCandidate - .magicFanoutErgInterestRankThresholdPredicate(config.statsReceiver), - MagicFanoutTargetingPredicatesEnum.TopicFollows -> MagicFanoutPredicatesForCandidate - .magicFanoutTopicFollowsTargetingPredicate( - config.statsReceiver, - config.interestsWithLookupContextStore), - MagicFanoutTargetingPredicatesEnum.UserFollows -> MagicFanoutPredicatesForCandidate - .followRankThreshold( - PushFeatureSwitchParams.MagicFanoutRealgraphRankThreshold - )(config.statsReceiver), - MagicFanoutTargetingPredicatesEnum.SimClusterAndERGAndTopicFollowsAndUserFollows -> - MagicFanoutTargetingPredicateWrappersForCandidate - .simClusterErgTopicFollowsUserFollowsTargetingPredicate( - config.statsReceiver, - config.interestsWithLookupContextStore - ) - ) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/CRTBasedNtabCaretClickFatiguePredicates.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/CRTBasedNtabCaretClickFatiguePredicates.docx new file mode 100644 index 000000000..2a26ad2d9 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/CRTBasedNtabCaretClickFatiguePredicates.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/CRTBasedNtabCaretClickFatiguePredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/CRTBasedNtabCaretClickFatiguePredicates.scala deleted file mode 100644 index 704f300a5..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/CRTBasedNtabCaretClickFatiguePredicates.scala +++ /dev/null @@ -1,973 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.notificationservice.thriftscala.GenericType -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.notificationservice.genericfeedbackstore.FeedbackPromptValue -import com.twitter.hermit.predicate.Predicate -import com.twitter.frigate.common.base.Candidate -import com.twitter.frigate.common.base.RecommendationType -import com.twitter.frigate.common.base.TargetInfo -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.SeeLessOftenType -import com.twitter.frigate.common.history.History -import com.twitter.frigate.common.predicate.FrigateHistoryFatiguePredicate.TimeSeries -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.common.predicate.ntab_caret_fatigue.NtabCaretClickFatiguePredicateHelper -import com.twitter.frigate.pushservice.predicate.CaretFeedbackHistoryFilter -import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.frigate.common.predicate.FatiguePredicate -import com.twitter.frigate.pushservice.util.PushCapUtil -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.util.PushDeviceUtil - -object CRTBasedNtabCaretClickFatiguePredicates { - - private val MagicRecsCategory = "MagicRecs" - - private val HighQualityRefreshableTypes: Set[Option[String]] = Set( - Some("MagicRecHighQualityTweet"), - ) - - private def getUserStateWeight(target: Target): Future[Double] = { - PushDeviceUtil.isNtabOnlyEligible.map { - case true => - target.params(PushFeatureSwitchParams.SeeLessOftenNtabOnlyNotifUserPushCapWeight) - case _ => 1.0 - } - } - - def crtToSeeLessOftenType( - crt: CommonRecommendationType, - candidate: Candidate - with RecommendationType - with TargetInfo[ - Target - ], - ): SeeLessOftenType = { - val crtToSeeLessOftenTypeMap: Map[CommonRecommendationType, SeeLessOftenType] = { - RecTypes.f1FirstDegreeTypes.map((_, SeeLessOftenType.F1Type)).toMap - } - - crtToSeeLessOftenTypeMap.getOrElse(crt, SeeLessOftenType.OtherTypes) - } - - def genericTypeToSeeLessOftenType( - genericType: GenericType, - candidate: Candidate - with RecommendationType - with TargetInfo[ - Target - ] - ): SeeLessOftenType = { - val genericTypeToSeeLessOftenTypeMap: Map[GenericType, SeeLessOftenType] = { - Map(GenericType.MagicRecFirstDegreeTweetRecent -> SeeLessOftenType.F1Type) - } - - genericTypeToSeeLessOftenTypeMap.getOrElse(genericType, SeeLessOftenType.OtherTypes) - } - - def getWeightForCaretFeedback( - dislikedType: SeeLessOftenType, - candidate: Candidate - with RecommendationType - with TargetInfo[ - Target - ] - ): Double = { - def getWeightFromDislikedAndCurrentType( - dislikedType: SeeLessOftenType, - currentType: SeeLessOftenType - ): Double = { - val weightMap: Map[(SeeLessOftenType, SeeLessOftenType), Double] = { - - Map( - (SeeLessOftenType.F1Type, SeeLessOftenType.F1Type) -> candidate.target.params( - PushFeatureSwitchParams.SeeLessOftenF1TriggerF1PushCapWeight), - (SeeLessOftenType.OtherTypes, SeeLessOftenType.OtherTypes) -> candidate.target.params( - PushFeatureSwitchParams.SeeLessOftenNonF1TriggerNonF1PushCapWeight), - (SeeLessOftenType.F1Type, SeeLessOftenType.OtherTypes) -> candidate.target.params( - PushFeatureSwitchParams.SeeLessOftenF1TriggerNonF1PushCapWeight), - (SeeLessOftenType.OtherTypes, SeeLessOftenType.F1Type) -> candidate.target.params( - PushFeatureSwitchParams.SeeLessOftenNonF1TriggerF1PushCapWeight) - ) - } - - weightMap - .getOrElse( - (dislikedType, currentType), - candidate.target.params(PushFeatureSwitchParams.SeeLessOftenDefaultPushCapWeight)) - } - - getWeightFromDislikedAndCurrentType( - dislikedType, - crtToSeeLessOftenType(candidate.commonRecType, candidate)) - } - - private def isOutsideCrtBasedNtabCaretClickFatiguePeriodContFn( - candidate: Candidate - with RecommendationType - with TargetInfo[ - Target - ], - history: History, - feedbackDetails: Seq[CaretFeedbackDetails], - filterHistory: TimeSeries => TimeSeries = - FatiguePredicate.recTypesOnlyFilter(RecTypes.sharedNTabCaretFatigueTypes), - filterCaretFeedbackHistory: Target => Seq[ - CaretFeedbackDetails - ] => Seq[CaretFeedbackDetails] = - CaretFeedbackHistoryFilter.caretFeedbackHistoryFilter(Seq(MagicRecsCategory)), - knobs: Seq[Double], - pushCapKnobs: Seq[Double], - powerKnobs: Seq[Double], - f1Weight: Double, - nonF1Weight: Double, - defaultPushCap: Int, - stats: StatsReceiver, - tripHqTweetWeight: Double = 0.0, - ): Boolean = { - val filteredFeedbackDetails = filterCaretFeedbackHistory(candidate.target)(feedbackDetails) - val weight = { - if (RecTypes.HighQualityTweetTypes.contains( - candidate.commonRecType) && (tripHqTweetWeight != 0)) { - tripHqTweetWeight - } else if (RecTypes.isF1Type(candidate.commonRecType)) { - f1Weight - } else { - nonF1Weight - } - } - val filteredHistory = History(filterHistory(history.history.toSeq).toMap) - isOutsideFatiguePeriod( - filteredHistory, - filteredFeedbackDetails, - Seq(), - ContinuousFunctionParam( - knobs, - pushCapKnobs, - powerKnobs, - weight, - defaultPushCap - ), - stats.scope( - if (RecTypes.isF1Type(candidate.commonRecType)) "mr_ntab_dislike_f1_candidate_fn" - else if (RecTypes.HighQualityTweetTypes.contains(candidate.commonRecType)) - "mr_ntab_dislike_high_quality_candidate_fn" - else "mr_ntab_dislike_nonf1_candidate_fn") - ) - } - - private def isOutsideFatiguePeriod( - history: History, - feedbackDetails: Seq[CaretFeedbackDetails], - feedbacks: Seq[FeedbackModel], - param: ContinuousFunctionParam, - stats: StatsReceiver - ): Boolean = { - val fatiguePeriod: Duration = - NtabCaretClickFatigueUtils.durationToFilterForFeedback( - feedbackDetails, - feedbacks, - param, - param.defaultValue, - stats - ) - - val hasRecentSent = - NtabCaretClickFatiguePredicateHelper.hasRecentSend(history, fatiguePeriod) - !hasRecentSent - - } - - def genericCRTBasedNtabCaretClickFnFatiguePredicate[ - Cand <: Candidate with RecommendationType with TargetInfo[ - Target - ] - ]( - filterHistory: TimeSeries => TimeSeries = - FatiguePredicate.recTypesOnlyFilter(RecTypes.sharedNTabCaretFatigueTypes), - filterCaretFeedbackHistory: Target => Seq[ - CaretFeedbackDetails - ] => Seq[CaretFeedbackDetails] = CaretFeedbackHistoryFilter - .caretFeedbackHistoryFilter(Seq(MagicRecsCategory)), - filterInlineFeedbackHistory: Seq[FeedbackModel] => Seq[FeedbackModel] = - NtabCaretClickFatigueUtils.feedbackModelFilterByCRT(RecTypes.sharedNTabCaretFatigueTypes) - )( - implicit stats: StatsReceiver - ): NamedPredicate[Cand] = { - val predicateName = "generic_crt_based_ntab_dislike_fatigue_fn" - Predicate - .fromAsync[Cand] { cand: Cand => - { - if (!cand.target.params(PushFeatureSwitchParams.EnableGenericCRTBasedFatiguePredicate)) { - Future.True - } else { - val scopedStats = stats.scope(predicateName) - val totalRequests = scopedStats.counter("mr_ntab_dislike_total") - val total90Day = - scopedStats.counter("mr_ntab_dislike_90day_dislike") - val totalDisabled = - scopedStats.counter("mr_ntab_dislike_not_90day_dislike") - val totalSuccess = scopedStats.counter("mr_ntab_dislike_success") - val totalFiltered = scopedStats.counter("mr_ntab_dislike_filtered") - val totalWithHistory = - scopedStats.counter("mr_ntab_dislike_with_history") - val totalWithoutHistory = - scopedStats.counter("mr_ntab_dislike_without_history") - totalRequests.incr() - - Future - .join( - cand.target.history, - cand.target.caretFeedbacks, - cand.target.dynamicPushcap, - cand.target.optoutAdjustedPushcap, - PushCapUtil.getDefaultPushCap(cand.target), - getUserStateWeight(cand.target) - ).map { - case ( - history, - Some(feedbackDetails), - dynamicPushcapOpt, - optoutAdjustedPushcapOpt, - defaultPushCap, - userStateWeight) => { - totalWithHistory.incr() - - val feedbackDetailsDeduped = - NtabCaretClickFatiguePredicateHelper.dedupFeedbackDetails( - filterCaretFeedbackHistory(cand.target)(feedbackDetails), - stats - ) - - val pushCap: Int = (dynamicPushcapOpt, optoutAdjustedPushcapOpt) match { - case (_, Some(optoutAdjustedPushcap)) => optoutAdjustedPushcap - case (Some(pushcapInfo), _) => pushcapInfo.pushcap - case _ => defaultPushCap - } - val filteredHistory = History(filterHistory(history.history.toSeq).toMap) - - val hasUserDislikeInLast90Days = - NtabCaretClickFatigueUtils.hasUserDislikeInLast90Days(feedbackDetailsDeduped) - val isF1TriggerFatigueEnabled = cand.target - .params(PushFeatureSwitchParams.EnableContFnF1TriggerSeeLessOftenFatigue) - val isNonF1TriggerFatigueEnabled = cand.target.params( - PushFeatureSwitchParams.EnableContFnNonF1TriggerSeeLessOftenFatigue) - - val isOutisdeSeeLessOftenFatigue = - if (hasUserDislikeInLast90Days && (isF1TriggerFatigueEnabled || isNonF1TriggerFatigueEnabled)) { - total90Day.incr() - - val feedbackDetailsGroupedBySeeLessOftenType: Map[Option[ - SeeLessOftenType - ], Seq[ - CaretFeedbackDetails - ]] = feedbackDetails.groupBy(feedbackDetail => - feedbackDetail.genericNotificationMetadata.map(x => - genericTypeToSeeLessOftenType(x.genericType, cand))) - - val isOutsideFatiguePeriodSeq = - for (elem <- feedbackDetailsGroupedBySeeLessOftenType if elem._1.isDefined) - yield { - val dislikedSeeLessOftenType: SeeLessOftenType = elem._1.get - val seqCaretFeedbackDetails: Seq[CaretFeedbackDetails] = elem._2 - - val weight = getWeightForCaretFeedback( - dislikedSeeLessOftenType, - cand) * userStateWeight - - if (isOutsideFatiguePeriod( - history = filteredHistory, - feedbackDetails = seqCaretFeedbackDetails, - feedbacks = Seq(), - param = ContinuousFunctionParam( - knobs = cand.target - .params(PushFeatureSwitchParams.SeeLessOftenListOfDayKnobs), - knobValues = cand.target - .params( - PushFeatureSwitchParams.SeeLessOftenListOfPushCapWeightKnobs).map( - _ * pushCap), - powers = cand.target - .params(PushFeatureSwitchParams.SeeLessOftenListOfPowerKnobs), - weight = weight, - defaultValue = pushCap - ), - scopedStats - )) { - true - } else { - false - } - } - - isOutsideFatiguePeriodSeq.forall(identity) - } else { - totalDisabled.incr() - true - } - - if (isOutisdeSeeLessOftenFatigue) { - totalSuccess.incr() - } else totalFiltered.incr() - - isOutisdeSeeLessOftenFatigue - } - - case _ => - totalSuccess.incr() - totalWithoutHistory.incr() - true - } - } - } - }.withStats(stats.scope(predicateName)) - .withName(predicateName) - } - - def f1TriggeredCRTBasedNtabCaretClickFnFatiguePredicate[ - Cand <: Candidate with RecommendationType with TargetInfo[ - Target - ] - ]( - filterHistory: TimeSeries => TimeSeries = - FatiguePredicate.recTypesOnlyFilter(RecTypes.sharedNTabCaretFatigueTypes), - filterCaretFeedbackHistory: Target => Seq[ - CaretFeedbackDetails - ] => Seq[CaretFeedbackDetails] = CaretFeedbackHistoryFilter - .caretFeedbackHistoryFilter(Seq(MagicRecsCategory)), - filterInlineFeedbackHistory: Seq[FeedbackModel] => Seq[FeedbackModel] = - NtabCaretClickFatigueUtils.feedbackModelFilterByCRT(RecTypes.sharedNTabCaretFatigueTypes) - )( - implicit stats: StatsReceiver - ): NamedPredicate[Cand] = { - val predicateName = "f1_triggered_crt_based_ntab_dislike_fatigue_fn" - Predicate - .fromAsync[Cand] { cand: Cand => - { - val scopedStats = stats.scope(predicateName) - val totalRequests = scopedStats.counter("mr_ntab_dislike_total") - val total90Day = - scopedStats.counter("mr_ntab_dislike_90day_dislike") - val totalDisabled = - scopedStats.counter("mr_ntab_dislike_not_90day_dislike") - val totalSuccess = scopedStats.counter("mr_ntab_dislike_success") - val totalFiltered = scopedStats.counter("mr_ntab_dislike_filtered") - val totalWithHistory = - scopedStats.counter("mr_ntab_dislike_with_history") - val totalWithoutHistory = - scopedStats.counter("mr_ntab_dislike_without_history") - totalRequests.incr() - - Future - .join( - cand.target.history, - cand.target.caretFeedbacks, - cand.target.dynamicPushcap, - cand.target.optoutAdjustedPushcap, - cand.target.notificationFeedbacks, - PushCapUtil.getDefaultPushCap(cand.target), - getUserStateWeight(cand.target) - ).map { - case ( - history, - Some(feedbackDetails), - dynamicPushcapOpt, - optoutAdjustedPushcapOpt, - Some(feedbacks), - defaultPushCap, - userStateWeight) => - totalWithHistory.incr() - - val feedbackDetailsDeduped = - NtabCaretClickFatiguePredicateHelper.dedupFeedbackDetails( - filterCaretFeedbackHistory(cand.target)(feedbackDetails), - stats - ) - - val pushCap: Int = (dynamicPushcapOpt, optoutAdjustedPushcapOpt) match { - case (_, Some(optoutAdjustedPushcap)) => optoutAdjustedPushcap - case (Some(pushcapInfo), _) => pushcapInfo.pushcap - case _ => defaultPushCap - } - val filteredHistory = History(filterHistory(history.history.toSeq).toMap) - - val isOutsideInlineDislikeFatigue = - if (cand.target - .params(PushFeatureSwitchParams.EnableContFnF1TriggerInlineFeedbackFatigue)) { - val weight = - if (RecTypes.isF1Type(cand.commonRecType)) { - cand.target - .params(PushFeatureSwitchParams.InlineFeedbackF1TriggerF1PushCapWeight) - } else { - cand.target - .params(PushFeatureSwitchParams.InlineFeedbackF1TriggerNonF1PushCapWeight) - } - - val inlineFeedbackFatigueParam = ContinuousFunctionParam( - cand.target - .params(PushFeatureSwitchParams.InlineFeedbackListOfDayKnobs), - cand.target - .params(PushFeatureSwitchParams.InlineFeedbackListOfPushCapWeightKnobs) - .map(_ * pushCap), - cand.target - .params(PushFeatureSwitchParams.InlineFeedbackListOfPowerKnobs), - weight, - pushCap - ) - - isInlineDislikeOutsideFatiguePeriod( - cand, - feedbacks - .collect { - case feedbackPromptValue: FeedbackPromptValue => - InlineFeedbackModel(feedbackPromptValue, None) - }, - filteredHistory, - Seq( - filterInlineFeedbackHistory, - NtabCaretClickFatigueUtils.feedbackModelFilterByCRT( - RecTypes.f1FirstDegreeTypes)), - inlineFeedbackFatigueParam, - scopedStats - ) - } else true - - lazy val isOutsidePromptDislikeFatigue = - if (cand.target - .params(PushFeatureSwitchParams.EnableContFnF1TriggerPromptFeedbackFatigue)) { - val weight = - if (RecTypes.isF1Type(cand.commonRecType)) { - cand.target - .params(PushFeatureSwitchParams.PromptFeedbackF1TriggerF1PushCapWeight) - } else { - cand.target - .params(PushFeatureSwitchParams.PromptFeedbackF1TriggerNonF1PushCapWeight) - } - - val promptFeedbackFatigueParam = ContinuousFunctionParam( - cand.target - .params(PushFeatureSwitchParams.PromptFeedbackListOfDayKnobs), - cand.target - .params(PushFeatureSwitchParams.PromptFeedbackListOfPushCapWeightKnobs) - .map(_ * pushCap), - cand.target - .params(PushFeatureSwitchParams.PromptFeedbackListOfPowerKnobs), - weight, - pushCap - ) - - isPromptDislikeOutsideFatiguePeriod( - feedbacks - .collect { - case feedbackPromptValue: FeedbackPromptValue => - PromptFeedbackModel(feedbackPromptValue, None) - }, - filteredHistory, - Seq( - filterInlineFeedbackHistory, - NtabCaretClickFatigueUtils.feedbackModelFilterByCRT( - RecTypes.f1FirstDegreeTypes)), - promptFeedbackFatigueParam, - scopedStats - ) - } else true - - isOutsideInlineDislikeFatigue && isOutsidePromptDislikeFatigue - - case _ => - totalSuccess.incr() - totalWithoutHistory.incr() - true - } - } - }.withStats(stats.scope(predicateName)) - .withName(predicateName) - } - - def nonF1TriggeredCRTBasedNtabCaretClickFnFatiguePredicate[ - Cand <: Candidate with RecommendationType with TargetInfo[ - Target - ] - ]( - filterHistory: TimeSeries => TimeSeries = - FatiguePredicate.recTypesOnlyFilter(RecTypes.sharedNTabCaretFatigueTypes), - filterCaretFeedbackHistory: Target => Seq[ - CaretFeedbackDetails - ] => Seq[CaretFeedbackDetails] = CaretFeedbackHistoryFilter - .caretFeedbackHistoryFilter(Seq(MagicRecsCategory)), - filterInlineFeedbackHistory: Seq[FeedbackModel] => Seq[FeedbackModel] = - NtabCaretClickFatigueUtils.feedbackModelFilterByCRT(RecTypes.sharedNTabCaretFatigueTypes) - )( - implicit stats: StatsReceiver - ): NamedPredicate[Cand] = { - val predicateName = "non_f1_triggered_crt_based_ntab_dislike_fatigue_fn" - Predicate - .fromAsync[Cand] { cand: Cand => - { - val scopedStats = stats.scope(predicateName) - val totalRequests = scopedStats.counter("mr_ntab_dislike_total") - val total90Day = - scopedStats.counter("mr_ntab_dislike_90day_dislike") - val totalDisabled = - scopedStats.counter("mr_ntab_dislike_not_90day_dislike") - val totalSuccess = scopedStats.counter("mr_ntab_dislike_success") - val totalFiltered = scopedStats.counter("mr_ntab_dislike_filtered") - val totalWithHistory = - scopedStats.counter("mr_ntab_dislike_with_history") - val totalWithoutHistory = - scopedStats.counter("mr_ntab_dislike_without_history") - val totalFeedbackSuccess = scopedStats.counter("mr_total_feedback_success") - totalRequests.incr() - - Future - .join( - cand.target.history, - cand.target.caretFeedbacks, - cand.target.dynamicPushcap, - cand.target.optoutAdjustedPushcap, - cand.target.notificationFeedbacks, - PushCapUtil.getDefaultPushCap(cand.target), - getUserStateWeight(cand.target), - ).map { - case ( - history, - Some(feedbackDetails), - dynamicPushcapOpt, - optoutAdjustedPushcapOpt, - Some(feedbacks), - defaultPushCap, - userStateWeight) => - totalWithHistory.incr() - - val filteredfeedbackDetails = - if (cand.target.params( - PushFeatureSwitchParams.AdjustTripHqTweetTriggeredNtabCaretClickFatigue)) { - val refreshableTypeFilter = CaretFeedbackHistoryFilter - .caretFeedbackHistoryFilterByRefreshableTypeDenyList( - HighQualityRefreshableTypes) - refreshableTypeFilter(cand.target)(feedbackDetails) - } else { - feedbackDetails - } - - val feedbackDetailsDeduped = - NtabCaretClickFatiguePredicateHelper.dedupFeedbackDetails( - filterCaretFeedbackHistory(cand.target)(filteredfeedbackDetails), - stats - ) - - val pushCap: Int = (dynamicPushcapOpt, optoutAdjustedPushcapOpt) match { - case (_, Some(optoutAdjustedPushcap)) => optoutAdjustedPushcap - case (Some(pushcapInfo), _) => pushcapInfo.pushcap - case _ => defaultPushCap - } - val filteredHistory = History(filterHistory(history.history.toSeq).toMap) - - val isOutsideInlineDislikeFatigue = - if (cand.target - .params( - PushFeatureSwitchParams.EnableContFnNonF1TriggerInlineFeedbackFatigue)) { - val weight = - if (RecTypes.isF1Type(cand.commonRecType)) - cand.target - .params(PushFeatureSwitchParams.InlineFeedbackNonF1TriggerF1PushCapWeight) - else - cand.target - .params( - PushFeatureSwitchParams.InlineFeedbackNonF1TriggerNonF1PushCapWeight) - - val inlineFeedbackFatigueParam = ContinuousFunctionParam( - cand.target - .params(PushFeatureSwitchParams.InlineFeedbackListOfDayKnobs), - cand.target - .params(PushFeatureSwitchParams.InlineFeedbackListOfPushCapWeightKnobs) - .map(_ * pushCap), - cand.target - .params(PushFeatureSwitchParams.InlineFeedbackListOfPowerKnobs), - weight, - pushCap - ) - - val excludedCRTs: Set[CommonRecommendationType] = - if (cand.target.params( - PushFeatureSwitchParams.AdjustTripHqTweetTriggeredNtabCaretClickFatigue)) { - RecTypes.f1FirstDegreeTypes ++ RecTypes.HighQualityTweetTypes - } else { - RecTypes.f1FirstDegreeTypes - } - - isInlineDislikeOutsideFatiguePeriod( - cand, - feedbacks - .collect { - case feedbackPromptValue: FeedbackPromptValue => - InlineFeedbackModel(feedbackPromptValue, None) - }, - filteredHistory, - Seq( - filterInlineFeedbackHistory, - NtabCaretClickFatigueUtils.feedbackModelExcludeCRT(excludedCRTs)), - inlineFeedbackFatigueParam, - scopedStats - ) - } else true - - lazy val isOutsidePromptDislikeFatigue = - if (cand.target - .params( - PushFeatureSwitchParams.EnableContFnNonF1TriggerPromptFeedbackFatigue)) { - val weight = - if (RecTypes.isF1Type(cand.commonRecType)) - cand.target - .params(PushFeatureSwitchParams.PromptFeedbackNonF1TriggerF1PushCapWeight) - else - cand.target - .params( - PushFeatureSwitchParams.PromptFeedbackNonF1TriggerNonF1PushCapWeight) - - val promptFeedbackFatigueParam = ContinuousFunctionParam( - cand.target - .params(PushFeatureSwitchParams.PromptFeedbackListOfDayKnobs), - cand.target - .params(PushFeatureSwitchParams.PromptFeedbackListOfPushCapWeightKnobs) - .map(_ * pushCap), - cand.target - .params(PushFeatureSwitchParams.PromptFeedbackListOfPowerKnobs), - weight, - pushCap - ) - - isPromptDislikeOutsideFatiguePeriod( - feedbacks - .collect { - case feedbackPromptValue: FeedbackPromptValue => - PromptFeedbackModel(feedbackPromptValue, None) - }, - filteredHistory, - Seq( - filterInlineFeedbackHistory, - NtabCaretClickFatigueUtils.feedbackModelExcludeCRT( - RecTypes.f1FirstDegreeTypes)), - promptFeedbackFatigueParam, - scopedStats - ) - } else true - - isOutsideInlineDislikeFatigue && isOutsidePromptDislikeFatigue - case _ => - totalFeedbackSuccess.incr() - totalWithoutHistory.incr() - true - } - } - }.withStats(stats.scope(predicateName)) - .withName(predicateName) - } - - def tripHqTweetTriggeredCRTBasedNtabCaretClickFnFatiguePredicate[ - Cand <: Candidate with RecommendationType with TargetInfo[ - Target - ] - ]( - filterHistory: TimeSeries => TimeSeries = - FatiguePredicate.recTypesOnlyFilter(RecTypes.sharedNTabCaretFatigueTypes), - filterCaretFeedbackHistory: Target => Seq[ - CaretFeedbackDetails - ] => Seq[CaretFeedbackDetails] = CaretFeedbackHistoryFilter - .caretFeedbackHistoryFilter(Seq(MagicRecsCategory)), - filterInlineFeedbackHistory: Seq[FeedbackModel] => Seq[FeedbackModel] = - NtabCaretClickFatigueUtils.feedbackModelFilterByCRT(RecTypes.sharedNTabCaretFatigueTypes) - )( - implicit stats: StatsReceiver - ): NamedPredicate[Cand] = { - val predicateName = "trip_hq_tweet_triggered_crt_based_ntab_dislike_fatigue_fn" - Predicate - .fromAsync[Cand] { cand: Cand => - { - val scopedStats = stats.scope(predicateName) - val totalRequests = scopedStats.counter("mr_ntab_dislike_total") - val total90Day = - scopedStats.counter("mr_ntab_dislike_90day_dislike") - val totalDisabled = - scopedStats.counter("mr_ntab_dislike_not_90day_dislike") - val totalSuccess = scopedStats.counter("mr_ntab_dislike_success") - val totalFiltered = scopedStats.counter("mr_ntab_dislike_filtered") - val totalWithHistory = - scopedStats.counter("mr_ntab_dislike_with_history") - val totalWithoutHistory = - scopedStats.counter("mr_ntab_dislike_without_history") - val totalFeedbackSuccess = scopedStats.counter("mr_total_feedback_success") - totalRequests.incr() - - Future - .join( - cand.target.history, - cand.target.caretFeedbacks, - cand.target.dynamicPushcap, - cand.target.optoutAdjustedPushcap, - cand.target.notificationFeedbacks, - PushCapUtil.getDefaultPushCap(cand.target), - getUserStateWeight(cand.target), - ).map { - case ( - history, - Some(feedbackDetails), - dynamicPushcapOpt, - optoutAdjustedPushcapOpt, - Some(feedbacks), - defaultPushCap, - userStateWeight) => - totalWithHistory.incr() - if (cand.target.params( - PushFeatureSwitchParams.AdjustTripHqTweetTriggeredNtabCaretClickFatigue)) { - - val refreshableTypeFilter = CaretFeedbackHistoryFilter - .caretFeedbackHistoryFilterByRefreshableType(HighQualityRefreshableTypes) - val filteredfeedbackDetails = refreshableTypeFilter(cand.target)(feedbackDetails) - - val feedbackDetailsDeduped = - NtabCaretClickFatiguePredicateHelper.dedupFeedbackDetails( - filterCaretFeedbackHistory(cand.target)(filteredfeedbackDetails), - stats - ) - - val pushCap: Int = (dynamicPushcapOpt, optoutAdjustedPushcapOpt) match { - case (_, Some(optoutAdjustedPushcap)) => optoutAdjustedPushcap - case (Some(pushcapInfo), _) => pushcapInfo.pushcap - case _ => defaultPushCap - } - val filteredHistory = History(filterHistory(history.history.toSeq).toMap) - - val isOutsideInlineDislikeFatigue = - if (cand.target - .params( - PushFeatureSwitchParams.EnableContFnNonF1TriggerInlineFeedbackFatigue)) { - val weight = { - if (RecTypes.HighQualityTweetTypes.contains(cand.commonRecType)) { - cand.target - .params( - PushFeatureSwitchParams.InlineFeedbackNonF1TriggerNonF1PushCapWeight) - } else { - cand.target - .params( - PushFeatureSwitchParams.InlineFeedbackNonF1TriggerF1PushCapWeight) - } - } - - val inlineFeedbackFatigueParam = ContinuousFunctionParam( - cand.target - .params(PushFeatureSwitchParams.InlineFeedbackListOfDayKnobs), - cand.target - .params(PushFeatureSwitchParams.InlineFeedbackListOfPushCapWeightKnobs) - .map(_ * pushCap), - cand.target - .params(PushFeatureSwitchParams.InlineFeedbackListOfPowerKnobs), - weight, - pushCap - ) - - val includedCRTs: Set[CommonRecommendationType] = - RecTypes.HighQualityTweetTypes - - isInlineDislikeOutsideFatiguePeriod( - cand, - feedbacks - .collect { - case feedbackPromptValue: FeedbackPromptValue => - InlineFeedbackModel(feedbackPromptValue, None) - }, - filteredHistory, - Seq( - filterInlineFeedbackHistory, - NtabCaretClickFatigueUtils.feedbackModelFilterByCRT(includedCRTs)), - inlineFeedbackFatigueParam, - scopedStats - ) - } else true - - lazy val isOutsidePromptDislikeFatigue = - if (cand.target - .params( - PushFeatureSwitchParams.EnableContFnNonF1TriggerPromptFeedbackFatigue)) { - val weight = - if (RecTypes.isF1Type(cand.commonRecType)) - cand.target - .params( - PushFeatureSwitchParams.PromptFeedbackNonF1TriggerF1PushCapWeight) - else - cand.target - .params( - PushFeatureSwitchParams.PromptFeedbackNonF1TriggerNonF1PushCapWeight) - - val promptFeedbackFatigueParam = ContinuousFunctionParam( - cand.target - .params(PushFeatureSwitchParams.PromptFeedbackListOfDayKnobs), - cand.target - .params(PushFeatureSwitchParams.PromptFeedbackListOfPushCapWeightKnobs) - .map(_ * pushCap), - cand.target - .params(PushFeatureSwitchParams.PromptFeedbackListOfPowerKnobs), - weight, - pushCap - ) - - isPromptDislikeOutsideFatiguePeriod( - feedbacks - .collect { - case feedbackPromptValue: FeedbackPromptValue => - PromptFeedbackModel(feedbackPromptValue, None) - }, - filteredHistory, - Seq( - filterInlineFeedbackHistory, - NtabCaretClickFatigueUtils.feedbackModelExcludeCRT( - RecTypes.f1FirstDegreeTypes)), - promptFeedbackFatigueParam, - scopedStats - ) - } else true - - isOutsideInlineDislikeFatigue && isOutsidePromptDislikeFatigue - } else { - true - } - case _ => - totalFeedbackSuccess.incr() - totalWithoutHistory.incr() - true - } - } - }.withStats(stats.scope(predicateName)) - .withName(predicateName) - } - - private def getDedupedInlineFeedbackByType( - inlineFeedbacks: Seq[FeedbackModel], - feedbackType: FeedbackTypeEnum.Value, - revertedFeedbackType: FeedbackTypeEnum.Value - ): Seq[FeedbackModel] = { - inlineFeedbacks - .filter(feedback => - feedback.feedbackTypeEnum == feedbackType || - feedback.feedbackTypeEnum == revertedFeedbackType) - .groupBy(feedback => feedback.notificationImpressionId.getOrElse("")) - .toSeq - .collect { - case (impressionId, feedbacks: Seq[FeedbackModel]) if (feedbacks.nonEmpty) => - val latestFeedback = feedbacks.maxBy(feedback => feedback.timestampMs) - if (latestFeedback.feedbackTypeEnum == feedbackType) - Some(latestFeedback) - else None - case _ => None - } - .flatten - } - - private def getDedupedInlineFeedback( - inlineFeedbacks: Seq[FeedbackModel], - target: Target - ): Seq[FeedbackModel] = { - val inlineDislikeFeedback = - if (target.params(PushFeatureSwitchParams.UseInlineDislikeForFatigue)) { - getDedupedInlineFeedbackByType( - inlineFeedbacks, - FeedbackTypeEnum.InlineDislike, - FeedbackTypeEnum.InlineRevertedDislike) - } else Seq() - val inlineDismissFeedback = - if (target.params(PushFeatureSwitchParams.UseInlineDismissForFatigue)) { - getDedupedInlineFeedbackByType( - inlineFeedbacks, - FeedbackTypeEnum.InlineDismiss, - FeedbackTypeEnum.InlineRevertedDismiss) - } else Seq() - val inlineSeeLessFeedback = - if (target.params(PushFeatureSwitchParams.UseInlineSeeLessForFatigue)) { - getDedupedInlineFeedbackByType( - inlineFeedbacks, - FeedbackTypeEnum.InlineSeeLess, - FeedbackTypeEnum.InlineRevertedSeeLess) - } else Seq() - val inlineNotRelevantFeedback = - if (target.params(PushFeatureSwitchParams.UseInlineNotRelevantForFatigue)) { - getDedupedInlineFeedbackByType( - inlineFeedbacks, - FeedbackTypeEnum.InlineNotRelevant, - FeedbackTypeEnum.InlineRevertedNotRelevant) - } else Seq() - - inlineDislikeFeedback ++ inlineDismissFeedback ++ inlineSeeLessFeedback ++ inlineNotRelevantFeedback - } - - private def isInlineDislikeOutsideFatiguePeriod( - candidate: Candidate - with RecommendationType - with TargetInfo[ - Target - ], - inlineFeedbacks: Seq[FeedbackModel], - filteredHistory: History, - feedbackFilters: Seq[Seq[FeedbackModel] => Seq[FeedbackModel]], - inlineFeedbackFatigueParam: ContinuousFunctionParam, - stats: StatsReceiver - ): Boolean = { - val scopedStats = stats.scope("inline_dislike_fatigue") - - val inlineNegativeFeedback = - getDedupedInlineFeedback(inlineFeedbacks, candidate.target) - - val hydratedInlineNegativeFeedback = FeedbackModelHydrator.HydrateNotification( - inlineNegativeFeedback, - filteredHistory.history.toSeq.map(_._2)) - - if (isOutsideFatiguePeriod( - filteredHistory, - Seq(), - feedbackFilters.foldLeft(hydratedInlineNegativeFeedback)((feedbacks, feedbackFilter) => - feedbackFilter(feedbacks)), - inlineFeedbackFatigueParam, - scopedStats - )) { - scopedStats.counter("feedback_inline_dislike_success").incr() - true - } else { - scopedStats.counter("feedback_inline_dislike_filtered").incr() - false - } - } - - private def isPromptDislikeOutsideFatiguePeriod( - feedbacks: Seq[FeedbackModel], - filteredHistory: History, - feedbackFilters: Seq[Seq[FeedbackModel] => Seq[FeedbackModel]], - inlineFeedbackFatigueParam: ContinuousFunctionParam, - stats: StatsReceiver - ): Boolean = { - val scopedStats = stats.scope("prompt_dislike_fatigue") - - val promptDislikeFeedback = feedbacks - .filter(feedback => feedback.feedbackTypeEnum == FeedbackTypeEnum.PromptIrrelevant) - val hydratedPromptDislikeFeedback = FeedbackModelHydrator.HydrateNotification( - promptDislikeFeedback, - filteredHistory.history.toSeq.map(_._2)) - - if (isOutsideFatiguePeriod( - filteredHistory, - Seq(), - feedbackFilters.foldLeft(hydratedPromptDislikeFeedback)((feedbacks, feedbackFilter) => - feedbackFilter(feedbacks)), - inlineFeedbackFatigueParam, - scopedStats - )) { - scopedStats.counter("feedback_prompt_dislike_success").incr() - true - } else { - scopedStats.counter("feedback_prompt_dislike_filtered").incr() - false - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/ContinuousFunction.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/ContinuousFunction.docx new file mode 100644 index 000000000..530f3f94f Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/ContinuousFunction.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/ContinuousFunction.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/ContinuousFunction.scala deleted file mode 100644 index 862541b63..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/ContinuousFunction.scala +++ /dev/null @@ -1,148 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue - -import com.twitter.finagle.stats.StatsReceiver - -case class ContinuousFunctionParam( - knobs: Seq[Double], - knobValues: Seq[Double], - powers: Seq[Double], - weight: Double, - defaultValue: Double) { - - def validateParams(): Boolean = { - knobs.size > 0 && knobs.size - 1 == powers.size && knobs.size == knobValues.size - } -} - -object ContinuousFunction { - - /** - * Evalutate the value for function f(x) = w(x - b)^power - * where w and b are decided by the start, startVal, end, endVal - * such that - * w(start - b) ^ power = startVal - * w(end - b) ^ power = endVal - * - * @param value the value at which we will evaluate the param - * @return weight * f(value) - */ - def evaluateFn( - value: Double, - start: Double, - startVal: Double, - end: Double, - endVal: Double, - power: Double, - weight: Double - ): Double = { - val b = - (math.pow(startVal / endVal, 1 / power) * end - start) / (math.pow( - startVal / endVal, - 1 / power) - 1) - val w = startVal / math.pow(start - b, power) - weight * w * math.pow(value - b, power) - } - - /** - * Evaluate value for function f(x), and return weight * f(x) - * - * f(x) is a piecewise function - * f(x) = w_i * (x - b_i)^powers[i] for knobs[i] <= x < knobs[i+1] - * such that - * w(knobs[i] - b) ^ power = knobVals[i] - * w(knobs[i+1] - b) ^ power = knobVals[i+1] - * - * @return Evaluate value for weight * f(x), for the function described above. If the any of the input is invalid, returns defaultVal - */ - def safeEvaluateFn( - value: Double, - knobs: Seq[Double], - knobVals: Seq[Double], - powers: Seq[Double], - weight: Double, - defaultVal: Double, - statsReceiver: StatsReceiver - ): Double = { - val totalStats = statsReceiver.counter("safe_evalfn_total") - val validStats = - statsReceiver.counter("safe_evalfn_valid") - val validEndCaseStats = - statsReceiver.counter("safe_evalfn_valid_endcase") - val invalidStats = statsReceiver.counter("safe_evalfn_invalid") - - totalStats.incr() - if (knobs.size <= 0 || knobs.size - 1 != powers.size || knobs.size != knobVals.size) { - invalidStats.incr() - defaultVal - } else { - val endIndex = knobs.indexWhere(knob => knob > value) - validStats.incr() - endIndex match { - case -1 => { - validEndCaseStats.incr() - knobVals(knobVals.size - 1) * weight - } - case 0 => { - validEndCaseStats.incr() - knobVals(0) * weight - } - case _ => { - val startIndex = endIndex - 1 - evaluateFn( - value, - knobs(startIndex), - knobVals(startIndex), - knobs(endIndex), - knobVals(endIndex), - powers(startIndex), - weight) - } - } - } - } - - def safeEvaluateFn( - value: Double, - fnParams: ContinuousFunctionParam, - statsReceiver: StatsReceiver - ): Double = { - val totalStats = statsReceiver.counter("safe_evalfn_total") - val validStats = - statsReceiver.counter("safe_evalfn_valid") - val validEndCaseStats = - statsReceiver.counter("safe_evalfn_valid_endcase") - val invalidStats = statsReceiver.counter("safe_evalfn_invalid") - - totalStats.incr() - - if (fnParams.validateParams()) { - val endIndex = fnParams.knobs.indexWhere(knob => knob > value) - validStats.incr() - endIndex match { - case -1 => { - validEndCaseStats.incr() - fnParams.knobValues(fnParams.knobValues.size - 1) * fnParams.weight - } - case 0 => { - validEndCaseStats.incr() - fnParams.knobValues(0) * fnParams.weight - } - case _ => { - val startIndex = endIndex - 1 - evaluateFn( - value, - fnParams.knobs(startIndex), - fnParams.knobValues(startIndex), - fnParams.knobs(endIndex), - fnParams.knobValues(endIndex), - fnParams.powers(startIndex), - fnParams.weight - ) - } - } - } else { - invalidStats.incr() - fnParams.defaultValue - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/FeedbackModel.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/FeedbackModel.docx new file mode 100644 index 000000000..bacba95fe Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/FeedbackModel.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/FeedbackModel.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/FeedbackModel.scala deleted file mode 100644 index 654889901..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/FeedbackModel.scala +++ /dev/null @@ -1,136 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue - -import com.twitter.notificationservice.thriftscala.GenericType -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.notificationservice.genericfeedbackstore.FeedbackPromptValue -import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails -import com.twitter.notificationservice.feedback.thriftscala.FeedbackMetadata -import com.twitter.notificationservice.feedback.thriftscala.InlineFeedback -import com.twitter.notificationservice.feedback.thriftscala.FeedbackValue -import com.twitter.notificationservice.feedback.thriftscala.YesOrNoAnswer - -object FeedbackTypeEnum extends Enumeration { - val Unknown = Value - val CaretDislike = Value - val InlineDislike = Value - val InlineLike = Value - val InlineRevertedLike = Value - val InlineRevertedDislike = Value - val PromptRelevant = Value - val PromptIrrelevant = Value - val InlineDismiss = Value - val InlineRevertedDismiss = Value - val InlineSeeLess = Value - val InlineRevertedSeeLess = Value - val InlineNotRelevant = Value - val InlineRevertedNotRelevant = Value - - def safeFindByName(name: String): Value = - values.find(_.toString.toLowerCase() == name.toLowerCase()).getOrElse(Unknown) -} - -trait FeedbackModel { - - def timestampMs: Long - - def feedbackTypeEnum: FeedbackTypeEnum.Value - - def notificationImpressionId: Option[String] - - def notification: Option[FrigateNotification] = None -} - -case class CaretFeedbackModel( - caretFeedbackDetails: CaretFeedbackDetails, - notificationOpt: Option[FrigateNotification] = None) - extends FeedbackModel { - - override def timestampMs: Long = caretFeedbackDetails.eventTimestamp - - override def feedbackTypeEnum: FeedbackTypeEnum.Value = FeedbackTypeEnum.CaretDislike - - override def notificationImpressionId: Option[String] = caretFeedbackDetails.impressionId - - override def notification: Option[FrigateNotification] = notificationOpt - - def notificationGenericType: Option[GenericType] = { - caretFeedbackDetails.genericNotificationMetadata match { - case Some(genericNotificationMetadata) => - Some(genericNotificationMetadata.genericType) - case None => None - } - } -} - -case class InlineFeedbackModel( - feedback: FeedbackPromptValue, - notificationOpt: Option[FrigateNotification] = None) - extends FeedbackModel { - - override def timestampMs: Long = feedback.createdAt.inMilliseconds - - override def feedbackTypeEnum: FeedbackTypeEnum.Value = { - feedback.feedbackValue match { - case FeedbackValue( - _, - _, - _, - Some(FeedbackMetadata.InlineFeedback(InlineFeedback(Some(answer))))) => - FeedbackTypeEnum.safeFindByName("inline" + answer) - case _ => FeedbackTypeEnum.Unknown - } - } - - override def notificationImpressionId: Option[String] = Some(feedback.feedbackValue.impressionId) - - override def notification: Option[FrigateNotification] = notificationOpt -} - -case class PromptFeedbackModel( - feedback: FeedbackPromptValue, - notificationOpt: Option[FrigateNotification] = None) - extends FeedbackModel { - - override def timestampMs: Long = feedback.createdAt.inMilliseconds - - override def feedbackTypeEnum: FeedbackTypeEnum.Value = { - feedback.feedbackValue match { - case FeedbackValue(_, _, _, Some(FeedbackMetadata.YesOrNoAnswer(answer))) => - answer match { - case YesOrNoAnswer.Yes => FeedbackTypeEnum.PromptRelevant - case YesOrNoAnswer.No => FeedbackTypeEnum.PromptIrrelevant - case _ => FeedbackTypeEnum.Unknown - } - case _ => FeedbackTypeEnum.Unknown - } - } - - override def notificationImpressionId: Option[String] = Some(feedback.feedbackValue.impressionId) - - override def notification: Option[FrigateNotification] = notificationOpt -} - -object FeedbackModelHydrator { - - def HydrateNotification( - feedbacks: Seq[FeedbackModel], - history: Seq[FrigateNotification] - ): Seq[FeedbackModel] = { - feedbacks.map { - case feedback @ (inlineFeedback: InlineFeedbackModel) => - inlineFeedback.copy(notificationOpt = history.find( - _.impressionId - .equals(feedback.notificationImpressionId))) - case feedback @ (caretFeedback: CaretFeedbackModel) => - caretFeedback.copy(notificationOpt = history.find( - _.impressionId - .equals(feedback.notificationImpressionId))) - case feedback @ (promptFeedback: PromptFeedbackModel) => - promptFeedback.copy(notificationOpt = history.find( - _.impressionId - .equals(feedback.notificationImpressionId))) - case feedback => feedback - } - - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/MagicFanoutNtabCaretFatiguePredicate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/MagicFanoutNtabCaretFatiguePredicate.docx new file mode 100644 index 000000000..4bc8fea94 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/MagicFanoutNtabCaretFatiguePredicate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/MagicFanoutNtabCaretFatiguePredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/MagicFanoutNtabCaretFatiguePredicate.scala deleted file mode 100644 index 040543660..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/MagicFanoutNtabCaretFatiguePredicate.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.predicate.ntab_caret_fatigue.NtabCaretClickFatiguePredicateHelper -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.hermit.predicate.NamedPredicate - -object MagicFanoutNtabCaretFatiguePredicate { - val name = "MagicFanoutNtabCaretFatiguePredicateForCandidate" - - private val MomentsCategory = "Moments" - private val MomentsViaMagicRecsCategory = "MomentsViaMagicRecs" - - def apply()(implicit globalStats: StatsReceiver): NamedPredicate[PushCandidate] = { - val scopedStats = globalStats.scope(name) - val genericTypeCategories = Seq(MomentsCategory, MomentsViaMagicRecsCategory) - val crts = RecTypes.magicFanoutEventTypes - RecTypeNtabCaretClickFatiguePredicate - .apply( - genericTypeCategories, - crts, - NtabCaretClickFatiguePredicateHelper.calculateFatiguePeriodMagicRecs, - useMostRecentDislikeTime = true, - name = name - ).withStats(scopedStats).withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickCandidateFatiguePredicate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickCandidateFatiguePredicate.docx new file mode 100644 index 000000000..f8fb50772 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickCandidateFatiguePredicate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickCandidateFatiguePredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickCandidateFatiguePredicate.scala deleted file mode 100644 index 376d9b11f..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickCandidateFatiguePredicate.scala +++ /dev/null @@ -1,87 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.predicate.FatiguePredicate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.frigate.common.base.Candidate -import com.twitter.frigate.common.base.TargetInfo -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.common.base.{RecommendationType => BaseRecommendationType} -import com.twitter.frigate.common.predicate.CandidateWithRecommendationTypeAndTargetInfoWithCaretFeedbackHistory -import com.twitter.frigate.common.predicate.FrigateHistoryFatiguePredicate.TimeSeries -import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.predicate.CaretFeedbackHistoryFilter - -object NtabCaretClickContFnFatiguePredicate { - - private val MagicRecsCategory = "MagicRecs" - - def ntabCaretClickContFnFatiguePredicates( - filterHistory: TimeSeries => TimeSeries = - FatiguePredicate.recTypesOnlyFilter(RecTypes.sharedNTabCaretFatigueTypes), - filterCaretFeedbackHistory: Target => Seq[ - CaretFeedbackDetails - ] => Seq[CaretFeedbackDetails] = - CaretFeedbackHistoryFilter.caretFeedbackHistoryFilter(Seq(MagicRecsCategory)), - filterInlineFeedbackHistory: Seq[FeedbackModel] => Seq[FeedbackModel] = - NtabCaretClickFatigueUtils.feedbackModelFilterByCRT(RecTypes.sharedNTabCaretFatigueTypes), - name: String = "NTabCaretClickFnCandidatePredicates" - )( - implicit globalStats: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val scopedStats = globalStats.scope(name) - CRTBasedNtabCaretClickFatiguePredicates - .f1TriggeredCRTBasedNtabCaretClickFnFatiguePredicate[ - Candidate with BaseRecommendationType with TargetInfo[ - Target - ] - ]( - filterHistory = filterHistory, - filterCaretFeedbackHistory = filterCaretFeedbackHistory, - filterInlineFeedbackHistory = filterInlineFeedbackHistory - ) - .applyOnlyToCandidateWithRecommendationTypeAndTargetWithCaretFeedbackHistory - .withName("f1_triggered_fn_seelessoften_fatigue") - .andThen( - CRTBasedNtabCaretClickFatiguePredicates - .nonF1TriggeredCRTBasedNtabCaretClickFnFatiguePredicate[ - Candidate with BaseRecommendationType with TargetInfo[ - Target - ] - ]( - filterHistory = filterHistory, - filterCaretFeedbackHistory = filterCaretFeedbackHistory, - filterInlineFeedbackHistory = filterInlineFeedbackHistory - ) - .applyOnlyToCandidateWithRecommendationTypeAndTargetWithCaretFeedbackHistory) - .withName("nonf1_triggered_fn_seelessoften_fatigue") - .andThen( - CRTBasedNtabCaretClickFatiguePredicates - .tripHqTweetTriggeredCRTBasedNtabCaretClickFnFatiguePredicate[ - Candidate with BaseRecommendationType with TargetInfo[ - Target - ] - ]( - filterHistory = filterHistory, - filterCaretFeedbackHistory = filterCaretFeedbackHistory, - filterInlineFeedbackHistory = filterInlineFeedbackHistory - ) - .applyOnlyToCandidateWithRecommendationTypeAndTargetWithCaretFeedbackHistory) - .withName("trip_hq_tweet_triggered_fn_seelessoften_fatigue") - .andThen( - CRTBasedNtabCaretClickFatiguePredicates - .genericCRTBasedNtabCaretClickFnFatiguePredicate[ - Candidate with BaseRecommendationType with TargetInfo[ - Target - ] - ]( - filterHistory = filterHistory, - filterCaretFeedbackHistory = filterCaretFeedbackHistory, - filterInlineFeedbackHistory = filterInlineFeedbackHistory) - .applyOnlyToCandidateWithRecommendationTypeAndTargetWithCaretFeedbackHistory - .withName("generic_fn_seelessoften_fatigue") - ) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickFatiguePredicate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickFatiguePredicate.docx new file mode 100644 index 000000000..1ec19d572 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickFatiguePredicate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickFatiguePredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickFatiguePredicate.scala deleted file mode 100644 index 579f4b25f..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickFatiguePredicate.scala +++ /dev/null @@ -1,47 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.predicate.ntab_caret_fatigue.NtabCaretClickFatiguePredicateHelper -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.util.Future - -object NtabCaretClickFatiguePredicate { - val name = "NtabCaretClickFatiguePredicate" - - def isSpacesTypeAndTeamMember(candidate: PushCandidate): Future[Boolean] = { - candidate.target.isTeamMember.map { isTeamMember => - val isSpacesType = RecTypes.isRecommendedSpacesType(candidate.commonRecType) - isTeamMember && isSpacesType - } - } - - def apply()(implicit globalStats: StatsReceiver): NamedPredicate[PushCandidate] = { - val scopedStats = globalStats.scope(name) - val genericTypeCategories = Seq("MagicRecs") - val crts = RecTypes.sharedNTabCaretFatigueTypes - val recTypeNtabCaretClickFatiguePredicate = - RecTypeNtabCaretClickFatiguePredicate.apply( - genericTypeCategories, - crts, - NtabCaretClickFatiguePredicateHelper.calculateFatiguePeriodMagicRecs, - useMostRecentDislikeTime = false - ) - Predicate - .fromAsync { candidate: PushCandidate => - isSpacesTypeAndTeamMember(candidate).flatMap { isSpacesTypeAndTeamMember => - if (RecTypes.sharedNTabCaretFatigueTypes( - candidate.commonRecType) && !isSpacesTypeAndTeamMember) { - recTypeNtabCaretClickFatiguePredicate - .apply(Seq(candidate)).map(_.headOption.getOrElse(false)) - } else { - Future.True - } - } - } - .withStats(scopedStats) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickFatigueUtils.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickFatigueUtils.docx new file mode 100644 index 000000000..06666246a Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickFatigueUtils.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickFatigueUtils.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickFatigueUtils.scala deleted file mode 100644 index cc2c0c072..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickFatigueUtils.scala +++ /dev/null @@ -1,108 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.predicate.ntab_caret_fatigue.NtabCaretClickFatiguePredicateHelper -import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails -import com.twitter.util.Duration -import com.twitter.conversions.DurationOps._ -import scala.math.min -import com.twitter.util.Time -import com.twitter.frigate.thriftscala.{CommonRecommendationType => CRT} - -object NtabCaretClickFatigueUtils { - - private def pushCapForFeedback( - feedbackDetails: Seq[CaretFeedbackDetails], - feedbacks: Seq[FeedbackModel], - param: ContinuousFunctionParam, - statsReceiver: StatsReceiver - ): Double = { - val stats = statsReceiver.scope("mr_seelessoften_contfn_pushcap") - val pushCapTotal = stats.counter("pushcap_total") - val pushCapInvalid = - stats.counter("pushcap_invalid") - - pushCapTotal.incr() - val timeSinceMostRecentDislikeMs = - NtabCaretClickFatiguePredicateHelper.getDurationSinceMostRecentDislike(feedbackDetails) - val mostRecentFeedbackTimestamp: Option[Long] = - feedbacks - .map { feedback => - feedback.timestampMs - }.reduceOption(_ max _) - val timeSinceMostRecentFeedback: Option[Duration] = - mostRecentFeedbackTimestamp.map(Time.now - Time.fromMilliseconds(_)) - - val nTabDislikePushCap = timeSinceMostRecentDislikeMs match { - case Some(lastDislikeTimeMs) => { - ContinuousFunction.safeEvaluateFn(lastDislikeTimeMs.inDays.toDouble, param, stats) - } - case _ => { - pushCapInvalid.incr() - param.defaultValue - } - } - val feedbackPushCap = timeSinceMostRecentFeedback match { - case Some(lastDislikeTimeVal) => { - ContinuousFunction.safeEvaluateFn(lastDislikeTimeVal.inDays.toDouble, param, stats) - } - case _ => { - pushCapInvalid.incr() - param.defaultValue - } - } - - min(nTabDislikePushCap, feedbackPushCap) - } - - def durationToFilterForFeedback( - feedbackDetails: Seq[CaretFeedbackDetails], - feedbacks: Seq[FeedbackModel], - param: ContinuousFunctionParam, - defaultPushCap: Double, - statsReceiver: StatsReceiver - ): Duration = { - val pushCap = min( - pushCapForFeedback(feedbackDetails, feedbacks, param, statsReceiver), - defaultPushCap - ) - if (pushCap <= 0) { - Duration.Top - } else { - 24.hours / pushCap - } - } - - def hasUserDislikeInLast90Days(feedbackDetails: Seq[CaretFeedbackDetails]): Boolean = { - val timeSinceMostRecentDislike = - NtabCaretClickFatiguePredicateHelper.getDurationSinceMostRecentDislike(feedbackDetails) - - timeSinceMostRecentDislike.exists(_ < 90.days) - } - - def feedbackModelFilterByCRT( - crts: Set[CRT] - ): Seq[FeedbackModel] => Seq[ - FeedbackModel - ] = { feedbacks => - feedbacks.filter { feedback => - feedback.notification match { - case Some(notification) => crts.contains(notification.commonRecommendationType) - case None => false - } - } - } - - def feedbackModelExcludeCRT( - crts: Set[CRT] - ): Seq[FeedbackModel] => Seq[ - FeedbackModel - ] = { feedbacks => - feedbacks.filter { feedback => - feedback.notification match { - case Some(notification) => !crts.contains(notification.commonRecommendationType) - case None => true - } - } - } -}