mirror of
https://github.com/twitter/the-algorithm.git
synced 2024-11-16 00:25:11 +01:00
[docx] split commit for file 3200
Signed-off-by: Ari Archer <ari.web.xyz@gmail.com>
This commit is contained in:
Ari Archer
parent
23cec12533
commit
f3c5ff35cb
Binary file not shown.
@ -1,57 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline
|
||||
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.MalformedCursor
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure
|
||||
import com.twitter.scrooge.BinaryThriftStructSerializer
|
||||
import com.twitter.scrooge.ThriftStruct
|
||||
import com.twitter.util.Return
|
||||
import com.twitter.util.Throw
|
||||
import com.twitter.util.Try
|
||||
|
||||
/**
|
||||
* Serializes a [[PipelineCursor]] into thrift and then into a base64 encoded string
|
||||
*/
|
||||
trait PipelineCursorSerializer[-Cursor <: PipelineCursor] {
|
||||
def serializeCursor(cursor: Cursor): String
|
||||
}
|
||||
|
||||
object PipelineCursorSerializer {
|
||||
|
||||
/**
|
||||
* Deserializes a cursor string into thrift and then into a [[PipelineCursor]]
|
||||
*
|
||||
* @param cursorString to deserialize, which is base64 encoded thrift
|
||||
* @param cursorThriftSerializer to deserialize the cursor string into thrift
|
||||
* @param deserializePf specifies how to transform the serialized thrift into a [[PipelineCursor]]
|
||||
* @return optional [[PipelineCursor]]. `None` may or may not be a failure depending on the
|
||||
* implementation of deserializePf.
|
||||
*
|
||||
* @note The "A" type of deserializePf cannot be inferred due to the thrift type not being present
|
||||
* on the PipelineCursorSerializer trait. Therefore invokers must often add an explicit type
|
||||
* on the deserializeCursor call to help out the compiler when passing deserializePf inline.
|
||||
* Alternatively, deserializePf can be declared as a val with a type annotation before it is
|
||||
* passed into this method.
|
||||
*/
|
||||
def deserializeCursor[Thrift <: ThriftStruct, Cursor <: PipelineCursor](
|
||||
cursorString: String,
|
||||
cursorThriftSerializer: BinaryThriftStructSerializer[Thrift],
|
||||
deserializePf: PartialFunction[Option[Thrift], Option[Cursor]]
|
||||
): Option[Cursor] = {
|
||||
val thriftCursor: Option[Thrift] =
|
||||
Try {
|
||||
cursorThriftSerializer.fromString(cursorString)
|
||||
} match {
|
||||
case Return(thriftCursor) => Some(thriftCursor)
|
||||
case Throw(_) => None
|
||||
}
|
||||
|
||||
// Add type annotation to help out the compiler since the type is lost due to the _ match
|
||||
val defaultDeserializePf: PartialFunction[Option[Thrift], Option[Cursor]] = {
|
||||
case _ =>
|
||||
// This case is the result of the client submitting a cursor we do not expect
|
||||
throw PipelineFailure(MalformedCursor, s"Unknown request cursor: $cursorString")
|
||||
}
|
||||
|
||||
(deserializePf orElse defaultDeserializePf)(thriftCursor)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,32 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline
|
||||
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.HasDebugOptions
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.HasProduct
|
||||
import com.twitter.timelines.configapi.HasParams
|
||||
import com.twitter.timelines.configapi.Param
|
||||
import com.twitter.util.Time
|
||||
|
||||
trait PipelineQuery extends HasParams with HasClientContext with HasProduct with HasDebugOptions {
|
||||
self =>
|
||||
|
||||
/** Set a query time val that is constant for the duration of the query lifecycle */
|
||||
val queryTime: Time = self.debugOptions.flatMap(_.requestTimeOverride).getOrElse(Time.now)
|
||||
|
||||
/** The requested max results is specified, or not specified, by the thrift client */
|
||||
def requestedMaxResults: Option[Int]
|
||||
|
||||
/** Retrieves the max results with a default Param, if not specified by the thrift client */
|
||||
def maxResults(defaultRequestedMaxResultParam: Param[Int]): Int =
|
||||
requestedMaxResults.getOrElse(params(defaultRequestedMaxResultParam))
|
||||
|
||||
/** Optional [[FeatureMap]], this may be updated later using [[withFeatureMap]] */
|
||||
def features: Option[FeatureMap]
|
||||
|
||||
/**
|
||||
* Since Query-Level features can be hydrated later, we need this method to update the PipelineQuery
|
||||
* usually this will be implemented via `copy(features = Some(features))`
|
||||
*/
|
||||
def withFeatureMap(features: FeatureMap): PipelineQuery
|
||||
}
|
||||
Binary file not shown.
@ -1,59 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline
|
||||
|
||||
import com.twitter.product_mixer.component_library.model.candidate.CursorCandidate
|
||||
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
|
||||
import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
|
||||
import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.ExecutionFailed
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure
|
||||
import com.twitter.util.Return
|
||||
import com.twitter.util.Throw
|
||||
import com.twitter.util.Try
|
||||
|
||||
/**
|
||||
* Pipelines return a PipelineResult.
|
||||
*
|
||||
* This allows us to return a single main result (optionally, incase the pipeline didn't execute successfully), but
|
||||
* still have a detailed response object to show how that result was produced.
|
||||
*/
|
||||
trait PipelineResult[ResultType] {
|
||||
val failure: Option[PipelineFailure]
|
||||
val result: Option[ResultType]
|
||||
|
||||
def withFailure(failure: PipelineFailure): PipelineResult[ResultType]
|
||||
def withResult(result: ResultType): PipelineResult[ResultType]
|
||||
|
||||
def resultSize(): Int
|
||||
|
||||
private[pipeline] def stopExecuting: Boolean = failure.isDefined || result.isDefined
|
||||
|
||||
final def toTry: Try[this.type] = (result, failure) match {
|
||||
case (_, Some(failure)) =>
|
||||
Throw(failure)
|
||||
case (_: Some[ResultType], _) =>
|
||||
Return(this)
|
||||
// Pipelines should always finish with either a result or a failure
|
||||
case _ => Throw(PipelineFailure(ExecutionFailed, "Pipeline did not execute"))
|
||||
}
|
||||
|
||||
final def toResultTry: Try[ResultType] = {
|
||||
// `.get` is safe here because `toTry` guarantees a value in the `Return` case
|
||||
toTry.map(_.result.get)
|
||||
}
|
||||
}
|
||||
|
||||
object PipelineResult {
|
||||
|
||||
/**
|
||||
* Track number of candidates returned by a Pipeline. Cursors are excluded from this
|
||||
* count and modules are counted as the sum of their candidates.
|
||||
*
|
||||
* @note this is a somewhat subjective measure of 'size' and it is spread across pipeline
|
||||
* definitions as well as selectors.
|
||||
*/
|
||||
def resultSize(results: Seq[CandidateWithDetails]): Int = results.map {
|
||||
case module: ModuleCandidateWithDetails => resultSize(module.candidates)
|
||||
case ItemCandidateWithDetails(_: CursorCandidate, _, _) => 0
|
||||
case _ => 1
|
||||
}.sum
|
||||
}
|
||||
@ -1,77 +0,0 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"configapi/configapi-core",
|
||||
"configapi/configapi-decider",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"finatra/inject/inject-core/src/main/scala/com/twitter/inject",
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/gate",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/async_feature_map_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_decorator_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_hydrator_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_source_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/filter_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/group_results_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util",
|
||||
"stitch/stitch-core",
|
||||
"util/util-core:scala",
|
||||
],
|
||||
exports = [
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"configapi/configapi-core",
|
||||
"configapi/configapi-decider",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/gate",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/async_feature_map_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_decorator_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_hydrator_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_source_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/filter_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/group_results_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor",
|
||||
"stitch/stitch-core",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,30 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.candidate
|
||||
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
|
||||
import com.twitter.product_mixer.core.pipeline.Pipeline
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Arrow
|
||||
|
||||
/**
|
||||
* A Candidate Pipeline
|
||||
*
|
||||
* This is an abstract class, as we only construct these via the [[CandidatePipelineBuilder]].
|
||||
*
|
||||
* A [[CandidatePipeline]] is capable of processing requests (queries) and returning candidates
|
||||
* in the form of a [[CandidatePipelineResult]]
|
||||
*
|
||||
* @tparam Query the domain model for the query or request
|
||||
*/
|
||||
abstract class CandidatePipeline[-Query <: PipelineQuery] private[candidate]
|
||||
extends Pipeline[CandidatePipeline.Inputs[Query], Seq[CandidateWithDetails]] {
|
||||
override private[core] val config: BaseCandidatePipelineConfig[Query, _, _, _]
|
||||
override val arrow: Arrow[CandidatePipeline.Inputs[Query], CandidatePipelineResult]
|
||||
override val identifier: CandidatePipelineIdentifier
|
||||
}
|
||||
|
||||
object CandidatePipeline {
|
||||
case class Inputs[+Query <: PipelineQuery](
|
||||
query: Query,
|
||||
existingCandidates: Seq[CandidateWithDetails])
|
||||
}
|
||||
Binary file not shown.
@ -1,735 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.candidate
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.feature.featuremap.asyncfeaturemap.AsyncFeatureMap
|
||||
import com.twitter.product_mixer.core.functional_component.common.alert.Alert
|
||||
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
|
||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.BaseCandidatePipelineQueryTransformer
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer
|
||||
import com.twitter.product_mixer.core.gate.ParamGate
|
||||
import com.twitter.product_mixer.core.gate.ParamGate.EnabledGateSuffix
|
||||
import com.twitter.product_mixer.core.gate.ParamGate.SupportedClientGateSuffix
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.Component
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifierStack
|
||||
import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
|
||||
import com.twitter.product_mixer.core.pipeline.InvalidStepStateException
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineBuilder
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.ClosedGate
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailureClassifier
|
||||
import com.twitter.product_mixer.core.service.Executor
|
||||
import com.twitter.product_mixer.core.service.async_feature_map_executor.AsyncFeatureMapExecutor
|
||||
import com.twitter.product_mixer.core.service.async_feature_map_executor.AsyncFeatureMapExecutorResults
|
||||
import com.twitter.product_mixer.core.service.candidate_decorator_executor.CandidateDecoratorExecutor
|
||||
import com.twitter.product_mixer.core.service.candidate_decorator_executor.CandidateDecoratorExecutorResult
|
||||
import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutor
|
||||
import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutorResult
|
||||
import com.twitter.product_mixer.core.service.candidate_source_executor.CandidateSourceExecutor
|
||||
import com.twitter.product_mixer.core.service.candidate_source_executor.CandidateSourceExecutorResult
|
||||
import com.twitter.product_mixer.core.service.candidate_source_executor.FetchedCandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.service.filter_executor.FilterExecutor
|
||||
import com.twitter.product_mixer.core.service.filter_executor.FilterExecutorResult
|
||||
import com.twitter.product_mixer.core.service.gate_executor.GateExecutor
|
||||
import com.twitter.product_mixer.core.service.gate_executor.GateExecutorResult
|
||||
import com.twitter.product_mixer.core.service.gate_executor.StoppedGateException
|
||||
import com.twitter.product_mixer.core.service.group_results_executor.GroupResultsExecutor
|
||||
import com.twitter.product_mixer.core.service.group_results_executor.GroupResultsExecutorInput
|
||||
import com.twitter.product_mixer.core.service.group_results_executor.GroupResultsExecutorResult
|
||||
import com.twitter.product_mixer.core.service.query_feature_hydrator_executor.QueryFeatureHydratorExecutor
|
||||
import com.twitter.stitch.Arrow
|
||||
import com.twitter.util.logging.Logging
|
||||
import javax.inject.Inject
|
||||
|
||||
class CandidatePipelineBuilder[
|
||||
Query <: PipelineQuery,
|
||||
CandidateSourceQuery,
|
||||
CandidateSourceResult,
|
||||
Result <: UniversalNoun[Any]] @Inject() (
|
||||
queryFeatureHydratorExecutor: QueryFeatureHydratorExecutor,
|
||||
asyncFeatureMapExecutor: AsyncFeatureMapExecutor,
|
||||
candidateDecoratorExecutor: CandidateDecoratorExecutor,
|
||||
candidateFeatureHydratorExecutor: CandidateFeatureHydratorExecutor,
|
||||
candidateSourceExecutor: CandidateSourceExecutor,
|
||||
groupResultsExecutor: GroupResultsExecutor,
|
||||
filterExecutor: FilterExecutor,
|
||||
gateExecutor: GateExecutor,
|
||||
override val statsReceiver: StatsReceiver)
|
||||
extends PipelineBuilder[CandidatePipeline.Inputs[Query]]
|
||||
with Logging {
|
||||
|
||||
override type UnderlyingResultType = Seq[CandidateWithDetails]
|
||||
override type PipelineResultType = IntermediateCandidatePipelineResult[Result]
|
||||
|
||||
def build(
|
||||
parentComponentIdentifierStack: ComponentIdentifierStack,
|
||||
config: BaseCandidatePipelineConfig[
|
||||
Query,
|
||||
CandidateSourceQuery,
|
||||
CandidateSourceResult,
|
||||
Result
|
||||
]
|
||||
): CandidatePipeline[Query] = {
|
||||
|
||||
val pipelineIdentifier = config.identifier
|
||||
val candidateSourceIdentifier = config.candidateSource.identifier
|
||||
|
||||
val context = Executor.Context(
|
||||
PipelineFailureClassifier(
|
||||
config.failureClassifier.orElse(StoppedGateException.classifier(ClosedGate))),
|
||||
parentComponentIdentifierStack.push(pipelineIdentifier)
|
||||
)
|
||||
|
||||
val enabledGateOpt = config.enabledDeciderParam.map { deciderParam =>
|
||||
ParamGate(pipelineIdentifier + EnabledGateSuffix, deciderParam)
|
||||
}
|
||||
val supportedClientGateOpt = config.supportedClientParam.map { param =>
|
||||
ParamGate(pipelineIdentifier + SupportedClientGateSuffix, param)
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluate enabled decider gate first since if it's off, there is no reason to proceed
|
||||
* Next evaluate supported client feature switch gate, followed by customer configured gates
|
||||
*/
|
||||
val allGates = enabledGateOpt.toSeq ++ supportedClientGateOpt.toSeq ++ config.gates
|
||||
|
||||
// Dynamically replace the identifier of both transformers if config used the inline constructor
|
||||
// which sets a default identifier. We need to do this to ensure uniqueness of identifiers.
|
||||
val queryTransformer = BaseCandidatePipelineQueryTransformer.copyWithUpdatedIdentifier(
|
||||
config.queryTransformer,
|
||||
pipelineIdentifier)
|
||||
|
||||
val resultsTransformer = CandidatePipelineResultsTransformer.copyWithUpdatedIdentifier(
|
||||
config.resultTransformer,
|
||||
pipelineIdentifier)
|
||||
|
||||
val decorator = config.decorator.map(decorator =>
|
||||
CandidateDecorator.copyWithUpdatedIdentifier(decorator, pipelineIdentifier))
|
||||
|
||||
val GatesStep = new Step[Query, GateExecutorResult] {
|
||||
override def identifier: PipelineStepIdentifier = CandidatePipelineConfig.gatesStep
|
||||
|
||||
override def executorArrow: Arrow[Query, GateExecutorResult] = {
|
||||
gateExecutor.arrow(allGates, context)
|
||||
}
|
||||
|
||||
override def inputAdaptor(
|
||||
query: CandidatePipeline.Inputs[Query],
|
||||
previousResult: IntermediateCandidatePipelineResult[Result]
|
||||
): Query =
|
||||
query.query
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: IntermediateCandidatePipelineResult[Result],
|
||||
executorResult: GateExecutorResult
|
||||
): IntermediateCandidatePipelineResult[Result] =
|
||||
previousPipelineResult.copy(underlyingResult =
|
||||
previousPipelineResult.underlyingResult.copy(gateResult = Some(executorResult)))
|
||||
}
|
||||
|
||||
def queryFeatureHydrationStep(
|
||||
queryFeatureHydrators: Seq[BaseQueryFeatureHydrator[Query, _]],
|
||||
stepIdentifier: PipelineStepIdentifier,
|
||||
updater: ResultUpdater[CandidatePipelineResult, QueryFeatureHydratorExecutor.Result]
|
||||
): Step[Query, QueryFeatureHydratorExecutor.Result] =
|
||||
new Step[Query, QueryFeatureHydratorExecutor.Result] {
|
||||
override def identifier: PipelineStepIdentifier = stepIdentifier
|
||||
|
||||
override def executorArrow: Arrow[Query, QueryFeatureHydratorExecutor.Result] =
|
||||
queryFeatureHydratorExecutor.arrow(
|
||||
queryFeatureHydrators,
|
||||
CandidatePipelineConfig.stepsAsyncFeatureHydrationCanBeCompletedBy,
|
||||
context)
|
||||
|
||||
override def inputAdaptor(
|
||||
query: CandidatePipeline.Inputs[Query],
|
||||
previousResult: IntermediateCandidatePipelineResult[Result]
|
||||
): Query = query.query
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: IntermediateCandidatePipelineResult[Result],
|
||||
executorResult: QueryFeatureHydratorExecutor.Result
|
||||
): IntermediateCandidatePipelineResult[Result] =
|
||||
previousPipelineResult.copy(
|
||||
underlyingResult = updater(previousPipelineResult.underlyingResult, executorResult))
|
||||
|
||||
override def queryUpdater(
|
||||
query: CandidatePipeline.Inputs[Query],
|
||||
executorResult: QueryFeatureHydratorExecutor.Result
|
||||
): CandidatePipeline.Inputs[Query] =
|
||||
CandidatePipeline.Inputs(
|
||||
query.query
|
||||
.withFeatureMap(
|
||||
query.query.features.getOrElse(
|
||||
FeatureMap.empty) ++ executorResult.featureMap).asInstanceOf[Query],
|
||||
query.existingCandidates)
|
||||
}
|
||||
|
||||
def asyncFeaturesStep(
|
||||
stepToHydrateFor: PipelineStepIdentifier,
|
||||
context: Executor.Context
|
||||
): Step[AsyncFeatureMap, AsyncFeatureMapExecutorResults] =
|
||||
new Step[AsyncFeatureMap, AsyncFeatureMapExecutorResults] {
|
||||
override def identifier: PipelineStepIdentifier =
|
||||
CandidatePipelineConfig.asyncFeaturesStep(stepToHydrateFor)
|
||||
|
||||
override def executorArrow: Arrow[AsyncFeatureMap, AsyncFeatureMapExecutorResults] =
|
||||
asyncFeatureMapExecutor.arrow(stepToHydrateFor, identifier, context)
|
||||
|
||||
override def inputAdaptor(
|
||||
query: CandidatePipeline.Inputs[Query],
|
||||
previousResult: IntermediateCandidatePipelineResult[Result]
|
||||
): AsyncFeatureMap =
|
||||
previousResult.underlyingResult.mergedAsyncQueryFeatures
|
||||
.getOrElse(
|
||||
throw InvalidStepStateException(identifier, "MergedAsyncQueryFeatures")
|
||||
)
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: IntermediateCandidatePipelineResult[Result],
|
||||
executorResult: AsyncFeatureMapExecutorResults
|
||||
): IntermediateCandidatePipelineResult[Result] =
|
||||
previousPipelineResult.copy(
|
||||
underlyingResult =
|
||||
previousPipelineResult.underlyingResult.copy(asyncFeatureHydrationResults =
|
||||
previousPipelineResult.underlyingResult.asyncFeatureHydrationResults match {
|
||||
case Some(existingResults) => Some(existingResults ++ executorResult)
|
||||
case None => Some(executorResult)
|
||||
}))
|
||||
|
||||
override def queryUpdater(
|
||||
query: CandidatePipeline.Inputs[Query],
|
||||
executorResult: AsyncFeatureMapExecutorResults
|
||||
): CandidatePipeline.Inputs[Query] =
|
||||
if (executorResult.featureMapsByStep
|
||||
.getOrElse(stepToHydrateFor, FeatureMap.empty).isEmpty) {
|
||||
query
|
||||
} else {
|
||||
val updatedQuery = query.query
|
||||
.withFeatureMap(
|
||||
query.query.features
|
||||
.getOrElse(FeatureMap.empty) ++ executorResult.featureMapsByStep(
|
||||
stepToHydrateFor)).asInstanceOf[Query]
|
||||
CandidatePipeline.Inputs(updatedQuery, query.existingCandidates)
|
||||
}
|
||||
}
|
||||
|
||||
val CandidateSourceStep =
|
||||
new Step[Query, CandidateSourceExecutorResult[Result]] {
|
||||
override def identifier: PipelineStepIdentifier =
|
||||
CandidatePipelineConfig.candidateSourceStep
|
||||
|
||||
override def executorArrow: Arrow[
|
||||
Query,
|
||||
CandidateSourceExecutorResult[Result]
|
||||
] =
|
||||
candidateSourceExecutor
|
||||
.arrow(
|
||||
config.candidateSource,
|
||||
queryTransformer,
|
||||
resultsTransformer,
|
||||
config.featuresFromCandidateSourceTransformers,
|
||||
context
|
||||
)
|
||||
|
||||
override def inputAdaptor(
|
||||
query: CandidatePipeline.Inputs[Query],
|
||||
previousResult: IntermediateCandidatePipelineResult[Result]
|
||||
): Query =
|
||||
query.query
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: IntermediateCandidatePipelineResult[Result],
|
||||
executorResult: CandidateSourceExecutorResult[Result]
|
||||
): IntermediateCandidatePipelineResult[Result] =
|
||||
previousPipelineResult.copy(underlyingResult =
|
||||
previousPipelineResult.underlyingResult.copy(
|
||||
candidateSourceResult =
|
||||
Some(executorResult.asInstanceOf[CandidateSourceExecutorResult[UniversalNoun[Any]]])
|
||||
))
|
||||
|
||||
override def queryUpdater(
|
||||
query: CandidatePipeline.Inputs[Query],
|
||||
executorResult: CandidateSourceExecutorResult[Result]
|
||||
): CandidatePipeline.Inputs[Query] = {
|
||||
val updatedFeatureMap =
|
||||
query.query.features
|
||||
.getOrElse(FeatureMap.empty) ++ executorResult.candidateSourceFeatureMap
|
||||
val updatedQuery = query.query
|
||||
.withFeatureMap(updatedFeatureMap).asInstanceOf[Query]
|
||||
CandidatePipeline.Inputs(updatedQuery, query.existingCandidates)
|
||||
}
|
||||
}
|
||||
|
||||
val PreFilterFeatureHydrationPhase1Step =
|
||||
new Step[
|
||||
CandidateFeatureHydratorExecutor.Inputs[Query, Result],
|
||||
CandidateFeatureHydratorExecutorResult[Result]
|
||||
] {
|
||||
override def identifier: PipelineStepIdentifier =
|
||||
CandidatePipelineConfig.preFilterFeatureHydrationPhase1Step
|
||||
|
||||
override def executorArrow: Arrow[
|
||||
CandidateFeatureHydratorExecutor.Inputs[Query, Result],
|
||||
CandidateFeatureHydratorExecutorResult[Result]
|
||||
] =
|
||||
candidateFeatureHydratorExecutor.arrow(config.preFilterFeatureHydrationPhase1, context)
|
||||
|
||||
override def inputAdaptor(
|
||||
query: CandidatePipeline.Inputs[Query],
|
||||
previousResult: IntermediateCandidatePipelineResult[Result]
|
||||
): CandidateFeatureHydratorExecutor.Inputs[Query, Result] = {
|
||||
val candidateSourceExecutorResult =
|
||||
previousResult.underlyingResult.candidateSourceResult.getOrElse {
|
||||
throw InvalidStepStateException(identifier, "CandidateSourceResult")
|
||||
}
|
||||
CandidateFeatureHydratorExecutor.Inputs(
|
||||
query.query,
|
||||
candidateSourceExecutorResult.candidates
|
||||
.asInstanceOf[Seq[CandidateWithFeatures[Result]]])
|
||||
}
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: IntermediateCandidatePipelineResult[Result],
|
||||
executorResult: CandidateFeatureHydratorExecutorResult[Result]
|
||||
): IntermediateCandidatePipelineResult[Result] = {
|
||||
val candidateSourceExecutorResult =
|
||||
previousPipelineResult.underlyingResult.candidateSourceResult.getOrElse {
|
||||
throw InvalidStepStateException(identifier, "CandidateSourceResult")
|
||||
}
|
||||
|
||||
val featureMapsFromPreFilter = executorResult.results.map { result =>
|
||||
result.candidate -> result.features
|
||||
}.toMap
|
||||
|
||||
val mergedFeatureMaps = candidateSourceExecutorResult.candidates.map { candidate =>
|
||||
val candidateFeatureMap = candidate.features
|
||||
val preFilterFeatureMap =
|
||||
featureMapsFromPreFilter.getOrElse(
|
||||
candidate.candidate.asInstanceOf[Result],
|
||||
FeatureMap.empty)
|
||||
|
||||
candidate.candidate.asInstanceOf[Result] -> (candidateFeatureMap ++ preFilterFeatureMap)
|
||||
}.toMap
|
||||
|
||||
previousPipelineResult.copy(
|
||||
underlyingResult = previousPipelineResult.underlyingResult.copy(
|
||||
preFilterHydrationResult = Some(
|
||||
executorResult
|
||||
.asInstanceOf[CandidateFeatureHydratorExecutorResult[UniversalNoun[Any]]])
|
||||
),
|
||||
featureMaps = Some(mergedFeatureMaps)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
val PreFilterFeatureHydrationPhase2Step =
|
||||
new Step[
|
||||
CandidateFeatureHydratorExecutor.Inputs[Query, Result],
|
||||
CandidateFeatureHydratorExecutorResult[Result]
|
||||
] {
|
||||
override def identifier: PipelineStepIdentifier =
|
||||
CandidatePipelineConfig.preFilterFeatureHydrationPhase2Step
|
||||
|
||||
override def executorArrow: Arrow[
|
||||
CandidateFeatureHydratorExecutor.Inputs[Query, Result],
|
||||
CandidateFeatureHydratorExecutorResult[Result]
|
||||
] =
|
||||
candidateFeatureHydratorExecutor.arrow(config.preFilterFeatureHydrationPhase2, context)
|
||||
|
||||
override def inputAdaptor(
|
||||
query: CandidatePipeline.Inputs[Query],
|
||||
previousResult: IntermediateCandidatePipelineResult[Result]
|
||||
): CandidateFeatureHydratorExecutor.Inputs[Query, Result] = {
|
||||
val candidates = previousResult.underlyingResult.preFilterHydrationResult.getOrElse {
|
||||
throw InvalidStepStateException(identifier, "PreFilterHydrationResult")
|
||||
}.results
|
||||
CandidateFeatureHydratorExecutor.Inputs(
|
||||
query.query,
|
||||
candidates.asInstanceOf[Seq[CandidateWithFeatures[Result]]]
|
||||
)
|
||||
}
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: IntermediateCandidatePipelineResult[Result],
|
||||
executorResult: CandidateFeatureHydratorExecutorResult[Result]
|
||||
): IntermediateCandidatePipelineResult[Result] = {
|
||||
|
||||
val featureMapsFromPreFilterPhase2 = executorResult.results.map { result =>
|
||||
result.candidate -> result.features
|
||||
}.toMap
|
||||
|
||||
val mergedFeatureMaps = previousPipelineResult.featureMaps
|
||||
.getOrElse(throw InvalidStepStateException(identifier, "FeatureMaps"))
|
||||
.map {
|
||||
case (candidate, featureMap) =>
|
||||
val preFilterPhase2FeatureMap =
|
||||
featureMapsFromPreFilterPhase2.getOrElse(candidate, FeatureMap.empty)
|
||||
|
||||
candidate -> (featureMap ++ preFilterPhase2FeatureMap)
|
||||
}
|
||||
|
||||
previousPipelineResult.copy(
|
||||
underlyingResult = previousPipelineResult.underlyingResult.copy(
|
||||
preFilterHydrationResultPhase2 = Some(
|
||||
executorResult
|
||||
.asInstanceOf[CandidateFeatureHydratorExecutorResult[UniversalNoun[Any]]])
|
||||
),
|
||||
featureMaps = Some(mergedFeatureMaps)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
val FiltersStep =
|
||||
new Step[(Query, Seq[CandidateWithFeatures[Result]]), FilterExecutorResult[Result]] {
|
||||
override def identifier: PipelineStepIdentifier = CandidatePipelineConfig.filtersStep
|
||||
|
||||
override def executorArrow: Arrow[
|
||||
(Query, Seq[CandidateWithFeatures[Result]]),
|
||||
FilterExecutorResult[
|
||||
Result
|
||||
]
|
||||
] =
|
||||
filterExecutor.arrow(config.filters, context)
|
||||
|
||||
override def inputAdaptor(
|
||||
query: CandidatePipeline.Inputs[Query],
|
||||
previousResult: IntermediateCandidatePipelineResult[Result]
|
||||
): (Query, Seq[CandidateWithFeatures[Result]]) = {
|
||||
val candidates =
|
||||
previousResult.underlyingResult.candidateSourceResult
|
||||
.getOrElse {
|
||||
throw InvalidStepStateException(identifier, "CandidateSourceResult")
|
||||
}.candidates.map(_.candidate).asInstanceOf[Seq[Result]]
|
||||
|
||||
val featureMaps = previousResult.featureMaps
|
||||
.getOrElse(throw InvalidStepStateException(identifier, "FeatureMaps"))
|
||||
|
||||
(
|
||||
query.query,
|
||||
candidates.map(candidate =>
|
||||
CandidateWithFeaturesImpl(
|
||||
candidate,
|
||||
featureMaps.getOrElse(candidate, FeatureMap.empty))))
|
||||
}
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: IntermediateCandidatePipelineResult[Result],
|
||||
executorResult: FilterExecutorResult[Result]
|
||||
): IntermediateCandidatePipelineResult[Result] =
|
||||
previousPipelineResult.copy(underlyingResult =
|
||||
previousPipelineResult.underlyingResult.copy(
|
||||
filterResult =
|
||||
Some(executorResult.asInstanceOf[FilterExecutorResult[UniversalNoun[Any]]])
|
||||
))
|
||||
}
|
||||
|
||||
val PostFilterFeatureHydrationStep =
|
||||
new Step[
|
||||
CandidateFeatureHydratorExecutor.Inputs[Query, Result],
|
||||
CandidateFeatureHydratorExecutorResult[Result]
|
||||
] {
|
||||
override def identifier: PipelineStepIdentifier =
|
||||
CandidatePipelineConfig.postFilterFeatureHydrationStep
|
||||
|
||||
override def executorArrow: Arrow[
|
||||
CandidateFeatureHydratorExecutor.Inputs[Query, Result],
|
||||
CandidateFeatureHydratorExecutorResult[Result]
|
||||
] =
|
||||
candidateFeatureHydratorExecutor.arrow(config.postFilterFeatureHydration, context)
|
||||
|
||||
override def inputAdaptor(
|
||||
query: CandidatePipeline.Inputs[Query],
|
||||
previousResult: IntermediateCandidatePipelineResult[Result]
|
||||
): CandidateFeatureHydratorExecutor.Inputs[Query, Result] = {
|
||||
val filterResult = previousResult.underlyingResult.filterResult
|
||||
.getOrElse(
|
||||
throw InvalidStepStateException(identifier, "FilterResult")
|
||||
).result.asInstanceOf[Seq[Result]]
|
||||
|
||||
val featureMaps = previousResult.featureMaps.getOrElse(
|
||||
throw InvalidStepStateException(identifier, "FeatureMaps")
|
||||
)
|
||||
|
||||
val filteredCandidates = filterResult.map { candidate =>
|
||||
CandidateWithFeaturesImpl(candidate, featureMaps.getOrElse(candidate, FeatureMap.empty))
|
||||
}
|
||||
CandidateFeatureHydratorExecutor.Inputs(query.query, filteredCandidates)
|
||||
}
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: IntermediateCandidatePipelineResult[Result],
|
||||
executorResult: CandidateFeatureHydratorExecutorResult[Result]
|
||||
): IntermediateCandidatePipelineResult[Result] = {
|
||||
val filterResult = previousPipelineResult.underlyingResult.filterResult
|
||||
.getOrElse(
|
||||
throw InvalidStepStateException(identifier, "FilterResult")
|
||||
).result.asInstanceOf[Seq[Result]]
|
||||
|
||||
val featureMaps = previousPipelineResult.featureMaps.getOrElse(
|
||||
throw InvalidStepStateException(identifier, "FeatureMaps")
|
||||
)
|
||||
|
||||
val postFilterFeatureMaps = executorResult.results.map { result =>
|
||||
result.candidate -> result.features
|
||||
}.toMap
|
||||
|
||||
val mergedFeatureMaps = filterResult.map { candidate =>
|
||||
candidate ->
|
||||
(featureMaps
|
||||
.getOrElse(candidate, FeatureMap.empty) ++ postFilterFeatureMaps.getOrElse(
|
||||
candidate,
|
||||
FeatureMap.empty))
|
||||
}.toMap
|
||||
|
||||
previousPipelineResult.copy(
|
||||
underlyingResult = previousPipelineResult.underlyingResult.copy(
|
||||
postFilterHydrationResult = Some(
|
||||
executorResult
|
||||
.asInstanceOf[CandidateFeatureHydratorExecutorResult[UniversalNoun[Any]]])
|
||||
),
|
||||
featureMaps = Some(mergedFeatureMaps)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
val ScorersStep =
|
||||
new Step[
|
||||
CandidateFeatureHydratorExecutor.Inputs[Query, Result],
|
||||
CandidateFeatureHydratorExecutorResult[Result]
|
||||
] {
|
||||
override def identifier: PipelineStepIdentifier = CandidatePipelineConfig.scorersStep
|
||||
|
||||
override def executorArrow: Arrow[
|
||||
CandidateFeatureHydratorExecutor.Inputs[Query, Result],
|
||||
CandidateFeatureHydratorExecutorResult[Result]
|
||||
] =
|
||||
candidateFeatureHydratorExecutor.arrow(config.scorers, context)
|
||||
|
||||
override def inputAdaptor(
|
||||
query: CandidatePipeline.Inputs[Query],
|
||||
previousResult: IntermediateCandidatePipelineResult[Result]
|
||||
): CandidateFeatureHydratorExecutor.Inputs[Query, Result] = {
|
||||
val filterResult = previousResult.underlyingResult.filterResult
|
||||
.getOrElse(
|
||||
throw InvalidStepStateException(identifier, "FilterResult")
|
||||
).result.asInstanceOf[Seq[Result]]
|
||||
|
||||
val featureMaps = previousResult.featureMaps.getOrElse(
|
||||
throw InvalidStepStateException(identifier, "FeatureMaps")
|
||||
)
|
||||
|
||||
val filteredCandidates = filterResult.map { candidate =>
|
||||
CandidateWithFeaturesImpl(candidate, featureMaps.getOrElse(candidate, FeatureMap.empty))
|
||||
}
|
||||
CandidateFeatureHydratorExecutor.Inputs(query.query, filteredCandidates)
|
||||
}
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: IntermediateCandidatePipelineResult[Result],
|
||||
executorResult: CandidateFeatureHydratorExecutorResult[Result]
|
||||
): IntermediateCandidatePipelineResult[Result] = {
|
||||
val filterResult = previousPipelineResult.underlyingResult.filterResult
|
||||
.getOrElse(
|
||||
throw InvalidStepStateException(identifier, "FilterResult")
|
||||
).result.asInstanceOf[Seq[Result]]
|
||||
|
||||
val featureMaps = previousPipelineResult.featureMaps.getOrElse(
|
||||
throw InvalidStepStateException(identifier, "FeatureMaps")
|
||||
)
|
||||
|
||||
val scoringFeatureMaps = executorResult.results.map { result =>
|
||||
result.candidate -> result.features
|
||||
}.toMap
|
||||
|
||||
val mergedFeatureMaps = filterResult.map { candidate =>
|
||||
candidate ->
|
||||
(featureMaps
|
||||
.getOrElse(candidate, FeatureMap.empty) ++ scoringFeatureMaps.getOrElse(
|
||||
candidate,
|
||||
FeatureMap.empty))
|
||||
}.toMap
|
||||
|
||||
previousPipelineResult.copy(
|
||||
underlyingResult = previousPipelineResult.underlyingResult.copy(
|
||||
scorersResult = Some(
|
||||
executorResult
|
||||
.asInstanceOf[CandidateFeatureHydratorExecutorResult[UniversalNoun[Any]]])
|
||||
),
|
||||
featureMaps = Some(mergedFeatureMaps)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
val DecorationStep =
|
||||
new Step[(Query, Seq[CandidateWithFeatures[Result]]), CandidateDecoratorExecutorResult] {
|
||||
override def identifier: PipelineStepIdentifier = CandidatePipelineConfig.decoratorStep
|
||||
|
||||
override def executorArrow: Arrow[
|
||||
(Query, Seq[CandidateWithFeatures[Result]]),
|
||||
CandidateDecoratorExecutorResult
|
||||
] =
|
||||
candidateDecoratorExecutor.arrow(decorator, context)
|
||||
|
||||
override def inputAdaptor(
|
||||
query: CandidatePipeline.Inputs[Query],
|
||||
previousResult: IntermediateCandidatePipelineResult[Result]
|
||||
): (Query, Seq[CandidateWithFeatures[Result]]) = {
|
||||
val keptCandidates = previousResult.underlyingResult.filterResult
|
||||
.getOrElse {
|
||||
throw InvalidStepStateException(identifier, "FilterResult")
|
||||
}.result.asInstanceOf[Seq[Result]]
|
||||
|
||||
val featureMaps = previousResult.featureMaps.getOrElse {
|
||||
throw InvalidStepStateException(identifier, "FeatureMaps")
|
||||
}
|
||||
|
||||
(
|
||||
query.query,
|
||||
keptCandidates.map(candidate =>
|
||||
CandidateWithFeaturesImpl(
|
||||
candidate,
|
||||
featureMaps.getOrElse(candidate, FeatureMap.empty))))
|
||||
}
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: IntermediateCandidatePipelineResult[Result],
|
||||
executorResult: CandidateDecoratorExecutorResult
|
||||
): IntermediateCandidatePipelineResult[Result] =
|
||||
previousPipelineResult.copy(underlyingResult =
|
||||
previousPipelineResult.underlyingResult.copy(
|
||||
candidateDecoratorResult = Some(executorResult)
|
||||
))
|
||||
}
|
||||
|
||||
/**
|
||||
* ResultStep is a synchronous step that basically takes the outputs from the other steps, groups modules,
|
||||
* and puts things into the final result object
|
||||
*/
|
||||
val ResultStep = new Step[GroupResultsExecutorInput[Result], GroupResultsExecutorResult] {
|
||||
override def identifier: PipelineStepIdentifier = CandidatePipelineConfig.resultStep
|
||||
|
||||
override def executorArrow: Arrow[
|
||||
GroupResultsExecutorInput[Result],
|
||||
GroupResultsExecutorResult
|
||||
] = groupResultsExecutor.arrow(pipelineIdentifier, candidateSourceIdentifier, context)
|
||||
|
||||
override def inputAdaptor(
|
||||
query: CandidatePipeline.Inputs[Query],
|
||||
previousResult: IntermediateCandidatePipelineResult[Result]
|
||||
): GroupResultsExecutorInput[Result] = {
|
||||
|
||||
val underlying = previousResult.underlyingResult
|
||||
|
||||
val keptCandidates = underlying.filterResult
|
||||
.getOrElse(
|
||||
throw InvalidStepStateException(identifier, "FilterResult")
|
||||
).result.asInstanceOf[Seq[Result]]
|
||||
|
||||
val decorations = underlying.candidateDecoratorResult
|
||||
.getOrElse(
|
||||
throw InvalidStepStateException(identifier, "DecorationResult")
|
||||
).result.map(decoration => decoration.candidate -> decoration.presentation).toMap
|
||||
|
||||
val combinedFeatureMaps: Map[Result, FeatureMap] = previousResult.featureMaps.getOrElse(
|
||||
throw InvalidStepStateException(identifier, "FeatureMaps"))
|
||||
|
||||
val filteredCandidates = keptCandidates.map { candidate =>
|
||||
val updatedMap = combinedFeatureMaps
|
||||
.get(candidate).getOrElse(FeatureMap.empty)
|
||||
FetchedCandidateWithFeatures(candidate, updatedMap)
|
||||
}
|
||||
|
||||
GroupResultsExecutorInput(
|
||||
candidates = filteredCandidates,
|
||||
decorations = decorations
|
||||
)
|
||||
}
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: IntermediateCandidatePipelineResult[Result],
|
||||
executorResult: GroupResultsExecutorResult
|
||||
): IntermediateCandidatePipelineResult[Result] =
|
||||
previousPipelineResult.copy(underlyingResult = previousPipelineResult.underlyingResult
|
||||
.copy(result = Some(executorResult.candidatesWithDetails)))
|
||||
}
|
||||
|
||||
val builtSteps = Seq(
|
||||
GatesStep,
|
||||
queryFeatureHydrationStep(
|
||||
config.queryFeatureHydration,
|
||||
CandidatePipelineConfig.fetchQueryFeaturesStep,
|
||||
(pipelineResult, executorResult) =>
|
||||
pipelineResult.copy(queryFeatures = Some(executorResult))
|
||||
),
|
||||
queryFeatureHydrationStep(
|
||||
config.queryFeatureHydrationPhase2,
|
||||
CandidatePipelineConfig.fetchQueryFeaturesPhase2Step,
|
||||
(pipelineResult, executorResult) =>
|
||||
pipelineResult.copy(
|
||||
queryFeaturesPhase2 = Some(executorResult),
|
||||
mergedAsyncQueryFeatures = Some(
|
||||
pipelineResult.queryFeatures
|
||||
.getOrElse(
|
||||
throw InvalidStepStateException(
|
||||
CandidatePipelineConfig.fetchQueryFeaturesPhase2Step,
|
||||
"QueryFeatures")
|
||||
).asyncFeatureMap ++ executorResult.asyncFeatureMap)
|
||||
)
|
||||
),
|
||||
asyncFeaturesStep(CandidatePipelineConfig.candidateSourceStep, context),
|
||||
CandidateSourceStep,
|
||||
asyncFeaturesStep(CandidatePipelineConfig.preFilterFeatureHydrationPhase1Step, context),
|
||||
PreFilterFeatureHydrationPhase1Step,
|
||||
asyncFeaturesStep(CandidatePipelineConfig.preFilterFeatureHydrationPhase2Step, context),
|
||||
PreFilterFeatureHydrationPhase2Step,
|
||||
asyncFeaturesStep(CandidatePipelineConfig.filtersStep, context),
|
||||
FiltersStep,
|
||||
asyncFeaturesStep(CandidatePipelineConfig.postFilterFeatureHydrationStep, context),
|
||||
PostFilterFeatureHydrationStep,
|
||||
asyncFeaturesStep(CandidatePipelineConfig.scorersStep, context),
|
||||
ScorersStep,
|
||||
asyncFeaturesStep(CandidatePipelineConfig.decoratorStep, context),
|
||||
DecorationStep,
|
||||
ResultStep
|
||||
)
|
||||
|
||||
/** The main execution logic for this Candidate Pipeline. */
|
||||
val finalArrow: Arrow[CandidatePipeline.Inputs[Query], CandidatePipelineResult] =
|
||||
buildCombinedArrowFromSteps(
|
||||
steps = builtSteps,
|
||||
context = context,
|
||||
initialEmptyResult =
|
||||
IntermediateCandidatePipelineResult.empty[Result](config.candidateSource.identifier),
|
||||
stepsInOrderFromConfig = CandidatePipelineConfig.stepsInOrder
|
||||
).map(_.underlyingResult)
|
||||
|
||||
val configFromBuilder = config
|
||||
new CandidatePipeline[Query] {
|
||||
override private[core] val config: BaseCandidatePipelineConfig[Query, _, _, _] =
|
||||
configFromBuilder
|
||||
override val arrow: Arrow[CandidatePipeline.Inputs[Query], CandidatePipelineResult] =
|
||||
finalArrow
|
||||
override val identifier: CandidatePipelineIdentifier = pipelineIdentifier
|
||||
override val alerts: Seq[Alert] = config.alerts
|
||||
override val children: Seq[Component] =
|
||||
allGates ++
|
||||
config.queryFeatureHydration ++
|
||||
Seq(queryTransformer, config.candidateSource, resultsTransformer) ++
|
||||
config.featuresFromCandidateSourceTransformers ++
|
||||
decorator.toSeq ++
|
||||
config.preFilterFeatureHydrationPhase1 ++
|
||||
config.filters ++
|
||||
config.postFilterFeatureHydration ++
|
||||
config.scorers
|
||||
}
|
||||
}
|
||||
|
||||
private case class CandidateWithFeaturesImpl(candidate: Result, features: FeatureMap)
|
||||
extends CandidateWithFeatures[Result]
|
||||
}
|
||||
Binary file not shown.
@ -1,56 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.candidate
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.service.async_feature_map_executor.AsyncFeatureMapExecutor
|
||||
import com.twitter.product_mixer.core.service.candidate_decorator_executor.CandidateDecoratorExecutor
|
||||
import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutor
|
||||
import com.twitter.product_mixer.core.service.candidate_source_executor.CandidateSourceExecutor
|
||||
import com.twitter.product_mixer.core.service.filter_executor.FilterExecutor
|
||||
import com.twitter.product_mixer.core.service.gate_executor.GateExecutor
|
||||
import com.twitter.product_mixer.core.service.group_results_executor.GroupResultsExecutor
|
||||
import com.twitter.product_mixer.core.service.query_feature_hydrator_executor.QueryFeatureHydratorExecutor
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class CandidatePipelineBuilderFactory @Inject() (
|
||||
queryFeatureHydratorExecutor: QueryFeatureHydratorExecutor,
|
||||
asyncFeatureMapExecutor: AsyncFeatureMapExecutor,
|
||||
candidateDecoratorExecutor: CandidateDecoratorExecutor,
|
||||
candidateFeatureHydratorExecutor: CandidateFeatureHydratorExecutor,
|
||||
candidateSourceExecutor: CandidateSourceExecutor,
|
||||
groupResultsExecutor: GroupResultsExecutor,
|
||||
filterExecutor: FilterExecutor,
|
||||
gateExecutor: GateExecutor,
|
||||
statsReceiver: StatsReceiver) {
|
||||
def get[
|
||||
Query <: PipelineQuery,
|
||||
CandidateSourceQuery,
|
||||
CandidateSourceResult,
|
||||
Result <: UniversalNoun[Any]
|
||||
]: CandidatePipelineBuilder[
|
||||
Query,
|
||||
CandidateSourceQuery,
|
||||
CandidateSourceResult,
|
||||
Result
|
||||
] = {
|
||||
new CandidatePipelineBuilder[
|
||||
Query,
|
||||
CandidateSourceQuery,
|
||||
CandidateSourceResult,
|
||||
Result
|
||||
](
|
||||
queryFeatureHydratorExecutor,
|
||||
asyncFeatureMapExecutor,
|
||||
candidateDecoratorExecutor,
|
||||
candidateFeatureHydratorExecutor,
|
||||
candidateSourceExecutor,
|
||||
groupResultsExecutor,
|
||||
filterExecutor,
|
||||
gateExecutor,
|
||||
statsReceiver
|
||||
)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,264 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.candidate
|
||||
|
||||
import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource
|
||||
import com.twitter.product_mixer.core.functional_component.common.alert.Alert
|
||||
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
|
||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator
|
||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator
|
||||
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||
import com.twitter.product_mixer.core.functional_component.gate.BaseGate
|
||||
import com.twitter.product_mixer.core.functional_component.gate.Gate
|
||||
import com.twitter.product_mixer.core.functional_component.scorer.Scorer
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer
|
||||
import com.twitter.product_mixer.core.functional_component.transformer._
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifierStack
|
||||
import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineConfig
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineConfigCompanion
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure
|
||||
import com.twitter.timelines.configapi.FSParam
|
||||
import com.twitter.timelines.configapi.decider.DeciderParam
|
||||
|
||||
sealed trait BaseCandidatePipelineConfig[
|
||||
-Query <: PipelineQuery,
|
||||
CandidateSourceQuery,
|
||||
CandidateSourceResult,
|
||||
Result <: UniversalNoun[Any]]
|
||||
extends PipelineConfig {
|
||||
|
||||
val identifier: CandidatePipelineIdentifier
|
||||
|
||||
/**
|
||||
* A candidate pipeline can fetch query-level features for use within the candidate source. It's
|
||||
* generally recommended to set a hydrator in the parent recos or mixer pipeline if multiple
|
||||
* candidate pipelines share the same feature but if a specific query feature hydrator is used
|
||||
* by one pipeline and you don't want to block the others, you could explicitly set it here.
|
||||
* If a feature is hydrated both in the parent pipeline or here, this one takes priority.
|
||||
*/
|
||||
def queryFeatureHydration: Seq[BaseQueryFeatureHydrator[Query, _]] = Seq.empty
|
||||
|
||||
/**
|
||||
* For query-level features that are dependent on query-level features from [[queryFeatureHydration]]
|
||||
*/
|
||||
def queryFeatureHydrationPhase2: Seq[BaseQueryFeatureHydrator[Query, _]] = Seq.empty
|
||||
|
||||
/**
|
||||
* When these Params are defined, they will automatically be added as Gates in the pipeline
|
||||
* by the CandidatePipelineBuilder
|
||||
*
|
||||
* The enabled decider param can to be used to quickly disable a Candidate Pipeline via Decider
|
||||
*/
|
||||
val enabledDeciderParam: Option[DeciderParam[Boolean]] = None
|
||||
|
||||
/**
|
||||
* This supported client feature switch param can be used with a Feature Switch to control the
|
||||
* rollout of a new Candidate Pipeline from dogfood to experiment to production
|
||||
*/
|
||||
val supportedClientParam: Option[FSParam[Boolean]] = None
|
||||
|
||||
/** [[Gate]]s that are applied sequentially, the pipeline will only run if all the Gates are open */
|
||||
def gates: Seq[BaseGate[Query]] = Seq.empty
|
||||
|
||||
/**
|
||||
* A pair of transforms to adapt the underlying candidate source to the pipeline's query and result types
|
||||
* Complex use cases such as those that need access to features should construct their own transformer, but
|
||||
* for simple use cases, you can pass in an anonymous function.
|
||||
* @example
|
||||
* {{{ override val queryTransformer: CandidatePipelineQueryTransformer[Query, CandidateSourceQuery] = { query =>
|
||||
* query.toExampleThrift
|
||||
* }
|
||||
* }}}
|
||||
*/
|
||||
def queryTransformer: BaseCandidatePipelineQueryTransformer[
|
||||
Query,
|
||||
CandidateSourceQuery
|
||||
]
|
||||
|
||||
/** Source for Candidates for this Pipeline */
|
||||
def candidateSource: BaseCandidateSource[CandidateSourceQuery, CandidateSourceResult]
|
||||
|
||||
/**
|
||||
* [[CandidateFeatureTransformer]] allow you to define [[com.twitter.product_mixer.core.feature.Feature]] extraction logic from your [[CandidateSource]] results.
|
||||
* If your candidate sources return [[com.twitter.product_mixer.core.feature.Feature]]s alongside the candidate that might be useful later on,
|
||||
* add transformers for constructing feature maps.
|
||||
*
|
||||
* @note If multiple transformers extract the same feature, the last one takes priority and is kept.
|
||||
*/
|
||||
def featuresFromCandidateSourceTransformers: Seq[
|
||||
CandidateFeatureTransformer[CandidateSourceResult]
|
||||
] = Seq.empty
|
||||
|
||||
/**
|
||||
* a result Transformer may throw PipelineFailure for candidates that are malformed and
|
||||
* should be removed. This should be exceptional behavior, and not a replacement for adding a Filter.
|
||||
* Complex use cases such as those that need access to features should construct their own transformer, but
|
||||
* for simple use cases, you can pass in an anonymous function.
|
||||
* @example
|
||||
* {{{ override val queryTransformer: CandidatePipelineResultsTransformer[CandidateSourceResult, Result] = { sourceResult =>
|
||||
* ExampleCandidate(sourceResult.id)
|
||||
* }
|
||||
* }}}
|
||||
*
|
||||
*/
|
||||
val resultTransformer: CandidatePipelineResultsTransformer[CandidateSourceResult, Result]
|
||||
|
||||
/**
|
||||
* Before filters are run, you can fetch features for each candidate.
|
||||
*
|
||||
* Uses Stitch, so you're encouraged to use a working Stitch Adaptor to batch between candidates.
|
||||
*
|
||||
* The existing features (from the candidate source) are passed in as an input. You are not expected
|
||||
* to put them into the resulting feature map yourself - they will be merged for you by the platform.
|
||||
*
|
||||
* This API is likely to change when Product Mixer does managed feature hydration
|
||||
*/
|
||||
val preFilterFeatureHydrationPhase1: Seq[BaseCandidateFeatureHydrator[Query, Result, _]] =
|
||||
Seq.empty
|
||||
|
||||
/**
|
||||
* A second phase of feature hydration that can be run before filtering and after the first phase
|
||||
* of [[preFilterFeatureHydrationPhase1]]. You are not expected to put them into the resulting
|
||||
* feature map yourself - they will be merged for you by the platform.
|
||||
*/
|
||||
val preFilterFeatureHydrationPhase2: Seq[BaseCandidateFeatureHydrator[Query, Result, _]] =
|
||||
Seq.empty
|
||||
|
||||
/** A list of filters to apply. Filters will be applied in sequential order. */
|
||||
def filters: Seq[Filter[Query, Result]] = Seq.empty
|
||||
|
||||
/**
|
||||
* After filters are run, you can fetch features for each candidate.
|
||||
*
|
||||
* Uses Stitch, so you're encouraged to use a working Stitch Adaptor to batch between candidates.
|
||||
*
|
||||
* The existing features (from the candidate source) & pre-filtering are passed in as an input.
|
||||
* You are not expected to put them into the resulting feature map yourself -
|
||||
* they will be merged for you by the platform.
|
||||
*
|
||||
* This API is likely to change when Product Mixer does managed feature hydration
|
||||
*/
|
||||
val postFilterFeatureHydration: Seq[BaseCandidateFeatureHydrator[Query, Result, _]] = Seq.empty
|
||||
|
||||
/**
|
||||
* Decorators allow for adding Presentations to candidates. While the Presentation can contain any
|
||||
* arbitrary data, Decorators are often used to add a UrtItemPresentation for URT item support, or
|
||||
* a UrtModulePresentation for grouping the candidates in a URT module.
|
||||
*/
|
||||
val decorator: Option[CandidateDecorator[Query, Result]] = None
|
||||
|
||||
/**
|
||||
* A candidate pipeline can define a partial function to rescue failures here. They will be treated as failures
|
||||
* from a monitoring standpoint, and cancellation exceptions will always be propagated (they cannot be caught here).
|
||||
*/
|
||||
def failureClassifier: PartialFunction[Throwable, PipelineFailure] = PartialFunction.empty
|
||||
|
||||
/**
|
||||
* Scorers for candidates. Scorers are executed in parallel. Order does not matter.
|
||||
*/
|
||||
def scorers: Seq[Scorer[Query, Result]] = Seq.empty
|
||||
|
||||
/**
|
||||
* Alerts can be used to indicate the pipeline's service level objectives. Alerts and
|
||||
* dashboards will be automatically created based on this information.
|
||||
*/
|
||||
val alerts: Seq[Alert] = Seq.empty
|
||||
|
||||
/**
|
||||
* This method is used by the product mixer framework to build the pipeline.
|
||||
*/
|
||||
private[core] final def build(
|
||||
parentComponentIdentifierStack: ComponentIdentifierStack,
|
||||
factory: CandidatePipelineBuilderFactory
|
||||
): CandidatePipeline[Query] = {
|
||||
factory.get.build(parentComponentIdentifierStack, this)
|
||||
}
|
||||
}
|
||||
|
||||
trait CandidatePipelineConfig[
|
||||
-Query <: PipelineQuery,
|
||||
CandidateSourceQuery,
|
||||
CandidateSourceResult,
|
||||
Result <: UniversalNoun[Any]]
|
||||
extends BaseCandidatePipelineConfig[
|
||||
Query,
|
||||
CandidateSourceQuery,
|
||||
CandidateSourceResult,
|
||||
Result
|
||||
] {
|
||||
override val gates: Seq[Gate[Query]] = Seq.empty
|
||||
|
||||
override val queryTransformer: CandidatePipelineQueryTransformer[
|
||||
Query,
|
||||
CandidateSourceQuery
|
||||
]
|
||||
}
|
||||
|
||||
trait DependentCandidatePipelineConfig[
|
||||
-Query <: PipelineQuery,
|
||||
CandidateSourceQuery,
|
||||
CandidateSourceResult,
|
||||
Result <: UniversalNoun[Any]]
|
||||
extends BaseCandidatePipelineConfig[
|
||||
Query,
|
||||
CandidateSourceQuery,
|
||||
CandidateSourceResult,
|
||||
Result
|
||||
]
|
||||
|
||||
/**
|
||||
* Contains [[PipelineStepIdentifier]]s for the Steps that are available for all [[BaseCandidatePipelineConfig]]s
|
||||
*/
|
||||
object CandidatePipelineConfig extends PipelineConfigCompanion {
|
||||
val gatesStep: PipelineStepIdentifier = PipelineStepIdentifier("Gates")
|
||||
val fetchQueryFeaturesStep: PipelineStepIdentifier = PipelineStepIdentifier("FetchQueryFeatures")
|
||||
val fetchQueryFeaturesPhase2Step: PipelineStepIdentifier = PipelineStepIdentifier(
|
||||
"FetchQueryFeaturesPhase2")
|
||||
val candidateSourceStep: PipelineStepIdentifier = PipelineStepIdentifier("CandidateSource")
|
||||
val preFilterFeatureHydrationPhase1Step: PipelineStepIdentifier =
|
||||
PipelineStepIdentifier("PreFilterFeatureHydration")
|
||||
val preFilterFeatureHydrationPhase2Step: PipelineStepIdentifier =
|
||||
PipelineStepIdentifier("PreFilterFeatureHydrationPhase2")
|
||||
val filtersStep: PipelineStepIdentifier = PipelineStepIdentifier("Filters")
|
||||
val postFilterFeatureHydrationStep: PipelineStepIdentifier =
|
||||
PipelineStepIdentifier("PostFilterFeatureHydration")
|
||||
val scorersStep: PipelineStepIdentifier = PipelineStepIdentifier("Scorer")
|
||||
val decoratorStep: PipelineStepIdentifier = PipelineStepIdentifier("Decorator")
|
||||
val resultStep: PipelineStepIdentifier = PipelineStepIdentifier("Result")
|
||||
|
||||
/** All the steps which are executed by a [[CandidatePipeline]] in the order in which they are run */
|
||||
override val stepsInOrder: Seq[PipelineStepIdentifier] = Seq(
|
||||
gatesStep,
|
||||
fetchQueryFeaturesStep,
|
||||
fetchQueryFeaturesPhase2Step,
|
||||
asyncFeaturesStep(candidateSourceStep),
|
||||
candidateSourceStep,
|
||||
asyncFeaturesStep(preFilterFeatureHydrationPhase1Step),
|
||||
preFilterFeatureHydrationPhase1Step,
|
||||
asyncFeaturesStep(preFilterFeatureHydrationPhase2Step),
|
||||
preFilterFeatureHydrationPhase2Step,
|
||||
asyncFeaturesStep(filtersStep),
|
||||
filtersStep,
|
||||
asyncFeaturesStep(postFilterFeatureHydrationStep),
|
||||
postFilterFeatureHydrationStep,
|
||||
asyncFeaturesStep(scorersStep),
|
||||
scorersStep,
|
||||
asyncFeaturesStep(decoratorStep),
|
||||
decoratorStep,
|
||||
resultStep
|
||||
)
|
||||
|
||||
override val stepsAsyncFeatureHydrationCanBeCompletedBy: Set[PipelineStepIdentifier] = Set(
|
||||
candidateSourceStep,
|
||||
preFilterFeatureHydrationPhase1Step,
|
||||
preFilterFeatureHydrationPhase2Step,
|
||||
filtersStep,
|
||||
postFilterFeatureHydrationStep,
|
||||
scorersStep,
|
||||
decoratorStep
|
||||
)
|
||||
}
|
||||
Binary file not shown.
@ -1,93 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.candidate
|
||||
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.feature.featuremap.asyncfeaturemap.AsyncFeatureMap
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineResult
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure
|
||||
import com.twitter.product_mixer.core.service.async_feature_map_executor.AsyncFeatureMapExecutorResults
|
||||
import com.twitter.product_mixer.core.service.candidate_decorator_executor.CandidateDecoratorExecutorResult
|
||||
import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutorResult
|
||||
import com.twitter.product_mixer.core.service.candidate_source_executor.CandidateSourceExecutorResult
|
||||
import com.twitter.product_mixer.core.service.filter_executor.FilterExecutorResult
|
||||
import com.twitter.product_mixer.core.service.gate_executor.GateExecutorResult
|
||||
import com.twitter.product_mixer.core.service.query_feature_hydrator_executor.QueryFeatureHydratorExecutor
|
||||
|
||||
case class CandidatePipelineResult(
|
||||
candidateSourceIdentifier: CandidateSourceIdentifier,
|
||||
gateResult: Option[GateExecutorResult],
|
||||
queryFeatures: Option[QueryFeatureHydratorExecutor.Result],
|
||||
queryFeaturesPhase2: Option[QueryFeatureHydratorExecutor.Result],
|
||||
mergedAsyncQueryFeatures: Option[AsyncFeatureMap],
|
||||
candidateSourceResult: Option[CandidateSourceExecutorResult[UniversalNoun[Any]]],
|
||||
preFilterHydrationResult: Option[CandidateFeatureHydratorExecutorResult[UniversalNoun[Any]]],
|
||||
preFilterHydrationResultPhase2: Option[
|
||||
CandidateFeatureHydratorExecutorResult[UniversalNoun[Any]]
|
||||
],
|
||||
filterResult: Option[FilterExecutorResult[UniversalNoun[Any]]],
|
||||
postFilterHydrationResult: Option[CandidateFeatureHydratorExecutorResult[UniversalNoun[Any]]],
|
||||
candidateDecoratorResult: Option[CandidateDecoratorExecutorResult],
|
||||
scorersResult: Option[CandidateFeatureHydratorExecutorResult[UniversalNoun[Any]]],
|
||||
asyncFeatureHydrationResults: Option[AsyncFeatureMapExecutorResults],
|
||||
failure: Option[PipelineFailure],
|
||||
result: Option[Seq[CandidateWithDetails]])
|
||||
extends PipelineResult[Seq[CandidateWithDetails]] {
|
||||
|
||||
override def withFailure(failure: PipelineFailure): CandidatePipelineResult =
|
||||
copy(failure = Some(failure))
|
||||
|
||||
override def withResult(
|
||||
result: Seq[CandidateWithDetails]
|
||||
): CandidatePipelineResult = copy(result = Some(result))
|
||||
|
||||
override val resultSize: Int = result.map(PipelineResult.resultSize).getOrElse(0)
|
||||
}
|
||||
|
||||
private[candidate] object IntermediateCandidatePipelineResult {
|
||||
def empty[Candidate <: UniversalNoun[Any]](
|
||||
candidateSourceIdentifier: CandidateSourceIdentifier
|
||||
): IntermediateCandidatePipelineResult[Candidate] = {
|
||||
IntermediateCandidatePipelineResult(
|
||||
CandidatePipelineResult(
|
||||
candidateSourceIdentifier = candidateSourceIdentifier,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None
|
||||
),
|
||||
None
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private[candidate] case class IntermediateCandidatePipelineResult[Candidate <: UniversalNoun[Any]](
|
||||
underlyingResult: CandidatePipelineResult,
|
||||
featureMaps: Option[Map[Candidate, FeatureMap]])
|
||||
extends PipelineResult[Seq[CandidateWithDetails]] {
|
||||
override val failure: Option[PipelineFailure] = underlyingResult.failure
|
||||
override val result: Option[Seq[CandidateWithDetails]] = underlyingResult.result
|
||||
|
||||
override def withFailure(
|
||||
failure: PipelineFailure
|
||||
): IntermediateCandidatePipelineResult[Candidate] =
|
||||
copy(underlyingResult = underlyingResult.withFailure(failure))
|
||||
|
||||
override def withResult(
|
||||
result: Seq[CandidateWithDetails]
|
||||
): IntermediateCandidatePipelineResult[Candidate] =
|
||||
copy(underlyingResult = underlyingResult.withResult(result))
|
||||
|
||||
override def resultSize(): Int = underlyingResult.resultSize
|
||||
}
|
||||
Binary file not shown.
@ -1,47 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.candidate
|
||||
|
||||
import com.twitter.product_mixer.core.functional_component.candidate_source.PassthroughCandidateSource
|
||||
import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateExtractor
|
||||
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
|
||||
object PassthroughCandidatePipelineConfig {
|
||||
|
||||
/**
|
||||
* Build a [[PassthroughCandidatePipelineConfig]] with a [[PassthroughCandidateSource]] built from
|
||||
* a [[CandidateExtractor]]
|
||||
*/
|
||||
def apply[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]](
|
||||
identifier: CandidatePipelineIdentifier,
|
||||
extractor: CandidateExtractor[Query, Candidate],
|
||||
decorator: Option[CandidateDecorator[Query, Candidate]] = None
|
||||
): PassthroughCandidatePipelineConfig[Query, Candidate] = {
|
||||
|
||||
// Renaming variables to keep the interface clean, but avoid naming collisions when creating
|
||||
// the anonymous class.
|
||||
val _identifier = identifier
|
||||
val _extractor = extractor
|
||||
val _decorator = decorator
|
||||
|
||||
new PassthroughCandidatePipelineConfig[Query, Candidate] {
|
||||
override val identifier = _identifier
|
||||
override val candidateSource =
|
||||
PassthroughCandidateSource(CandidateSourceIdentifier(_identifier.name), _extractor)
|
||||
override val decorator = _decorator
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
trait PassthroughCandidatePipelineConfig[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]
|
||||
extends CandidatePipelineConfig[Query, Query, Candidate, Candidate] {
|
||||
|
||||
override val queryTransformer: CandidatePipelineQueryTransformer[Query, Query] = identity
|
||||
|
||||
override val resultTransformer: CandidatePipelineResultsTransformer[Candidate, Candidate] =
|
||||
identity
|
||||
}
|
||||
Binary file not shown.
@ -1,51 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.candidate
|
||||
|
||||
import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource
|
||||
import com.twitter.product_mixer.core.functional_component.candidate_source.StaticCandidateSource
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
|
||||
|
||||
object StaticCandidatePipelineConfig {
|
||||
|
||||
/**
|
||||
* Build a [[StaticCandidatePipelineConfig]] with a [[CandidateSource]] that returns the [[candidate]]
|
||||
*/
|
||||
def apply[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]](
|
||||
identifier: CandidatePipelineIdentifier,
|
||||
candidate: Candidate,
|
||||
decorator: Option[CandidateDecorator[Query, Candidate]] = None
|
||||
): StaticCandidatePipelineConfig[Query, Candidate] = {
|
||||
|
||||
// Renaming variables to keep the interface clean, but avoid naming collisions when creating
|
||||
// the anonymous class.
|
||||
val _identifier = identifier
|
||||
val _candidate = candidate
|
||||
val _decorator = decorator
|
||||
|
||||
new StaticCandidatePipelineConfig[Query, Candidate] {
|
||||
override val identifier = _identifier
|
||||
override val candidate = _candidate
|
||||
override val decorator = _decorator
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
trait StaticCandidatePipelineConfig[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]
|
||||
extends CandidatePipelineConfig[Query, Unit, Unit, Candidate] {
|
||||
|
||||
val candidate: Candidate
|
||||
|
||||
override def candidateSource: CandidateSource[Unit, Unit] = StaticCandidateSource[Unit](
|
||||
identifier = CandidateSourceIdentifier(identifier.name),
|
||||
result = Seq(()))
|
||||
|
||||
override val queryTransformer: CandidatePipelineQueryTransformer[Query, Unit] = _ => Unit
|
||||
|
||||
override val resultTransformer: CandidatePipelineResultsTransformer[Unit, Candidate] = _ =>
|
||||
candidate
|
||||
}
|
||||
@ -1,62 +0,0 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/premarshaller",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/side_effect",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/async_feature_map_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_pipeline_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/domain_marshaller_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_result_side_effect_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/quality_factor_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transport_marshaller_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util",
|
||||
"stitch/stitch-core",
|
||||
],
|
||||
exports = [
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/premarshaller",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/side_effect",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_pipeline_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/domain_marshaller_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_result_side_effect_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/quality_factor_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transport_marshaller_executor",
|
||||
"stitch/stitch-core",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,24 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.mixer
|
||||
|
||||
import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.Pipeline
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Arrow
|
||||
|
||||
/**
|
||||
* A Mixer Pipeline
|
||||
*
|
||||
* This is an abstract class, as we only construct these via the [[MixerPipelineBuilder]].
|
||||
*
|
||||
* A [[MixerPipeline]] is capable of processing requests (queries) and returning responses (results)
|
||||
* in the correct format to directly send to users.
|
||||
*
|
||||
* @tparam Query the domain model for the query or request
|
||||
* @tparam Result the final marshalled result type
|
||||
*/
|
||||
abstract class MixerPipeline[Query <: PipelineQuery, Result] private[mixer]
|
||||
extends Pipeline[Query, Result] {
|
||||
override private[core] val config: MixerPipelineConfig[Query, _, Result]
|
||||
override val arrow: Arrow[Query, MixerPipelineResult[Result]]
|
||||
override val identifier: MixerPipelineIdentifier
|
||||
}
|
||||
Binary file not shown.
@ -1,582 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.mixer
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.feature.featuremap.asyncfeaturemap.AsyncFeatureMap
|
||||
import com.twitter.product_mixer.core.functional_component.common.alert.Alert
|
||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
|
||||
import com.twitter.product_mixer.core.functional_component.gate.Gate
|
||||
import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller
|
||||
import com.twitter.product_mixer.core.functional_component.selector.Selector
|
||||
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
|
||||
import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller
|
||||
import com.twitter.product_mixer.core.model.common.Component
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifierStack
|
||||
import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier
|
||||
import com.twitter.product_mixer.core.model.marshalling.HasMarshalling
|
||||
import com.twitter.product_mixer.core.pipeline.FailOpenPolicy
|
||||
import com.twitter.product_mixer.core.pipeline.InvalidStepStateException
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineBuilder
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipeline
|
||||
import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineBuilderFactory
|
||||
import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig
|
||||
import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailureClassifier
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.ProductDisabled
|
||||
import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus
|
||||
import com.twitter.product_mixer.core.quality_factor.QualityFactorObserver
|
||||
import com.twitter.product_mixer.core.quality_factor.QualityFactorStatus
|
||||
import com.twitter.product_mixer.core.service.Executor
|
||||
import com.twitter.product_mixer.core.service.async_feature_map_executor.AsyncFeatureMapExecutor
|
||||
import com.twitter.product_mixer.core.service.async_feature_map_executor.AsyncFeatureMapExecutorResults
|
||||
import com.twitter.product_mixer.core.service.candidate_pipeline_executor.CandidatePipelineExecutor
|
||||
import com.twitter.product_mixer.core.service.candidate_pipeline_executor.CandidatePipelineExecutorResult
|
||||
import com.twitter.product_mixer.core.service.domain_marshaller_executor.DomainMarshallerExecutor
|
||||
import com.twitter.product_mixer.core.service.gate_executor.GateExecutor
|
||||
import com.twitter.product_mixer.core.service.gate_executor.GateExecutorResult
|
||||
import com.twitter.product_mixer.core.service.gate_executor.StoppedGateException
|
||||
import com.twitter.product_mixer.core.service.pipeline_result_side_effect_executor.PipelineResultSideEffectExecutor
|
||||
import com.twitter.product_mixer.core.service.quality_factor_executor.QualityFactorExecutorResult
|
||||
import com.twitter.product_mixer.core.service.query_feature_hydrator_executor.QueryFeatureHydratorExecutor
|
||||
import com.twitter.product_mixer.core.service.selector_executor.SelectorExecutor
|
||||
import com.twitter.product_mixer.core.service.selector_executor.SelectorExecutorResult
|
||||
import com.twitter.product_mixer.core.service.transport_marshaller_executor.TransportMarshallerExecutor
|
||||
import com.twitter.stitch.Arrow
|
||||
import com.twitter.util.logging.Logging
|
||||
|
||||
/**
|
||||
* MixerPipelineBuilder builds [[MixerPipeline]]s from [[MixerPipelineConfig]]s.
|
||||
*
|
||||
* You should inject a [[MixerPipelineBuilderFactory]] and call `.get` to build these.
|
||||
*
|
||||
* @see [[MixerPipelineConfig]] for the description of the type parameters
|
||||
*/
|
||||
class MixerPipelineBuilder[Query <: PipelineQuery, DomainResultType <: HasMarshalling, Result](
|
||||
candidatePipelineExecutor: CandidatePipelineExecutor,
|
||||
gateExecutor: GateExecutor,
|
||||
selectorExecutor: SelectorExecutor,
|
||||
queryFeatureHydratorExecutor: QueryFeatureHydratorExecutor,
|
||||
asyncFeatureMapExecutor: AsyncFeatureMapExecutor,
|
||||
domainMarshallerExecutor: DomainMarshallerExecutor,
|
||||
transportMarshallerExecutor: TransportMarshallerExecutor,
|
||||
pipelineResultSideEffectExecutor: PipelineResultSideEffectExecutor,
|
||||
candidatePipelineBuilderFactory: CandidatePipelineBuilderFactory,
|
||||
override val statsReceiver: StatsReceiver)
|
||||
extends PipelineBuilder[Query]
|
||||
with Logging {
|
||||
|
||||
override type UnderlyingResultType = Result
|
||||
override type PipelineResultType = MixerPipelineResult[Result]
|
||||
|
||||
def qualityFactorStep(
|
||||
qualityFactorStatus: QualityFactorStatus
|
||||
): Step[Query, QualityFactorExecutorResult] =
|
||||
new Step[Query, QualityFactorExecutorResult] {
|
||||
override def identifier: PipelineStepIdentifier = MixerPipelineConfig.qualityFactorStep
|
||||
|
||||
override def executorArrow: Arrow[Query, QualityFactorExecutorResult] =
|
||||
Arrow
|
||||
.map[Query, QualityFactorExecutorResult] { _ =>
|
||||
QualityFactorExecutorResult(
|
||||
pipelineQualityFactors =
|
||||
qualityFactorStatus.qualityFactorByPipeline.mapValues(_.currentValue)
|
||||
)
|
||||
}
|
||||
|
||||
override def inputAdaptor(
|
||||
query: Query,
|
||||
previousResult: MixerPipelineResult[Result]
|
||||
): Query = query
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: MixerPipelineResult[Result],
|
||||
executorResult: QualityFactorExecutorResult
|
||||
): MixerPipelineResult[Result] =
|
||||
previousPipelineResult.copy(qualityFactorResult = Some(executorResult))
|
||||
|
||||
override def queryUpdater(
|
||||
query: Query,
|
||||
executorResult: QualityFactorExecutorResult
|
||||
): Query = {
|
||||
query match {
|
||||
case queryWithQualityFactor: HasQualityFactorStatus =>
|
||||
queryWithQualityFactor
|
||||
.withQualityFactorStatus(
|
||||
queryWithQualityFactor.qualityFactorStatus.getOrElse(QualityFactorStatus.empty) ++
|
||||
qualityFactorStatus
|
||||
).asInstanceOf[Query]
|
||||
case _ =>
|
||||
query
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def gatesStep(
|
||||
gates: Seq[Gate[Query]],
|
||||
context: Executor.Context
|
||||
): Step[Query, GateExecutorResult] = new Step[Query, GateExecutorResult] {
|
||||
override def identifier: PipelineStepIdentifier = MixerPipelineConfig.gatesStep
|
||||
|
||||
override def executorArrow: Arrow[Query, GateExecutorResult] =
|
||||
gateExecutor.arrow(gates, context)
|
||||
|
||||
override def inputAdaptor(query: Query, previousResult: MixerPipelineResult[Result]): Query =
|
||||
query
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: MixerPipelineResult[Result],
|
||||
executorResult: GateExecutorResult
|
||||
): MixerPipelineResult[Result] =
|
||||
previousPipelineResult.copy(gateResult = Some(executorResult))
|
||||
}
|
||||
|
||||
def fetchQueryFeaturesStep(
|
||||
queryFeatureHydrators: Seq[QueryFeatureHydrator[Query]],
|
||||
stepIdentifier: PipelineStepIdentifier,
|
||||
updater: ResultUpdater[MixerPipelineResult[Result], QueryFeatureHydratorExecutor.Result],
|
||||
context: Executor.Context
|
||||
): Step[Query, QueryFeatureHydratorExecutor.Result] =
|
||||
new Step[Query, QueryFeatureHydratorExecutor.Result] {
|
||||
override def identifier: PipelineStepIdentifier = stepIdentifier
|
||||
|
||||
override def executorArrow: Arrow[Query, QueryFeatureHydratorExecutor.Result] =
|
||||
queryFeatureHydratorExecutor.arrow(
|
||||
queryFeatureHydrators,
|
||||
MixerPipelineConfig.stepsAsyncFeatureHydrationCanBeCompletedBy,
|
||||
context)
|
||||
|
||||
override def inputAdaptor(
|
||||
query: Query,
|
||||
previousResult: MixerPipelineResult[Result]
|
||||
): Query = query
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: MixerPipelineResult[Result],
|
||||
executorResult: QueryFeatureHydratorExecutor.Result
|
||||
): MixerPipelineResult[Result] =
|
||||
updater(previousPipelineResult, executorResult)
|
||||
|
||||
override def queryUpdater(
|
||||
query: Query,
|
||||
executorResult: QueryFeatureHydratorExecutor.Result
|
||||
): Query =
|
||||
query
|
||||
.withFeatureMap(
|
||||
query.features
|
||||
.getOrElse(FeatureMap.empty) ++ executorResult.featureMap).asInstanceOf[Query]
|
||||
}
|
||||
|
||||
def asyncFeaturesStep(
|
||||
stepToHydrateFor: PipelineStepIdentifier,
|
||||
context: Executor.Context
|
||||
): Step[AsyncFeatureMap, AsyncFeatureMapExecutorResults] =
|
||||
new Step[AsyncFeatureMap, AsyncFeatureMapExecutorResults] {
|
||||
override def identifier: PipelineStepIdentifier =
|
||||
MixerPipelineConfig.asyncFeaturesStep(stepToHydrateFor)
|
||||
|
||||
override def executorArrow: Arrow[AsyncFeatureMap, AsyncFeatureMapExecutorResults] =
|
||||
asyncFeatureMapExecutor.arrow(
|
||||
stepToHydrateFor,
|
||||
identifier,
|
||||
context
|
||||
)
|
||||
|
||||
override def inputAdaptor(
|
||||
query: Query,
|
||||
previousResult: MixerPipelineResult[Result]
|
||||
): AsyncFeatureMap =
|
||||
previousResult.mergedAsyncQueryFeatures
|
||||
.getOrElse(
|
||||
throw InvalidStepStateException(identifier, "MergedAsyncQueryFeatures")
|
||||
)
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: MixerPipelineResult[Result],
|
||||
executorResult: AsyncFeatureMapExecutorResults
|
||||
): MixerPipelineResult[Result] = previousPipelineResult.copy(
|
||||
asyncFeatureHydrationResults = previousPipelineResult.asyncFeatureHydrationResults match {
|
||||
case Some(existingResults) => Some(existingResults ++ executorResult)
|
||||
case None => Some(executorResult)
|
||||
})
|
||||
|
||||
override def queryUpdater(
|
||||
query: Query,
|
||||
executorResult: AsyncFeatureMapExecutorResults
|
||||
): Query =
|
||||
if (executorResult.featureMapsByStep
|
||||
.getOrElse(stepToHydrateFor, FeatureMap.empty).isEmpty) {
|
||||
query
|
||||
} else {
|
||||
query
|
||||
.withFeatureMap(
|
||||
query.features
|
||||
.getOrElse(FeatureMap.empty) ++ executorResult.featureMapsByStep(
|
||||
stepToHydrateFor)).asInstanceOf[Query]
|
||||
}
|
||||
}
|
||||
|
||||
def candidatePipelinesStep(
|
||||
candidatePipelines: Seq[CandidatePipeline[Query]],
|
||||
defaultFailOpenPolicy: FailOpenPolicy,
|
||||
failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy],
|
||||
qualityFactorObserverByPipeline: Map[ComponentIdentifier, QualityFactorObserver],
|
||||
context: Executor.Context
|
||||
): Step[CandidatePipeline.Inputs[Query], CandidatePipelineExecutorResult] =
|
||||
new Step[CandidatePipeline.Inputs[Query], CandidatePipelineExecutorResult] {
|
||||
override def identifier: PipelineStepIdentifier = MixerPipelineConfig.candidatePipelinesStep
|
||||
|
||||
override def executorArrow: Arrow[CandidatePipeline.Inputs[
|
||||
Query
|
||||
], CandidatePipelineExecutorResult] =
|
||||
candidatePipelineExecutor
|
||||
.arrow(
|
||||
candidatePipelines,
|
||||
defaultFailOpenPolicy,
|
||||
failOpenPolicies,
|
||||
qualityFactorObserverByPipeline,
|
||||
context
|
||||
)
|
||||
|
||||
override def inputAdaptor(
|
||||
query: Query,
|
||||
previousResult: MixerPipelineResult[Result]
|
||||
): CandidatePipeline.Inputs[Query] = CandidatePipeline.Inputs[Query](query, Seq.empty)
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: MixerPipelineResult[Result],
|
||||
executorResult: CandidatePipelineExecutorResult
|
||||
): MixerPipelineResult[Result] =
|
||||
previousPipelineResult.copy(candidatePipelineResults = Some(executorResult))
|
||||
|
||||
override def queryUpdater(
|
||||
query: Query,
|
||||
executorResult: CandidatePipelineExecutorResult
|
||||
): Query = {
|
||||
val updatedFeatureMap = query.features
|
||||
.getOrElse(FeatureMap.empty) ++ executorResult.queryFeatureMap
|
||||
query
|
||||
.withFeatureMap(updatedFeatureMap).asInstanceOf[Query]
|
||||
}
|
||||
}
|
||||
|
||||
def dependentCandidatePipelinesStep(
|
||||
candidatePipelines: Seq[CandidatePipeline[Query]],
|
||||
defaultFailOpenPolicy: FailOpenPolicy,
|
||||
failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy],
|
||||
qualityFactorObserverByPipeline: Map[ComponentIdentifier, QualityFactorObserver],
|
||||
context: Executor.Context
|
||||
): Step[CandidatePipeline.Inputs[Query], CandidatePipelineExecutorResult] =
|
||||
new Step[CandidatePipeline.Inputs[Query], CandidatePipelineExecutorResult] {
|
||||
override def identifier: PipelineStepIdentifier =
|
||||
MixerPipelineConfig.dependentCandidatePipelinesStep
|
||||
|
||||
override def executorArrow: Arrow[CandidatePipeline.Inputs[
|
||||
Query
|
||||
], CandidatePipelineExecutorResult] =
|
||||
candidatePipelineExecutor
|
||||
.arrow(
|
||||
candidatePipelines,
|
||||
defaultFailOpenPolicy,
|
||||
failOpenPolicies,
|
||||
qualityFactorObserverByPipeline,
|
||||
context
|
||||
)
|
||||
|
||||
override def inputAdaptor(
|
||||
query: Query,
|
||||
previousResult: MixerPipelineResult[Result]
|
||||
): CandidatePipeline.Inputs[Query] = {
|
||||
val previousCandidates = previousResult.candidatePipelineResults
|
||||
.getOrElse {
|
||||
throw InvalidStepStateException(identifier, "Candidates")
|
||||
}.candidatePipelineResults.flatMap(_.result.getOrElse(Seq.empty))
|
||||
CandidatePipeline.Inputs[Query](query, previousCandidates)
|
||||
}
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: MixerPipelineResult[Result],
|
||||
executorResult: CandidatePipelineExecutorResult
|
||||
): MixerPipelineResult[Result] =
|
||||
previousPipelineResult.copy(dependentCandidatePipelineResults = Some(executorResult))
|
||||
|
||||
override def queryUpdater(
|
||||
query: Query,
|
||||
executorResult: CandidatePipelineExecutorResult
|
||||
): Query = {
|
||||
val updatedFeatureMap = query.features
|
||||
.getOrElse(FeatureMap.empty) ++ executorResult.queryFeatureMap
|
||||
query
|
||||
.withFeatureMap(updatedFeatureMap).asInstanceOf[Query]
|
||||
}
|
||||
}
|
||||
|
||||
def resultSelectorsStep(
|
||||
selectors: Seq[Selector[Query]],
|
||||
context: Executor.Context
|
||||
): Step[SelectorExecutor.Inputs[Query], SelectorExecutorResult] =
|
||||
new Step[SelectorExecutor.Inputs[Query], SelectorExecutorResult] {
|
||||
override def identifier: PipelineStepIdentifier = MixerPipelineConfig.resultSelectorsStep
|
||||
|
||||
override def executorArrow: Arrow[SelectorExecutor.Inputs[Query], SelectorExecutorResult] =
|
||||
selectorExecutor.arrow(selectors, context)
|
||||
|
||||
override def inputAdaptor(
|
||||
query: Query,
|
||||
previousResult: MixerPipelineResult[Result]
|
||||
): SelectorExecutor.Inputs[Query] = {
|
||||
val candidates = previousResult.candidatePipelineResults
|
||||
.getOrElse {
|
||||
throw InvalidStepStateException(identifier, "Candidates")
|
||||
}.candidatePipelineResults.flatMap(_.result.getOrElse(Seq.empty))
|
||||
|
||||
val dependentCandidates =
|
||||
previousResult.dependentCandidatePipelineResults
|
||||
.getOrElse {
|
||||
throw InvalidStepStateException(identifier, "DependentCandidates")
|
||||
}.candidatePipelineResults.flatMap(_.result.getOrElse(Seq.empty))
|
||||
|
||||
SelectorExecutor.Inputs(
|
||||
query = query,
|
||||
candidatesWithDetails = candidates ++ dependentCandidates
|
||||
)
|
||||
}
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: MixerPipelineResult[Result],
|
||||
executorResult: SelectorExecutorResult
|
||||
): MixerPipelineResult[Result] =
|
||||
previousPipelineResult.copy(resultSelectorResults = Some(executorResult))
|
||||
}
|
||||
|
||||
def domainMarshallingStep(
|
||||
domainMarshaller: DomainMarshaller[Query, DomainResultType],
|
||||
context: Executor.Context
|
||||
): Step[DomainMarshallerExecutor.Inputs[Query], DomainMarshallerExecutor.Result[
|
||||
DomainResultType
|
||||
]] =
|
||||
new Step[DomainMarshallerExecutor.Inputs[Query], DomainMarshallerExecutor.Result[
|
||||
DomainResultType
|
||||
]] {
|
||||
override def identifier: PipelineStepIdentifier = MixerPipelineConfig.domainMarshallerStep
|
||||
|
||||
override def executorArrow: Arrow[
|
||||
DomainMarshallerExecutor.Inputs[Query],
|
||||
DomainMarshallerExecutor.Result[DomainResultType]
|
||||
] =
|
||||
domainMarshallerExecutor.arrow(domainMarshaller, context)
|
||||
|
||||
override def inputAdaptor(
|
||||
query: Query,
|
||||
previousResult: MixerPipelineResult[Result]
|
||||
): DomainMarshallerExecutor.Inputs[Query] = {
|
||||
val selectorResults = previousResult.resultSelectorResults.getOrElse {
|
||||
throw InvalidStepStateException(identifier, "SelectorResults")
|
||||
}
|
||||
|
||||
DomainMarshallerExecutor.Inputs(
|
||||
query = query,
|
||||
candidatesWithDetails = selectorResults.selectedCandidates
|
||||
)
|
||||
}
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: MixerPipelineResult[Result],
|
||||
executorResult: DomainMarshallerExecutor.Result[DomainResultType]
|
||||
): MixerPipelineResult[Result] = previousPipelineResult.copy(
|
||||
domainMarshallerResults = Some(executorResult)
|
||||
)
|
||||
}
|
||||
|
||||
def resultSideEffectsStep(
|
||||
sideEffects: Seq[PipelineResultSideEffect[Query, DomainResultType]],
|
||||
context: Executor.Context
|
||||
): Step[
|
||||
PipelineResultSideEffect.Inputs[Query, DomainResultType],
|
||||
PipelineResultSideEffectExecutor.Result
|
||||
] = new Step[
|
||||
PipelineResultSideEffect.Inputs[Query, DomainResultType],
|
||||
PipelineResultSideEffectExecutor.Result
|
||||
] {
|
||||
override def identifier: PipelineStepIdentifier = MixerPipelineConfig.resultSideEffectsStep
|
||||
|
||||
override def executorArrow: Arrow[
|
||||
PipelineResultSideEffect.Inputs[Query, DomainResultType],
|
||||
PipelineResultSideEffectExecutor.Result
|
||||
] = pipelineResultSideEffectExecutor.arrow(sideEffects, context)
|
||||
|
||||
override def inputAdaptor(
|
||||
query: Query,
|
||||
previousResult: MixerPipelineResult[Result]
|
||||
): PipelineResultSideEffect.Inputs[Query, DomainResultType] = {
|
||||
|
||||
val selectorResults = previousResult.resultSelectorResults.getOrElse {
|
||||
throw InvalidStepStateException(identifier, "SelectorResults")
|
||||
}
|
||||
|
||||
val domainMarshallerResults = previousResult.domainMarshallerResults.getOrElse {
|
||||
throw InvalidStepStateException(identifier, "DomainMarshallerResults")
|
||||
}
|
||||
|
||||
PipelineResultSideEffect.Inputs[Query, DomainResultType](
|
||||
query = query,
|
||||
selectedCandidates = selectorResults.selectedCandidates,
|
||||
remainingCandidates = selectorResults.remainingCandidates,
|
||||
droppedCandidates = selectorResults.droppedCandidates,
|
||||
response = domainMarshallerResults.result.asInstanceOf[DomainResultType]
|
||||
)
|
||||
}
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: MixerPipelineResult[Result],
|
||||
executorResult: PipelineResultSideEffectExecutor.Result
|
||||
): MixerPipelineResult[Result] =
|
||||
previousPipelineResult.copy(resultSideEffectResults = Some(executorResult))
|
||||
}
|
||||
|
||||
def transportMarshallingStep(
|
||||
transportMarshaller: TransportMarshaller[DomainResultType, Result],
|
||||
context: Executor.Context
|
||||
): Step[
|
||||
TransportMarshallerExecutor.Inputs[DomainResultType],
|
||||
TransportMarshallerExecutor.Result[Result]
|
||||
] = new Step[TransportMarshallerExecutor.Inputs[
|
||||
DomainResultType
|
||||
], TransportMarshallerExecutor.Result[Result]] {
|
||||
override def identifier: PipelineStepIdentifier = MixerPipelineConfig.transportMarshallerStep
|
||||
|
||||
override def executorArrow: Arrow[TransportMarshallerExecutor.Inputs[
|
||||
DomainResultType
|
||||
], TransportMarshallerExecutor.Result[Result]] =
|
||||
transportMarshallerExecutor.arrow(transportMarshaller, context)
|
||||
|
||||
override def inputAdaptor(
|
||||
query: Query,
|
||||
previousResult: MixerPipelineResult[Result]
|
||||
): TransportMarshallerExecutor.Inputs[DomainResultType] = {
|
||||
val domainMarshallingResults = previousResult.domainMarshallerResults.getOrElse {
|
||||
throw InvalidStepStateException(identifier, "DomainMarshallerResults")
|
||||
}
|
||||
|
||||
// Since the PipelineResult just uses HasMarshalling
|
||||
val domainResult = domainMarshallingResults.result.asInstanceOf[DomainResultType]
|
||||
|
||||
TransportMarshallerExecutor.Inputs(domainResult)
|
||||
}
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: MixerPipelineResult[Result],
|
||||
executorResult: TransportMarshallerExecutor.Result[Result]
|
||||
): MixerPipelineResult[Result] = previousPipelineResult.copy(
|
||||
transportMarshallerResults = Some(executorResult),
|
||||
result = Some(executorResult.result)
|
||||
)
|
||||
}
|
||||
|
||||
def build(
|
||||
parentComponentIdentifierStack: ComponentIdentifierStack,
|
||||
config: MixerPipelineConfig[Query, DomainResultType, Result]
|
||||
): MixerPipeline[Query, Result] = {
|
||||
|
||||
val pipelineIdentifier = config.identifier
|
||||
|
||||
val context = Executor.Context(
|
||||
PipelineFailureClassifier(
|
||||
config.failureClassifier.orElse(StoppedGateException.classifier(ProductDisabled))),
|
||||
parentComponentIdentifierStack.push(pipelineIdentifier)
|
||||
)
|
||||
|
||||
val qualityFactorStatus: QualityFactorStatus =
|
||||
QualityFactorStatus.build(config.qualityFactorConfigs)
|
||||
|
||||
val qualityFactorObserverByPipeline =
|
||||
qualityFactorStatus.qualityFactorByPipeline.mapValues { qualityFactor =>
|
||||
qualityFactor.buildObserver()
|
||||
}
|
||||
|
||||
buildGaugesForQualityFactor(pipelineIdentifier, qualityFactorStatus, statsReceiver)
|
||||
|
||||
val candidatePipelines: Seq[CandidatePipeline[Query]] = config.candidatePipelines.map {
|
||||
pipelineConfig: CandidatePipelineConfig[Query, _, _, _] =>
|
||||
pipelineConfig.build(context.componentStack, candidatePipelineBuilderFactory)
|
||||
}
|
||||
|
||||
val dependentCandidatePipelines: Seq[CandidatePipeline[Query]] =
|
||||
config.dependentCandidatePipelines.map {
|
||||
pipelineConfig: DependentCandidatePipelineConfig[Query, _, _, _] =>
|
||||
pipelineConfig.build(context.componentStack, candidatePipelineBuilderFactory)
|
||||
}
|
||||
|
||||
val builtSteps = Seq(
|
||||
qualityFactorStep(qualityFactorStatus),
|
||||
gatesStep(config.gates, context),
|
||||
fetchQueryFeaturesStep(
|
||||
config.fetchQueryFeatures,
|
||||
MixerPipelineConfig.fetchQueryFeaturesStep,
|
||||
(previousPipelineResult, executorResult) =>
|
||||
previousPipelineResult.copy(queryFeatures = Some(executorResult)),
|
||||
context
|
||||
),
|
||||
fetchQueryFeaturesStep(
|
||||
config.fetchQueryFeaturesPhase2,
|
||||
MixerPipelineConfig.fetchQueryFeaturesPhase2Step,
|
||||
(previousPipelineResult, executorResult) =>
|
||||
previousPipelineResult.copy(
|
||||
queryFeaturesPhase2 = Some(executorResult),
|
||||
mergedAsyncQueryFeatures = Some(
|
||||
previousPipelineResult.queryFeatures
|
||||
.getOrElse(throw InvalidStepStateException(
|
||||
MixerPipelineConfig.fetchQueryFeaturesPhase2Step,
|
||||
"QueryFeatures"))
|
||||
.asyncFeatureMap ++ executorResult.asyncFeatureMap)
|
||||
),
|
||||
context
|
||||
),
|
||||
asyncFeaturesStep(MixerPipelineConfig.candidatePipelinesStep, context),
|
||||
candidatePipelinesStep(
|
||||
candidatePipelines,
|
||||
config.defaultFailOpenPolicy,
|
||||
config.failOpenPolicies,
|
||||
qualityFactorObserverByPipeline,
|
||||
context),
|
||||
asyncFeaturesStep(MixerPipelineConfig.dependentCandidatePipelinesStep, context),
|
||||
dependentCandidatePipelinesStep(
|
||||
dependentCandidatePipelines,
|
||||
config.defaultFailOpenPolicy,
|
||||
config.failOpenPolicies,
|
||||
qualityFactorObserverByPipeline,
|
||||
context),
|
||||
asyncFeaturesStep(MixerPipelineConfig.resultSelectorsStep, context),
|
||||
resultSelectorsStep(config.resultSelectors, context),
|
||||
domainMarshallingStep(config.domainMarshaller, context),
|
||||
asyncFeaturesStep(MixerPipelineConfig.resultSideEffectsStep, context),
|
||||
resultSideEffectsStep(config.resultSideEffects, context),
|
||||
transportMarshallingStep(config.transportMarshaller, context)
|
||||
)
|
||||
|
||||
val finalArrow = buildCombinedArrowFromSteps(
|
||||
steps = builtSteps,
|
||||
context = context,
|
||||
initialEmptyResult = MixerPipelineResult.empty,
|
||||
stepsInOrderFromConfig = MixerPipelineConfig.stepsInOrder
|
||||
)
|
||||
|
||||
val configFromBuilder = config
|
||||
new MixerPipeline[Query, Result] {
|
||||
override private[core] val config: MixerPipelineConfig[Query, _, Result] = configFromBuilder
|
||||
override val arrow: Arrow[Query, MixerPipelineResult[Result]] = finalArrow
|
||||
override val identifier: MixerPipelineIdentifier = pipelineIdentifier
|
||||
override val alerts: Seq[Alert] = config.alerts
|
||||
override val children: Seq[Component] =
|
||||
config.gates ++
|
||||
config.fetchQueryFeatures ++
|
||||
candidatePipelines ++
|
||||
dependentCandidatePipelines ++
|
||||
config.resultSideEffects ++
|
||||
Seq(config.domainMarshaller, config.transportMarshaller)
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,49 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.mixer
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.product_mixer.core.model.marshalling.HasMarshalling
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineBuilderFactory
|
||||
import com.twitter.product_mixer.core.service.candidate_pipeline_executor.CandidatePipelineExecutor
|
||||
import com.twitter.product_mixer.core.service.domain_marshaller_executor.DomainMarshallerExecutor
|
||||
import com.twitter.product_mixer.core.service.gate_executor.GateExecutor
|
||||
import com.twitter.product_mixer.core.service.pipeline_result_side_effect_executor.PipelineResultSideEffectExecutor
|
||||
import com.twitter.product_mixer.core.service.async_feature_map_executor.AsyncFeatureMapExecutor
|
||||
import com.twitter.product_mixer.core.service.query_feature_hydrator_executor.QueryFeatureHydratorExecutor
|
||||
import com.twitter.product_mixer.core.service.selector_executor.SelectorExecutor
|
||||
import com.twitter.product_mixer.core.service.transport_marshaller_executor.TransportMarshallerExecutor
|
||||
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class MixerPipelineBuilderFactory @Inject() (
|
||||
candidatePipelineExecutor: CandidatePipelineExecutor,
|
||||
gateExecutor: GateExecutor,
|
||||
selectorExecutor: SelectorExecutor,
|
||||
queryFeatureHydratorExecutor: QueryFeatureHydratorExecutor,
|
||||
asyncFeatureMapExecutor: AsyncFeatureMapExecutor,
|
||||
domainMarshallerExecutor: DomainMarshallerExecutor,
|
||||
transportMarshallerExecutor: TransportMarshallerExecutor,
|
||||
pipelineResultSideEffectExecutor: PipelineResultSideEffectExecutor,
|
||||
candidatePipelineBuilderFactory: CandidatePipelineBuilderFactory,
|
||||
statsReceiver: StatsReceiver) {
|
||||
def get[
|
||||
Query <: PipelineQuery,
|
||||
DomainResultType <: HasMarshalling,
|
||||
Result
|
||||
]: MixerPipelineBuilder[Query, DomainResultType, Result] = {
|
||||
new MixerPipelineBuilder[Query, DomainResultType, Result](
|
||||
candidatePipelineExecutor,
|
||||
gateExecutor,
|
||||
selectorExecutor,
|
||||
queryFeatureHydratorExecutor,
|
||||
asyncFeatureMapExecutor,
|
||||
domainMarshallerExecutor,
|
||||
transportMarshallerExecutor,
|
||||
pipelineResultSideEffectExecutor,
|
||||
candidatePipelineBuilderFactory,
|
||||
statsReceiver
|
||||
)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,175 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.mixer
|
||||
|
||||
import com.twitter.product_mixer.core.functional_component.common.alert.Alert
|
||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
|
||||
import com.twitter.product_mixer.core.functional_component.gate.Gate
|
||||
import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller
|
||||
import com.twitter.product_mixer.core.functional_component.selector.Selector
|
||||
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
|
||||
import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifierStack
|
||||
import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier
|
||||
import com.twitter.product_mixer.core.model.marshalling.HasMarshalling
|
||||
import com.twitter.product_mixer.core.pipeline.FailOpenPolicy
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineConfig
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineConfigCompanion
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig
|
||||
import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.ClosedGate
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure
|
||||
import com.twitter.product_mixer.core.quality_factor.QualityFactorConfig
|
||||
|
||||
/**
|
||||
* This is the configuration necessary to generate a Mixer Pipeline. Product code should create a
|
||||
* MixerPipelineConfig, and then use a MixerPipelineBuilder to get the final MixerPipeline which can
|
||||
* process requests.
|
||||
*
|
||||
* @tparam Query - The domain model for the query or request
|
||||
* @tparam UnmarshalledResultType - The result type of the pipeline, but before marshalling to a wire protocol like URT
|
||||
* @tparam Result - The final result that will be served to users
|
||||
*/
|
||||
trait MixerPipelineConfig[Query <: PipelineQuery, UnmarshalledResultType <: HasMarshalling, Result]
|
||||
extends PipelineConfig {
|
||||
|
||||
override val identifier: MixerPipelineIdentifier
|
||||
|
||||
/**
|
||||
* Mixer Pipeline Gates will be executed before any other step (including retrieval from candidate
|
||||
* pipelines). They're executed sequentially, and any "Stop" result will prevent pipeline execution.
|
||||
*/
|
||||
def gates: Seq[Gate[Query]] = Seq.empty
|
||||
|
||||
/**
|
||||
* A mixer pipeline can fetch query-level features before candidate pipelines are executed.
|
||||
*/
|
||||
def fetchQueryFeatures: Seq[QueryFeatureHydrator[Query]] = Seq.empty
|
||||
|
||||
/**
|
||||
* For query-level features that are dependent on query-level features from [[fetchQueryFeatures]]
|
||||
*/
|
||||
def fetchQueryFeaturesPhase2: Seq[QueryFeatureHydrator[Query]] = Seq.empty
|
||||
|
||||
/**
|
||||
* Candidate pipelines retrieve candidates for possible inclusion in the result
|
||||
*/
|
||||
def candidatePipelines: Seq[CandidatePipelineConfig[Query, _, _, _]]
|
||||
|
||||
/**
|
||||
* Dependent candidate pipelines to retrieve candidates that depend on the result of [[candidatePipelines]]
|
||||
* [[DependentCandidatePipelineConfig]] have access to the list of previously retrieved & decorated
|
||||
* candidates for use in constructing the query object.
|
||||
*/
|
||||
def dependentCandidatePipelines: Seq[DependentCandidatePipelineConfig[Query, _, _, _]] = Seq.empty
|
||||
|
||||
/**
|
||||
* [[defaultFailOpenPolicy]] is the [[FailOpenPolicy]] that will be applied to any candidate
|
||||
* pipeline that isn't in the [[failOpenPolicies]] map. By default Candidate Pipelines will fail
|
||||
* open for Closed Gates only.
|
||||
*/
|
||||
def defaultFailOpenPolicy: FailOpenPolicy = FailOpenPolicy(Set(ClosedGate))
|
||||
|
||||
/**
|
||||
* [[failOpenPolicies]] associates [[FailOpenPolicy]]s to specific candidate pipelines using
|
||||
* [[CandidatePipelineIdentifier]].
|
||||
*
|
||||
* @note these [[FailOpenPolicy]]s override the [[defaultFailOpenPolicy]] for a mapped
|
||||
* Candidate Pipeline.
|
||||
*/
|
||||
def failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = Map.empty
|
||||
|
||||
/**
|
||||
** [[qualityFactorConfigs]] associates [[QualityFactorConfig]]s to specific candidate pipelines
|
||||
* using [[CandidatePipelineIdentifier]].
|
||||
*/
|
||||
def qualityFactorConfigs: Map[CandidatePipelineIdentifier, QualityFactorConfig] =
|
||||
Map.empty
|
||||
|
||||
/**
|
||||
* Selectors are executed in sequential order to combine the candidates into a result
|
||||
*/
|
||||
def resultSelectors: Seq[Selector[Query]]
|
||||
|
||||
/**
|
||||
* Mixer result side effects that are executed after selection and domain marshalling
|
||||
*/
|
||||
def resultSideEffects: Seq[PipelineResultSideEffect[Query, UnmarshalledResultType]] = Seq()
|
||||
|
||||
/**
|
||||
* Domain marshaller transforms the selections into the model expected by the marshaller
|
||||
*/
|
||||
def domainMarshaller: DomainMarshaller[Query, UnmarshalledResultType]
|
||||
|
||||
/**
|
||||
* Transport marshaller transforms the model into our line-level API like URT or JSON
|
||||
*/
|
||||
def transportMarshaller: TransportMarshaller[UnmarshalledResultType, Result]
|
||||
|
||||
/**
|
||||
* A pipeline can define a partial function to rescue failures here. They will be treated as failures
|
||||
* from a monitoring standpoint, and cancellation exceptions will always be propagated (they cannot be caught here).
|
||||
*/
|
||||
def failureClassifier: PartialFunction[Throwable, PipelineFailure] = PartialFunction.empty
|
||||
|
||||
/**
|
||||
* Alert can be used to indicate the pipeline's service level objectives. Alerts and
|
||||
* dashboards will be automatically created based on this information.
|
||||
*/
|
||||
val alerts: Seq[Alert] = Seq.empty
|
||||
|
||||
/**
|
||||
* This method is used by the product mixer framework to build the pipeline.
|
||||
*/
|
||||
private[core] final def build(
|
||||
parentComponentIdentifierStack: ComponentIdentifierStack,
|
||||
builder: MixerPipelineBuilderFactory
|
||||
): MixerPipeline[Query, Result] =
|
||||
builder.get.build(parentComponentIdentifierStack, this)
|
||||
}
|
||||
|
||||
object MixerPipelineConfig extends PipelineConfigCompanion {
|
||||
val qualityFactorStep: PipelineStepIdentifier = PipelineStepIdentifier("QualityFactor")
|
||||
val gatesStep: PipelineStepIdentifier = PipelineStepIdentifier("Gates")
|
||||
val fetchQueryFeaturesStep: PipelineStepIdentifier = PipelineStepIdentifier("FetchQueryFeatures")
|
||||
val fetchQueryFeaturesPhase2Step: PipelineStepIdentifier =
|
||||
PipelineStepIdentifier("FetchQueryFeaturesPhase2")
|
||||
val candidatePipelinesStep: PipelineStepIdentifier = PipelineStepIdentifier("CandidatePipelines")
|
||||
val dependentCandidatePipelinesStep: PipelineStepIdentifier =
|
||||
PipelineStepIdentifier("DependentCandidatePipelines")
|
||||
val resultSelectorsStep: PipelineStepIdentifier = PipelineStepIdentifier("ResultSelectors")
|
||||
val domainMarshallerStep: PipelineStepIdentifier = PipelineStepIdentifier("DomainMarshaller")
|
||||
val resultSideEffectsStep: PipelineStepIdentifier = PipelineStepIdentifier("ResultSideEffects")
|
||||
val transportMarshallerStep: PipelineStepIdentifier = PipelineStepIdentifier(
|
||||
"TransportMarshaller")
|
||||
|
||||
/** All the Steps which are executed by a [[MixerPipeline]] in the order in which they are run */
|
||||
override val stepsInOrder: Seq[PipelineStepIdentifier] = Seq(
|
||||
qualityFactorStep,
|
||||
gatesStep,
|
||||
fetchQueryFeaturesStep,
|
||||
fetchQueryFeaturesPhase2Step,
|
||||
asyncFeaturesStep(candidatePipelinesStep),
|
||||
candidatePipelinesStep,
|
||||
asyncFeaturesStep(dependentCandidatePipelinesStep),
|
||||
dependentCandidatePipelinesStep,
|
||||
asyncFeaturesStep(resultSelectorsStep),
|
||||
resultSelectorsStep,
|
||||
domainMarshallerStep,
|
||||
asyncFeaturesStep(resultSideEffectsStep),
|
||||
resultSideEffectsStep,
|
||||
transportMarshallerStep
|
||||
)
|
||||
|
||||
/**
|
||||
* All the Steps which an [[com.twitter.product_mixer.core.functional_component.feature_hydrator.AsyncHydrator AsyncHydrator]]
|
||||
* can be configured to [[com.twitter.product_mixer.core.functional_component.feature_hydrator.AsyncHydrator.hydrateBefore hydrateBefore]]
|
||||
*/
|
||||
override val stepsAsyncFeatureHydrationCanBeCompletedBy: Set[PipelineStepIdentifier] = Set(
|
||||
candidatePipelinesStep,
|
||||
dependentCandidatePipelinesStep,
|
||||
resultSelectorsStep,
|
||||
resultSideEffectsStep
|
||||
)
|
||||
}
|
||||
Binary file not shown.
@ -1,70 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.mixer
|
||||
|
||||
import com.twitter.product_mixer.core.feature.featuremap.asyncfeaturemap.AsyncFeatureMap
|
||||
import com.twitter.product_mixer.core.model.marshalling.HasMarshalling
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineResult
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure
|
||||
import com.twitter.product_mixer.core.service.async_feature_map_executor.AsyncFeatureMapExecutorResults
|
||||
import com.twitter.product_mixer.core.service.candidate_pipeline_executor.CandidatePipelineExecutorResult
|
||||
import com.twitter.product_mixer.core.service.domain_marshaller_executor.DomainMarshallerExecutor
|
||||
import com.twitter.product_mixer.core.service.gate_executor.GateExecutorResult
|
||||
import com.twitter.product_mixer.core.service.pipeline_result_side_effect_executor.PipelineResultSideEffectExecutor
|
||||
import com.twitter.product_mixer.core.service.quality_factor_executor.QualityFactorExecutorResult
|
||||
import com.twitter.product_mixer.core.service.query_feature_hydrator_executor.QueryFeatureHydratorExecutor
|
||||
import com.twitter.product_mixer.core.service.selector_executor.SelectorExecutorResult
|
||||
import com.twitter.product_mixer.core.service.transport_marshaller_executor.TransportMarshallerExecutor
|
||||
|
||||
/**
|
||||
* A [[MixerPipelineResult]] includes both the user-visible [[PipelineResult]] and all the
|
||||
* Execution details possible - intermediate results, what components did, etc.
|
||||
*/
|
||||
case class MixerPipelineResult[Result](
|
||||
qualityFactorResult: Option[QualityFactorExecutorResult],
|
||||
gateResult: Option[GateExecutorResult],
|
||||
queryFeatures: Option[QueryFeatureHydratorExecutor.Result],
|
||||
queryFeaturesPhase2: Option[QueryFeatureHydratorExecutor.Result],
|
||||
mergedAsyncQueryFeatures: Option[AsyncFeatureMap],
|
||||
candidatePipelineResults: Option[CandidatePipelineExecutorResult],
|
||||
dependentCandidatePipelineResults: Option[CandidatePipelineExecutorResult],
|
||||
resultSelectorResults: Option[SelectorExecutorResult],
|
||||
domainMarshallerResults: Option[DomainMarshallerExecutor.Result[HasMarshalling]],
|
||||
resultSideEffectResults: Option[PipelineResultSideEffectExecutor.Result],
|
||||
asyncFeatureHydrationResults: Option[AsyncFeatureMapExecutorResults],
|
||||
transportMarshallerResults: Option[TransportMarshallerExecutor.Result[Result]],
|
||||
failure: Option[PipelineFailure],
|
||||
result: Option[Result])
|
||||
extends PipelineResult[Result] {
|
||||
|
||||
override def withFailure(failure: PipelineFailure): PipelineResult[Result] =
|
||||
copy(failure = Some(failure))
|
||||
|
||||
override def withResult(result: Result): PipelineResult[Result] = copy(result = Some(result))
|
||||
|
||||
/**
|
||||
* resultSize is calculated based on the selector results rather than the marshalled results. The
|
||||
* structure of the marshalled format is unknown, making operating on selector results more
|
||||
* convenient. This will implicitly excluded cursors built during marshalling but cursors don't
|
||||
* contribute to the result size anyway.
|
||||
*/
|
||||
override val resultSize: Int =
|
||||
resultSelectorResults.map(_.selectedCandidates).map(PipelineResult.resultSize).getOrElse(0)
|
||||
}
|
||||
|
||||
object MixerPipelineResult {
|
||||
def empty[A]: MixerPipelineResult[A] = MixerPipelineResult(
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None
|
||||
)
|
||||
}
|
||||
@ -1,14 +0,0 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier",
|
||||
"util/util-jackson/src/main/scala/com/twitter/util/jackson",
|
||||
],
|
||||
exports = [
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,49 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.pipeline_failure
|
||||
|
||||
import com.fasterxml.jackson.databind.annotation.JsonSerialize
|
||||
import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifierStack
|
||||
import scala.util.control.NoStackTrace
|
||||
|
||||
/**
|
||||
* Pipeline Failures represent pipeline requests that were not able to complete.
|
||||
*
|
||||
* A pipeline result will always define either a result or a failure.
|
||||
*
|
||||
* The reason field should not be displayed to end-users, and is free to change over time.
|
||||
* It should always be free of private user data such that we can log it.
|
||||
*
|
||||
* The pipeline can classify it's own failures into categories (timeouts, invalid arguments,
|
||||
* rate limited, etc) such that the caller can choose how to handle it.
|
||||
*
|
||||
* @note [[componentStack]] should only be set by the product mixer framework,
|
||||
* it should **NOT** be set when making a [[PipelineFailure]]
|
||||
*/
|
||||
@JsonSerialize(using = classOf[PipelineFailureSerializer])
|
||||
case class PipelineFailure(
|
||||
category: PipelineFailureCategory,
|
||||
reason: String,
|
||||
underlying: Option[Throwable] = None,
|
||||
componentStack: Option[ComponentIdentifierStack] = None)
|
||||
extends Exception(
|
||||
"PipelineFailure(" +
|
||||
s"category = $category, " +
|
||||
s"reason = $reason, " +
|
||||
s"underlying = $underlying, " +
|
||||
s"componentStack = $componentStack)",
|
||||
underlying.orNull
|
||||
) {
|
||||
override def toString: String = getMessage
|
||||
|
||||
/** Returns an updated copy of this [[PipelineFailure]] with the same exception stacktrace */
|
||||
def copy(
|
||||
category: PipelineFailureCategory = this.category,
|
||||
reason: String = this.reason,
|
||||
underlying: Option[Throwable] = this.underlying,
|
||||
componentStack: Option[ComponentIdentifierStack] = this.componentStack
|
||||
): PipelineFailure = {
|
||||
val newPipelineFailure =
|
||||
new PipelineFailure(category, reason, underlying, componentStack) with NoStackTrace
|
||||
newPipelineFailure.setStackTrace(this.getStackTrace)
|
||||
newPipelineFailure
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,190 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.pipeline_failure
|
||||
|
||||
/**
|
||||
* Failures are grouped into categories based on which party is 'responsible' for the issue. This
|
||||
* is important for generating accurate SLOs and ensuring that the correct team is alerted.
|
||||
*/
|
||||
sealed trait PipelineFailureCategory {
|
||||
val categoryName: String
|
||||
val failureName: String
|
||||
}
|
||||
|
||||
/**
|
||||
* Client Failures are failures where the client is deemed responsible for the issue. Such as by
|
||||
* issuing an invalid request or not having the right permissions.
|
||||
*
|
||||
* A failure might belong in this category if it relates to behaviour on the client and is not
|
||||
* actionable by the team which owns the product.
|
||||
*/
|
||||
trait ClientFailure extends PipelineFailureCategory {
|
||||
override val categoryName: String = "ClientFailure"
|
||||
}
|
||||
|
||||
/**
|
||||
* The requested product is disabled so the request cannot be served.
|
||||
*/
|
||||
case object ProductDisabled extends ClientFailure {
|
||||
override val failureName: String = "ProductDisabled"
|
||||
}
|
||||
|
||||
/**
|
||||
* The request was deemed invalid by this or a backing service.
|
||||
*/
|
||||
case object BadRequest extends ClientFailure {
|
||||
override val failureName: String = "BadRequest"
|
||||
}
|
||||
|
||||
/**
|
||||
* Credentials proving the identity of the caller were missing, not trusted, or expired.
|
||||
* For example, an auth cookie might be expired and in need of refreshing.
|
||||
*
|
||||
* Do not confuse this with Authorization, where the credentials are believed but not allowed to perform the operation.
|
||||
*/
|
||||
case object Authentication extends ClientFailure {
|
||||
override val failureName: String = "Authentication"
|
||||
}
|
||||
|
||||
/**
|
||||
* The operation was forbidden (often, but not always, by a Strato access control policy).
|
||||
*
|
||||
* Do not confuse this with Authentication, where the given credentials were missing, not trusted, or expired.
|
||||
*/
|
||||
case object Unauthorized extends ClientFailure {
|
||||
override val failureName: String = "Unauthorized"
|
||||
}
|
||||
|
||||
/**
|
||||
* The operation returned a Not Found response.
|
||||
*/
|
||||
case object NotFound extends ClientFailure {
|
||||
override val failureName: String = "NotFound"
|
||||
}
|
||||
|
||||
/**
|
||||
* An invalid input is included in a cursor field.
|
||||
*/
|
||||
case object MalformedCursor extends ClientFailure {
|
||||
override val failureName: String = "MalformedCursor"
|
||||
}
|
||||
|
||||
/**
|
||||
* The operation did not succeed due to a closed gate
|
||||
*/
|
||||
case object ClosedGate extends ClientFailure {
|
||||
override val failureName: String = "ClosedGate"
|
||||
}
|
||||
|
||||
/**
|
||||
* Server Failures are failures for which the owner of the product is responsible. Typically this
|
||||
* means the request was valid but an issue within Product Mixer or a dependent service prevented
|
||||
* it from succeeding.
|
||||
*
|
||||
* Server Failures contribute to the success rate SLO for the product.
|
||||
*/
|
||||
trait ServerFailure extends PipelineFailureCategory {
|
||||
override val categoryName: String = "ServerFailure"
|
||||
}
|
||||
|
||||
/**
|
||||
* Unclassified failures occur when product code throws an exception that Product Mixer does not
|
||||
* know how to classify.
|
||||
*
|
||||
* They can be used in failOpen policies, etc - but it's always preferred to instead add additional
|
||||
* classification logic and to keep Unclassified failures at 0.
|
||||
*/
|
||||
case object UncategorizedServerFailure extends ServerFailure {
|
||||
override val failureName: String = "UncategorizedServerFailure"
|
||||
}
|
||||
|
||||
/**
|
||||
* A hydrator or transformer returned a misconfigured feature map, this indicates a customer
|
||||
* configuration error. The owner of the component should make sure the hydrator always returns a
|
||||
* [[FeatureMap]] with the all features defined in the hydrator also set in the map, it should not have
|
||||
* any unregistered features nor should registered features be absent.
|
||||
*/
|
||||
case object MisconfiguredFeatureMapFailure extends ServerFailure {
|
||||
override val failureName: String = "MisconfiguredFeatureMapFailure"
|
||||
}
|
||||
|
||||
/**
|
||||
* A PipelineSelector returned an invalid ComponentIdentifier.
|
||||
*
|
||||
* A pipeline selector should choose the identifier of a pipeline that is contained by the 'pipelines'
|
||||
* sequence of the ProductPipelineConfig.
|
||||
*/
|
||||
case object InvalidPipelineSelected extends ServerFailure {
|
||||
override val failureName: String = "InvalidPipelineSelected"
|
||||
}
|
||||
|
||||
/**
|
||||
* Failures that occur when product code reaches an unexpected or otherwise illegal state.
|
||||
*/
|
||||
case object IllegalStateFailure extends ServerFailure {
|
||||
override val failureName: String = "IllegalStateFailure"
|
||||
}
|
||||
|
||||
/**
|
||||
* An unexpected candidate was returned in a candidate source that was unable to be transformed.
|
||||
*/
|
||||
case object UnexpectedCandidateResult extends ServerFailure {
|
||||
override val failureName: String = "UnexpectedCandidateResult"
|
||||
}
|
||||
|
||||
/**
|
||||
* An unexpected Candidate was returned in a marshaller
|
||||
*/
|
||||
case object UnexpectedCandidateInMarshaller extends ServerFailure {
|
||||
override val failureName: String = "UnexpectedCandidateInMarshaller"
|
||||
}
|
||||
|
||||
/**
|
||||
* Pipeline execution failed due to an incorrectly configured quality factor (e.g, accessing
|
||||
* quality factor state for a pipeline that does not have quality factor configured)
|
||||
*/
|
||||
case object MisconfiguredQualityFactor extends ServerFailure {
|
||||
override val failureName: String = "MisconfiguredQualityFactor"
|
||||
}
|
||||
|
||||
/**
|
||||
* Pipeline execution failed due to an incorrectly configured decorator (e.g, decorator
|
||||
* returned the wrong type or tried to decorate an already decorated candidate)
|
||||
*/
|
||||
case object MisconfiguredDecorator extends ServerFailure {
|
||||
override val failureName: String = "MisconfiguredDecorator"
|
||||
}
|
||||
|
||||
/**
|
||||
* Candidate Source Pipeline execution failed due to a timeout.
|
||||
*/
|
||||
case object CandidateSourceTimeout extends ServerFailure {
|
||||
override val failureName: String = "CandidateSourceTimeout"
|
||||
}
|
||||
|
||||
/**
|
||||
* Platform Failures are issues in the core Product Mixer logic itself which prevent a pipeline from
|
||||
* properly executing. These failures are the responsibility of the Product Mixer team.
|
||||
*/
|
||||
trait PlatformFailure extends PipelineFailureCategory {
|
||||
override val categoryName: String = "PlatformFailure"
|
||||
}
|
||||
|
||||
/**
|
||||
* Pipeline execution failed due to an unexpected error in Product Mixer.
|
||||
*
|
||||
* ExecutionFailed indicates a bug with the core Product Mixer execution logic rather than with a
|
||||
* specific product. For example, a bug in PipelineBuilder leading to us returning a
|
||||
* ProductPipelineResult that neither succeeded nor failed.
|
||||
*/
|
||||
case object ExecutionFailed extends PlatformFailure {
|
||||
override val failureName: String = "ExecutionFailed"
|
||||
}
|
||||
|
||||
/**
|
||||
* Pipeline execution failed due to a feature hydration failure.
|
||||
*
|
||||
* FeatureHydrationFailed indicates that the underlying hydration for a feature defined in a hydrator
|
||||
* failed (e.g, typically from a RPC call failing).
|
||||
*/
|
||||
case object FeatureHydrationFailed extends PlatformFailure {
|
||||
override val failureName: String = "FeatureHydrationFailed"
|
||||
}
|
||||
Binary file not shown.
@ -1,13 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.pipeline_failure
|
||||
|
||||
/** Represents a way to classify a given [[Throwable]] to a [[PipelineFailure]] */
|
||||
case class PipelineFailureClassifier(
|
||||
classifier: PartialFunction[Throwable, PipelineFailure])
|
||||
extends PartialFunction[Throwable, PipelineFailure] {
|
||||
override def isDefinedAt(throwable: Throwable): Boolean = classifier.isDefinedAt(throwable)
|
||||
override def apply(throwable: Throwable): PipelineFailure = classifier.apply(throwable)
|
||||
}
|
||||
|
||||
private[core] object PipelineFailureClassifier {
|
||||
val Empty: PipelineFailureClassifier = PipelineFailureClassifier(PartialFunction.empty)
|
||||
}
|
||||
Binary file not shown.
@ -1,67 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.pipeline_failure
|
||||
|
||||
import com.fasterxml.jackson.core.JsonGenerator
|
||||
import com.fasterxml.jackson.databind.JsonSerializer
|
||||
import com.fasterxml.jackson.databind.SerializerProvider
|
||||
import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifierStack
|
||||
|
||||
private[pipeline_failure] class PipelineFailureSerializer()
|
||||
extends JsonSerializer[PipelineFailure] {
|
||||
|
||||
private sealed trait BaseSerializableException
|
||||
|
||||
private case class SerializableException(
|
||||
`class`: String,
|
||||
message: String,
|
||||
stackTrace: Seq[String],
|
||||
cause: Option[BaseSerializableException])
|
||||
extends BaseSerializableException
|
||||
|
||||
private case class SerializablePipelineFailure(
|
||||
category: String,
|
||||
reason: String,
|
||||
underlying: Option[BaseSerializableException],
|
||||
componentStack: Option[ComponentIdentifierStack],
|
||||
stackTrace: Seq[String])
|
||||
extends BaseSerializableException
|
||||
|
||||
private def serializeStackTrace(stackTrace: Array[StackTraceElement]): Seq[String] =
|
||||
stackTrace.map(stackTraceElement => "at " + stackTraceElement.toString)
|
||||
|
||||
private def mkSerializableException(
|
||||
t: Throwable,
|
||||
recursionDepth: Int = 0
|
||||
): Option[BaseSerializableException] = {
|
||||
t match {
|
||||
case _ if recursionDepth > 4 =>
|
||||
// in the unfortunate case of a super deep chain of exceptions, stop if we get too deep
|
||||
None
|
||||
case pipelineFailure: PipelineFailure =>
|
||||
Some(
|
||||
SerializablePipelineFailure(
|
||||
category =
|
||||
pipelineFailure.category.categoryName + "/" + pipelineFailure.category.failureName,
|
||||
reason = pipelineFailure.reason,
|
||||
underlying =
|
||||
pipelineFailure.underlying.flatMap(mkSerializableException(_, recursionDepth + 1)),
|
||||
componentStack = pipelineFailure.componentStack,
|
||||
stackTrace = serializeStackTrace(pipelineFailure.getStackTrace)
|
||||
))
|
||||
case t =>
|
||||
Some(
|
||||
SerializableException(
|
||||
`class` = t.getClass.getName,
|
||||
message = t.getMessage,
|
||||
stackTrace = serializeStackTrace(t.getStackTrace),
|
||||
cause = Option(t.getCause).flatMap(mkSerializableException(_, recursionDepth + 1))
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
override def serialize(
|
||||
pipelineFailure: PipelineFailure,
|
||||
gen: JsonGenerator,
|
||||
serializers: SerializerProvider
|
||||
): Unit = serializers.defaultSerializeValue(mkSerializableException(pipelineFailure), gen)
|
||||
}
|
||||
@ -1,51 +0,0 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"configapi/configapi-core",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/gate",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_execution_logger",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_selector_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/quality_factor_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util",
|
||||
"stitch/stitch-core",
|
||||
"stringcenter/client/src/main/scala/com/twitter/stringcenter/client",
|
||||
"stringcenter/client/src/main/scala/com/twitter/stringcenter/client/stitch",
|
||||
],
|
||||
exports = [
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"configapi/configapi-core",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/gate",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_execution_logger",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_selector_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/quality_factor_executor",
|
||||
"stitch/stitch-core",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,28 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.product
|
||||
|
||||
import com.twitter.product_mixer.core.functional_component.common.access_policy.WithDebugAccessPolicies
|
||||
import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.Request
|
||||
import com.twitter.product_mixer.core.pipeline.Pipeline
|
||||
import com.twitter.stitch.Arrow
|
||||
|
||||
/**
|
||||
* A Product Pipeline
|
||||
*
|
||||
* This is an abstract class, as we only construct these via the [[ProductPipelineBuilder]].
|
||||
*
|
||||
* A [[ProductPipeline]] is capable of processing a [[Request]] and returning a response.
|
||||
*
|
||||
* @tparam RequestType the domain model for the query or request
|
||||
* @tparam ResponseType the final marshalled result type
|
||||
*/
|
||||
abstract class ProductPipeline[RequestType <: Request, ResponseType] private[product]
|
||||
extends Pipeline[ProductPipelineRequest[RequestType], ResponseType]
|
||||
with WithDebugAccessPolicies {
|
||||
override private[core] val config: ProductPipelineConfig[RequestType, _, ResponseType]
|
||||
override val arrow: Arrow[
|
||||
ProductPipelineRequest[RequestType],
|
||||
ProductPipelineResult[ResponseType]
|
||||
]
|
||||
override val identifier: ProductPipelineIdentifier
|
||||
}
|
||||
Binary file not shown.
@ -1,385 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.product
|
||||
|
||||
import com.twitter.finagle.mtls.authentication.ServiceIdentifier
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.finagle.tracing.Trace
|
||||
import com.twitter.finagle.transport.Transport
|
||||
import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy
|
||||
import com.twitter.product_mixer.core.functional_component.common.alert.Alert
|
||||
import com.twitter.product_mixer.core.functional_component.gate.Gate
|
||||
import com.twitter.product_mixer.core.gate.DenyLoggedOutUsersGate
|
||||
import com.twitter.product_mixer.core.gate.ParamGate
|
||||
import com.twitter.product_mixer.core.gate.ParamGate.EnabledGateSuffix
|
||||
import com.twitter.product_mixer.core.gate.ParamGate.SupportedClientGateSuffix
|
||||
import com.twitter.product_mixer.core.model.common.Component
|
||||
import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifierStack
|
||||
import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.Request
|
||||
import com.twitter.product_mixer.core.pipeline.InvalidStepStateException
|
||||
import com.twitter.product_mixer.core.pipeline.Pipeline
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineBuilder
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineBuilderFactory
|
||||
import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig
|
||||
import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineResult
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailureClassifier
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.ProductDisabled
|
||||
import com.twitter.product_mixer.core.pipeline.recommendation.RecommendationPipelineBuilderFactory
|
||||
import com.twitter.product_mixer.core.pipeline.recommendation.RecommendationPipelineConfig
|
||||
import com.twitter.product_mixer.core.pipeline.recommendation.RecommendationPipelineResult
|
||||
import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus
|
||||
import com.twitter.product_mixer.core.quality_factor.QualityFactorObserver
|
||||
import com.twitter.product_mixer.core.quality_factor.QualityFactorStatus
|
||||
import com.twitter.product_mixer.core.service.Executor
|
||||
import com.twitter.product_mixer.core.service.gate_executor.GateExecutor
|
||||
import com.twitter.product_mixer.core.service.gate_executor.GateExecutorResult
|
||||
import com.twitter.product_mixer.core.service.gate_executor.StoppedGateException
|
||||
import com.twitter.product_mixer.core.service.pipeline_execution_logger.PipelineExecutionLogger
|
||||
import com.twitter.product_mixer.core.service.pipeline_executor.PipelineExecutor
|
||||
import com.twitter.product_mixer.core.service.pipeline_executor.PipelineExecutorRequest
|
||||
import com.twitter.product_mixer.core.service.pipeline_executor.PipelineExecutorResult
|
||||
import com.twitter.product_mixer.core.service.pipeline_selector_executor.PipelineSelectorExecutor
|
||||
import com.twitter.product_mixer.core.service.pipeline_selector_executor.PipelineSelectorExecutorResult
|
||||
import com.twitter.product_mixer.core.service.quality_factor_executor.QualityFactorExecutorResult
|
||||
import com.twitter.stitch.Arrow
|
||||
import com.twitter.stringcenter.client.StringCenterRequestContext
|
||||
import com.twitter.stringcenter.client.stitch.StringCenterRequestContextLetter
|
||||
import com.twitter.timelines.configapi.Params
|
||||
import com.twitter.util.logging.Logging
|
||||
import org.slf4j.MDC
|
||||
|
||||
class ProductPipelineBuilder[TRequest <: Request, Query <: PipelineQuery, Response](
|
||||
gateExecutor: GateExecutor,
|
||||
pipelineSelectorExecutor: PipelineSelectorExecutor,
|
||||
pipelineExecutor: PipelineExecutor,
|
||||
mixerPipelineBuilderFactory: MixerPipelineBuilderFactory,
|
||||
recommendationPipelineBuilderFactory: RecommendationPipelineBuilderFactory,
|
||||
override val statsReceiver: StatsReceiver,
|
||||
pipelineExecutionLogger: PipelineExecutionLogger)
|
||||
extends PipelineBuilder[ProductPipelineRequest[TRequest]]
|
||||
with Logging { builder =>
|
||||
|
||||
override type UnderlyingResultType = Response
|
||||
override type PipelineResultType = ProductPipelineResult[Response]
|
||||
|
||||
/**
|
||||
* Query Transformer Step is implemented inline instead of using an executor.
|
||||
*
|
||||
* It's a simple, synchronous step that executes the query transformer.
|
||||
*
|
||||
* Since the output of the transformer is used in multiple other steps (Gate, Pipeline Execution),
|
||||
* we've promoted the transformer to a step so that it's outputs can be reused easily.
|
||||
*/
|
||||
def pipelineQueryTransformerStep(
|
||||
queryTransformer: (TRequest, Params) => Query,
|
||||
context: Executor.Context
|
||||
): Step[ProductPipelineRequest[TRequest], Query] =
|
||||
new Step[ProductPipelineRequest[TRequest], Query] {
|
||||
|
||||
override def identifier: PipelineStepIdentifier =
|
||||
ProductPipelineConfig.pipelineQueryTransformerStep
|
||||
|
||||
override def executorArrow: Arrow[ProductPipelineRequest[TRequest], Query] = {
|
||||
wrapWithErrorHandling(context, identifier)(
|
||||
Arrow.map[ProductPipelineRequest[TRequest], Query] {
|
||||
case ProductPipelineRequest(request, params) => queryTransformer(request, params)
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
override def inputAdaptor(
|
||||
query: ProductPipelineRequest[TRequest],
|
||||
previousResult: ProductPipelineResult[Response]
|
||||
): ProductPipelineRequest[TRequest] = query
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: ProductPipelineResult[Response],
|
||||
executorResult: Query
|
||||
): ProductPipelineResult[Response] =
|
||||
previousPipelineResult.copy(transformedQuery = Some(executorResult))
|
||||
}
|
||||
|
||||
def qualityFactorStep(
|
||||
qualityFactorStatus: QualityFactorStatus
|
||||
): Step[Query, QualityFactorExecutorResult] = {
|
||||
new Step[Query, QualityFactorExecutorResult] {
|
||||
override def identifier: PipelineStepIdentifier = ProductPipelineConfig.qualityFactorStep
|
||||
|
||||
override def executorArrow: Arrow[Query, QualityFactorExecutorResult] =
|
||||
Arrow
|
||||
.map[Query, QualityFactorExecutorResult] { _ =>
|
||||
QualityFactorExecutorResult(
|
||||
pipelineQualityFactors =
|
||||
qualityFactorStatus.qualityFactorByPipeline.mapValues(_.currentValue)
|
||||
)
|
||||
}
|
||||
|
||||
override def inputAdaptor(
|
||||
query: ProductPipelineRequest[TRequest],
|
||||
previousResult: ProductPipelineResult[Response]
|
||||
): Query = previousResult.transformedQuery
|
||||
.getOrElse {
|
||||
throw InvalidStepStateException(identifier, "TransformedQuery")
|
||||
}.asInstanceOf[Query]
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: ProductPipelineResult[Response],
|
||||
executorResult: QualityFactorExecutorResult
|
||||
): ProductPipelineResult[Response] = {
|
||||
previousPipelineResult.copy(
|
||||
transformedQuery = previousPipelineResult.transformedQuery.map {
|
||||
case queryWithQualityFactor: HasQualityFactorStatus =>
|
||||
queryWithQualityFactor
|
||||
.withQualityFactorStatus(qualityFactorStatus).asInstanceOf[Query]
|
||||
case query =>
|
||||
query
|
||||
},
|
||||
qualityFactorResult = Some(executorResult)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def gatesStep(
|
||||
gates: Seq[Gate[Query]],
|
||||
context: Executor.Context
|
||||
): Step[Query, GateExecutorResult] = new Step[Query, GateExecutorResult] {
|
||||
override def identifier: PipelineStepIdentifier = ProductPipelineConfig.gatesStep
|
||||
|
||||
override def executorArrow: Arrow[Query, GateExecutorResult] = {
|
||||
gateExecutor.arrow(gates, context)
|
||||
}
|
||||
|
||||
override def inputAdaptor(
|
||||
query: ProductPipelineRequest[TRequest],
|
||||
previousResult: ProductPipelineResult[Response]
|
||||
): Query = previousResult.transformedQuery
|
||||
.getOrElse {
|
||||
throw InvalidStepStateException(identifier, "TransformedQuery")
|
||||
}.asInstanceOf[Query]
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: ProductPipelineResult[Response],
|
||||
executorResult: GateExecutorResult
|
||||
): ProductPipelineResult[Response] =
|
||||
previousPipelineResult.copy(gateResult = Some(executorResult))
|
||||
}
|
||||
|
||||
def pipelineSelectorStep(
|
||||
pipelineByIdentifer: Map[ComponentIdentifier, Pipeline[Query, Response]],
|
||||
pipelineSelector: Query => ComponentIdentifier,
|
||||
context: Executor.Context
|
||||
): Step[Query, PipelineSelectorExecutorResult] =
|
||||
new Step[Query, PipelineSelectorExecutorResult] {
|
||||
override def identifier: PipelineStepIdentifier = ProductPipelineConfig.pipelineSelectorStep
|
||||
|
||||
override def executorArrow: Arrow[
|
||||
Query,
|
||||
PipelineSelectorExecutorResult
|
||||
] = pipelineSelectorExecutor.arrow(pipelineByIdentifer, pipelineSelector, context)
|
||||
|
||||
override def inputAdaptor(
|
||||
query: ProductPipelineRequest[TRequest],
|
||||
previousResult: ProductPipelineResult[Response]
|
||||
): Query =
|
||||
previousResult.transformedQuery
|
||||
.getOrElse(throw InvalidStepStateException(identifier, "TransformedQuery")).asInstanceOf[
|
||||
Query]
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: ProductPipelineResult[Response],
|
||||
executorResult: PipelineSelectorExecutorResult
|
||||
): ProductPipelineResult[Response] =
|
||||
previousPipelineResult.copy(pipelineSelectorResult = Some(executorResult))
|
||||
}
|
||||
|
||||
def pipelineExecutionStep(
|
||||
pipelineByIdentifier: Map[ComponentIdentifier, Pipeline[Query, Response]],
|
||||
qualityFactorObserverByPipeline: Map[ComponentIdentifier, QualityFactorObserver],
|
||||
context: Executor.Context
|
||||
): Step[PipelineExecutorRequest[Query], PipelineExecutorResult[Response]] =
|
||||
new Step[PipelineExecutorRequest[Query], PipelineExecutorResult[Response]] {
|
||||
override def identifier: PipelineStepIdentifier = ProductPipelineConfig.pipelineExecutionStep
|
||||
|
||||
override def executorArrow: Arrow[
|
||||
PipelineExecutorRequest[Query],
|
||||
PipelineExecutorResult[Response]
|
||||
] = {
|
||||
pipelineExecutor.arrow(pipelineByIdentifier, qualityFactorObserverByPipeline, context)
|
||||
}
|
||||
|
||||
override def inputAdaptor(
|
||||
request: ProductPipelineRequest[TRequest],
|
||||
previousResult: ProductPipelineResult[Response]
|
||||
): PipelineExecutorRequest[Query] = {
|
||||
val query = previousResult.transformedQuery
|
||||
.getOrElse {
|
||||
throw InvalidStepStateException(identifier, "TransformedQuery")
|
||||
}.asInstanceOf[Query]
|
||||
|
||||
val pipelineIdentifier = previousResult.pipelineSelectorResult
|
||||
.map(_.pipelineIdentifier).getOrElse {
|
||||
throw InvalidStepStateException(identifier, "PipelineSelectorResult")
|
||||
}
|
||||
|
||||
PipelineExecutorRequest(query, pipelineIdentifier)
|
||||
}
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: ProductPipelineResult[Response],
|
||||
executorResult: PipelineExecutorResult[Response]
|
||||
): ProductPipelineResult[Response] = {
|
||||
|
||||
val mixerPipelineResult = executorResult.pipelineResult match {
|
||||
case mixerPipelineResult: MixerPipelineResult[Response] @unchecked =>
|
||||
Some(mixerPipelineResult)
|
||||
case _ =>
|
||||
None
|
||||
}
|
||||
|
||||
val recommendationPipelineResult = executorResult.pipelineResult match {
|
||||
case recommendationPipelineResult: RecommendationPipelineResult[
|
||||
_,
|
||||
Response
|
||||
] @unchecked =>
|
||||
Some(recommendationPipelineResult)
|
||||
case _ =>
|
||||
None
|
||||
}
|
||||
|
||||
previousPipelineResult.copy(
|
||||
mixerPipelineResult = mixerPipelineResult,
|
||||
recommendationPipelineResult = recommendationPipelineResult,
|
||||
traceId = Trace.idOption.map(_.traceId.toString()),
|
||||
result = executorResult.pipelineResult.result
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
def build(
|
||||
parentComponentIdentifierStack: ComponentIdentifierStack,
|
||||
config: ProductPipelineConfig[TRequest, Query, Response]
|
||||
): ProductPipeline[TRequest, Response] = {
|
||||
|
||||
val pipelineIdentifier = config.identifier
|
||||
|
||||
val context = Executor.Context(
|
||||
PipelineFailureClassifier(
|
||||
config.failureClassifier.orElse(StoppedGateException.classifier(ProductDisabled))),
|
||||
parentComponentIdentifierStack.push(pipelineIdentifier)
|
||||
)
|
||||
|
||||
val denyLoggedOutUsersGate = if (config.denyLoggedOutUsers) {
|
||||
Some(DenyLoggedOutUsersGate(pipelineIdentifier))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
val enabledGate: ParamGate =
|
||||
ParamGate(pipelineIdentifier + EnabledGateSuffix, config.paramConfig.EnabledDeciderParam)
|
||||
val supportedClientGate =
|
||||
ParamGate(
|
||||
pipelineIdentifier + SupportedClientGateSuffix,
|
||||
config.paramConfig.SupportedClientParam)
|
||||
|
||||
/**
|
||||
* Evaluate enabled decider gate first since if it's off, there is no reason to proceed
|
||||
* Next evaluate supported client feature switch gate, followed by customer configured gates
|
||||
*/
|
||||
val allGates =
|
||||
denyLoggedOutUsersGate.toSeq ++: enabledGate +: supportedClientGate +: config.gates
|
||||
|
||||
val childPipelines: Seq[Pipeline[Query, Response]] =
|
||||
config.pipelines.map {
|
||||
case mixerConfig: MixerPipelineConfig[Query, _, Response] =>
|
||||
mixerConfig.build(context.componentStack, mixerPipelineBuilderFactory)
|
||||
case recommendationConfig: RecommendationPipelineConfig[Query, _, _, Response] =>
|
||||
recommendationConfig.build(context.componentStack, recommendationPipelineBuilderFactory)
|
||||
case other =>
|
||||
throw new IllegalArgumentException(
|
||||
s"Product Pipelines only support Mixer and Recommendation pipelines, not $other")
|
||||
}
|
||||
|
||||
val pipelineByIdentifier: Map[ComponentIdentifier, Pipeline[Query, Response]] =
|
||||
childPipelines.map { pipeline =>
|
||||
(pipeline.identifier, pipeline)
|
||||
}.toMap
|
||||
|
||||
val qualityFactorStatus: QualityFactorStatus =
|
||||
QualityFactorStatus.build(config.qualityFactorConfigs)
|
||||
|
||||
val qualityFactorObserverByPipeline = qualityFactorStatus.qualityFactorByPipeline.mapValues {
|
||||
qualityFactor =>
|
||||
qualityFactor.buildObserver()
|
||||
}
|
||||
|
||||
buildGaugesForQualityFactor(pipelineIdentifier, qualityFactorStatus, statsReceiver)
|
||||
|
||||
/**
|
||||
* Initialize MDC with access logging with everything we have at request time. We can put
|
||||
* more stuff into MDC later down the pipeline, but at risk of exceptions/errors preventing
|
||||
* them from being added
|
||||
*/
|
||||
val mdcInitArrow =
|
||||
Arrow.map[ProductPipelineRequest[TRequest], ProductPipelineRequest[TRequest]] { request =>
|
||||
val serviceIdentifier = ServiceIdentifier.fromCertificate(Transport.peerCertificate)
|
||||
MDC.put("product", config.product.identifier.name)
|
||||
MDC.put("serviceIdentifier", ServiceIdentifier.asString(serviceIdentifier))
|
||||
request
|
||||
}
|
||||
|
||||
val builtSteps = Seq(
|
||||
pipelineQueryTransformerStep(config.pipelineQueryTransformer, context),
|
||||
qualityFactorStep(qualityFactorStatus),
|
||||
gatesStep(allGates, context),
|
||||
pipelineSelectorStep(pipelineByIdentifier, config.pipelineSelector, context),
|
||||
pipelineExecutionStep(pipelineByIdentifier, qualityFactorObserverByPipeline, context)
|
||||
)
|
||||
|
||||
val underlying: Arrow[ProductPipelineRequest[TRequest], ProductPipelineResult[Response]] =
|
||||
buildCombinedArrowFromSteps(
|
||||
steps = builtSteps,
|
||||
context = context,
|
||||
initialEmptyResult = ProductPipelineResult.empty,
|
||||
stepsInOrderFromConfig = ProductPipelineConfig.stepsInOrder
|
||||
)
|
||||
|
||||
/**
|
||||
* Unlike other components and pipelines, [[ProductPipeline]] must be observed in the
|
||||
* [[ProductPipelineBuilder]] directly because the resulting [[ProductPipeline.arrow]]
|
||||
* is run directly without an executor so must contain all stats.
|
||||
*/
|
||||
val observed =
|
||||
wrapProductPipelineWithExecutorBookkeeping[
|
||||
ProductPipelineRequest[TRequest],
|
||||
ProductPipelineResult[Response]
|
||||
](context, pipelineIdentifier)(underlying)
|
||||
|
||||
val finalArrow: Arrow[ProductPipelineRequest[TRequest], ProductPipelineResult[Response]] =
|
||||
Arrow
|
||||
.letWithArg[
|
||||
ProductPipelineRequest[TRequest],
|
||||
ProductPipelineResult[Response],
|
||||
StringCenterRequestContext](StringCenterRequestContextLetter)(request =>
|
||||
StringCenterRequestContext(
|
||||
request.request.clientContext.languageCode,
|
||||
request.request.clientContext.countryCode
|
||||
))(
|
||||
mdcInitArrow
|
||||
.andThen(observed)
|
||||
.onSuccess(result => result.transformedQuery.map(pipelineExecutionLogger(_, result))))
|
||||
|
||||
val configFromBuilder = config
|
||||
new ProductPipeline[TRequest, Response] {
|
||||
override private[core] val config: ProductPipelineConfig[TRequest, _, Response] =
|
||||
configFromBuilder
|
||||
override val arrow: Arrow[ProductPipelineRequest[TRequest], ProductPipelineResult[Response]] =
|
||||
finalArrow
|
||||
override val identifier: ProductPipelineIdentifier = pipelineIdentifier
|
||||
override val alerts: Seq[Alert] = config.alerts
|
||||
override val debugAccessPolicies: Set[AccessPolicy] = config.debugAccessPolicies
|
||||
override val children: Seq[Component] = allGates ++ childPipelines
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,39 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.product
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.Request
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineBuilderFactory
|
||||
import com.twitter.product_mixer.core.pipeline.recommendation.RecommendationPipelineBuilderFactory
|
||||
import com.twitter.product_mixer.core.service.gate_executor.GateExecutor
|
||||
import com.twitter.product_mixer.core.service.pipeline_execution_logger.PipelineExecutionLogger
|
||||
import com.twitter.product_mixer.core.service.pipeline_executor.PipelineExecutor
|
||||
import com.twitter.product_mixer.core.service.pipeline_selector_executor.PipelineSelectorExecutor
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class ProductPipelineBuilderFactory @Inject() (
|
||||
gateExecutor: GateExecutor,
|
||||
pipelineSelectorExecutor: PipelineSelectorExecutor,
|
||||
pipelineExecutor: PipelineExecutor,
|
||||
mixerPipelineBuilderFactory: MixerPipelineBuilderFactory,
|
||||
recommendationPipelineBuilderFactory: RecommendationPipelineBuilderFactory,
|
||||
statsReceiver: StatsReceiver,
|
||||
pipelineExecutionLogger: PipelineExecutionLogger) {
|
||||
def get[
|
||||
TRequest <: Request,
|
||||
Query <: PipelineQuery,
|
||||
Response
|
||||
]: ProductPipelineBuilder[TRequest, Query, Response] = {
|
||||
new ProductPipelineBuilder[TRequest, Query, Response](
|
||||
gateExecutor,
|
||||
pipelineSelectorExecutor,
|
||||
pipelineExecutor,
|
||||
mixerPipelineBuilderFactory,
|
||||
recommendationPipelineBuilderFactory,
|
||||
statsReceiver,
|
||||
pipelineExecutionLogger
|
||||
)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,107 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.product
|
||||
|
||||
import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy
|
||||
import com.twitter.product_mixer.core.functional_component.common.alert.Alert
|
||||
import com.twitter.product_mixer.core.functional_component.gate.Gate
|
||||
import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.Product
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.Request
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineConfig
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineConfigCompanion
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure
|
||||
import com.twitter.product_mixer.core.product.ProductParamConfig
|
||||
import com.twitter.product_mixer.core.quality_factor.QualityFactorConfig
|
||||
import com.twitter.timelines.configapi.Params
|
||||
|
||||
trait ProductPipelineConfig[TRequest <: Request, Query <: PipelineQuery, Response]
|
||||
extends PipelineConfig {
|
||||
|
||||
override val identifier: ProductPipelineIdentifier
|
||||
|
||||
val product: Product
|
||||
val paramConfig: ProductParamConfig
|
||||
|
||||
/**
|
||||
* Product Pipeline Gates will be executed before any other step (including retrieval from mixer
|
||||
* pipelines). They're executed sequentially, and any "Stop" result will prevent pipeline execution.
|
||||
*/
|
||||
def gates: Seq[Gate[Query]] = Seq.empty
|
||||
|
||||
def pipelineQueryTransformer(request: TRequest, params: Params): Query
|
||||
|
||||
/**
|
||||
* A list of all pipelines that power this product directly (there is no need to include pipelines
|
||||
* called by those pipelines).
|
||||
*
|
||||
* Only pipeline from this list should referenced from the pipelineSelector
|
||||
*/
|
||||
def pipelines: Seq[PipelineConfig]
|
||||
|
||||
/**
|
||||
* A pipeline selector selects a pipeline (from the list in `def pipelines`) to handle the
|
||||
* current request.
|
||||
*/
|
||||
def pipelineSelector(query: Query): ComponentIdentifier
|
||||
|
||||
/**
|
||||
** [[qualityFactorConfigs]] associates [[QualityFactorConfig]]s to specific pipelines
|
||||
* using [[ComponentIdentifier]].
|
||||
*/
|
||||
def qualityFactorConfigs: Map[ComponentIdentifier, QualityFactorConfig] =
|
||||
Map.empty
|
||||
|
||||
/**
|
||||
* By default (for safety), product mixer pipelines do not allow logged out requests.
|
||||
* A "DenyLoggedOutUsersGate" will be generated and added to the pipeline.
|
||||
*
|
||||
* You can disable this behavior by overriding `denyLoggedOutUsers` with False.
|
||||
*/
|
||||
val denyLoggedOutUsers: Boolean = true
|
||||
|
||||
/**
|
||||
* A pipeline can define a partial function to rescue failures here. They will be treated as failures
|
||||
* from a monitoring standpoint, and cancellation exceptions will always be propagated (they cannot be caught here).
|
||||
*/
|
||||
def failureClassifier: PartialFunction[Throwable, PipelineFailure] = PartialFunction.empty
|
||||
|
||||
/**
|
||||
* Alerts can be used to indicate the pipeline's service level objectives. Alerts and
|
||||
* dashboards will be automatically created based on this information.
|
||||
*/
|
||||
val alerts: Seq[Alert] = Seq.empty
|
||||
|
||||
/**
|
||||
* Access Policies can be used to gate who can query a product from Product Mixer's query tool
|
||||
* (go/turntable).
|
||||
*
|
||||
* This will typically be gated by an LDAP group associated with your team. For example:
|
||||
*
|
||||
* {{{
|
||||
* override val debugAccessPolicies: Set[AccessPolicy] = Set(AllowedLdapGroups("NAME"))
|
||||
* }}}
|
||||
*
|
||||
* You can disable all queries by using the [[com.twitter.product_mixer.core.functional_component.common.access_policy.BlockEverything]] policy.
|
||||
*/
|
||||
val debugAccessPolicies: Set[AccessPolicy]
|
||||
}
|
||||
|
||||
object ProductPipelineConfig extends PipelineConfigCompanion {
|
||||
val pipelineQueryTransformerStep: PipelineStepIdentifier = PipelineStepIdentifier(
|
||||
"PipelineQueryTransformer")
|
||||
val qualityFactorStep: PipelineStepIdentifier = PipelineStepIdentifier("QualityFactor")
|
||||
val gatesStep: PipelineStepIdentifier = PipelineStepIdentifier("Gates")
|
||||
val pipelineSelectorStep: PipelineStepIdentifier = PipelineStepIdentifier("PipelineSelector")
|
||||
val pipelineExecutionStep: PipelineStepIdentifier = PipelineStepIdentifier("PipelineExecution")
|
||||
|
||||
/** All the Steps which are executed by a [[ProductPipeline]] in the order in which they are run */
|
||||
override val stepsInOrder: Seq[PipelineStepIdentifier] = Seq(
|
||||
pipelineQueryTransformerStep,
|
||||
qualityFactorStep,
|
||||
gatesStep,
|
||||
pipelineSelectorStep,
|
||||
pipelineExecutionStep
|
||||
)
|
||||
}
|
||||
Binary file not shown.
@ -1,5 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.product
|
||||
|
||||
import com.twitter.timelines.configapi.Params
|
||||
|
||||
case class ProductPipelineRequest[RequestType](request: RequestType, params: Params)
|
||||
Binary file not shown.
@ -1,62 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.product
|
||||
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineResult
|
||||
import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineResult
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure
|
||||
import com.twitter.product_mixer.core.pipeline.recommendation.RecommendationPipelineResult
|
||||
import com.twitter.product_mixer.core.service.gate_executor.GateExecutorResult
|
||||
import com.twitter.product_mixer.core.service.pipeline_selector_executor.PipelineSelectorExecutorResult
|
||||
import com.twitter.product_mixer.core.service.quality_factor_executor.QualityFactorExecutorResult
|
||||
|
||||
case class ProductPipelineResult[Result](
|
||||
transformedQuery: Option[PipelineQuery],
|
||||
qualityFactorResult: Option[QualityFactorExecutorResult],
|
||||
gateResult: Option[GateExecutorResult],
|
||||
pipelineSelectorResult: Option[PipelineSelectorExecutorResult],
|
||||
mixerPipelineResult: Option[MixerPipelineResult[Result]],
|
||||
recommendationPipelineResult: Option[RecommendationPipelineResult[_, Result]],
|
||||
traceId: Option[String],
|
||||
failure: Option[PipelineFailure],
|
||||
result: Option[Result])
|
||||
extends PipelineResult[Result] {
|
||||
|
||||
override val resultSize: Int = {
|
||||
if (mixerPipelineResult.isDefined) {
|
||||
mixerPipelineResult.map(_.resultSize).getOrElse(0)
|
||||
} else {
|
||||
recommendationPipelineResult.map(_.resultSize).getOrElse(0)
|
||||
}
|
||||
}
|
||||
|
||||
override def withFailure(failure: PipelineFailure): PipelineResult[Result] =
|
||||
copy(failure = Some(failure))
|
||||
|
||||
override def withResult(result: Result): PipelineResult[Result] = copy(result = Some(result))
|
||||
}
|
||||
|
||||
object ProductPipelineResult {
|
||||
def empty[A]: ProductPipelineResult[A] = ProductPipelineResult(
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None
|
||||
)
|
||||
|
||||
def fromResult[A](result: A): ProductPipelineResult[A] = ProductPipelineResult(
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
Some(result)
|
||||
)
|
||||
}
|
||||
@ -1,62 +0,0 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector:insert_append_results",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/premarshaller",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/side_effect",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/async_feature_map_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_pipeline_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/filter_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/quality_factor_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/scoring_pipeline_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util",
|
||||
"stitch/stitch-core",
|
||||
],
|
||||
exports = [
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/premarshaller",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/side_effect",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/async_feature_map_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_pipeline_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/filter_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/quality_factor_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/scoring_pipeline_executor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor",
|
||||
"stitch/stitch-core",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,29 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.recommendation
|
||||
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.model.common.identifier.RecommendationPipelineIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.Pipeline
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Arrow
|
||||
|
||||
/**
|
||||
* A Recommendation Pipeline
|
||||
*
|
||||
* This is an abstract class, as we only construct these via the [[RecommendationPipelineBuilder]].
|
||||
*
|
||||
* A [[RecommendationPipeline]] is capable of processing requests (queries) and returning responses (results)
|
||||
* in the correct format to directly send to users.
|
||||
*
|
||||
* @tparam Query the domain model for the query or request
|
||||
* @tparam Candidate the type of the candidates
|
||||
* @tparam Result the final marshalled result type
|
||||
*/
|
||||
abstract class RecommendationPipeline[
|
||||
Query <: PipelineQuery,
|
||||
Candidate <: UniversalNoun[Any],
|
||||
Result]
|
||||
extends Pipeline[Query, Result] {
|
||||
override private[core] val config: RecommendationPipelineConfig[Query, Candidate, _, Result]
|
||||
override val arrow: Arrow[Query, RecommendationPipelineResult[Candidate, Result]]
|
||||
override val identifier: RecommendationPipelineIdentifier
|
||||
}
|
||||
Binary file not shown.
File diff suppressed because it is too large
Load Diff
Binary file not shown.
@ -1,67 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.recommendation
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.model.marshalling.HasMarshalling
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineBuilderFactory
|
||||
import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipelineBuilderFactory
|
||||
import com.twitter.product_mixer.core.service.candidate_decorator_executor.CandidateDecoratorExecutor
|
||||
import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutor
|
||||
import com.twitter.product_mixer.core.service.candidate_pipeline_executor.CandidatePipelineExecutor
|
||||
import com.twitter.product_mixer.core.service.domain_marshaller_executor.DomainMarshallerExecutor
|
||||
import com.twitter.product_mixer.core.service.filter_executor.FilterExecutor
|
||||
import com.twitter.product_mixer.core.service.gate_executor.GateExecutor
|
||||
import com.twitter.product_mixer.core.service.pipeline_result_side_effect_executor.PipelineResultSideEffectExecutor
|
||||
import com.twitter.product_mixer.core.service.async_feature_map_executor.AsyncFeatureMapExecutor
|
||||
import com.twitter.product_mixer.core.service.query_feature_hydrator_executor.QueryFeatureHydratorExecutor
|
||||
import com.twitter.product_mixer.core.service.scoring_pipeline_executor.ScoringPipelineExecutor
|
||||
import com.twitter.product_mixer.core.service.selector_executor.SelectorExecutor
|
||||
import com.twitter.product_mixer.core.service.transport_marshaller_executor.TransportMarshallerExecutor
|
||||
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class RecommendationPipelineBuilderFactory @Inject() (
|
||||
candidatePipelineExecutor: CandidatePipelineExecutor,
|
||||
gateExecutor: GateExecutor,
|
||||
selectorExecutor: SelectorExecutor,
|
||||
queryFeatureHydratorExecutor: QueryFeatureHydratorExecutor,
|
||||
asyncFeatureMapExecutor: AsyncFeatureMapExecutor,
|
||||
candidateFeatureHydratorExecutor: CandidateFeatureHydratorExecutor,
|
||||
filterExecutor: FilterExecutor,
|
||||
scoringPipelineExecutor: ScoringPipelineExecutor,
|
||||
candidateDecoratorExecutor: CandidateDecoratorExecutor,
|
||||
domainMarshallerExecutor: DomainMarshallerExecutor,
|
||||
transportMarshallerExecutor: TransportMarshallerExecutor,
|
||||
pipelineResultSideEffectExecutor: PipelineResultSideEffectExecutor,
|
||||
candidatePipelineBuilderFactory: CandidatePipelineBuilderFactory,
|
||||
scoringPipelineBuilderFactory: ScoringPipelineBuilderFactory,
|
||||
statsReceiver: StatsReceiver) {
|
||||
|
||||
def get[
|
||||
Query <: PipelineQuery,
|
||||
Candidate <: UniversalNoun[Any],
|
||||
DomainResultType <: HasMarshalling,
|
||||
Result
|
||||
]: RecommendationPipelineBuilder[Query, Candidate, DomainResultType, Result] = {
|
||||
new RecommendationPipelineBuilder[Query, Candidate, DomainResultType, Result](
|
||||
candidatePipelineExecutor,
|
||||
gateExecutor,
|
||||
selectorExecutor,
|
||||
queryFeatureHydratorExecutor,
|
||||
asyncFeatureMapExecutor,
|
||||
candidateFeatureHydratorExecutor,
|
||||
filterExecutor,
|
||||
scoringPipelineExecutor,
|
||||
candidateDecoratorExecutor,
|
||||
domainMarshallerExecutor,
|
||||
transportMarshallerExecutor,
|
||||
pipelineResultSideEffectExecutor,
|
||||
candidatePipelineBuilderFactory,
|
||||
scoringPipelineBuilderFactory,
|
||||
statsReceiver
|
||||
)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,262 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.recommendation
|
||||
|
||||
import com.twitter.product_mixer.component_library.selector.InsertAppendResults
|
||||
import com.twitter.product_mixer.core.functional_component.common.AllPipelines
|
||||
import com.twitter.product_mixer.core.functional_component.common.alert.Alert
|
||||
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
|
||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator
|
||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator
|
||||
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||
import com.twitter.product_mixer.core.functional_component.gate.Gate
|
||||
import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller
|
||||
import com.twitter.product_mixer.core.functional_component.selector.Selector
|
||||
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
|
||||
import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifierStack
|
||||
import com.twitter.product_mixer.core.model.common.identifier.RecommendationPipelineIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier
|
||||
import com.twitter.product_mixer.core.model.marshalling.HasMarshalling
|
||||
import com.twitter.product_mixer.core.pipeline.FailOpenPolicy
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineConfig
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineConfigCompanion
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig
|
||||
import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.ClosedGate
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure
|
||||
import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipelineConfig
|
||||
import com.twitter.product_mixer.core.quality_factor.QualityFactorConfig
|
||||
|
||||
/**
|
||||
* This is the configuration necessary to generate a Recommendation Pipeline. Product code should create a
|
||||
* RecommendationPipelineConfig, and then use a RecommendationPipelineBuilder to get the final RecommendationPipeline which can
|
||||
* process requests.
|
||||
*
|
||||
* @tparam Query - The domain model for the query or request
|
||||
* @tparam Candidate - The type of the candidates that the Candidate Pipelines are generating
|
||||
* @tparam UnmarshalledResultType - The result type of the pipeline, but before marshalling to a wire protocol like URT
|
||||
* @tparam Result - The final result that will be served to users
|
||||
*/
|
||||
trait RecommendationPipelineConfig[
|
||||
Query <: PipelineQuery,
|
||||
Candidate <: UniversalNoun[Any],
|
||||
UnmarshalledResultType <: HasMarshalling,
|
||||
Result]
|
||||
extends PipelineConfig {
|
||||
|
||||
override val identifier: RecommendationPipelineIdentifier
|
||||
|
||||
/**
|
||||
* Recommendation Pipeline Gates will be executed before any other step (including retrieval from candidate
|
||||
* pipelines). They're executed sequentially, and any "Stop" result will prevent pipeline execution.
|
||||
*/
|
||||
def gates: Seq[Gate[Query]] = Seq.empty
|
||||
|
||||
/**
|
||||
* A recommendation pipeline can fetch query-level features before candidate pipelines are executed.
|
||||
*/
|
||||
def fetchQueryFeatures: Seq[BaseQueryFeatureHydrator[Query, _]] = Seq.empty
|
||||
|
||||
/**
|
||||
* Candidate pipelines retrieve candidates for possible inclusion in the result
|
||||
*/
|
||||
def fetchQueryFeaturesPhase2: Seq[BaseQueryFeatureHydrator[Query, _]] = Seq.empty
|
||||
|
||||
/**
|
||||
* What candidate pipelines should this Recommendations Pipeline get candidate from?
|
||||
*/
|
||||
def candidatePipelines: Seq[CandidatePipelineConfig[Query, _, _, _]]
|
||||
|
||||
/**
|
||||
* Dependent candidate pipelines to retrieve candidates that depend on the result of [[candidatePipelines]]
|
||||
* [[DependentCandidatePipelineConfig]] have access to the list of previously retrieved & decorated
|
||||
* candidates for use in constructing the query object.
|
||||
*/
|
||||
def dependentCandidatePipelines: Seq[DependentCandidatePipelineConfig[Query, _, _, _]] = Seq.empty
|
||||
|
||||
/**
|
||||
* Takes final ranked list of candidates & apply any business logic (e.g, deduplicating and merging
|
||||
* candidates before scoring).
|
||||
*/
|
||||
def postCandidatePipelinesSelectors: Seq[Selector[Query]] = Seq(InsertAppendResults(AllPipelines))
|
||||
|
||||
/**
|
||||
* After selectors are run, you can fetch features for each candidate.
|
||||
* The existing features from previous hydrations are passed in as inputs. You are not expected to
|
||||
* put them into the resulting feature map yourself - they will be merged for you by the platform.
|
||||
*/
|
||||
def postCandidatePipelinesFeatureHydration: Seq[
|
||||
BaseCandidateFeatureHydrator[Query, Candidate, _]
|
||||
] =
|
||||
Seq.empty
|
||||
|
||||
/**
|
||||
* Global filters to run on all candidates.
|
||||
*/
|
||||
def globalFilters: Seq[Filter[Query, Candidate]] = Seq.empty
|
||||
|
||||
/**
|
||||
* By default, a Recommendation Pipeline will fail closed - if any candidate or scoring
|
||||
* pipeline fails to return a result, then the Recommendation Pipeline will not return a result.
|
||||
* You can adjust this default policy, or provide specific policies to specific pipelines.
|
||||
* Those specific policies will take priority.
|
||||
*
|
||||
* FailOpenPolicy.All will always fail open (the RecommendationPipeline will continue without that pipeline)
|
||||
* FailOpenPolicy.Never will always fail closed (the RecommendationPipeline will fail if that pipeline fails)
|
||||
*
|
||||
* There's a default policy, and a specific Map of policies that takes precedence.
|
||||
*/
|
||||
def defaultFailOpenPolicy: FailOpenPolicy = FailOpenPolicy(Set(ClosedGate))
|
||||
def candidatePipelineFailOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] =
|
||||
Map.empty
|
||||
def scoringPipelineFailOpenPolicies: Map[ScoringPipelineIdentifier, FailOpenPolicy] = Map.empty
|
||||
|
||||
/**
|
||||
** [[qualityFactorConfigs]] associates [[QualityFactorConfig]]s to specific candidate pipelines
|
||||
* using [[ComponentIdentifier]].
|
||||
*/
|
||||
def qualityFactorConfigs: Map[ComponentIdentifier, QualityFactorConfig] =
|
||||
Map.empty
|
||||
|
||||
/**
|
||||
* Scoring pipelines for scoring candidates.
|
||||
* @note These do not drop or re-order candidates, you should do those in the sub-sequent selectors
|
||||
* step based off of the scores on candidates set in those [[ScoringPipeline]]s.
|
||||
*/
|
||||
def scoringPipelines: Seq[ScoringPipelineConfig[Query, Candidate]]
|
||||
|
||||
/**
|
||||
* Takes final ranked list of candidates & apply any business logic (e.g, capping number
|
||||
* of ad accounts or pacing ad accounts).
|
||||
*/
|
||||
def resultSelectors: Seq[Selector[Query]]
|
||||
|
||||
/**
|
||||
* Takes the final selected list of candidates and applies a final list of filters.
|
||||
* Useful for doing very expensive filtering at the end of your pipeline.
|
||||
*/
|
||||
def postSelectionFilters: Seq[Filter[Query, Candidate]] = Seq.empty
|
||||
|
||||
/**
|
||||
* Decorators allow for adding Presentations to candidates. While the Presentation can contain any
|
||||
* arbitrary data, Decorators are often used to add a UrtItemPresentation for URT item support. Most
|
||||
* customers will prefer to set a decorator in their respective candidate pipeline, however, a final
|
||||
* global one is available for those that do global decoration as late possible to avoid unnecessary hydrations.
|
||||
* @note This decorator can only return an ItemPresentation.
|
||||
* @note This decorator cannot decorate an already decorated candidate from the prior decorator
|
||||
* step in candidate pipelines.
|
||||
*/
|
||||
def decorator: Option[CandidateDecorator[Query, Candidate]] = None
|
||||
|
||||
/**
|
||||
* Domain marshaller transforms the selections into the model expected by the marshaller
|
||||
*/
|
||||
def domainMarshaller: DomainMarshaller[Query, UnmarshalledResultType]
|
||||
|
||||
/**
|
||||
* Mixer result side effects that are executed after selection and domain marshalling
|
||||
*/
|
||||
def resultSideEffects: Seq[PipelineResultSideEffect[Query, UnmarshalledResultType]] = Seq()
|
||||
|
||||
/**
|
||||
* Transport marshaller transforms the model into our line-level API like URT or JSON
|
||||
*/
|
||||
def transportMarshaller: TransportMarshaller[UnmarshalledResultType, Result]
|
||||
|
||||
/**
|
||||
* A pipeline can define a partial function to rescue failures here. They will be treated as failures
|
||||
* from a monitoring standpoint, and cancellation exceptions will always be propagated (they cannot be caught here).
|
||||
*/
|
||||
def failureClassifier: PartialFunction[Throwable, PipelineFailure] = PartialFunction.empty
|
||||
|
||||
/**
|
||||
* Alerts can be used to indicate the pipeline's service level objectives. Alerts and
|
||||
* dashboards will be automatically created based on this information.
|
||||
*/
|
||||
val alerts: Seq[Alert] = Seq.empty
|
||||
|
||||
/**
|
||||
* This method is used by the product mixer framework to build the pipeline.
|
||||
*/
|
||||
private[core] final def build(
|
||||
parentComponentIdentifierStack: ComponentIdentifierStack,
|
||||
builder: RecommendationPipelineBuilderFactory
|
||||
): RecommendationPipeline[Query, Candidate, Result] =
|
||||
builder.get.build(parentComponentIdentifierStack, this)
|
||||
}
|
||||
|
||||
object RecommendationPipelineConfig extends PipelineConfigCompanion {
|
||||
val qualityFactorStep: PipelineStepIdentifier = PipelineStepIdentifier("QualityFactor")
|
||||
val gatesStep: PipelineStepIdentifier = PipelineStepIdentifier("Gates")
|
||||
val fetchQueryFeaturesStep: PipelineStepIdentifier = PipelineStepIdentifier("FetchQueryFeatures")
|
||||
val fetchQueryFeaturesPhase2Step: PipelineStepIdentifier = PipelineStepIdentifier(
|
||||
"FetchQueryFeaturesPhase2")
|
||||
val candidatePipelinesStep: PipelineStepIdentifier = PipelineStepIdentifier("CandidatePipelines")
|
||||
val dependentCandidatePipelinesStep: PipelineStepIdentifier =
|
||||
PipelineStepIdentifier("DependentCandidatePipelines")
|
||||
val postCandidatePipelinesSelectorsStep: PipelineStepIdentifier =
|
||||
PipelineStepIdentifier("PostCandidatePipelinesSelectors")
|
||||
val postCandidatePipelinesFeatureHydrationStep: PipelineStepIdentifier =
|
||||
PipelineStepIdentifier("PostCandidatePipelinesFeatureHydration")
|
||||
val globalFiltersStep: PipelineStepIdentifier = PipelineStepIdentifier("GlobalFilters")
|
||||
val scoringPipelinesStep: PipelineStepIdentifier = PipelineStepIdentifier("ScoringPipelines")
|
||||
val resultSelectorsStep: PipelineStepIdentifier = PipelineStepIdentifier("ResultSelectors")
|
||||
val postSelectionFiltersStep: PipelineStepIdentifier = PipelineStepIdentifier(
|
||||
"PostSelectionFilters")
|
||||
val decoratorStep: PipelineStepIdentifier = PipelineStepIdentifier("Decorator")
|
||||
val domainMarshallerStep: PipelineStepIdentifier = PipelineStepIdentifier("DomainMarshaller")
|
||||
val resultSideEffectsStep: PipelineStepIdentifier = PipelineStepIdentifier("ResultSideEffects")
|
||||
val transportMarshallerStep: PipelineStepIdentifier = PipelineStepIdentifier(
|
||||
"TransportMarshaller")
|
||||
|
||||
/** All the Steps which are executed by a [[RecommendationPipeline]] in the order in which they are run */
|
||||
override val stepsInOrder: Seq[PipelineStepIdentifier] = Seq(
|
||||
qualityFactorStep,
|
||||
gatesStep,
|
||||
fetchQueryFeaturesStep,
|
||||
fetchQueryFeaturesPhase2Step,
|
||||
asyncFeaturesStep(candidatePipelinesStep),
|
||||
candidatePipelinesStep,
|
||||
asyncFeaturesStep(dependentCandidatePipelinesStep),
|
||||
dependentCandidatePipelinesStep,
|
||||
asyncFeaturesStep(postCandidatePipelinesSelectorsStep),
|
||||
postCandidatePipelinesSelectorsStep,
|
||||
asyncFeaturesStep(postCandidatePipelinesFeatureHydrationStep),
|
||||
postCandidatePipelinesFeatureHydrationStep,
|
||||
asyncFeaturesStep(globalFiltersStep),
|
||||
globalFiltersStep,
|
||||
asyncFeaturesStep(scoringPipelinesStep),
|
||||
scoringPipelinesStep,
|
||||
asyncFeaturesStep(resultSelectorsStep),
|
||||
resultSelectorsStep,
|
||||
asyncFeaturesStep(postSelectionFiltersStep),
|
||||
postSelectionFiltersStep,
|
||||
asyncFeaturesStep(decoratorStep),
|
||||
decoratorStep,
|
||||
domainMarshallerStep,
|
||||
asyncFeaturesStep(resultSideEffectsStep),
|
||||
resultSideEffectsStep,
|
||||
transportMarshallerStep
|
||||
)
|
||||
|
||||
/**
|
||||
* All the Steps which an [[com.twitter.product_mixer.core.functional_component.feature_hydrator.AsyncHydrator AsyncHydrator]]
|
||||
* can be configured to [[com.twitter.product_mixer.core.functional_component.feature_hydrator.AsyncHydrator.hydrateBefore hydrateBefore]]
|
||||
*/
|
||||
override val stepsAsyncFeatureHydrationCanBeCompletedBy: Set[PipelineStepIdentifier] = Set(
|
||||
candidatePipelinesStep,
|
||||
dependentCandidatePipelinesStep,
|
||||
postCandidatePipelinesSelectorsStep,
|
||||
postCandidatePipelinesFeatureHydrationStep,
|
||||
globalFiltersStep,
|
||||
scoringPipelinesStep,
|
||||
resultSelectorsStep,
|
||||
postSelectionFiltersStep,
|
||||
decoratorStep,
|
||||
resultSideEffectsStep,
|
||||
)
|
||||
}
|
||||
Binary file not shown.
@ -1,84 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.recommendation
|
||||
|
||||
import com.twitter.product_mixer.core.feature.featuremap.asyncfeaturemap.AsyncFeatureMap
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.model.marshalling.HasMarshalling
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineResult
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure
|
||||
import com.twitter.product_mixer.core.service.async_feature_map_executor.AsyncFeatureMapExecutorResults
|
||||
import com.twitter.product_mixer.core.service.candidate_decorator_executor.CandidateDecoratorExecutorResult
|
||||
import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutorResult
|
||||
import com.twitter.product_mixer.core.service.candidate_pipeline_executor.CandidatePipelineExecutorResult
|
||||
import com.twitter.product_mixer.core.service.domain_marshaller_executor.DomainMarshallerExecutor
|
||||
import com.twitter.product_mixer.core.service.filter_executor.FilterExecutorResult
|
||||
import com.twitter.product_mixer.core.service.gate_executor.GateExecutorResult
|
||||
import com.twitter.product_mixer.core.service.pipeline_result_side_effect_executor.PipelineResultSideEffectExecutor
|
||||
import com.twitter.product_mixer.core.service.quality_factor_executor.QualityFactorExecutorResult
|
||||
import com.twitter.product_mixer.core.service.query_feature_hydrator_executor.QueryFeatureHydratorExecutor
|
||||
import com.twitter.product_mixer.core.service.scoring_pipeline_executor.ScoringPipelineExecutorResult
|
||||
import com.twitter.product_mixer.core.service.selector_executor.SelectorExecutorResult
|
||||
import com.twitter.product_mixer.core.service.transport_marshaller_executor.TransportMarshallerExecutor
|
||||
|
||||
case class RecommendationPipelineResult[Candidate <: UniversalNoun[Any], ResultType](
|
||||
qualityFactorResult: Option[QualityFactorExecutorResult],
|
||||
gateResult: Option[GateExecutorResult],
|
||||
queryFeatures: Option[QueryFeatureHydratorExecutor.Result],
|
||||
queryFeaturesPhase2: Option[QueryFeatureHydratorExecutor.Result],
|
||||
mergedAsyncQueryFeatures: Option[AsyncFeatureMap],
|
||||
candidatePipelineResults: Option[CandidatePipelineExecutorResult],
|
||||
dependentCandidatePipelineResults: Option[CandidatePipelineExecutorResult],
|
||||
postCandidatePipelinesSelectorResults: Option[SelectorExecutorResult],
|
||||
postCandidatePipelinesFeatureHydrationResults: Option[
|
||||
CandidateFeatureHydratorExecutorResult[Candidate]
|
||||
],
|
||||
globalFilterResults: Option[FilterExecutorResult[Candidate]],
|
||||
scoringPipelineResults: Option[ScoringPipelineExecutorResult[Candidate]],
|
||||
resultSelectorResults: Option[SelectorExecutorResult],
|
||||
postSelectionFilterResults: Option[FilterExecutorResult[Candidate]],
|
||||
candidateDecoratorResult: Option[CandidateDecoratorExecutorResult],
|
||||
domainMarshallerResults: Option[DomainMarshallerExecutor.Result[HasMarshalling]],
|
||||
resultSideEffectResults: Option[PipelineResultSideEffectExecutor.Result],
|
||||
asyncFeatureHydrationResults: Option[AsyncFeatureMapExecutorResults],
|
||||
transportMarshallerResults: Option[TransportMarshallerExecutor.Result[ResultType]],
|
||||
failure: Option[PipelineFailure],
|
||||
result: Option[ResultType])
|
||||
extends PipelineResult[ResultType] {
|
||||
override val resultSize: Int = result match {
|
||||
case Some(seqResult @ Seq(_)) => seqResult.length
|
||||
case Some(_) => 1
|
||||
case None => 0
|
||||
}
|
||||
|
||||
override def withFailure(
|
||||
failure: PipelineFailure
|
||||
): RecommendationPipelineResult[Candidate, ResultType] =
|
||||
copy(failure = Some(failure))
|
||||
override def withResult(result: ResultType): RecommendationPipelineResult[Candidate, ResultType] =
|
||||
copy(result = Some(result))
|
||||
}
|
||||
|
||||
object RecommendationPipelineResult {
|
||||
def empty[A <: UniversalNoun[Any], B]: RecommendationPipelineResult[A, B] =
|
||||
RecommendationPipelineResult(
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None
|
||||
)
|
||||
}
|
||||
@ -1,36 +0,0 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector:insert_append_results",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/candidate_feature_hydrator",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/gate",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/scorer",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/selector",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util",
|
||||
"stitch/stitch-core",
|
||||
"util/util-core:scala",
|
||||
],
|
||||
exports = [
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/candidate_feature_hydrator",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/gate",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/scorer",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/selector",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice",
|
||||
"stitch/stitch-core",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,202 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.scoring
|
||||
|
||||
import com.twitter.product_mixer.core.functional_component.common.alert.Alert
|
||||
import com.twitter.product_mixer.core.functional_component.decorator.Decoration
|
||||
import com.twitter.product_mixer.core.functional_component.scorer.ScoredCandidateResult
|
||||
import com.twitter.product_mixer.core.gate.ParamGate
|
||||
import com.twitter.product_mixer.core.gate.ParamGate._
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.Component
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifierStack
|
||||
import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
|
||||
import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
|
||||
import com.twitter.product_mixer.core.pipeline.NewPipelineBuilder
|
||||
import com.twitter.product_mixer.core.pipeline.NewPipelineArrowBuilder
|
||||
import com.twitter.product_mixer.core.pipeline.NewPipelineResult
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.ClosedGate
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailureClassifier
|
||||
import com.twitter.product_mixer.core.pipeline.state.HasCandidatesWithDetails
|
||||
import com.twitter.product_mixer.core.pipeline.state.HasCandidatesWithFeatures
|
||||
import com.twitter.product_mixer.core.pipeline.state.HasExecutorResults
|
||||
import com.twitter.product_mixer.core.pipeline.state.HasQuery
|
||||
import com.twitter.product_mixer.core.pipeline.state.HasResult
|
||||
import com.twitter.product_mixer.core.pipeline.step.candidate_feature_hydrator.CandidateFeatureHydratorStep
|
||||
import com.twitter.product_mixer.core.pipeline.step.gate.GateStep
|
||||
import com.twitter.product_mixer.core.pipeline.step.scorer.ScorerStep
|
||||
import com.twitter.product_mixer.core.pipeline.step.selector.SelectorStep
|
||||
import com.twitter.product_mixer.core.service.Executor
|
||||
import com.twitter.product_mixer.core.service.ExecutorResult
|
||||
import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutorResult
|
||||
import com.twitter.product_mixer.core.service.gate_executor.GateExecutorResult
|
||||
import com.twitter.product_mixer.core.service.gate_executor.StoppedGateException
|
||||
import com.twitter.product_mixer.core.service.selector_executor.SelectorExecutorResult
|
||||
import com.twitter.stitch.Arrow
|
||||
import javax.inject.Inject
|
||||
import scala.collection.immutable.ListMap
|
||||
|
||||
/**
|
||||
* NewScoringPipelineBuilder builds [[ScoringPipeline]]s from [[ScoringPipelineConfig]]s.
|
||||
* New because it's meant to eventually replace [[ScoringPipelineBuilder]]
|
||||
* You should inject a [[ScoringPipelineBuilderFactory]] and call `.get` to build these.
|
||||
*
|
||||
* @see [[ScoringPipelineConfig]] for the description of the type parameters
|
||||
* @tparam Query the type of query these accept.
|
||||
* @tparam Candidate the domain model for the candidate being scored
|
||||
*/
|
||||
class NewScoringPipelineBuilder[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]] @Inject() (
|
||||
selectionStep: SelectorStep[Query, ScoringPipelineState[Query, Candidate]],
|
||||
gateStep: GateStep[Query, ScoringPipelineState[Query, Candidate]],
|
||||
candidateFeatureHydrationStep: CandidateFeatureHydratorStep[
|
||||
Query,
|
||||
Candidate,
|
||||
ScoringPipelineState[Query, Candidate]
|
||||
],
|
||||
scorerStep: ScorerStep[Query, Candidate, ScoringPipelineState[Query, Candidate]])
|
||||
extends NewPipelineBuilder[ScoringPipelineConfig[Query, Candidate], Seq[
|
||||
CandidateWithFeatures[Candidate]
|
||||
], ScoringPipelineState[Query, Candidate], ScoringPipeline[Query, Candidate]] {
|
||||
|
||||
override def build(
|
||||
parentComponentIdentifierStack: ComponentIdentifierStack,
|
||||
arrowBuilder: NewPipelineArrowBuilder[ArrowResult, ArrowState],
|
||||
scoringPipelineConfig: ScoringPipelineConfig[Query, Candidate]
|
||||
): ScoringPipeline[Query, Candidate] = {
|
||||
val pipelineIdentifier = scoringPipelineConfig.identifier
|
||||
|
||||
val context = Executor.Context(
|
||||
PipelineFailureClassifier(
|
||||
scoringPipelineConfig.failureClassifier.orElse(
|
||||
StoppedGateException.classifier(ClosedGate))),
|
||||
parentComponentIdentifierStack.push(pipelineIdentifier)
|
||||
)
|
||||
|
||||
val enabledGateOpt = scoringPipelineConfig.enabledDeciderParam.map { deciderParam =>
|
||||
ParamGate(pipelineIdentifier + EnabledGateSuffix, deciderParam)
|
||||
}
|
||||
val supportedClientGateOpt = scoringPipelineConfig.supportedClientParam.map { param =>
|
||||
ParamGate(pipelineIdentifier + SupportedClientGateSuffix, param)
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluate enabled decider gate first since if it's off, there is no reason to proceed
|
||||
* Next evaluate supported client feature switch gate, followed by customer configured gates
|
||||
*/
|
||||
val allGates =
|
||||
enabledGateOpt.toSeq ++ supportedClientGateOpt.toSeq ++ scoringPipelineConfig.gates
|
||||
|
||||
val underlyingArrow = arrowBuilder
|
||||
.add(ScoringPipelineConfig.gatesStep, gateStep, allGates)
|
||||
.add(ScoringPipelineConfig.selectorsStep, selectionStep, scoringPipelineConfig.selectors)
|
||||
.add(
|
||||
ScoringPipelineConfig.preScoringFeatureHydrationPhase1Step,
|
||||
candidateFeatureHydrationStep,
|
||||
scoringPipelineConfig.preScoringFeatureHydrationPhase1)
|
||||
.add(
|
||||
ScoringPipelineConfig.preScoringFeatureHydrationPhase2Step,
|
||||
candidateFeatureHydrationStep,
|
||||
scoringPipelineConfig.preScoringFeatureHydrationPhase2)
|
||||
.add(ScoringPipelineConfig.scorersStep, scorerStep, scoringPipelineConfig.scorers).buildArrow(
|
||||
context)
|
||||
|
||||
val finalArrow = Arrow
|
||||
.map { inputs: ScoringPipeline.Inputs[Query] =>
|
||||
ScoringPipelineState[Query, Candidate](inputs.query, inputs.candidates, ListMap.empty)
|
||||
}.andThen(underlyingArrow).map { pipelineResult =>
|
||||
ScoringPipelineResult(
|
||||
gateResults = pipelineResult.executorResultsByPipelineStep
|
||||
.get(ScoringPipelineConfig.gatesStep)
|
||||
.map(_.asInstanceOf[GateExecutorResult]),
|
||||
selectorResults = pipelineResult.executorResultsByPipelineStep
|
||||
.get(ScoringPipelineConfig.selectorsStep)
|
||||
.map(_.asInstanceOf[SelectorExecutorResult]),
|
||||
preScoringHydrationPhase1Result = pipelineResult.executorResultsByPipelineStep
|
||||
.get(ScoringPipelineConfig.preScoringFeatureHydrationPhase1Step)
|
||||
.map(_.asInstanceOf[CandidateFeatureHydratorExecutorResult[Candidate]]),
|
||||
preScoringHydrationPhase2Result = pipelineResult.executorResultsByPipelineStep
|
||||
.get(ScoringPipelineConfig.preScoringFeatureHydrationPhase2Step)
|
||||
.map(_.asInstanceOf[CandidateFeatureHydratorExecutorResult[Candidate]]),
|
||||
scorerResults = pipelineResult.executorResultsByPipelineStep
|
||||
.get(ScoringPipelineConfig.scorersStep)
|
||||
.map(_.asInstanceOf[CandidateFeatureHydratorExecutorResult[Candidate]]),
|
||||
failure = pipelineResult match {
|
||||
case failure: NewPipelineResult.Failure =>
|
||||
Some(failure.failure)
|
||||
case _ => None
|
||||
},
|
||||
result = pipelineResult match {
|
||||
case result: NewPipelineResult.Success[Seq[CandidateWithFeatures[Candidate]]] =>
|
||||
Some(result.result.map { candidateWithFeatures =>
|
||||
ScoredCandidateResult(
|
||||
candidateWithFeatures.candidate,
|
||||
candidateWithFeatures.features)
|
||||
})
|
||||
case _ => None
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
new ScoringPipeline[Query, Candidate] {
|
||||
override val arrow: Arrow[ScoringPipeline.Inputs[Query], ScoringPipelineResult[Candidate]] =
|
||||
finalArrow
|
||||
|
||||
override val identifier: ScoringPipelineIdentifier = scoringPipelineConfig.identifier
|
||||
|
||||
override val alerts: Seq[Alert] = scoringPipelineConfig.alerts
|
||||
|
||||
override val children: Seq[Component] =
|
||||
allGates ++ scoringPipelineConfig.preScoringFeatureHydrationPhase1 ++ scoringPipelineConfig.preScoringFeatureHydrationPhase2 ++ scoringPipelineConfig.scorers
|
||||
|
||||
override private[core] val config = scoringPipelineConfig
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
case class ScoringPipelineState[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]](
|
||||
override val query: Query,
|
||||
candidates: Seq[ItemCandidateWithDetails],
|
||||
override val executorResultsByPipelineStep: ListMap[PipelineStepIdentifier, ExecutorResult])
|
||||
extends HasQuery[Query, ScoringPipelineState[Query, Candidate]]
|
||||
with HasCandidatesWithDetails[ScoringPipelineState[Query, Candidate]]
|
||||
with HasCandidatesWithFeatures[Candidate, ScoringPipelineState[Query, Candidate]]
|
||||
with HasExecutorResults[ScoringPipelineState[Query, Candidate]]
|
||||
with HasResult[Seq[CandidateWithFeatures[Candidate]]] {
|
||||
|
||||
override val candidatesWithDetails: Seq[CandidateWithDetails] = candidates
|
||||
|
||||
override val candidatesWithFeatures: Seq[CandidateWithFeatures[Candidate]] =
|
||||
candidates.asInstanceOf[Seq[CandidateWithFeatures[Candidate]]]
|
||||
|
||||
override val buildResult: Seq[CandidateWithFeatures[Candidate]] = candidatesWithFeatures
|
||||
|
||||
override def updateCandidatesWithDetails(
|
||||
newCandidates: Seq[CandidateWithDetails]
|
||||
): ScoringPipelineState[Query, Candidate] = {
|
||||
this.copy(candidates = newCandidates.asInstanceOf[Seq[ItemCandidateWithDetails]])
|
||||
}
|
||||
|
||||
override def updateQuery(newQuery: Query): ScoringPipelineState[Query, Candidate] =
|
||||
this.copy(query = newQuery)
|
||||
|
||||
override def updateDecorations(
|
||||
decoration: Seq[Decoration]
|
||||
): ScoringPipelineState[Query, Candidate] = ???
|
||||
|
||||
override def updateCandidatesWithFeatures(
|
||||
newCandidates: Seq[CandidateWithFeatures[Candidate]]
|
||||
): ScoringPipelineState[Query, Candidate] = {
|
||||
val updatedCandidates = candidates.zip(newCandidates).map {
|
||||
case (itemCandidateWithDetails, newCandidate) =>
|
||||
itemCandidateWithDetails.copy(features =
|
||||
itemCandidateWithDetails.features ++ newCandidate.features)
|
||||
}
|
||||
this.copy(query, updatedCandidates)
|
||||
}
|
||||
|
||||
override private[pipeline] def setExecutorResults(
|
||||
newMap: ListMap[PipelineStepIdentifier, ExecutorResult]
|
||||
) = this.copy(executorResultsByPipelineStep = newMap)
|
||||
}
|
||||
Binary file not shown.
@ -1,32 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.scoring
|
||||
|
||||
import com.twitter.product_mixer.core.functional_component.scorer.ScoredCandidateResult
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
|
||||
import com.twitter.product_mixer.core.pipeline.Pipeline
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Arrow
|
||||
|
||||
/**
|
||||
* A Scoring Pipeline
|
||||
*
|
||||
* This is an abstract class, as we only construct these via the [[ScoringPipelineBuilder]].
|
||||
*
|
||||
* A [[ScoringPipeline]] is capable of pre-filtering candidates for scoring, performing the scoring
|
||||
* then running selection heuristics (ranking, dropping, etc) based off of the score.
|
||||
* @tparam Query the domain model for the query or request
|
||||
* @tparam Candidate the domain model for the candidate being scored
|
||||
*/
|
||||
abstract class ScoringPipeline[-Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]
|
||||
extends Pipeline[ScoringPipeline.Inputs[Query], Seq[ScoredCandidateResult[Candidate]]] {
|
||||
override private[core] val config: ScoringPipelineConfig[Query, Candidate]
|
||||
override val arrow: Arrow[ScoringPipeline.Inputs[Query], ScoringPipelineResult[Candidate]]
|
||||
override val identifier: ScoringPipelineIdentifier
|
||||
}
|
||||
|
||||
object ScoringPipeline {
|
||||
case class Inputs[+Query <: PipelineQuery](
|
||||
query: Query,
|
||||
candidates: Seq[ItemCandidateWithDetails])
|
||||
}
|
||||
Binary file not shown.
@ -1,367 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.scoring
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.functional_component.common.alert.Alert
|
||||
import com.twitter.product_mixer.core.functional_component.scorer.ScoredCandidateResult
|
||||
import com.twitter.product_mixer.core.gate.ParamGate
|
||||
import com.twitter.product_mixer.core.gate.ParamGate.EnabledGateSuffix
|
||||
import com.twitter.product_mixer.core.gate.ParamGate.SupportedClientGateSuffix
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.Component
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifierStack
|
||||
import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
|
||||
import com.twitter.product_mixer.core.pipeline.InvalidStepStateException
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineBuilder
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.ClosedGate
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailureClassifier
|
||||
import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipeline.Inputs
|
||||
import com.twitter.product_mixer.core.service.Executor
|
||||
import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutor
|
||||
import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutorResult
|
||||
import com.twitter.product_mixer.core.service.gate_executor.GateExecutor
|
||||
import com.twitter.product_mixer.core.service.gate_executor.GateExecutorResult
|
||||
import com.twitter.product_mixer.core.service.gate_executor.StoppedGateException
|
||||
import com.twitter.product_mixer.core.service.selector_executor.SelectorExecutor
|
||||
import com.twitter.product_mixer.core.service.selector_executor.SelectorExecutorResult
|
||||
import com.twitter.stitch.Arrow
|
||||
import javax.inject.Inject
|
||||
|
||||
/**
|
||||
* ScoringPipelineBuilder builds [[ScoringPipeline]]s from [[ScoringPipelineConfig]]s.
|
||||
*
|
||||
* You should inject a [[ScoringPipelineBuilderFactory]] and call `.get` to build these.
|
||||
*
|
||||
* @see [[ScoringPipelineConfig]] for the description of the type parameters
|
||||
* @tparam Query the type of query these accept.
|
||||
* @tparam Candidate the domain model for the candidate being scored
|
||||
*/
|
||||
class ScoringPipelineBuilder[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]] @Inject() (
|
||||
gateExecutor: GateExecutor,
|
||||
selectorExecutor: SelectorExecutor,
|
||||
candidateFeatureHydratorExecutor: CandidateFeatureHydratorExecutor,
|
||||
override val statsReceiver: StatsReceiver)
|
||||
extends PipelineBuilder[Inputs[Query]] {
|
||||
|
||||
override type UnderlyingResultType = Seq[ScoredCandidateResult[Candidate]]
|
||||
override type PipelineResultType = ScoringPipelineResult[Candidate]
|
||||
|
||||
def build(
|
||||
parentComponentIdentifierStack: ComponentIdentifierStack,
|
||||
config: ScoringPipelineConfig[Query, Candidate]
|
||||
): ScoringPipeline[Query, Candidate] = {
|
||||
|
||||
val pipelineIdentifier = config.identifier
|
||||
|
||||
val context = Executor.Context(
|
||||
PipelineFailureClassifier(
|
||||
config.failureClassifier.orElse(StoppedGateException.classifier(ClosedGate))),
|
||||
parentComponentIdentifierStack.push(pipelineIdentifier)
|
||||
)
|
||||
|
||||
val enabledGateOpt = config.enabledDeciderParam.map { deciderParam =>
|
||||
ParamGate(pipelineIdentifier + EnabledGateSuffix, deciderParam)
|
||||
}
|
||||
val supportedClientGateOpt = config.supportedClientParam.map { param =>
|
||||
ParamGate(pipelineIdentifier + SupportedClientGateSuffix, param)
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluate enabled decider gate first since if it's off, there is no reason to proceed
|
||||
* Next evaluate supported client feature switch gate, followed by customer configured gates
|
||||
*/
|
||||
val allGates = enabledGateOpt.toSeq ++ supportedClientGateOpt.toSeq ++ config.gates
|
||||
|
||||
val GatesStep = new Step[Query, GateExecutorResult] {
|
||||
override def identifier: PipelineStepIdentifier = ScoringPipelineConfig.gatesStep
|
||||
|
||||
override lazy val executorArrow: Arrow[Query, GateExecutorResult] =
|
||||
gateExecutor.arrow(allGates, context)
|
||||
|
||||
override def inputAdaptor(
|
||||
query: ScoringPipeline.Inputs[Query],
|
||||
previousResult: ScoringPipelineResult[Candidate]
|
||||
): Query = {
|
||||
query.query
|
||||
}
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: ScoringPipelineResult[Candidate],
|
||||
executorResult: GateExecutorResult
|
||||
): ScoringPipelineResult[Candidate] =
|
||||
previousPipelineResult.copy(gateResults = Some(executorResult))
|
||||
}
|
||||
|
||||
val SelectorsStep = new Step[SelectorExecutor.Inputs[Query], SelectorExecutorResult] {
|
||||
override def identifier: PipelineStepIdentifier = ScoringPipelineConfig.selectorsStep
|
||||
|
||||
override def executorArrow: Arrow[SelectorExecutor.Inputs[Query], SelectorExecutorResult] =
|
||||
selectorExecutor.arrow(config.selectors, context)
|
||||
|
||||
override def inputAdaptor(
|
||||
query: ScoringPipeline.Inputs[Query],
|
||||
previousResult: ScoringPipelineResult[Candidate]
|
||||
): SelectorExecutor.Inputs[Query] = SelectorExecutor.Inputs(query.query, query.candidates)
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: ScoringPipelineResult[Candidate],
|
||||
executorResult: SelectorExecutorResult
|
||||
): ScoringPipelineResult[Candidate] =
|
||||
previousPipelineResult.copy(selectorResults = Some(executorResult))
|
||||
}
|
||||
|
||||
val PreScoringFeatureHydrationPhase1Step =
|
||||
new Step[
|
||||
CandidateFeatureHydratorExecutor.Inputs[Query, Candidate],
|
||||
CandidateFeatureHydratorExecutorResult[Candidate]
|
||||
] {
|
||||
override def identifier: PipelineStepIdentifier =
|
||||
ScoringPipelineConfig.preScoringFeatureHydrationPhase1Step
|
||||
|
||||
override def executorArrow: Arrow[
|
||||
CandidateFeatureHydratorExecutor.Inputs[Query, Candidate],
|
||||
CandidateFeatureHydratorExecutorResult[Candidate]
|
||||
] =
|
||||
candidateFeatureHydratorExecutor.arrow(config.preScoringFeatureHydrationPhase1, context)
|
||||
|
||||
override def inputAdaptor(
|
||||
query: ScoringPipeline.Inputs[Query],
|
||||
previousResult: ScoringPipelineResult[Candidate]
|
||||
): CandidateFeatureHydratorExecutor.Inputs[Query, Candidate] = {
|
||||
val selectedCandidatesResult = previousResult.selectorResults.getOrElse {
|
||||
throw InvalidStepStateException(identifier, "SelectorResults")
|
||||
}.selectedCandidates
|
||||
|
||||
CandidateFeatureHydratorExecutor.Inputs(
|
||||
query.query,
|
||||
selectedCandidatesResult.asInstanceOf[Seq[CandidateWithFeatures[Candidate]]])
|
||||
}
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: ScoringPipelineResult[Candidate],
|
||||
executorResult: CandidateFeatureHydratorExecutorResult[Candidate]
|
||||
): ScoringPipelineResult[Candidate] = previousPipelineResult.copy(
|
||||
preScoringHydrationPhase1Result = Some(executorResult)
|
||||
)
|
||||
}
|
||||
|
||||
val PreScoringFeatureHydrationPhase2Step =
|
||||
new Step[
|
||||
CandidateFeatureHydratorExecutor.Inputs[Query, Candidate],
|
||||
CandidateFeatureHydratorExecutorResult[Candidate]
|
||||
] {
|
||||
override def identifier: PipelineStepIdentifier =
|
||||
ScoringPipelineConfig.preScoringFeatureHydrationPhase2Step
|
||||
|
||||
override def executorArrow: Arrow[
|
||||
CandidateFeatureHydratorExecutor.Inputs[Query, Candidate],
|
||||
CandidateFeatureHydratorExecutorResult[Candidate]
|
||||
] =
|
||||
candidateFeatureHydratorExecutor.arrow(config.preScoringFeatureHydrationPhase2, context)
|
||||
|
||||
override def inputAdaptor(
|
||||
query: ScoringPipeline.Inputs[Query],
|
||||
previousResult: ScoringPipelineResult[Candidate]
|
||||
): CandidateFeatureHydratorExecutor.Inputs[Query, Candidate] = {
|
||||
val preScoringHydrationPhase1FeatureMaps: Seq[FeatureMap] =
|
||||
previousResult.preScoringHydrationPhase1Result
|
||||
.getOrElse(
|
||||
throw InvalidStepStateException(identifier, "PreScoringHydrationPhase1Result"))
|
||||
.results.map(_.features)
|
||||
|
||||
val itemCandidates = previousResult.selectorResults
|
||||
.getOrElse(throw InvalidStepStateException(identifier, "SelectionResults"))
|
||||
.selectedCandidates.collect {
|
||||
case itemCandidate: ItemCandidateWithDetails => itemCandidate
|
||||
}
|
||||
// If there is no feature hydration (empty results), no need to attempt merging.
|
||||
val candidates = if (preScoringHydrationPhase1FeatureMaps.isEmpty) {
|
||||
itemCandidates
|
||||
} else {
|
||||
itemCandidates.zip(preScoringHydrationPhase1FeatureMaps).map {
|
||||
case (itemCandidate, featureMap) =>
|
||||
itemCandidate.copy(features = itemCandidate.features ++ featureMap)
|
||||
}
|
||||
}
|
||||
|
||||
CandidateFeatureHydratorExecutor.Inputs(
|
||||
query.query,
|
||||
candidates.asInstanceOf[Seq[CandidateWithFeatures[Candidate]]])
|
||||
}
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: ScoringPipelineResult[Candidate],
|
||||
executorResult: CandidateFeatureHydratorExecutorResult[Candidate]
|
||||
): ScoringPipelineResult[Candidate] = previousPipelineResult.copy(
|
||||
preScoringHydrationPhase2Result = Some(executorResult)
|
||||
)
|
||||
}
|
||||
|
||||
def getMergedPreScoringFeatureMap(
|
||||
stepIdentifier: PipelineStepIdentifier,
|
||||
previousResult: ScoringPipelineResult[Candidate]
|
||||
): Seq[FeatureMap] = {
|
||||
val preScoringHydrationPhase1FeatureMaps: Seq[FeatureMap] =
|
||||
previousResult.preScoringHydrationPhase1Result
|
||||
.getOrElse(
|
||||
throw InvalidStepStateException(
|
||||
stepIdentifier,
|
||||
"PreScoringHydrationPhase1Result")).results.map(_.features)
|
||||
|
||||
val preScoringHydrationPhase2FeatureMaps: Seq[FeatureMap] =
|
||||
previousResult.preScoringHydrationPhase2Result
|
||||
.getOrElse(
|
||||
throw InvalidStepStateException(
|
||||
stepIdentifier,
|
||||
"PreScoringHydrationPhase2Result")).results.map(_.features)
|
||||
/*
|
||||
* If either pre-scoring hydration phase feature map is empty, no need to merge them,
|
||||
* we can just take all non-empty ones.
|
||||
*/
|
||||
if (preScoringHydrationPhase1FeatureMaps.isEmpty) {
|
||||
preScoringHydrationPhase2FeatureMaps
|
||||
} else if (preScoringHydrationPhase2FeatureMaps.isEmpty) {
|
||||
preScoringHydrationPhase1FeatureMaps
|
||||
} else {
|
||||
// No need to check the size in both, since the inputs to both hydration phases are the
|
||||
// same and each phase ensures the number of candidates and ordering matches the input.
|
||||
preScoringHydrationPhase1FeatureMaps.zip(preScoringHydrationPhase2FeatureMaps).map {
|
||||
case (preScoringHydrationPhase1FeatureMap, preScoringHydrationPhasesFeatureMap) =>
|
||||
preScoringHydrationPhase1FeatureMap ++ preScoringHydrationPhasesFeatureMap
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val ScorersStep =
|
||||
new Step[
|
||||
CandidateFeatureHydratorExecutor.Inputs[Query, Candidate],
|
||||
CandidateFeatureHydratorExecutorResult[Candidate]
|
||||
] {
|
||||
override def identifier: PipelineStepIdentifier = ScoringPipelineConfig.scorersStep
|
||||
|
||||
override def inputAdaptor(
|
||||
query: ScoringPipeline.Inputs[Query],
|
||||
previousResult: ScoringPipelineResult[Candidate]
|
||||
): CandidateFeatureHydratorExecutor.Inputs[Query, Candidate] = {
|
||||
|
||||
val mergedPreScoringFeatureHydrationFeatures: Seq[FeatureMap] =
|
||||
getMergedPreScoringFeatureMap(ScoringPipelineConfig.scorersStep, previousResult)
|
||||
|
||||
val itemCandidates = previousResult.selectorResults
|
||||
.getOrElse(throw InvalidStepStateException(identifier, "SelectionResults"))
|
||||
.selectedCandidates.collect {
|
||||
case itemCandidate: ItemCandidateWithDetails => itemCandidate
|
||||
}
|
||||
|
||||
// If there was no pre-scoring features hydration, no need to re-merge feature maps
|
||||
// and construct a new item candidate
|
||||
val updatedCandidates = if (mergedPreScoringFeatureHydrationFeatures.isEmpty) {
|
||||
itemCandidates
|
||||
} else {
|
||||
itemCandidates.zip(mergedPreScoringFeatureHydrationFeatures).map {
|
||||
case (itemCandidate, preScoringFeatureMap) =>
|
||||
itemCandidate.copy(features = itemCandidate.features ++ preScoringFeatureMap)
|
||||
}
|
||||
}
|
||||
CandidateFeatureHydratorExecutor.Inputs(
|
||||
query.query,
|
||||
updatedCandidates.asInstanceOf[Seq[CandidateWithFeatures[Candidate]]])
|
||||
}
|
||||
|
||||
override lazy val executorArrow: Arrow[
|
||||
CandidateFeatureHydratorExecutor.Inputs[Query, Candidate],
|
||||
CandidateFeatureHydratorExecutorResult[
|
||||
Candidate
|
||||
]
|
||||
] = candidateFeatureHydratorExecutor.arrow(config.scorers.toSeq, context)
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: ScoringPipelineResult[Candidate],
|
||||
executorResult: CandidateFeatureHydratorExecutorResult[Candidate]
|
||||
): ScoringPipelineResult[Candidate] =
|
||||
previousPipelineResult.copy(scorerResults = Some(executorResult))
|
||||
}
|
||||
|
||||
val ResultStep =
|
||||
new Step[Seq[CandidateWithFeatures[UniversalNoun[Any]]], Seq[
|
||||
CandidateWithFeatures[UniversalNoun[Any]]
|
||||
]] {
|
||||
override def identifier: PipelineStepIdentifier = ScoringPipelineConfig.resultStep
|
||||
|
||||
override def executorArrow: Arrow[Seq[CandidateWithFeatures[UniversalNoun[Any]]], Seq[
|
||||
CandidateWithFeatures[UniversalNoun[Any]]
|
||||
]] = Arrow.identity
|
||||
|
||||
override def inputAdaptor(
|
||||
query: Inputs[Query],
|
||||
previousResult: ScoringPipelineResult[Candidate]
|
||||
): Seq[CandidateWithFeatures[UniversalNoun[Any]]] = previousResult.selectorResults
|
||||
.getOrElse(throw InvalidStepStateException(identifier, "SelectionResults"))
|
||||
.selectedCandidates.collect {
|
||||
case itemCandidate: ItemCandidateWithDetails => itemCandidate
|
||||
}
|
||||
|
||||
override def resultUpdater(
|
||||
previousPipelineResult: ScoringPipelineResult[Candidate],
|
||||
executorResult: Seq[CandidateWithFeatures[UniversalNoun[Any]]]
|
||||
): ScoringPipelineResult[Candidate] = {
|
||||
val scorerResults: Seq[FeatureMap] = previousPipelineResult.scorerResults
|
||||
.getOrElse(throw InvalidStepStateException(identifier, "ScorerResult")).results.map(
|
||||
_.features)
|
||||
|
||||
val mergedPreScoringFeatureHydrationFeatureMaps: Seq[FeatureMap] =
|
||||
getMergedPreScoringFeatureMap(ScoringPipelineConfig.resultStep, previousPipelineResult)
|
||||
|
||||
val itemCandidates = executorResult.asInstanceOf[Seq[ItemCandidateWithDetails]]
|
||||
val finalFeatureMap = if (mergedPreScoringFeatureHydrationFeatureMaps.isEmpty) {
|
||||
scorerResults
|
||||
} else {
|
||||
scorerResults
|
||||
.zip(mergedPreScoringFeatureHydrationFeatureMaps).map {
|
||||
case (preScoringFeatureMap, scoringFeatureMap) =>
|
||||
preScoringFeatureMap ++ scoringFeatureMap
|
||||
}
|
||||
}
|
||||
|
||||
val finalResults = itemCandidates.zip(finalFeatureMap).map {
|
||||
case (itemCandidate, featureMap) =>
|
||||
ScoredCandidateResult(itemCandidate.candidate.asInstanceOf[Candidate], featureMap)
|
||||
}
|
||||
previousPipelineResult.withResult(finalResults)
|
||||
}
|
||||
}
|
||||
|
||||
val builtSteps = Seq(
|
||||
GatesStep,
|
||||
SelectorsStep,
|
||||
PreScoringFeatureHydrationPhase1Step,
|
||||
PreScoringFeatureHydrationPhase2Step,
|
||||
ScorersStep,
|
||||
ResultStep
|
||||
)
|
||||
|
||||
/** The main execution logic for this Candidate Pipeline. */
|
||||
val finalArrow: Arrow[ScoringPipeline.Inputs[Query], ScoringPipelineResult[Candidate]] =
|
||||
buildCombinedArrowFromSteps(
|
||||
steps = builtSteps,
|
||||
context = context,
|
||||
initialEmptyResult = ScoringPipelineResult.empty,
|
||||
stepsInOrderFromConfig = ScoringPipelineConfig.stepsInOrder
|
||||
)
|
||||
|
||||
val configFromBuilder = config
|
||||
new ScoringPipeline[Query, Candidate] {
|
||||
override private[core] val config: ScoringPipelineConfig[Query, Candidate] = configFromBuilder
|
||||
override val arrow: Arrow[ScoringPipeline.Inputs[Query], ScoringPipelineResult[Candidate]] =
|
||||
finalArrow
|
||||
override val identifier: ScoringPipelineIdentifier = pipelineIdentifier
|
||||
override val alerts: Seq[Alert] = config.alerts
|
||||
override val children: Seq[Component] =
|
||||
allGates ++ config.preScoringFeatureHydrationPhase1 ++ config.preScoringFeatureHydrationPhase2 ++ config.scorers
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,30 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.scoring
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutor
|
||||
import com.twitter.product_mixer.core.service.gate_executor.GateExecutor
|
||||
import com.twitter.product_mixer.core.service.selector_executor.SelectorExecutor
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class ScoringPipelineBuilderFactory @Inject() (
|
||||
gateExecutor: GateExecutor,
|
||||
selectorExecutor: SelectorExecutor,
|
||||
candidateFeatureHydratorExecutor: CandidateFeatureHydratorExecutor,
|
||||
statsReceiver: StatsReceiver) {
|
||||
|
||||
def get[
|
||||
Query <: PipelineQuery,
|
||||
Candidate <: UniversalNoun[Any]
|
||||
]: ScoringPipelineBuilder[Query, Candidate] = {
|
||||
new ScoringPipelineBuilder[Query, Candidate](
|
||||
gateExecutor,
|
||||
selectorExecutor,
|
||||
candidateFeatureHydratorExecutor,
|
||||
statsReceiver
|
||||
)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,131 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.scoring
|
||||
|
||||
import com.twitter.product_mixer.component_library.selector.InsertAppendResults
|
||||
import com.twitter.product_mixer.core.functional_component.common.AllPipelines
|
||||
import com.twitter.product_mixer.core.functional_component.common.alert.Alert
|
||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator
|
||||
import com.twitter.product_mixer.core.functional_component.gate.BaseGate
|
||||
import com.twitter.product_mixer.core.functional_component.scorer.Scorer
|
||||
import com.twitter.product_mixer.core.functional_component.selector.Selector
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifierStack
|
||||
import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineConfig
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineConfigCompanion
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure
|
||||
import com.twitter.timelines.configapi.FSParam
|
||||
import com.twitter.timelines.configapi.decider.DeciderParam
|
||||
|
||||
/**
|
||||
* This is the configuration necessary to generate a Scoring Pipeline. Product code should create a
|
||||
* ScoringPipelineConfig, and then use a ScoringPipelineBuilder to get the final ScoringPipeline which can
|
||||
* process requests.
|
||||
*
|
||||
* @tparam Query - The domain model for the query or request
|
||||
* @tparam Candidate the domain model for the candidate being scored
|
||||
*/
|
||||
trait ScoringPipelineConfig[-Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]
|
||||
extends PipelineConfig {
|
||||
|
||||
override val identifier: ScoringPipelineIdentifier
|
||||
|
||||
/**
|
||||
* When these Params are defined, they will automatically be added as Gates in the pipeline
|
||||
* by the CandidatePipelineBuilder
|
||||
*
|
||||
* The enabled decider param can to be used to quickly disable a Candidate Pipeline via Decider
|
||||
*/
|
||||
val enabledDeciderParam: Option[DeciderParam[Boolean]] = None
|
||||
|
||||
/**
|
||||
* This supported client feature switch param can be used with a Feature Switch to control the
|
||||
* rollout of a new Candidate Pipeline from dogfood to experiment to production
|
||||
*/
|
||||
val supportedClientParam: Option[FSParam[Boolean]] = None
|
||||
|
||||
/** [[BaseGate]]s that are applied sequentially, the pipeline will only run if all the Gates are open */
|
||||
def gates: Seq[BaseGate[Query]] = Seq.empty
|
||||
|
||||
/**
|
||||
* Logic for selecting which candidates to score. Note, this doesn't drop the candidates from
|
||||
* the final result, just whether to score it in this pipeline or not.
|
||||
*/
|
||||
def selectors: Seq[Selector[Query]]
|
||||
|
||||
/**
|
||||
* After selectors are run, you can fetch features for each candidate.
|
||||
* The existing features from previous hydrations are passed in as inputs. You are not expected to
|
||||
* put them into the resulting feature map yourself - they will be merged for you by the platform.
|
||||
*/
|
||||
def preScoringFeatureHydrationPhase1: Seq[BaseCandidateFeatureHydrator[Query, Candidate, _]] =
|
||||
Seq.empty
|
||||
|
||||
/**
|
||||
* A second phase of feature hydration that can be run after selection and after the first phase
|
||||
* of pre-scoring feature hydration. You are not expected to put them into the resulting
|
||||
* feature map yourself - they will be merged for you by the platform.
|
||||
*/
|
||||
def preScoringFeatureHydrationPhase2: Seq[BaseCandidateFeatureHydrator[Query, Candidate, _]] =
|
||||
Seq.empty
|
||||
|
||||
/**
|
||||
* Ranker Function for candidates. Scorers are executed in parallel.
|
||||
* Note: Order does not matter, this could be a Set if Set was covariant over it's type.
|
||||
*/
|
||||
def scorers: Seq[Scorer[Query, Candidate]]
|
||||
|
||||
/**
|
||||
* A pipeline can define a partial function to rescue failures here. They will be treated as failures
|
||||
* from a monitoring standpoint, and cancellation exceptions will always be propagated (they cannot be caught here).
|
||||
*/
|
||||
def failureClassifier: PartialFunction[Throwable, PipelineFailure] = PartialFunction.empty
|
||||
|
||||
/**
|
||||
* Alerts can be used to indicate the pipeline's service level objectives. Alerts and
|
||||
* dashboards will be automatically created based on this information.
|
||||
*/
|
||||
val alerts: Seq[Alert] = Seq.empty
|
||||
|
||||
/**
|
||||
* This method is used by the product mixer framework to build the pipeline.
|
||||
*/
|
||||
private[core] final def build(
|
||||
parentComponentIdentifierStack: ComponentIdentifierStack,
|
||||
builder: ScoringPipelineBuilderFactory
|
||||
): ScoringPipeline[Query, Candidate] =
|
||||
builder.get.build(parentComponentIdentifierStack, this)
|
||||
}
|
||||
|
||||
object ScoringPipelineConfig extends PipelineConfigCompanion {
|
||||
def apply[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]](
|
||||
scorer: Scorer[Query, Candidate]
|
||||
): ScoringPipelineConfig[Query, Candidate] = new ScoringPipelineConfig[Query, Candidate] {
|
||||
override val identifier: ScoringPipelineIdentifier = ScoringPipelineIdentifier(
|
||||
s"ScoreAll${scorer.identifier.name}")
|
||||
|
||||
override val selectors: Seq[Selector[Query]] = Seq(InsertAppendResults(AllPipelines))
|
||||
|
||||
override val scorers: Seq[Scorer[Query, Candidate]] = Seq(scorer)
|
||||
}
|
||||
|
||||
val gatesStep: PipelineStepIdentifier = PipelineStepIdentifier("Gates")
|
||||
val selectorsStep: PipelineStepIdentifier = PipelineStepIdentifier("Selectors")
|
||||
val preScoringFeatureHydrationPhase1Step: PipelineStepIdentifier =
|
||||
PipelineStepIdentifier("PreScoringFeatureHydrationPhase1")
|
||||
val preScoringFeatureHydrationPhase2Step: PipelineStepIdentifier =
|
||||
PipelineStepIdentifier("PreScoringFeatureHydrationPhase2")
|
||||
val scorersStep: PipelineStepIdentifier = PipelineStepIdentifier("Scorers")
|
||||
val resultStep: PipelineStepIdentifier = PipelineStepIdentifier("Result")
|
||||
|
||||
/** All the Steps which are executed by a [[ScoringPipeline]] in the order in which they are run */
|
||||
override val stepsInOrder: Seq[PipelineStepIdentifier] = Seq(
|
||||
gatesStep,
|
||||
selectorsStep,
|
||||
preScoringFeatureHydrationPhase1Step,
|
||||
preScoringFeatureHydrationPhase2Step,
|
||||
scorersStep,
|
||||
resultStep
|
||||
)
|
||||
}
|
||||
Binary file not shown.
@ -1,51 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.scoring
|
||||
|
||||
import com.twitter.product_mixer.core.functional_component.scorer.ScoredCandidateResult
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineResult
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure
|
||||
import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutorResult
|
||||
import com.twitter.product_mixer.core.service.gate_executor.GateExecutorResult
|
||||
import com.twitter.product_mixer.core.service.selector_executor.SelectorExecutorResult
|
||||
|
||||
/**
|
||||
* The Results of every step during the ScoringPipeline process. The end result contains
|
||||
* only the candidates that were actually scored (e.g, not dropped by a filter) with an updated,
|
||||
* combined feature map of all features that were passed in with the candidate plus all features
|
||||
* returned as part of scoring.
|
||||
*/
|
||||
case class ScoringPipelineResult[Candidate <: UniversalNoun[Any]](
|
||||
gateResults: Option[GateExecutorResult],
|
||||
selectorResults: Option[SelectorExecutorResult],
|
||||
preScoringHydrationPhase1Result: Option[CandidateFeatureHydratorExecutorResult[Candidate]],
|
||||
preScoringHydrationPhase2Result: Option[CandidateFeatureHydratorExecutorResult[Candidate]],
|
||||
scorerResults: Option[CandidateFeatureHydratorExecutorResult[
|
||||
Candidate
|
||||
]],
|
||||
failure: Option[PipelineFailure],
|
||||
result: Option[Seq[ScoredCandidateResult[Candidate]]])
|
||||
extends PipelineResult[Seq[ScoredCandidateResult[Candidate]]] {
|
||||
override val resultSize: Int = result.map(_.size).getOrElse(0)
|
||||
|
||||
override def withFailure(
|
||||
failure: PipelineFailure
|
||||
): ScoringPipelineResult[Candidate] =
|
||||
copy(failure = Some(failure))
|
||||
override def withResult(
|
||||
result: Seq[ScoredCandidateResult[Candidate]]
|
||||
): ScoringPipelineResult[Candidate] =
|
||||
copy(result = Some(result))
|
||||
}
|
||||
|
||||
object ScoringPipelineResult {
|
||||
def empty[Candidate <: UniversalNoun[Any]]: ScoringPipelineResult[Candidate] =
|
||||
ScoringPipelineResult(
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None
|
||||
)
|
||||
}
|
||||
@ -1,33 +0,0 @@
|
||||
scala_library(
|
||||
sources = [
|
||||
"*.scala",
|
||||
],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator:decoration",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service",
|
||||
],
|
||||
exports = [
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator:decoration",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,9 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.state
|
||||
|
||||
import com.twitter.product_mixer.core.feature.featuremap.asyncfeaturemap.AsyncFeatureMap
|
||||
|
||||
trait HasAsyncFeatureMap[State] {
|
||||
def asyncFeatureMap: AsyncFeatureMap
|
||||
|
||||
private[core] def addAsyncFeatureMap(newFeatureMap: AsyncFeatureMap): State
|
||||
}
|
||||
Binary file not shown.
@ -1,8 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.state
|
||||
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
|
||||
trait HasCandidates[Candidate <: UniversalNoun[Any], T] {
|
||||
def candidates: Seq[Candidate]
|
||||
def updateCandidates(newCandidates: Seq[Candidate]): T
|
||||
}
|
||||
Binary file not shown.
@ -1,11 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.state
|
||||
|
||||
import com.twitter.product_mixer.core.functional_component.decorator.Decoration
|
||||
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
|
||||
|
||||
trait HasCandidatesWithDetails[T] {
|
||||
def candidatesWithDetails: Seq[CandidateWithDetails]
|
||||
def updateCandidatesWithDetails(newCandidates: Seq[CandidateWithDetails]): T
|
||||
|
||||
def updateDecorations(decoration: Seq[Decoration]): T
|
||||
}
|
||||
Binary file not shown.
@ -1,9 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.state
|
||||
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
|
||||
trait HasCandidatesWithFeatures[Candidate <: UniversalNoun[Any], T] {
|
||||
def candidatesWithFeatures: Seq[CandidateWithFeatures[Candidate]]
|
||||
def updateCandidatesWithFeatures(newCandidates: Seq[CandidateWithFeatures[Candidate]]): T
|
||||
}
|
||||
Binary file not shown.
@ -1,13 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.state
|
||||
|
||||
import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier
|
||||
import com.twitter.product_mixer.core.service.ExecutorResult
|
||||
import scala.collection.immutable.ListMap
|
||||
|
||||
trait HasExecutorResults[State] {
|
||||
// We use a list map to maintain the insertion order
|
||||
val executorResultsByPipelineStep: ListMap[PipelineStepIdentifier, ExecutorResult]
|
||||
private[pipeline] def setExecutorResults(
|
||||
newMap: ListMap[PipelineStepIdentifier, ExecutorResult]
|
||||
): State
|
||||
}
|
||||
Binary file not shown.
@ -1,7 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.state
|
||||
|
||||
import com.twitter.timelines.configapi.Params
|
||||
|
||||
trait HasParams {
|
||||
def params: Params
|
||||
}
|
||||
Binary file not shown.
@ -1,8 +0,0 @@
|
||||
package com.twitter.product_mixer.core.pipeline.state
|
||||
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
|
||||
trait HasQuery[Query <: PipelineQuery, T] {
|
||||
def query: Query
|
||||
def updateQuery(query: Query): T
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user