diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/PipelineCursorSerializer.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/PipelineCursorSerializer.docx new file mode 100644 index 000000000..c1d311b86 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/PipelineCursorSerializer.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/PipelineCursorSerializer.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/PipelineCursorSerializer.scala deleted file mode 100644 index 6ed6a9ce2..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/PipelineCursorSerializer.scala +++ /dev/null @@ -1,57 +0,0 @@ -package com.twitter.product_mixer.core.pipeline - -import com.twitter.product_mixer.core.pipeline.pipeline_failure.MalformedCursor -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.scrooge.BinaryThriftStructSerializer -import com.twitter.scrooge.ThriftStruct -import com.twitter.util.Return -import com.twitter.util.Throw -import com.twitter.util.Try - -/** - * Serializes a [[PipelineCursor]] into thrift and then into a base64 encoded string - */ -trait PipelineCursorSerializer[-Cursor <: PipelineCursor] { - def serializeCursor(cursor: Cursor): String -} - -object PipelineCursorSerializer { - - /** - * Deserializes a cursor string into thrift and then into a [[PipelineCursor]] - * - * @param cursorString to deserialize, which is base64 encoded thrift - * @param cursorThriftSerializer to deserialize the cursor string into thrift - * @param deserializePf specifies how to transform the serialized thrift into a [[PipelineCursor]] - * @return optional [[PipelineCursor]]. `None` may or may not be a failure depending on the - * implementation of deserializePf. - * - * @note The "A" type of deserializePf cannot be inferred due to the thrift type not being present - * on the PipelineCursorSerializer trait. Therefore invokers must often add an explicit type - * on the deserializeCursor call to help out the compiler when passing deserializePf inline. - * Alternatively, deserializePf can be declared as a val with a type annotation before it is - * passed into this method. - */ - def deserializeCursor[Thrift <: ThriftStruct, Cursor <: PipelineCursor]( - cursorString: String, - cursorThriftSerializer: BinaryThriftStructSerializer[Thrift], - deserializePf: PartialFunction[Option[Thrift], Option[Cursor]] - ): Option[Cursor] = { - val thriftCursor: Option[Thrift] = - Try { - cursorThriftSerializer.fromString(cursorString) - } match { - case Return(thriftCursor) => Some(thriftCursor) - case Throw(_) => None - } - - // Add type annotation to help out the compiler since the type is lost due to the _ match - val defaultDeserializePf: PartialFunction[Option[Thrift], Option[Cursor]] = { - case _ => - // This case is the result of the client submitting a cursor we do not expect - throw PipelineFailure(MalformedCursor, s"Unknown request cursor: $cursorString") - } - - (deserializePf orElse defaultDeserializePf)(thriftCursor) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/PipelineQuery.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/PipelineQuery.docx new file mode 100644 index 000000000..8bbe273b5 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/PipelineQuery.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/PipelineQuery.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/PipelineQuery.scala deleted file mode 100644 index ae6add7e1..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/PipelineQuery.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.product_mixer.core.pipeline - -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.product_mixer.core.model.marshalling.request.HasDebugOptions -import com.twitter.product_mixer.core.model.marshalling.request.HasProduct -import com.twitter.timelines.configapi.HasParams -import com.twitter.timelines.configapi.Param -import com.twitter.util.Time - -trait PipelineQuery extends HasParams with HasClientContext with HasProduct with HasDebugOptions { - self => - - /** Set a query time val that is constant for the duration of the query lifecycle */ - val queryTime: Time = self.debugOptions.flatMap(_.requestTimeOverride).getOrElse(Time.now) - - /** The requested max results is specified, or not specified, by the thrift client */ - def requestedMaxResults: Option[Int] - - /** Retrieves the max results with a default Param, if not specified by the thrift client */ - def maxResults(defaultRequestedMaxResultParam: Param[Int]): Int = - requestedMaxResults.getOrElse(params(defaultRequestedMaxResultParam)) - - /** Optional [[FeatureMap]], this may be updated later using [[withFeatureMap]] */ - def features: Option[FeatureMap] - - /** - * Since Query-Level features can be hydrated later, we need this method to update the PipelineQuery - * usually this will be implemented via `copy(features = Some(features))` - */ - def withFeatureMap(features: FeatureMap): PipelineQuery -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/PipelineResult.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/PipelineResult.docx new file mode 100644 index 000000000..8ed8faa1a Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/PipelineResult.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/PipelineResult.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/PipelineResult.scala deleted file mode 100644 index f7e378917..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/PipelineResult.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.twitter.product_mixer.core.pipeline - -import com.twitter.product_mixer.component_library.model.candidate.CursorCandidate -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails -import com.twitter.product_mixer.core.pipeline.pipeline_failure.ExecutionFailed -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.util.Return -import com.twitter.util.Throw -import com.twitter.util.Try - -/** - * Pipelines return a PipelineResult. - * - * This allows us to return a single main result (optionally, incase the pipeline didn't execute successfully), but - * still have a detailed response object to show how that result was produced. - */ -trait PipelineResult[ResultType] { - val failure: Option[PipelineFailure] - val result: Option[ResultType] - - def withFailure(failure: PipelineFailure): PipelineResult[ResultType] - def withResult(result: ResultType): PipelineResult[ResultType] - - def resultSize(): Int - - private[pipeline] def stopExecuting: Boolean = failure.isDefined || result.isDefined - - final def toTry: Try[this.type] = (result, failure) match { - case (_, Some(failure)) => - Throw(failure) - case (_: Some[ResultType], _) => - Return(this) - // Pipelines should always finish with either a result or a failure - case _ => Throw(PipelineFailure(ExecutionFailed, "Pipeline did not execute")) - } - - final def toResultTry: Try[ResultType] = { - // `.get` is safe here because `toTry` guarantees a value in the `Return` case - toTry.map(_.result.get) - } -} - -object PipelineResult { - - /** - * Track number of candidates returned by a Pipeline. Cursors are excluded from this - * count and modules are counted as the sum of their candidates. - * - * @note this is a somewhat subjective measure of 'size' and it is spread across pipeline - * definitions as well as selectors. - */ - def resultSize(results: Seq[CandidateWithDetails]): Int = results.map { - case module: ModuleCandidateWithDetails => resultSize(module.candidates) - case ItemCandidateWithDetails(_: CursorCandidate, _, _) => 0 - case _ => 1 - }.sum -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/BUILD deleted file mode 100644 index 642e7c74f..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/BUILD +++ /dev/null @@ -1,77 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "configapi/configapi-core", - "configapi/configapi-decider", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/gate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/async_feature_map_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_decorator_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_hydrator_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_source_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/filter_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/group_results_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "stitch/stitch-core", - "util/util-core:scala", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "configapi/configapi-core", - "configapi/configapi-decider", - "finatra/inject/inject-core/src/main/scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/gate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/async_feature_map_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_decorator_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_hydrator_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_source_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/filter_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/group_results_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/BUILD.docx new file mode 100644 index 000000000..60e0362b8 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipeline.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipeline.docx new file mode 100644 index 000000000..604721b66 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipeline.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipeline.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipeline.scala deleted file mode 100644 index 2c52636b6..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipeline.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.candidate - -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.Pipeline -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Arrow - -/** - * A Candidate Pipeline - * - * This is an abstract class, as we only construct these via the [[CandidatePipelineBuilder]]. - * - * A [[CandidatePipeline]] is capable of processing requests (queries) and returning candidates - * in the form of a [[CandidatePipelineResult]] - * - * @tparam Query the domain model for the query or request - */ -abstract class CandidatePipeline[-Query <: PipelineQuery] private[candidate] - extends Pipeline[CandidatePipeline.Inputs[Query], Seq[CandidateWithDetails]] { - override private[core] val config: BaseCandidatePipelineConfig[Query, _, _, _] - override val arrow: Arrow[CandidatePipeline.Inputs[Query], CandidatePipelineResult] - override val identifier: CandidatePipelineIdentifier -} - -object CandidatePipeline { - case class Inputs[+Query <: PipelineQuery]( - query: Query, - existingCandidates: Seq[CandidateWithDetails]) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipelineBuilder.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipelineBuilder.docx new file mode 100644 index 000000000..6d301cfbf Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipelineBuilder.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipelineBuilder.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipelineBuilder.scala deleted file mode 100644 index 2d3df3440..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipelineBuilder.scala +++ /dev/null @@ -1,735 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.candidate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.asyncfeaturemap.AsyncFeatureMap -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator -import com.twitter.product_mixer.core.functional_component.transformer.BaseCandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.gate.ParamGate -import com.twitter.product_mixer.core.gate.ParamGate.EnabledGateSuffix -import com.twitter.product_mixer.core.gate.ParamGate.SupportedClientGateSuffix -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.Component -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifierStack -import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.InvalidStepStateException -import com.twitter.product_mixer.core.pipeline.PipelineBuilder -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.pipeline_failure.ClosedGate -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailureClassifier -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.async_feature_map_executor.AsyncFeatureMapExecutor -import com.twitter.product_mixer.core.service.async_feature_map_executor.AsyncFeatureMapExecutorResults -import com.twitter.product_mixer.core.service.candidate_decorator_executor.CandidateDecoratorExecutor -import com.twitter.product_mixer.core.service.candidate_decorator_executor.CandidateDecoratorExecutorResult -import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutor -import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutorResult -import com.twitter.product_mixer.core.service.candidate_source_executor.CandidateSourceExecutor -import com.twitter.product_mixer.core.service.candidate_source_executor.CandidateSourceExecutorResult -import com.twitter.product_mixer.core.service.candidate_source_executor.FetchedCandidateWithFeatures -import com.twitter.product_mixer.core.service.filter_executor.FilterExecutor -import com.twitter.product_mixer.core.service.filter_executor.FilterExecutorResult -import com.twitter.product_mixer.core.service.gate_executor.GateExecutor -import com.twitter.product_mixer.core.service.gate_executor.GateExecutorResult -import com.twitter.product_mixer.core.service.gate_executor.StoppedGateException -import com.twitter.product_mixer.core.service.group_results_executor.GroupResultsExecutor -import com.twitter.product_mixer.core.service.group_results_executor.GroupResultsExecutorInput -import com.twitter.product_mixer.core.service.group_results_executor.GroupResultsExecutorResult -import com.twitter.product_mixer.core.service.query_feature_hydrator_executor.QueryFeatureHydratorExecutor -import com.twitter.stitch.Arrow -import com.twitter.util.logging.Logging -import javax.inject.Inject - -class CandidatePipelineBuilder[ - Query <: PipelineQuery, - CandidateSourceQuery, - CandidateSourceResult, - Result <: UniversalNoun[Any]] @Inject() ( - queryFeatureHydratorExecutor: QueryFeatureHydratorExecutor, - asyncFeatureMapExecutor: AsyncFeatureMapExecutor, - candidateDecoratorExecutor: CandidateDecoratorExecutor, - candidateFeatureHydratorExecutor: CandidateFeatureHydratorExecutor, - candidateSourceExecutor: CandidateSourceExecutor, - groupResultsExecutor: GroupResultsExecutor, - filterExecutor: FilterExecutor, - gateExecutor: GateExecutor, - override val statsReceiver: StatsReceiver) - extends PipelineBuilder[CandidatePipeline.Inputs[Query]] - with Logging { - - override type UnderlyingResultType = Seq[CandidateWithDetails] - override type PipelineResultType = IntermediateCandidatePipelineResult[Result] - - def build( - parentComponentIdentifierStack: ComponentIdentifierStack, - config: BaseCandidatePipelineConfig[ - Query, - CandidateSourceQuery, - CandidateSourceResult, - Result - ] - ): CandidatePipeline[Query] = { - - val pipelineIdentifier = config.identifier - val candidateSourceIdentifier = config.candidateSource.identifier - - val context = Executor.Context( - PipelineFailureClassifier( - config.failureClassifier.orElse(StoppedGateException.classifier(ClosedGate))), - parentComponentIdentifierStack.push(pipelineIdentifier) - ) - - val enabledGateOpt = config.enabledDeciderParam.map { deciderParam => - ParamGate(pipelineIdentifier + EnabledGateSuffix, deciderParam) - } - val supportedClientGateOpt = config.supportedClientParam.map { param => - ParamGate(pipelineIdentifier + SupportedClientGateSuffix, param) - } - - /** - * Evaluate enabled decider gate first since if it's off, there is no reason to proceed - * Next evaluate supported client feature switch gate, followed by customer configured gates - */ - val allGates = enabledGateOpt.toSeq ++ supportedClientGateOpt.toSeq ++ config.gates - - // Dynamically replace the identifier of both transformers if config used the inline constructor - // which sets a default identifier. We need to do this to ensure uniqueness of identifiers. - val queryTransformer = BaseCandidatePipelineQueryTransformer.copyWithUpdatedIdentifier( - config.queryTransformer, - pipelineIdentifier) - - val resultsTransformer = CandidatePipelineResultsTransformer.copyWithUpdatedIdentifier( - config.resultTransformer, - pipelineIdentifier) - - val decorator = config.decorator.map(decorator => - CandidateDecorator.copyWithUpdatedIdentifier(decorator, pipelineIdentifier)) - - val GatesStep = new Step[Query, GateExecutorResult] { - override def identifier: PipelineStepIdentifier = CandidatePipelineConfig.gatesStep - - override def executorArrow: Arrow[Query, GateExecutorResult] = { - gateExecutor.arrow(allGates, context) - } - - override def inputAdaptor( - query: CandidatePipeline.Inputs[Query], - previousResult: IntermediateCandidatePipelineResult[Result] - ): Query = - query.query - - override def resultUpdater( - previousPipelineResult: IntermediateCandidatePipelineResult[Result], - executorResult: GateExecutorResult - ): IntermediateCandidatePipelineResult[Result] = - previousPipelineResult.copy(underlyingResult = - previousPipelineResult.underlyingResult.copy(gateResult = Some(executorResult))) - } - - def queryFeatureHydrationStep( - queryFeatureHydrators: Seq[BaseQueryFeatureHydrator[Query, _]], - stepIdentifier: PipelineStepIdentifier, - updater: ResultUpdater[CandidatePipelineResult, QueryFeatureHydratorExecutor.Result] - ): Step[Query, QueryFeatureHydratorExecutor.Result] = - new Step[Query, QueryFeatureHydratorExecutor.Result] { - override def identifier: PipelineStepIdentifier = stepIdentifier - - override def executorArrow: Arrow[Query, QueryFeatureHydratorExecutor.Result] = - queryFeatureHydratorExecutor.arrow( - queryFeatureHydrators, - CandidatePipelineConfig.stepsAsyncFeatureHydrationCanBeCompletedBy, - context) - - override def inputAdaptor( - query: CandidatePipeline.Inputs[Query], - previousResult: IntermediateCandidatePipelineResult[Result] - ): Query = query.query - - override def resultUpdater( - previousPipelineResult: IntermediateCandidatePipelineResult[Result], - executorResult: QueryFeatureHydratorExecutor.Result - ): IntermediateCandidatePipelineResult[Result] = - previousPipelineResult.copy( - underlyingResult = updater(previousPipelineResult.underlyingResult, executorResult)) - - override def queryUpdater( - query: CandidatePipeline.Inputs[Query], - executorResult: QueryFeatureHydratorExecutor.Result - ): CandidatePipeline.Inputs[Query] = - CandidatePipeline.Inputs( - query.query - .withFeatureMap( - query.query.features.getOrElse( - FeatureMap.empty) ++ executorResult.featureMap).asInstanceOf[Query], - query.existingCandidates) - } - - def asyncFeaturesStep( - stepToHydrateFor: PipelineStepIdentifier, - context: Executor.Context - ): Step[AsyncFeatureMap, AsyncFeatureMapExecutorResults] = - new Step[AsyncFeatureMap, AsyncFeatureMapExecutorResults] { - override def identifier: PipelineStepIdentifier = - CandidatePipelineConfig.asyncFeaturesStep(stepToHydrateFor) - - override def executorArrow: Arrow[AsyncFeatureMap, AsyncFeatureMapExecutorResults] = - asyncFeatureMapExecutor.arrow(stepToHydrateFor, identifier, context) - - override def inputAdaptor( - query: CandidatePipeline.Inputs[Query], - previousResult: IntermediateCandidatePipelineResult[Result] - ): AsyncFeatureMap = - previousResult.underlyingResult.mergedAsyncQueryFeatures - .getOrElse( - throw InvalidStepStateException(identifier, "MergedAsyncQueryFeatures") - ) - - override def resultUpdater( - previousPipelineResult: IntermediateCandidatePipelineResult[Result], - executorResult: AsyncFeatureMapExecutorResults - ): IntermediateCandidatePipelineResult[Result] = - previousPipelineResult.copy( - underlyingResult = - previousPipelineResult.underlyingResult.copy(asyncFeatureHydrationResults = - previousPipelineResult.underlyingResult.asyncFeatureHydrationResults match { - case Some(existingResults) => Some(existingResults ++ executorResult) - case None => Some(executorResult) - })) - - override def queryUpdater( - query: CandidatePipeline.Inputs[Query], - executorResult: AsyncFeatureMapExecutorResults - ): CandidatePipeline.Inputs[Query] = - if (executorResult.featureMapsByStep - .getOrElse(stepToHydrateFor, FeatureMap.empty).isEmpty) { - query - } else { - val updatedQuery = query.query - .withFeatureMap( - query.query.features - .getOrElse(FeatureMap.empty) ++ executorResult.featureMapsByStep( - stepToHydrateFor)).asInstanceOf[Query] - CandidatePipeline.Inputs(updatedQuery, query.existingCandidates) - } - } - - val CandidateSourceStep = - new Step[Query, CandidateSourceExecutorResult[Result]] { - override def identifier: PipelineStepIdentifier = - CandidatePipelineConfig.candidateSourceStep - - override def executorArrow: Arrow[ - Query, - CandidateSourceExecutorResult[Result] - ] = - candidateSourceExecutor - .arrow( - config.candidateSource, - queryTransformer, - resultsTransformer, - config.featuresFromCandidateSourceTransformers, - context - ) - - override def inputAdaptor( - query: CandidatePipeline.Inputs[Query], - previousResult: IntermediateCandidatePipelineResult[Result] - ): Query = - query.query - - override def resultUpdater( - previousPipelineResult: IntermediateCandidatePipelineResult[Result], - executorResult: CandidateSourceExecutorResult[Result] - ): IntermediateCandidatePipelineResult[Result] = - previousPipelineResult.copy(underlyingResult = - previousPipelineResult.underlyingResult.copy( - candidateSourceResult = - Some(executorResult.asInstanceOf[CandidateSourceExecutorResult[UniversalNoun[Any]]]) - )) - - override def queryUpdater( - query: CandidatePipeline.Inputs[Query], - executorResult: CandidateSourceExecutorResult[Result] - ): CandidatePipeline.Inputs[Query] = { - val updatedFeatureMap = - query.query.features - .getOrElse(FeatureMap.empty) ++ executorResult.candidateSourceFeatureMap - val updatedQuery = query.query - .withFeatureMap(updatedFeatureMap).asInstanceOf[Query] - CandidatePipeline.Inputs(updatedQuery, query.existingCandidates) - } - } - - val PreFilterFeatureHydrationPhase1Step = - new Step[ - CandidateFeatureHydratorExecutor.Inputs[Query, Result], - CandidateFeatureHydratorExecutorResult[Result] - ] { - override def identifier: PipelineStepIdentifier = - CandidatePipelineConfig.preFilterFeatureHydrationPhase1Step - - override def executorArrow: Arrow[ - CandidateFeatureHydratorExecutor.Inputs[Query, Result], - CandidateFeatureHydratorExecutorResult[Result] - ] = - candidateFeatureHydratorExecutor.arrow(config.preFilterFeatureHydrationPhase1, context) - - override def inputAdaptor( - query: CandidatePipeline.Inputs[Query], - previousResult: IntermediateCandidatePipelineResult[Result] - ): CandidateFeatureHydratorExecutor.Inputs[Query, Result] = { - val candidateSourceExecutorResult = - previousResult.underlyingResult.candidateSourceResult.getOrElse { - throw InvalidStepStateException(identifier, "CandidateSourceResult") - } - CandidateFeatureHydratorExecutor.Inputs( - query.query, - candidateSourceExecutorResult.candidates - .asInstanceOf[Seq[CandidateWithFeatures[Result]]]) - } - - override def resultUpdater( - previousPipelineResult: IntermediateCandidatePipelineResult[Result], - executorResult: CandidateFeatureHydratorExecutorResult[Result] - ): IntermediateCandidatePipelineResult[Result] = { - val candidateSourceExecutorResult = - previousPipelineResult.underlyingResult.candidateSourceResult.getOrElse { - throw InvalidStepStateException(identifier, "CandidateSourceResult") - } - - val featureMapsFromPreFilter = executorResult.results.map { result => - result.candidate -> result.features - }.toMap - - val mergedFeatureMaps = candidateSourceExecutorResult.candidates.map { candidate => - val candidateFeatureMap = candidate.features - val preFilterFeatureMap = - featureMapsFromPreFilter.getOrElse( - candidate.candidate.asInstanceOf[Result], - FeatureMap.empty) - - candidate.candidate.asInstanceOf[Result] -> (candidateFeatureMap ++ preFilterFeatureMap) - }.toMap - - previousPipelineResult.copy( - underlyingResult = previousPipelineResult.underlyingResult.copy( - preFilterHydrationResult = Some( - executorResult - .asInstanceOf[CandidateFeatureHydratorExecutorResult[UniversalNoun[Any]]]) - ), - featureMaps = Some(mergedFeatureMaps) - ) - } - } - - val PreFilterFeatureHydrationPhase2Step = - new Step[ - CandidateFeatureHydratorExecutor.Inputs[Query, Result], - CandidateFeatureHydratorExecutorResult[Result] - ] { - override def identifier: PipelineStepIdentifier = - CandidatePipelineConfig.preFilterFeatureHydrationPhase2Step - - override def executorArrow: Arrow[ - CandidateFeatureHydratorExecutor.Inputs[Query, Result], - CandidateFeatureHydratorExecutorResult[Result] - ] = - candidateFeatureHydratorExecutor.arrow(config.preFilterFeatureHydrationPhase2, context) - - override def inputAdaptor( - query: CandidatePipeline.Inputs[Query], - previousResult: IntermediateCandidatePipelineResult[Result] - ): CandidateFeatureHydratorExecutor.Inputs[Query, Result] = { - val candidates = previousResult.underlyingResult.preFilterHydrationResult.getOrElse { - throw InvalidStepStateException(identifier, "PreFilterHydrationResult") - }.results - CandidateFeatureHydratorExecutor.Inputs( - query.query, - candidates.asInstanceOf[Seq[CandidateWithFeatures[Result]]] - ) - } - - override def resultUpdater( - previousPipelineResult: IntermediateCandidatePipelineResult[Result], - executorResult: CandidateFeatureHydratorExecutorResult[Result] - ): IntermediateCandidatePipelineResult[Result] = { - - val featureMapsFromPreFilterPhase2 = executorResult.results.map { result => - result.candidate -> result.features - }.toMap - - val mergedFeatureMaps = previousPipelineResult.featureMaps - .getOrElse(throw InvalidStepStateException(identifier, "FeatureMaps")) - .map { - case (candidate, featureMap) => - val preFilterPhase2FeatureMap = - featureMapsFromPreFilterPhase2.getOrElse(candidate, FeatureMap.empty) - - candidate -> (featureMap ++ preFilterPhase2FeatureMap) - } - - previousPipelineResult.copy( - underlyingResult = previousPipelineResult.underlyingResult.copy( - preFilterHydrationResultPhase2 = Some( - executorResult - .asInstanceOf[CandidateFeatureHydratorExecutorResult[UniversalNoun[Any]]]) - ), - featureMaps = Some(mergedFeatureMaps) - ) - } - } - - val FiltersStep = - new Step[(Query, Seq[CandidateWithFeatures[Result]]), FilterExecutorResult[Result]] { - override def identifier: PipelineStepIdentifier = CandidatePipelineConfig.filtersStep - - override def executorArrow: Arrow[ - (Query, Seq[CandidateWithFeatures[Result]]), - FilterExecutorResult[ - Result - ] - ] = - filterExecutor.arrow(config.filters, context) - - override def inputAdaptor( - query: CandidatePipeline.Inputs[Query], - previousResult: IntermediateCandidatePipelineResult[Result] - ): (Query, Seq[CandidateWithFeatures[Result]]) = { - val candidates = - previousResult.underlyingResult.candidateSourceResult - .getOrElse { - throw InvalidStepStateException(identifier, "CandidateSourceResult") - }.candidates.map(_.candidate).asInstanceOf[Seq[Result]] - - val featureMaps = previousResult.featureMaps - .getOrElse(throw InvalidStepStateException(identifier, "FeatureMaps")) - - ( - query.query, - candidates.map(candidate => - CandidateWithFeaturesImpl( - candidate, - featureMaps.getOrElse(candidate, FeatureMap.empty)))) - } - - override def resultUpdater( - previousPipelineResult: IntermediateCandidatePipelineResult[Result], - executorResult: FilterExecutorResult[Result] - ): IntermediateCandidatePipelineResult[Result] = - previousPipelineResult.copy(underlyingResult = - previousPipelineResult.underlyingResult.copy( - filterResult = - Some(executorResult.asInstanceOf[FilterExecutorResult[UniversalNoun[Any]]]) - )) - } - - val PostFilterFeatureHydrationStep = - new Step[ - CandidateFeatureHydratorExecutor.Inputs[Query, Result], - CandidateFeatureHydratorExecutorResult[Result] - ] { - override def identifier: PipelineStepIdentifier = - CandidatePipelineConfig.postFilterFeatureHydrationStep - - override def executorArrow: Arrow[ - CandidateFeatureHydratorExecutor.Inputs[Query, Result], - CandidateFeatureHydratorExecutorResult[Result] - ] = - candidateFeatureHydratorExecutor.arrow(config.postFilterFeatureHydration, context) - - override def inputAdaptor( - query: CandidatePipeline.Inputs[Query], - previousResult: IntermediateCandidatePipelineResult[Result] - ): CandidateFeatureHydratorExecutor.Inputs[Query, Result] = { - val filterResult = previousResult.underlyingResult.filterResult - .getOrElse( - throw InvalidStepStateException(identifier, "FilterResult") - ).result.asInstanceOf[Seq[Result]] - - val featureMaps = previousResult.featureMaps.getOrElse( - throw InvalidStepStateException(identifier, "FeatureMaps") - ) - - val filteredCandidates = filterResult.map { candidate => - CandidateWithFeaturesImpl(candidate, featureMaps.getOrElse(candidate, FeatureMap.empty)) - } - CandidateFeatureHydratorExecutor.Inputs(query.query, filteredCandidates) - } - - override def resultUpdater( - previousPipelineResult: IntermediateCandidatePipelineResult[Result], - executorResult: CandidateFeatureHydratorExecutorResult[Result] - ): IntermediateCandidatePipelineResult[Result] = { - val filterResult = previousPipelineResult.underlyingResult.filterResult - .getOrElse( - throw InvalidStepStateException(identifier, "FilterResult") - ).result.asInstanceOf[Seq[Result]] - - val featureMaps = previousPipelineResult.featureMaps.getOrElse( - throw InvalidStepStateException(identifier, "FeatureMaps") - ) - - val postFilterFeatureMaps = executorResult.results.map { result => - result.candidate -> result.features - }.toMap - - val mergedFeatureMaps = filterResult.map { candidate => - candidate -> - (featureMaps - .getOrElse(candidate, FeatureMap.empty) ++ postFilterFeatureMaps.getOrElse( - candidate, - FeatureMap.empty)) - }.toMap - - previousPipelineResult.copy( - underlyingResult = previousPipelineResult.underlyingResult.copy( - postFilterHydrationResult = Some( - executorResult - .asInstanceOf[CandidateFeatureHydratorExecutorResult[UniversalNoun[Any]]]) - ), - featureMaps = Some(mergedFeatureMaps) - ) - } - } - - val ScorersStep = - new Step[ - CandidateFeatureHydratorExecutor.Inputs[Query, Result], - CandidateFeatureHydratorExecutorResult[Result] - ] { - override def identifier: PipelineStepIdentifier = CandidatePipelineConfig.scorersStep - - override def executorArrow: Arrow[ - CandidateFeatureHydratorExecutor.Inputs[Query, Result], - CandidateFeatureHydratorExecutorResult[Result] - ] = - candidateFeatureHydratorExecutor.arrow(config.scorers, context) - - override def inputAdaptor( - query: CandidatePipeline.Inputs[Query], - previousResult: IntermediateCandidatePipelineResult[Result] - ): CandidateFeatureHydratorExecutor.Inputs[Query, Result] = { - val filterResult = previousResult.underlyingResult.filterResult - .getOrElse( - throw InvalidStepStateException(identifier, "FilterResult") - ).result.asInstanceOf[Seq[Result]] - - val featureMaps = previousResult.featureMaps.getOrElse( - throw InvalidStepStateException(identifier, "FeatureMaps") - ) - - val filteredCandidates = filterResult.map { candidate => - CandidateWithFeaturesImpl(candidate, featureMaps.getOrElse(candidate, FeatureMap.empty)) - } - CandidateFeatureHydratorExecutor.Inputs(query.query, filteredCandidates) - } - - override def resultUpdater( - previousPipelineResult: IntermediateCandidatePipelineResult[Result], - executorResult: CandidateFeatureHydratorExecutorResult[Result] - ): IntermediateCandidatePipelineResult[Result] = { - val filterResult = previousPipelineResult.underlyingResult.filterResult - .getOrElse( - throw InvalidStepStateException(identifier, "FilterResult") - ).result.asInstanceOf[Seq[Result]] - - val featureMaps = previousPipelineResult.featureMaps.getOrElse( - throw InvalidStepStateException(identifier, "FeatureMaps") - ) - - val scoringFeatureMaps = executorResult.results.map { result => - result.candidate -> result.features - }.toMap - - val mergedFeatureMaps = filterResult.map { candidate => - candidate -> - (featureMaps - .getOrElse(candidate, FeatureMap.empty) ++ scoringFeatureMaps.getOrElse( - candidate, - FeatureMap.empty)) - }.toMap - - previousPipelineResult.copy( - underlyingResult = previousPipelineResult.underlyingResult.copy( - scorersResult = Some( - executorResult - .asInstanceOf[CandidateFeatureHydratorExecutorResult[UniversalNoun[Any]]]) - ), - featureMaps = Some(mergedFeatureMaps) - ) - } - } - - val DecorationStep = - new Step[(Query, Seq[CandidateWithFeatures[Result]]), CandidateDecoratorExecutorResult] { - override def identifier: PipelineStepIdentifier = CandidatePipelineConfig.decoratorStep - - override def executorArrow: Arrow[ - (Query, Seq[CandidateWithFeatures[Result]]), - CandidateDecoratorExecutorResult - ] = - candidateDecoratorExecutor.arrow(decorator, context) - - override def inputAdaptor( - query: CandidatePipeline.Inputs[Query], - previousResult: IntermediateCandidatePipelineResult[Result] - ): (Query, Seq[CandidateWithFeatures[Result]]) = { - val keptCandidates = previousResult.underlyingResult.filterResult - .getOrElse { - throw InvalidStepStateException(identifier, "FilterResult") - }.result.asInstanceOf[Seq[Result]] - - val featureMaps = previousResult.featureMaps.getOrElse { - throw InvalidStepStateException(identifier, "FeatureMaps") - } - - ( - query.query, - keptCandidates.map(candidate => - CandidateWithFeaturesImpl( - candidate, - featureMaps.getOrElse(candidate, FeatureMap.empty)))) - } - - override def resultUpdater( - previousPipelineResult: IntermediateCandidatePipelineResult[Result], - executorResult: CandidateDecoratorExecutorResult - ): IntermediateCandidatePipelineResult[Result] = - previousPipelineResult.copy(underlyingResult = - previousPipelineResult.underlyingResult.copy( - candidateDecoratorResult = Some(executorResult) - )) - } - - /** - * ResultStep is a synchronous step that basically takes the outputs from the other steps, groups modules, - * and puts things into the final result object - */ - val ResultStep = new Step[GroupResultsExecutorInput[Result], GroupResultsExecutorResult] { - override def identifier: PipelineStepIdentifier = CandidatePipelineConfig.resultStep - - override def executorArrow: Arrow[ - GroupResultsExecutorInput[Result], - GroupResultsExecutorResult - ] = groupResultsExecutor.arrow(pipelineIdentifier, candidateSourceIdentifier, context) - - override def inputAdaptor( - query: CandidatePipeline.Inputs[Query], - previousResult: IntermediateCandidatePipelineResult[Result] - ): GroupResultsExecutorInput[Result] = { - - val underlying = previousResult.underlyingResult - - val keptCandidates = underlying.filterResult - .getOrElse( - throw InvalidStepStateException(identifier, "FilterResult") - ).result.asInstanceOf[Seq[Result]] - - val decorations = underlying.candidateDecoratorResult - .getOrElse( - throw InvalidStepStateException(identifier, "DecorationResult") - ).result.map(decoration => decoration.candidate -> decoration.presentation).toMap - - val combinedFeatureMaps: Map[Result, FeatureMap] = previousResult.featureMaps.getOrElse( - throw InvalidStepStateException(identifier, "FeatureMaps")) - - val filteredCandidates = keptCandidates.map { candidate => - val updatedMap = combinedFeatureMaps - .get(candidate).getOrElse(FeatureMap.empty) - FetchedCandidateWithFeatures(candidate, updatedMap) - } - - GroupResultsExecutorInput( - candidates = filteredCandidates, - decorations = decorations - ) - } - - override def resultUpdater( - previousPipelineResult: IntermediateCandidatePipelineResult[Result], - executorResult: GroupResultsExecutorResult - ): IntermediateCandidatePipelineResult[Result] = - previousPipelineResult.copy(underlyingResult = previousPipelineResult.underlyingResult - .copy(result = Some(executorResult.candidatesWithDetails))) - } - - val builtSteps = Seq( - GatesStep, - queryFeatureHydrationStep( - config.queryFeatureHydration, - CandidatePipelineConfig.fetchQueryFeaturesStep, - (pipelineResult, executorResult) => - pipelineResult.copy(queryFeatures = Some(executorResult)) - ), - queryFeatureHydrationStep( - config.queryFeatureHydrationPhase2, - CandidatePipelineConfig.fetchQueryFeaturesPhase2Step, - (pipelineResult, executorResult) => - pipelineResult.copy( - queryFeaturesPhase2 = Some(executorResult), - mergedAsyncQueryFeatures = Some( - pipelineResult.queryFeatures - .getOrElse( - throw InvalidStepStateException( - CandidatePipelineConfig.fetchQueryFeaturesPhase2Step, - "QueryFeatures") - ).asyncFeatureMap ++ executorResult.asyncFeatureMap) - ) - ), - asyncFeaturesStep(CandidatePipelineConfig.candidateSourceStep, context), - CandidateSourceStep, - asyncFeaturesStep(CandidatePipelineConfig.preFilterFeatureHydrationPhase1Step, context), - PreFilterFeatureHydrationPhase1Step, - asyncFeaturesStep(CandidatePipelineConfig.preFilterFeatureHydrationPhase2Step, context), - PreFilterFeatureHydrationPhase2Step, - asyncFeaturesStep(CandidatePipelineConfig.filtersStep, context), - FiltersStep, - asyncFeaturesStep(CandidatePipelineConfig.postFilterFeatureHydrationStep, context), - PostFilterFeatureHydrationStep, - asyncFeaturesStep(CandidatePipelineConfig.scorersStep, context), - ScorersStep, - asyncFeaturesStep(CandidatePipelineConfig.decoratorStep, context), - DecorationStep, - ResultStep - ) - - /** The main execution logic for this Candidate Pipeline. */ - val finalArrow: Arrow[CandidatePipeline.Inputs[Query], CandidatePipelineResult] = - buildCombinedArrowFromSteps( - steps = builtSteps, - context = context, - initialEmptyResult = - IntermediateCandidatePipelineResult.empty[Result](config.candidateSource.identifier), - stepsInOrderFromConfig = CandidatePipelineConfig.stepsInOrder - ).map(_.underlyingResult) - - val configFromBuilder = config - new CandidatePipeline[Query] { - override private[core] val config: BaseCandidatePipelineConfig[Query, _, _, _] = - configFromBuilder - override val arrow: Arrow[CandidatePipeline.Inputs[Query], CandidatePipelineResult] = - finalArrow - override val identifier: CandidatePipelineIdentifier = pipelineIdentifier - override val alerts: Seq[Alert] = config.alerts - override val children: Seq[Component] = - allGates ++ - config.queryFeatureHydration ++ - Seq(queryTransformer, config.candidateSource, resultsTransformer) ++ - config.featuresFromCandidateSourceTransformers ++ - decorator.toSeq ++ - config.preFilterFeatureHydrationPhase1 ++ - config.filters ++ - config.postFilterFeatureHydration ++ - config.scorers - } - } - - private case class CandidateWithFeaturesImpl(candidate: Result, features: FeatureMap) - extends CandidateWithFeatures[Result] -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipelineBuilderFactory.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipelineBuilderFactory.docx new file mode 100644 index 000000000..e3cc99eae Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipelineBuilderFactory.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipelineBuilderFactory.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipelineBuilderFactory.scala deleted file mode 100644 index f488b341d..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipelineBuilderFactory.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.candidate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.service.async_feature_map_executor.AsyncFeatureMapExecutor -import com.twitter.product_mixer.core.service.candidate_decorator_executor.CandidateDecoratorExecutor -import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutor -import com.twitter.product_mixer.core.service.candidate_source_executor.CandidateSourceExecutor -import com.twitter.product_mixer.core.service.filter_executor.FilterExecutor -import com.twitter.product_mixer.core.service.gate_executor.GateExecutor -import com.twitter.product_mixer.core.service.group_results_executor.GroupResultsExecutor -import com.twitter.product_mixer.core.service.query_feature_hydrator_executor.QueryFeatureHydratorExecutor -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class CandidatePipelineBuilderFactory @Inject() ( - queryFeatureHydratorExecutor: QueryFeatureHydratorExecutor, - asyncFeatureMapExecutor: AsyncFeatureMapExecutor, - candidateDecoratorExecutor: CandidateDecoratorExecutor, - candidateFeatureHydratorExecutor: CandidateFeatureHydratorExecutor, - candidateSourceExecutor: CandidateSourceExecutor, - groupResultsExecutor: GroupResultsExecutor, - filterExecutor: FilterExecutor, - gateExecutor: GateExecutor, - statsReceiver: StatsReceiver) { - def get[ - Query <: PipelineQuery, - CandidateSourceQuery, - CandidateSourceResult, - Result <: UniversalNoun[Any] - ]: CandidatePipelineBuilder[ - Query, - CandidateSourceQuery, - CandidateSourceResult, - Result - ] = { - new CandidatePipelineBuilder[ - Query, - CandidateSourceQuery, - CandidateSourceResult, - Result - ]( - queryFeatureHydratorExecutor, - asyncFeatureMapExecutor, - candidateDecoratorExecutor, - candidateFeatureHydratorExecutor, - candidateSourceExecutor, - groupResultsExecutor, - filterExecutor, - gateExecutor, - statsReceiver - ) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipelineConfig.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipelineConfig.docx new file mode 100644 index 000000000..1f6908489 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipelineConfig.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipelineConfig.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipelineConfig.scala deleted file mode 100644 index bbdcb2d50..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipelineConfig.scala +++ /dev/null @@ -1,264 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.candidate - -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.gate.BaseGate -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.functional_component.transformer._ -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifierStack -import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineConfig -import com.twitter.product_mixer.core.pipeline.PipelineConfigCompanion -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.decider.DeciderParam - -sealed trait BaseCandidatePipelineConfig[ - -Query <: PipelineQuery, - CandidateSourceQuery, - CandidateSourceResult, - Result <: UniversalNoun[Any]] - extends PipelineConfig { - - val identifier: CandidatePipelineIdentifier - - /** - * A candidate pipeline can fetch query-level features for use within the candidate source. It's - * generally recommended to set a hydrator in the parent recos or mixer pipeline if multiple - * candidate pipelines share the same feature but if a specific query feature hydrator is used - * by one pipeline and you don't want to block the others, you could explicitly set it here. - * If a feature is hydrated both in the parent pipeline or here, this one takes priority. - */ - def queryFeatureHydration: Seq[BaseQueryFeatureHydrator[Query, _]] = Seq.empty - - /** - * For query-level features that are dependent on query-level features from [[queryFeatureHydration]] - */ - def queryFeatureHydrationPhase2: Seq[BaseQueryFeatureHydrator[Query, _]] = Seq.empty - - /** - * When these Params are defined, they will automatically be added as Gates in the pipeline - * by the CandidatePipelineBuilder - * - * The enabled decider param can to be used to quickly disable a Candidate Pipeline via Decider - */ - val enabledDeciderParam: Option[DeciderParam[Boolean]] = None - - /** - * This supported client feature switch param can be used with a Feature Switch to control the - * rollout of a new Candidate Pipeline from dogfood to experiment to production - */ - val supportedClientParam: Option[FSParam[Boolean]] = None - - /** [[Gate]]s that are applied sequentially, the pipeline will only run if all the Gates are open */ - def gates: Seq[BaseGate[Query]] = Seq.empty - - /** - * A pair of transforms to adapt the underlying candidate source to the pipeline's query and result types - * Complex use cases such as those that need access to features should construct their own transformer, but - * for simple use cases, you can pass in an anonymous function. - * @example - * {{{ override val queryTransformer: CandidatePipelineQueryTransformer[Query, CandidateSourceQuery] = { query => - * query.toExampleThrift - * } - * }}} - */ - def queryTransformer: BaseCandidatePipelineQueryTransformer[ - Query, - CandidateSourceQuery - ] - - /** Source for Candidates for this Pipeline */ - def candidateSource: BaseCandidateSource[CandidateSourceQuery, CandidateSourceResult] - - /** - * [[CandidateFeatureTransformer]] allow you to define [[com.twitter.product_mixer.core.feature.Feature]] extraction logic from your [[CandidateSource]] results. - * If your candidate sources return [[com.twitter.product_mixer.core.feature.Feature]]s alongside the candidate that might be useful later on, - * add transformers for constructing feature maps. - * - * @note If multiple transformers extract the same feature, the last one takes priority and is kept. - */ - def featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[CandidateSourceResult] - ] = Seq.empty - - /** - * a result Transformer may throw PipelineFailure for candidates that are malformed and - * should be removed. This should be exceptional behavior, and not a replacement for adding a Filter. - * Complex use cases such as those that need access to features should construct their own transformer, but - * for simple use cases, you can pass in an anonymous function. - * @example - * {{{ override val queryTransformer: CandidatePipelineResultsTransformer[CandidateSourceResult, Result] = { sourceResult => - * ExampleCandidate(sourceResult.id) - * } - * }}} - * - */ - val resultTransformer: CandidatePipelineResultsTransformer[CandidateSourceResult, Result] - - /** - * Before filters are run, you can fetch features for each candidate. - * - * Uses Stitch, so you're encouraged to use a working Stitch Adaptor to batch between candidates. - * - * The existing features (from the candidate source) are passed in as an input. You are not expected - * to put them into the resulting feature map yourself - they will be merged for you by the platform. - * - * This API is likely to change when Product Mixer does managed feature hydration - */ - val preFilterFeatureHydrationPhase1: Seq[BaseCandidateFeatureHydrator[Query, Result, _]] = - Seq.empty - - /** - * A second phase of feature hydration that can be run before filtering and after the first phase - * of [[preFilterFeatureHydrationPhase1]]. You are not expected to put them into the resulting - * feature map yourself - they will be merged for you by the platform. - */ - val preFilterFeatureHydrationPhase2: Seq[BaseCandidateFeatureHydrator[Query, Result, _]] = - Seq.empty - - /** A list of filters to apply. Filters will be applied in sequential order. */ - def filters: Seq[Filter[Query, Result]] = Seq.empty - - /** - * After filters are run, you can fetch features for each candidate. - * - * Uses Stitch, so you're encouraged to use a working Stitch Adaptor to batch between candidates. - * - * The existing features (from the candidate source) & pre-filtering are passed in as an input. - * You are not expected to put them into the resulting feature map yourself - - * they will be merged for you by the platform. - * - * This API is likely to change when Product Mixer does managed feature hydration - */ - val postFilterFeatureHydration: Seq[BaseCandidateFeatureHydrator[Query, Result, _]] = Seq.empty - - /** - * Decorators allow for adding Presentations to candidates. While the Presentation can contain any - * arbitrary data, Decorators are often used to add a UrtItemPresentation for URT item support, or - * a UrtModulePresentation for grouping the candidates in a URT module. - */ - val decorator: Option[CandidateDecorator[Query, Result]] = None - - /** - * A candidate pipeline can define a partial function to rescue failures here. They will be treated as failures - * from a monitoring standpoint, and cancellation exceptions will always be propagated (they cannot be caught here). - */ - def failureClassifier: PartialFunction[Throwable, PipelineFailure] = PartialFunction.empty - - /** - * Scorers for candidates. Scorers are executed in parallel. Order does not matter. - */ - def scorers: Seq[Scorer[Query, Result]] = Seq.empty - - /** - * Alerts can be used to indicate the pipeline's service level objectives. Alerts and - * dashboards will be automatically created based on this information. - */ - val alerts: Seq[Alert] = Seq.empty - - /** - * This method is used by the product mixer framework to build the pipeline. - */ - private[core] final def build( - parentComponentIdentifierStack: ComponentIdentifierStack, - factory: CandidatePipelineBuilderFactory - ): CandidatePipeline[Query] = { - factory.get.build(parentComponentIdentifierStack, this) - } -} - -trait CandidatePipelineConfig[ - -Query <: PipelineQuery, - CandidateSourceQuery, - CandidateSourceResult, - Result <: UniversalNoun[Any]] - extends BaseCandidatePipelineConfig[ - Query, - CandidateSourceQuery, - CandidateSourceResult, - Result - ] { - override val gates: Seq[Gate[Query]] = Seq.empty - - override val queryTransformer: CandidatePipelineQueryTransformer[ - Query, - CandidateSourceQuery - ] -} - -trait DependentCandidatePipelineConfig[ - -Query <: PipelineQuery, - CandidateSourceQuery, - CandidateSourceResult, - Result <: UniversalNoun[Any]] - extends BaseCandidatePipelineConfig[ - Query, - CandidateSourceQuery, - CandidateSourceResult, - Result - ] - -/** - * Contains [[PipelineStepIdentifier]]s for the Steps that are available for all [[BaseCandidatePipelineConfig]]s - */ -object CandidatePipelineConfig extends PipelineConfigCompanion { - val gatesStep: PipelineStepIdentifier = PipelineStepIdentifier("Gates") - val fetchQueryFeaturesStep: PipelineStepIdentifier = PipelineStepIdentifier("FetchQueryFeatures") - val fetchQueryFeaturesPhase2Step: PipelineStepIdentifier = PipelineStepIdentifier( - "FetchQueryFeaturesPhase2") - val candidateSourceStep: PipelineStepIdentifier = PipelineStepIdentifier("CandidateSource") - val preFilterFeatureHydrationPhase1Step: PipelineStepIdentifier = - PipelineStepIdentifier("PreFilterFeatureHydration") - val preFilterFeatureHydrationPhase2Step: PipelineStepIdentifier = - PipelineStepIdentifier("PreFilterFeatureHydrationPhase2") - val filtersStep: PipelineStepIdentifier = PipelineStepIdentifier("Filters") - val postFilterFeatureHydrationStep: PipelineStepIdentifier = - PipelineStepIdentifier("PostFilterFeatureHydration") - val scorersStep: PipelineStepIdentifier = PipelineStepIdentifier("Scorer") - val decoratorStep: PipelineStepIdentifier = PipelineStepIdentifier("Decorator") - val resultStep: PipelineStepIdentifier = PipelineStepIdentifier("Result") - - /** All the steps which are executed by a [[CandidatePipeline]] in the order in which they are run */ - override val stepsInOrder: Seq[PipelineStepIdentifier] = Seq( - gatesStep, - fetchQueryFeaturesStep, - fetchQueryFeaturesPhase2Step, - asyncFeaturesStep(candidateSourceStep), - candidateSourceStep, - asyncFeaturesStep(preFilterFeatureHydrationPhase1Step), - preFilterFeatureHydrationPhase1Step, - asyncFeaturesStep(preFilterFeatureHydrationPhase2Step), - preFilterFeatureHydrationPhase2Step, - asyncFeaturesStep(filtersStep), - filtersStep, - asyncFeaturesStep(postFilterFeatureHydrationStep), - postFilterFeatureHydrationStep, - asyncFeaturesStep(scorersStep), - scorersStep, - asyncFeaturesStep(decoratorStep), - decoratorStep, - resultStep - ) - - override val stepsAsyncFeatureHydrationCanBeCompletedBy: Set[PipelineStepIdentifier] = Set( - candidateSourceStep, - preFilterFeatureHydrationPhase1Step, - preFilterFeatureHydrationPhase2Step, - filtersStep, - postFilterFeatureHydrationStep, - scorersStep, - decoratorStep - ) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipelineResult.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipelineResult.docx new file mode 100644 index 000000000..bf69d8813 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipelineResult.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipelineResult.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipelineResult.scala deleted file mode 100644 index 7729cbcf8..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/CandidatePipelineResult.scala +++ /dev/null @@ -1,93 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.candidate - -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.asyncfeaturemap.AsyncFeatureMap -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineResult -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.service.async_feature_map_executor.AsyncFeatureMapExecutorResults -import com.twitter.product_mixer.core.service.candidate_decorator_executor.CandidateDecoratorExecutorResult -import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutorResult -import com.twitter.product_mixer.core.service.candidate_source_executor.CandidateSourceExecutorResult -import com.twitter.product_mixer.core.service.filter_executor.FilterExecutorResult -import com.twitter.product_mixer.core.service.gate_executor.GateExecutorResult -import com.twitter.product_mixer.core.service.query_feature_hydrator_executor.QueryFeatureHydratorExecutor - -case class CandidatePipelineResult( - candidateSourceIdentifier: CandidateSourceIdentifier, - gateResult: Option[GateExecutorResult], - queryFeatures: Option[QueryFeatureHydratorExecutor.Result], - queryFeaturesPhase2: Option[QueryFeatureHydratorExecutor.Result], - mergedAsyncQueryFeatures: Option[AsyncFeatureMap], - candidateSourceResult: Option[CandidateSourceExecutorResult[UniversalNoun[Any]]], - preFilterHydrationResult: Option[CandidateFeatureHydratorExecutorResult[UniversalNoun[Any]]], - preFilterHydrationResultPhase2: Option[ - CandidateFeatureHydratorExecutorResult[UniversalNoun[Any]] - ], - filterResult: Option[FilterExecutorResult[UniversalNoun[Any]]], - postFilterHydrationResult: Option[CandidateFeatureHydratorExecutorResult[UniversalNoun[Any]]], - candidateDecoratorResult: Option[CandidateDecoratorExecutorResult], - scorersResult: Option[CandidateFeatureHydratorExecutorResult[UniversalNoun[Any]]], - asyncFeatureHydrationResults: Option[AsyncFeatureMapExecutorResults], - failure: Option[PipelineFailure], - result: Option[Seq[CandidateWithDetails]]) - extends PipelineResult[Seq[CandidateWithDetails]] { - - override def withFailure(failure: PipelineFailure): CandidatePipelineResult = - copy(failure = Some(failure)) - - override def withResult( - result: Seq[CandidateWithDetails] - ): CandidatePipelineResult = copy(result = Some(result)) - - override val resultSize: Int = result.map(PipelineResult.resultSize).getOrElse(0) -} - -private[candidate] object IntermediateCandidatePipelineResult { - def empty[Candidate <: UniversalNoun[Any]]( - candidateSourceIdentifier: CandidateSourceIdentifier - ): IntermediateCandidatePipelineResult[Candidate] = { - IntermediateCandidatePipelineResult( - CandidatePipelineResult( - candidateSourceIdentifier = candidateSourceIdentifier, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None - ), - None - ) - } -} - -private[candidate] case class IntermediateCandidatePipelineResult[Candidate <: UniversalNoun[Any]]( - underlyingResult: CandidatePipelineResult, - featureMaps: Option[Map[Candidate, FeatureMap]]) - extends PipelineResult[Seq[CandidateWithDetails]] { - override val failure: Option[PipelineFailure] = underlyingResult.failure - override val result: Option[Seq[CandidateWithDetails]] = underlyingResult.result - - override def withFailure( - failure: PipelineFailure - ): IntermediateCandidatePipelineResult[Candidate] = - copy(underlyingResult = underlyingResult.withFailure(failure)) - - override def withResult( - result: Seq[CandidateWithDetails] - ): IntermediateCandidatePipelineResult[Candidate] = - copy(underlyingResult = underlyingResult.withResult(result)) - - override def resultSize(): Int = underlyingResult.resultSize -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/PassthroughCandidatePipelineConfig.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/PassthroughCandidatePipelineConfig.docx new file mode 100644 index 000000000..1824303f1 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/PassthroughCandidatePipelineConfig.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/PassthroughCandidatePipelineConfig.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/PassthroughCandidatePipelineConfig.scala deleted file mode 100644 index 73baa8abb..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/PassthroughCandidatePipelineConfig.scala +++ /dev/null @@ -1,47 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.candidate - -import com.twitter.product_mixer.core.functional_component.candidate_source.PassthroughCandidateSource -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateExtractor -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object PassthroughCandidatePipelineConfig { - - /** - * Build a [[PassthroughCandidatePipelineConfig]] with a [[PassthroughCandidateSource]] built from - * a [[CandidateExtractor]] - */ - def apply[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]( - identifier: CandidatePipelineIdentifier, - extractor: CandidateExtractor[Query, Candidate], - decorator: Option[CandidateDecorator[Query, Candidate]] = None - ): PassthroughCandidatePipelineConfig[Query, Candidate] = { - - // Renaming variables to keep the interface clean, but avoid naming collisions when creating - // the anonymous class. - val _identifier = identifier - val _extractor = extractor - val _decorator = decorator - - new PassthroughCandidatePipelineConfig[Query, Candidate] { - override val identifier = _identifier - override val candidateSource = - PassthroughCandidateSource(CandidateSourceIdentifier(_identifier.name), _extractor) - override val decorator = _decorator - } - } -} - -trait PassthroughCandidatePipelineConfig[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]] - extends CandidatePipelineConfig[Query, Query, Candidate, Candidate] { - - override val queryTransformer: CandidatePipelineQueryTransformer[Query, Query] = identity - - override val resultTransformer: CandidatePipelineResultsTransformer[Candidate, Candidate] = - identity -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/StaticCandidatePipelineConfig.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/StaticCandidatePipelineConfig.docx new file mode 100644 index 000000000..5b0295d84 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/StaticCandidatePipelineConfig.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/StaticCandidatePipelineConfig.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/StaticCandidatePipelineConfig.scala deleted file mode 100644 index 2829e8334..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate/StaticCandidatePipelineConfig.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.candidate - -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.functional_component.candidate_source.StaticCandidateSource -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator - -object StaticCandidatePipelineConfig { - - /** - * Build a [[StaticCandidatePipelineConfig]] with a [[CandidateSource]] that returns the [[candidate]] - */ - def apply[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]( - identifier: CandidatePipelineIdentifier, - candidate: Candidate, - decorator: Option[CandidateDecorator[Query, Candidate]] = None - ): StaticCandidatePipelineConfig[Query, Candidate] = { - - // Renaming variables to keep the interface clean, but avoid naming collisions when creating - // the anonymous class. - val _identifier = identifier - val _candidate = candidate - val _decorator = decorator - - new StaticCandidatePipelineConfig[Query, Candidate] { - override val identifier = _identifier - override val candidate = _candidate - override val decorator = _decorator - } - } -} - -trait StaticCandidatePipelineConfig[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]] - extends CandidatePipelineConfig[Query, Unit, Unit, Candidate] { - - val candidate: Candidate - - override def candidateSource: CandidateSource[Unit, Unit] = StaticCandidateSource[Unit]( - identifier = CandidateSourceIdentifier(identifier.name), - result = Seq(())) - - override val queryTransformer: CandidatePipelineQueryTransformer[Query, Unit] = _ => Unit - - override val resultTransformer: CandidatePipelineResultsTransformer[Unit, Candidate] = _ => - candidate -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/BUILD deleted file mode 100644 index bbda52b47..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/BUILD +++ /dev/null @@ -1,62 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/premarshaller", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/side_effect", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/async_feature_map_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_pipeline_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/domain_marshaller_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_result_side_effect_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/quality_factor_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transport_marshaller_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "stitch/stitch-core", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/premarshaller", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/side_effect", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_pipeline_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/domain_marshaller_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_result_side_effect_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/quality_factor_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transport_marshaller_executor", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/BUILD.docx new file mode 100644 index 000000000..700ab4025 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipeline.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipeline.docx new file mode 100644 index 000000000..cbbe04997 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipeline.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipeline.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipeline.scala deleted file mode 100644 index 645e20bea..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipeline.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.mixer - -import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier -import com.twitter.product_mixer.core.pipeline.Pipeline -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Arrow - -/** - * A Mixer Pipeline - * - * This is an abstract class, as we only construct these via the [[MixerPipelineBuilder]]. - * - * A [[MixerPipeline]] is capable of processing requests (queries) and returning responses (results) - * in the correct format to directly send to users. - * - * @tparam Query the domain model for the query or request - * @tparam Result the final marshalled result type - */ -abstract class MixerPipeline[Query <: PipelineQuery, Result] private[mixer] - extends Pipeline[Query, Result] { - override private[core] val config: MixerPipelineConfig[Query, _, Result] - override val arrow: Arrow[Query, MixerPipelineResult[Result]] - override val identifier: MixerPipelineIdentifier -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipelineBuilder.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipelineBuilder.docx new file mode 100644 index 000000000..517c9f359 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipelineBuilder.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipelineBuilder.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipelineBuilder.scala deleted file mode 100644 index 4cdb44552..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipelineBuilder.scala +++ /dev/null @@ -1,582 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.mixer - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.asyncfeaturemap.AsyncFeatureMap -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller -import com.twitter.product_mixer.core.model.common.Component -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifierStack -import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.FailOpenPolicy -import com.twitter.product_mixer.core.pipeline.InvalidStepStateException -import com.twitter.product_mixer.core.pipeline.PipelineBuilder -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipeline -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineBuilderFactory -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailureClassifier -import com.twitter.product_mixer.core.pipeline.pipeline_failure.ProductDisabled -import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus -import com.twitter.product_mixer.core.quality_factor.QualityFactorObserver -import com.twitter.product_mixer.core.quality_factor.QualityFactorStatus -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.async_feature_map_executor.AsyncFeatureMapExecutor -import com.twitter.product_mixer.core.service.async_feature_map_executor.AsyncFeatureMapExecutorResults -import com.twitter.product_mixer.core.service.candidate_pipeline_executor.CandidatePipelineExecutor -import com.twitter.product_mixer.core.service.candidate_pipeline_executor.CandidatePipelineExecutorResult -import com.twitter.product_mixer.core.service.domain_marshaller_executor.DomainMarshallerExecutor -import com.twitter.product_mixer.core.service.gate_executor.GateExecutor -import com.twitter.product_mixer.core.service.gate_executor.GateExecutorResult -import com.twitter.product_mixer.core.service.gate_executor.StoppedGateException -import com.twitter.product_mixer.core.service.pipeline_result_side_effect_executor.PipelineResultSideEffectExecutor -import com.twitter.product_mixer.core.service.quality_factor_executor.QualityFactorExecutorResult -import com.twitter.product_mixer.core.service.query_feature_hydrator_executor.QueryFeatureHydratorExecutor -import com.twitter.product_mixer.core.service.selector_executor.SelectorExecutor -import com.twitter.product_mixer.core.service.selector_executor.SelectorExecutorResult -import com.twitter.product_mixer.core.service.transport_marshaller_executor.TransportMarshallerExecutor -import com.twitter.stitch.Arrow -import com.twitter.util.logging.Logging - -/** - * MixerPipelineBuilder builds [[MixerPipeline]]s from [[MixerPipelineConfig]]s. - * - * You should inject a [[MixerPipelineBuilderFactory]] and call `.get` to build these. - * - * @see [[MixerPipelineConfig]] for the description of the type parameters - */ -class MixerPipelineBuilder[Query <: PipelineQuery, DomainResultType <: HasMarshalling, Result]( - candidatePipelineExecutor: CandidatePipelineExecutor, - gateExecutor: GateExecutor, - selectorExecutor: SelectorExecutor, - queryFeatureHydratorExecutor: QueryFeatureHydratorExecutor, - asyncFeatureMapExecutor: AsyncFeatureMapExecutor, - domainMarshallerExecutor: DomainMarshallerExecutor, - transportMarshallerExecutor: TransportMarshallerExecutor, - pipelineResultSideEffectExecutor: PipelineResultSideEffectExecutor, - candidatePipelineBuilderFactory: CandidatePipelineBuilderFactory, - override val statsReceiver: StatsReceiver) - extends PipelineBuilder[Query] - with Logging { - - override type UnderlyingResultType = Result - override type PipelineResultType = MixerPipelineResult[Result] - - def qualityFactorStep( - qualityFactorStatus: QualityFactorStatus - ): Step[Query, QualityFactorExecutorResult] = - new Step[Query, QualityFactorExecutorResult] { - override def identifier: PipelineStepIdentifier = MixerPipelineConfig.qualityFactorStep - - override def executorArrow: Arrow[Query, QualityFactorExecutorResult] = - Arrow - .map[Query, QualityFactorExecutorResult] { _ => - QualityFactorExecutorResult( - pipelineQualityFactors = - qualityFactorStatus.qualityFactorByPipeline.mapValues(_.currentValue) - ) - } - - override def inputAdaptor( - query: Query, - previousResult: MixerPipelineResult[Result] - ): Query = query - - override def resultUpdater( - previousPipelineResult: MixerPipelineResult[Result], - executorResult: QualityFactorExecutorResult - ): MixerPipelineResult[Result] = - previousPipelineResult.copy(qualityFactorResult = Some(executorResult)) - - override def queryUpdater( - query: Query, - executorResult: QualityFactorExecutorResult - ): Query = { - query match { - case queryWithQualityFactor: HasQualityFactorStatus => - queryWithQualityFactor - .withQualityFactorStatus( - queryWithQualityFactor.qualityFactorStatus.getOrElse(QualityFactorStatus.empty) ++ - qualityFactorStatus - ).asInstanceOf[Query] - case _ => - query - } - } - } - - def gatesStep( - gates: Seq[Gate[Query]], - context: Executor.Context - ): Step[Query, GateExecutorResult] = new Step[Query, GateExecutorResult] { - override def identifier: PipelineStepIdentifier = MixerPipelineConfig.gatesStep - - override def executorArrow: Arrow[Query, GateExecutorResult] = - gateExecutor.arrow(gates, context) - - override def inputAdaptor(query: Query, previousResult: MixerPipelineResult[Result]): Query = - query - - override def resultUpdater( - previousPipelineResult: MixerPipelineResult[Result], - executorResult: GateExecutorResult - ): MixerPipelineResult[Result] = - previousPipelineResult.copy(gateResult = Some(executorResult)) - } - - def fetchQueryFeaturesStep( - queryFeatureHydrators: Seq[QueryFeatureHydrator[Query]], - stepIdentifier: PipelineStepIdentifier, - updater: ResultUpdater[MixerPipelineResult[Result], QueryFeatureHydratorExecutor.Result], - context: Executor.Context - ): Step[Query, QueryFeatureHydratorExecutor.Result] = - new Step[Query, QueryFeatureHydratorExecutor.Result] { - override def identifier: PipelineStepIdentifier = stepIdentifier - - override def executorArrow: Arrow[Query, QueryFeatureHydratorExecutor.Result] = - queryFeatureHydratorExecutor.arrow( - queryFeatureHydrators, - MixerPipelineConfig.stepsAsyncFeatureHydrationCanBeCompletedBy, - context) - - override def inputAdaptor( - query: Query, - previousResult: MixerPipelineResult[Result] - ): Query = query - - override def resultUpdater( - previousPipelineResult: MixerPipelineResult[Result], - executorResult: QueryFeatureHydratorExecutor.Result - ): MixerPipelineResult[Result] = - updater(previousPipelineResult, executorResult) - - override def queryUpdater( - query: Query, - executorResult: QueryFeatureHydratorExecutor.Result - ): Query = - query - .withFeatureMap( - query.features - .getOrElse(FeatureMap.empty) ++ executorResult.featureMap).asInstanceOf[Query] - } - - def asyncFeaturesStep( - stepToHydrateFor: PipelineStepIdentifier, - context: Executor.Context - ): Step[AsyncFeatureMap, AsyncFeatureMapExecutorResults] = - new Step[AsyncFeatureMap, AsyncFeatureMapExecutorResults] { - override def identifier: PipelineStepIdentifier = - MixerPipelineConfig.asyncFeaturesStep(stepToHydrateFor) - - override def executorArrow: Arrow[AsyncFeatureMap, AsyncFeatureMapExecutorResults] = - asyncFeatureMapExecutor.arrow( - stepToHydrateFor, - identifier, - context - ) - - override def inputAdaptor( - query: Query, - previousResult: MixerPipelineResult[Result] - ): AsyncFeatureMap = - previousResult.mergedAsyncQueryFeatures - .getOrElse( - throw InvalidStepStateException(identifier, "MergedAsyncQueryFeatures") - ) - - override def resultUpdater( - previousPipelineResult: MixerPipelineResult[Result], - executorResult: AsyncFeatureMapExecutorResults - ): MixerPipelineResult[Result] = previousPipelineResult.copy( - asyncFeatureHydrationResults = previousPipelineResult.asyncFeatureHydrationResults match { - case Some(existingResults) => Some(existingResults ++ executorResult) - case None => Some(executorResult) - }) - - override def queryUpdater( - query: Query, - executorResult: AsyncFeatureMapExecutorResults - ): Query = - if (executorResult.featureMapsByStep - .getOrElse(stepToHydrateFor, FeatureMap.empty).isEmpty) { - query - } else { - query - .withFeatureMap( - query.features - .getOrElse(FeatureMap.empty) ++ executorResult.featureMapsByStep( - stepToHydrateFor)).asInstanceOf[Query] - } - } - - def candidatePipelinesStep( - candidatePipelines: Seq[CandidatePipeline[Query]], - defaultFailOpenPolicy: FailOpenPolicy, - failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy], - qualityFactorObserverByPipeline: Map[ComponentIdentifier, QualityFactorObserver], - context: Executor.Context - ): Step[CandidatePipeline.Inputs[Query], CandidatePipelineExecutorResult] = - new Step[CandidatePipeline.Inputs[Query], CandidatePipelineExecutorResult] { - override def identifier: PipelineStepIdentifier = MixerPipelineConfig.candidatePipelinesStep - - override def executorArrow: Arrow[CandidatePipeline.Inputs[ - Query - ], CandidatePipelineExecutorResult] = - candidatePipelineExecutor - .arrow( - candidatePipelines, - defaultFailOpenPolicy, - failOpenPolicies, - qualityFactorObserverByPipeline, - context - ) - - override def inputAdaptor( - query: Query, - previousResult: MixerPipelineResult[Result] - ): CandidatePipeline.Inputs[Query] = CandidatePipeline.Inputs[Query](query, Seq.empty) - - override def resultUpdater( - previousPipelineResult: MixerPipelineResult[Result], - executorResult: CandidatePipelineExecutorResult - ): MixerPipelineResult[Result] = - previousPipelineResult.copy(candidatePipelineResults = Some(executorResult)) - - override def queryUpdater( - query: Query, - executorResult: CandidatePipelineExecutorResult - ): Query = { - val updatedFeatureMap = query.features - .getOrElse(FeatureMap.empty) ++ executorResult.queryFeatureMap - query - .withFeatureMap(updatedFeatureMap).asInstanceOf[Query] - } - } - - def dependentCandidatePipelinesStep( - candidatePipelines: Seq[CandidatePipeline[Query]], - defaultFailOpenPolicy: FailOpenPolicy, - failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy], - qualityFactorObserverByPipeline: Map[ComponentIdentifier, QualityFactorObserver], - context: Executor.Context - ): Step[CandidatePipeline.Inputs[Query], CandidatePipelineExecutorResult] = - new Step[CandidatePipeline.Inputs[Query], CandidatePipelineExecutorResult] { - override def identifier: PipelineStepIdentifier = - MixerPipelineConfig.dependentCandidatePipelinesStep - - override def executorArrow: Arrow[CandidatePipeline.Inputs[ - Query - ], CandidatePipelineExecutorResult] = - candidatePipelineExecutor - .arrow( - candidatePipelines, - defaultFailOpenPolicy, - failOpenPolicies, - qualityFactorObserverByPipeline, - context - ) - - override def inputAdaptor( - query: Query, - previousResult: MixerPipelineResult[Result] - ): CandidatePipeline.Inputs[Query] = { - val previousCandidates = previousResult.candidatePipelineResults - .getOrElse { - throw InvalidStepStateException(identifier, "Candidates") - }.candidatePipelineResults.flatMap(_.result.getOrElse(Seq.empty)) - CandidatePipeline.Inputs[Query](query, previousCandidates) - } - - override def resultUpdater( - previousPipelineResult: MixerPipelineResult[Result], - executorResult: CandidatePipelineExecutorResult - ): MixerPipelineResult[Result] = - previousPipelineResult.copy(dependentCandidatePipelineResults = Some(executorResult)) - - override def queryUpdater( - query: Query, - executorResult: CandidatePipelineExecutorResult - ): Query = { - val updatedFeatureMap = query.features - .getOrElse(FeatureMap.empty) ++ executorResult.queryFeatureMap - query - .withFeatureMap(updatedFeatureMap).asInstanceOf[Query] - } - } - - def resultSelectorsStep( - selectors: Seq[Selector[Query]], - context: Executor.Context - ): Step[SelectorExecutor.Inputs[Query], SelectorExecutorResult] = - new Step[SelectorExecutor.Inputs[Query], SelectorExecutorResult] { - override def identifier: PipelineStepIdentifier = MixerPipelineConfig.resultSelectorsStep - - override def executorArrow: Arrow[SelectorExecutor.Inputs[Query], SelectorExecutorResult] = - selectorExecutor.arrow(selectors, context) - - override def inputAdaptor( - query: Query, - previousResult: MixerPipelineResult[Result] - ): SelectorExecutor.Inputs[Query] = { - val candidates = previousResult.candidatePipelineResults - .getOrElse { - throw InvalidStepStateException(identifier, "Candidates") - }.candidatePipelineResults.flatMap(_.result.getOrElse(Seq.empty)) - - val dependentCandidates = - previousResult.dependentCandidatePipelineResults - .getOrElse { - throw InvalidStepStateException(identifier, "DependentCandidates") - }.candidatePipelineResults.flatMap(_.result.getOrElse(Seq.empty)) - - SelectorExecutor.Inputs( - query = query, - candidatesWithDetails = candidates ++ dependentCandidates - ) - } - - override def resultUpdater( - previousPipelineResult: MixerPipelineResult[Result], - executorResult: SelectorExecutorResult - ): MixerPipelineResult[Result] = - previousPipelineResult.copy(resultSelectorResults = Some(executorResult)) - } - - def domainMarshallingStep( - domainMarshaller: DomainMarshaller[Query, DomainResultType], - context: Executor.Context - ): Step[DomainMarshallerExecutor.Inputs[Query], DomainMarshallerExecutor.Result[ - DomainResultType - ]] = - new Step[DomainMarshallerExecutor.Inputs[Query], DomainMarshallerExecutor.Result[ - DomainResultType - ]] { - override def identifier: PipelineStepIdentifier = MixerPipelineConfig.domainMarshallerStep - - override def executorArrow: Arrow[ - DomainMarshallerExecutor.Inputs[Query], - DomainMarshallerExecutor.Result[DomainResultType] - ] = - domainMarshallerExecutor.arrow(domainMarshaller, context) - - override def inputAdaptor( - query: Query, - previousResult: MixerPipelineResult[Result] - ): DomainMarshallerExecutor.Inputs[Query] = { - val selectorResults = previousResult.resultSelectorResults.getOrElse { - throw InvalidStepStateException(identifier, "SelectorResults") - } - - DomainMarshallerExecutor.Inputs( - query = query, - candidatesWithDetails = selectorResults.selectedCandidates - ) - } - - override def resultUpdater( - previousPipelineResult: MixerPipelineResult[Result], - executorResult: DomainMarshallerExecutor.Result[DomainResultType] - ): MixerPipelineResult[Result] = previousPipelineResult.copy( - domainMarshallerResults = Some(executorResult) - ) - } - - def resultSideEffectsStep( - sideEffects: Seq[PipelineResultSideEffect[Query, DomainResultType]], - context: Executor.Context - ): Step[ - PipelineResultSideEffect.Inputs[Query, DomainResultType], - PipelineResultSideEffectExecutor.Result - ] = new Step[ - PipelineResultSideEffect.Inputs[Query, DomainResultType], - PipelineResultSideEffectExecutor.Result - ] { - override def identifier: PipelineStepIdentifier = MixerPipelineConfig.resultSideEffectsStep - - override def executorArrow: Arrow[ - PipelineResultSideEffect.Inputs[Query, DomainResultType], - PipelineResultSideEffectExecutor.Result - ] = pipelineResultSideEffectExecutor.arrow(sideEffects, context) - - override def inputAdaptor( - query: Query, - previousResult: MixerPipelineResult[Result] - ): PipelineResultSideEffect.Inputs[Query, DomainResultType] = { - - val selectorResults = previousResult.resultSelectorResults.getOrElse { - throw InvalidStepStateException(identifier, "SelectorResults") - } - - val domainMarshallerResults = previousResult.domainMarshallerResults.getOrElse { - throw InvalidStepStateException(identifier, "DomainMarshallerResults") - } - - PipelineResultSideEffect.Inputs[Query, DomainResultType]( - query = query, - selectedCandidates = selectorResults.selectedCandidates, - remainingCandidates = selectorResults.remainingCandidates, - droppedCandidates = selectorResults.droppedCandidates, - response = domainMarshallerResults.result.asInstanceOf[DomainResultType] - ) - } - - override def resultUpdater( - previousPipelineResult: MixerPipelineResult[Result], - executorResult: PipelineResultSideEffectExecutor.Result - ): MixerPipelineResult[Result] = - previousPipelineResult.copy(resultSideEffectResults = Some(executorResult)) - } - - def transportMarshallingStep( - transportMarshaller: TransportMarshaller[DomainResultType, Result], - context: Executor.Context - ): Step[ - TransportMarshallerExecutor.Inputs[DomainResultType], - TransportMarshallerExecutor.Result[Result] - ] = new Step[TransportMarshallerExecutor.Inputs[ - DomainResultType - ], TransportMarshallerExecutor.Result[Result]] { - override def identifier: PipelineStepIdentifier = MixerPipelineConfig.transportMarshallerStep - - override def executorArrow: Arrow[TransportMarshallerExecutor.Inputs[ - DomainResultType - ], TransportMarshallerExecutor.Result[Result]] = - transportMarshallerExecutor.arrow(transportMarshaller, context) - - override def inputAdaptor( - query: Query, - previousResult: MixerPipelineResult[Result] - ): TransportMarshallerExecutor.Inputs[DomainResultType] = { - val domainMarshallingResults = previousResult.domainMarshallerResults.getOrElse { - throw InvalidStepStateException(identifier, "DomainMarshallerResults") - } - - // Since the PipelineResult just uses HasMarshalling - val domainResult = domainMarshallingResults.result.asInstanceOf[DomainResultType] - - TransportMarshallerExecutor.Inputs(domainResult) - } - - override def resultUpdater( - previousPipelineResult: MixerPipelineResult[Result], - executorResult: TransportMarshallerExecutor.Result[Result] - ): MixerPipelineResult[Result] = previousPipelineResult.copy( - transportMarshallerResults = Some(executorResult), - result = Some(executorResult.result) - ) - } - - def build( - parentComponentIdentifierStack: ComponentIdentifierStack, - config: MixerPipelineConfig[Query, DomainResultType, Result] - ): MixerPipeline[Query, Result] = { - - val pipelineIdentifier = config.identifier - - val context = Executor.Context( - PipelineFailureClassifier( - config.failureClassifier.orElse(StoppedGateException.classifier(ProductDisabled))), - parentComponentIdentifierStack.push(pipelineIdentifier) - ) - - val qualityFactorStatus: QualityFactorStatus = - QualityFactorStatus.build(config.qualityFactorConfigs) - - val qualityFactorObserverByPipeline = - qualityFactorStatus.qualityFactorByPipeline.mapValues { qualityFactor => - qualityFactor.buildObserver() - } - - buildGaugesForQualityFactor(pipelineIdentifier, qualityFactorStatus, statsReceiver) - - val candidatePipelines: Seq[CandidatePipeline[Query]] = config.candidatePipelines.map { - pipelineConfig: CandidatePipelineConfig[Query, _, _, _] => - pipelineConfig.build(context.componentStack, candidatePipelineBuilderFactory) - } - - val dependentCandidatePipelines: Seq[CandidatePipeline[Query]] = - config.dependentCandidatePipelines.map { - pipelineConfig: DependentCandidatePipelineConfig[Query, _, _, _] => - pipelineConfig.build(context.componentStack, candidatePipelineBuilderFactory) - } - - val builtSteps = Seq( - qualityFactorStep(qualityFactorStatus), - gatesStep(config.gates, context), - fetchQueryFeaturesStep( - config.fetchQueryFeatures, - MixerPipelineConfig.fetchQueryFeaturesStep, - (previousPipelineResult, executorResult) => - previousPipelineResult.copy(queryFeatures = Some(executorResult)), - context - ), - fetchQueryFeaturesStep( - config.fetchQueryFeaturesPhase2, - MixerPipelineConfig.fetchQueryFeaturesPhase2Step, - (previousPipelineResult, executorResult) => - previousPipelineResult.copy( - queryFeaturesPhase2 = Some(executorResult), - mergedAsyncQueryFeatures = Some( - previousPipelineResult.queryFeatures - .getOrElse(throw InvalidStepStateException( - MixerPipelineConfig.fetchQueryFeaturesPhase2Step, - "QueryFeatures")) - .asyncFeatureMap ++ executorResult.asyncFeatureMap) - ), - context - ), - asyncFeaturesStep(MixerPipelineConfig.candidatePipelinesStep, context), - candidatePipelinesStep( - candidatePipelines, - config.defaultFailOpenPolicy, - config.failOpenPolicies, - qualityFactorObserverByPipeline, - context), - asyncFeaturesStep(MixerPipelineConfig.dependentCandidatePipelinesStep, context), - dependentCandidatePipelinesStep( - dependentCandidatePipelines, - config.defaultFailOpenPolicy, - config.failOpenPolicies, - qualityFactorObserverByPipeline, - context), - asyncFeaturesStep(MixerPipelineConfig.resultSelectorsStep, context), - resultSelectorsStep(config.resultSelectors, context), - domainMarshallingStep(config.domainMarshaller, context), - asyncFeaturesStep(MixerPipelineConfig.resultSideEffectsStep, context), - resultSideEffectsStep(config.resultSideEffects, context), - transportMarshallingStep(config.transportMarshaller, context) - ) - - val finalArrow = buildCombinedArrowFromSteps( - steps = builtSteps, - context = context, - initialEmptyResult = MixerPipelineResult.empty, - stepsInOrderFromConfig = MixerPipelineConfig.stepsInOrder - ) - - val configFromBuilder = config - new MixerPipeline[Query, Result] { - override private[core] val config: MixerPipelineConfig[Query, _, Result] = configFromBuilder - override val arrow: Arrow[Query, MixerPipelineResult[Result]] = finalArrow - override val identifier: MixerPipelineIdentifier = pipelineIdentifier - override val alerts: Seq[Alert] = config.alerts - override val children: Seq[Component] = - config.gates ++ - config.fetchQueryFeatures ++ - candidatePipelines ++ - dependentCandidatePipelines ++ - config.resultSideEffects ++ - Seq(config.domainMarshaller, config.transportMarshaller) - } - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipelineBuilderFactory.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipelineBuilderFactory.docx new file mode 100644 index 000000000..3787212e9 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipelineBuilderFactory.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipelineBuilderFactory.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipelineBuilderFactory.scala deleted file mode 100644 index 4b8ebacbc..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipelineBuilderFactory.scala +++ /dev/null @@ -1,49 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.mixer - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineBuilderFactory -import com.twitter.product_mixer.core.service.candidate_pipeline_executor.CandidatePipelineExecutor -import com.twitter.product_mixer.core.service.domain_marshaller_executor.DomainMarshallerExecutor -import com.twitter.product_mixer.core.service.gate_executor.GateExecutor -import com.twitter.product_mixer.core.service.pipeline_result_side_effect_executor.PipelineResultSideEffectExecutor -import com.twitter.product_mixer.core.service.async_feature_map_executor.AsyncFeatureMapExecutor -import com.twitter.product_mixer.core.service.query_feature_hydrator_executor.QueryFeatureHydratorExecutor -import com.twitter.product_mixer.core.service.selector_executor.SelectorExecutor -import com.twitter.product_mixer.core.service.transport_marshaller_executor.TransportMarshallerExecutor - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class MixerPipelineBuilderFactory @Inject() ( - candidatePipelineExecutor: CandidatePipelineExecutor, - gateExecutor: GateExecutor, - selectorExecutor: SelectorExecutor, - queryFeatureHydratorExecutor: QueryFeatureHydratorExecutor, - asyncFeatureMapExecutor: AsyncFeatureMapExecutor, - domainMarshallerExecutor: DomainMarshallerExecutor, - transportMarshallerExecutor: TransportMarshallerExecutor, - pipelineResultSideEffectExecutor: PipelineResultSideEffectExecutor, - candidatePipelineBuilderFactory: CandidatePipelineBuilderFactory, - statsReceiver: StatsReceiver) { - def get[ - Query <: PipelineQuery, - DomainResultType <: HasMarshalling, - Result - ]: MixerPipelineBuilder[Query, DomainResultType, Result] = { - new MixerPipelineBuilder[Query, DomainResultType, Result]( - candidatePipelineExecutor, - gateExecutor, - selectorExecutor, - queryFeatureHydratorExecutor, - asyncFeatureMapExecutor, - domainMarshallerExecutor, - transportMarshallerExecutor, - pipelineResultSideEffectExecutor, - candidatePipelineBuilderFactory, - statsReceiver - ) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipelineConfig.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipelineConfig.docx new file mode 100644 index 000000000..6484365fa Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipelineConfig.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipelineConfig.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipelineConfig.scala deleted file mode 100644 index 238533143..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipelineConfig.scala +++ /dev/null @@ -1,175 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.mixer - -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifierStack -import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.FailOpenPolicy -import com.twitter.product_mixer.core.pipeline.PipelineConfig -import com.twitter.product_mixer.core.pipeline.PipelineConfigCompanion -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.pipeline_failure.ClosedGate -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.quality_factor.QualityFactorConfig - -/** - * This is the configuration necessary to generate a Mixer Pipeline. Product code should create a - * MixerPipelineConfig, and then use a MixerPipelineBuilder to get the final MixerPipeline which can - * process requests. - * - * @tparam Query - The domain model for the query or request - * @tparam UnmarshalledResultType - The result type of the pipeline, but before marshalling to a wire protocol like URT - * @tparam Result - The final result that will be served to users - */ -trait MixerPipelineConfig[Query <: PipelineQuery, UnmarshalledResultType <: HasMarshalling, Result] - extends PipelineConfig { - - override val identifier: MixerPipelineIdentifier - - /** - * Mixer Pipeline Gates will be executed before any other step (including retrieval from candidate - * pipelines). They're executed sequentially, and any "Stop" result will prevent pipeline execution. - */ - def gates: Seq[Gate[Query]] = Seq.empty - - /** - * A mixer pipeline can fetch query-level features before candidate pipelines are executed. - */ - def fetchQueryFeatures: Seq[QueryFeatureHydrator[Query]] = Seq.empty - - /** - * For query-level features that are dependent on query-level features from [[fetchQueryFeatures]] - */ - def fetchQueryFeaturesPhase2: Seq[QueryFeatureHydrator[Query]] = Seq.empty - - /** - * Candidate pipelines retrieve candidates for possible inclusion in the result - */ - def candidatePipelines: Seq[CandidatePipelineConfig[Query, _, _, _]] - - /** - * Dependent candidate pipelines to retrieve candidates that depend on the result of [[candidatePipelines]] - * [[DependentCandidatePipelineConfig]] have access to the list of previously retrieved & decorated - * candidates for use in constructing the query object. - */ - def dependentCandidatePipelines: Seq[DependentCandidatePipelineConfig[Query, _, _, _]] = Seq.empty - - /** - * [[defaultFailOpenPolicy]] is the [[FailOpenPolicy]] that will be applied to any candidate - * pipeline that isn't in the [[failOpenPolicies]] map. By default Candidate Pipelines will fail - * open for Closed Gates only. - */ - def defaultFailOpenPolicy: FailOpenPolicy = FailOpenPolicy(Set(ClosedGate)) - - /** - * [[failOpenPolicies]] associates [[FailOpenPolicy]]s to specific candidate pipelines using - * [[CandidatePipelineIdentifier]]. - * - * @note these [[FailOpenPolicy]]s override the [[defaultFailOpenPolicy]] for a mapped - * Candidate Pipeline. - */ - def failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = Map.empty - - /** - ** [[qualityFactorConfigs]] associates [[QualityFactorConfig]]s to specific candidate pipelines - * using [[CandidatePipelineIdentifier]]. - */ - def qualityFactorConfigs: Map[CandidatePipelineIdentifier, QualityFactorConfig] = - Map.empty - - /** - * Selectors are executed in sequential order to combine the candidates into a result - */ - def resultSelectors: Seq[Selector[Query]] - - /** - * Mixer result side effects that are executed after selection and domain marshalling - */ - def resultSideEffects: Seq[PipelineResultSideEffect[Query, UnmarshalledResultType]] = Seq() - - /** - * Domain marshaller transforms the selections into the model expected by the marshaller - */ - def domainMarshaller: DomainMarshaller[Query, UnmarshalledResultType] - - /** - * Transport marshaller transforms the model into our line-level API like URT or JSON - */ - def transportMarshaller: TransportMarshaller[UnmarshalledResultType, Result] - - /** - * A pipeline can define a partial function to rescue failures here. They will be treated as failures - * from a monitoring standpoint, and cancellation exceptions will always be propagated (they cannot be caught here). - */ - def failureClassifier: PartialFunction[Throwable, PipelineFailure] = PartialFunction.empty - - /** - * Alert can be used to indicate the pipeline's service level objectives. Alerts and - * dashboards will be automatically created based on this information. - */ - val alerts: Seq[Alert] = Seq.empty - - /** - * This method is used by the product mixer framework to build the pipeline. - */ - private[core] final def build( - parentComponentIdentifierStack: ComponentIdentifierStack, - builder: MixerPipelineBuilderFactory - ): MixerPipeline[Query, Result] = - builder.get.build(parentComponentIdentifierStack, this) -} - -object MixerPipelineConfig extends PipelineConfigCompanion { - val qualityFactorStep: PipelineStepIdentifier = PipelineStepIdentifier("QualityFactor") - val gatesStep: PipelineStepIdentifier = PipelineStepIdentifier("Gates") - val fetchQueryFeaturesStep: PipelineStepIdentifier = PipelineStepIdentifier("FetchQueryFeatures") - val fetchQueryFeaturesPhase2Step: PipelineStepIdentifier = - PipelineStepIdentifier("FetchQueryFeaturesPhase2") - val candidatePipelinesStep: PipelineStepIdentifier = PipelineStepIdentifier("CandidatePipelines") - val dependentCandidatePipelinesStep: PipelineStepIdentifier = - PipelineStepIdentifier("DependentCandidatePipelines") - val resultSelectorsStep: PipelineStepIdentifier = PipelineStepIdentifier("ResultSelectors") - val domainMarshallerStep: PipelineStepIdentifier = PipelineStepIdentifier("DomainMarshaller") - val resultSideEffectsStep: PipelineStepIdentifier = PipelineStepIdentifier("ResultSideEffects") - val transportMarshallerStep: PipelineStepIdentifier = PipelineStepIdentifier( - "TransportMarshaller") - - /** All the Steps which are executed by a [[MixerPipeline]] in the order in which they are run */ - override val stepsInOrder: Seq[PipelineStepIdentifier] = Seq( - qualityFactorStep, - gatesStep, - fetchQueryFeaturesStep, - fetchQueryFeaturesPhase2Step, - asyncFeaturesStep(candidatePipelinesStep), - candidatePipelinesStep, - asyncFeaturesStep(dependentCandidatePipelinesStep), - dependentCandidatePipelinesStep, - asyncFeaturesStep(resultSelectorsStep), - resultSelectorsStep, - domainMarshallerStep, - asyncFeaturesStep(resultSideEffectsStep), - resultSideEffectsStep, - transportMarshallerStep - ) - - /** - * All the Steps which an [[com.twitter.product_mixer.core.functional_component.feature_hydrator.AsyncHydrator AsyncHydrator]] - * can be configured to [[com.twitter.product_mixer.core.functional_component.feature_hydrator.AsyncHydrator.hydrateBefore hydrateBefore]] - */ - override val stepsAsyncFeatureHydrationCanBeCompletedBy: Set[PipelineStepIdentifier] = Set( - candidatePipelinesStep, - dependentCandidatePipelinesStep, - resultSelectorsStep, - resultSideEffectsStep - ) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipelineResult.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipelineResult.docx new file mode 100644 index 000000000..f18e7dfb4 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipelineResult.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipelineResult.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipelineResult.scala deleted file mode 100644 index 1ddf00bd4..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer/MixerPipelineResult.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.mixer - -import com.twitter.product_mixer.core.feature.featuremap.asyncfeaturemap.AsyncFeatureMap -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.PipelineResult -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.service.async_feature_map_executor.AsyncFeatureMapExecutorResults -import com.twitter.product_mixer.core.service.candidate_pipeline_executor.CandidatePipelineExecutorResult -import com.twitter.product_mixer.core.service.domain_marshaller_executor.DomainMarshallerExecutor -import com.twitter.product_mixer.core.service.gate_executor.GateExecutorResult -import com.twitter.product_mixer.core.service.pipeline_result_side_effect_executor.PipelineResultSideEffectExecutor -import com.twitter.product_mixer.core.service.quality_factor_executor.QualityFactorExecutorResult -import com.twitter.product_mixer.core.service.query_feature_hydrator_executor.QueryFeatureHydratorExecutor -import com.twitter.product_mixer.core.service.selector_executor.SelectorExecutorResult -import com.twitter.product_mixer.core.service.transport_marshaller_executor.TransportMarshallerExecutor - -/** - * A [[MixerPipelineResult]] includes both the user-visible [[PipelineResult]] and all the - * Execution details possible - intermediate results, what components did, etc. - */ -case class MixerPipelineResult[Result]( - qualityFactorResult: Option[QualityFactorExecutorResult], - gateResult: Option[GateExecutorResult], - queryFeatures: Option[QueryFeatureHydratorExecutor.Result], - queryFeaturesPhase2: Option[QueryFeatureHydratorExecutor.Result], - mergedAsyncQueryFeatures: Option[AsyncFeatureMap], - candidatePipelineResults: Option[CandidatePipelineExecutorResult], - dependentCandidatePipelineResults: Option[CandidatePipelineExecutorResult], - resultSelectorResults: Option[SelectorExecutorResult], - domainMarshallerResults: Option[DomainMarshallerExecutor.Result[HasMarshalling]], - resultSideEffectResults: Option[PipelineResultSideEffectExecutor.Result], - asyncFeatureHydrationResults: Option[AsyncFeatureMapExecutorResults], - transportMarshallerResults: Option[TransportMarshallerExecutor.Result[Result]], - failure: Option[PipelineFailure], - result: Option[Result]) - extends PipelineResult[Result] { - - override def withFailure(failure: PipelineFailure): PipelineResult[Result] = - copy(failure = Some(failure)) - - override def withResult(result: Result): PipelineResult[Result] = copy(result = Some(result)) - - /** - * resultSize is calculated based on the selector results rather than the marshalled results. The - * structure of the marshalled format is unknown, making operating on selector results more - * convenient. This will implicitly excluded cursors built during marshalling but cursors don't - * contribute to the result size anyway. - */ - override val resultSize: Int = - resultSelectorResults.map(_.selectedCandidates).map(PipelineResult.resultSize).getOrElse(0) -} - -object MixerPipelineResult { - def empty[A]: MixerPipelineResult[A] = MixerPipelineResult( - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None - ) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/BUILD deleted file mode 100644 index 68dfa5b8f..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "util/util-jackson/src/main/scala/com/twitter/util/jackson", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/BUILD.docx new file mode 100644 index 000000000..085ce90fe Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/PipelineFailure.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/PipelineFailure.docx new file mode 100644 index 000000000..9adfda191 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/PipelineFailure.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/PipelineFailure.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/PipelineFailure.scala deleted file mode 100644 index eef0db9b4..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/PipelineFailure.scala +++ /dev/null @@ -1,49 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.pipeline_failure - -import com.fasterxml.jackson.databind.annotation.JsonSerialize -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifierStack -import scala.util.control.NoStackTrace - -/** - * Pipeline Failures represent pipeline requests that were not able to complete. - * - * A pipeline result will always define either a result or a failure. - * - * The reason field should not be displayed to end-users, and is free to change over time. - * It should always be free of private user data such that we can log it. - * - * The pipeline can classify it's own failures into categories (timeouts, invalid arguments, - * rate limited, etc) such that the caller can choose how to handle it. - * - * @note [[componentStack]] should only be set by the product mixer framework, - * it should **NOT** be set when making a [[PipelineFailure]] - */ -@JsonSerialize(using = classOf[PipelineFailureSerializer]) -case class PipelineFailure( - category: PipelineFailureCategory, - reason: String, - underlying: Option[Throwable] = None, - componentStack: Option[ComponentIdentifierStack] = None) - extends Exception( - "PipelineFailure(" + - s"category = $category, " + - s"reason = $reason, " + - s"underlying = $underlying, " + - s"componentStack = $componentStack)", - underlying.orNull - ) { - override def toString: String = getMessage - - /** Returns an updated copy of this [[PipelineFailure]] with the same exception stacktrace */ - def copy( - category: PipelineFailureCategory = this.category, - reason: String = this.reason, - underlying: Option[Throwable] = this.underlying, - componentStack: Option[ComponentIdentifierStack] = this.componentStack - ): PipelineFailure = { - val newPipelineFailure = - new PipelineFailure(category, reason, underlying, componentStack) with NoStackTrace - newPipelineFailure.setStackTrace(this.getStackTrace) - newPipelineFailure - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/PipelineFailureCategory.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/PipelineFailureCategory.docx new file mode 100644 index 000000000..bd9cc74c4 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/PipelineFailureCategory.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/PipelineFailureCategory.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/PipelineFailureCategory.scala deleted file mode 100644 index f0cb57fe0..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/PipelineFailureCategory.scala +++ /dev/null @@ -1,190 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.pipeline_failure - -/** - * Failures are grouped into categories based on which party is 'responsible' for the issue. This - * is important for generating accurate SLOs and ensuring that the correct team is alerted. - */ -sealed trait PipelineFailureCategory { - val categoryName: String - val failureName: String -} - -/** - * Client Failures are failures where the client is deemed responsible for the issue. Such as by - * issuing an invalid request or not having the right permissions. - * - * A failure might belong in this category if it relates to behaviour on the client and is not - * actionable by the team which owns the product. - */ -trait ClientFailure extends PipelineFailureCategory { - override val categoryName: String = "ClientFailure" -} - -/** - * The requested product is disabled so the request cannot be served. - */ -case object ProductDisabled extends ClientFailure { - override val failureName: String = "ProductDisabled" -} - -/** - * The request was deemed invalid by this or a backing service. - */ -case object BadRequest extends ClientFailure { - override val failureName: String = "BadRequest" -} - -/** - * Credentials proving the identity of the caller were missing, not trusted, or expired. - * For example, an auth cookie might be expired and in need of refreshing. - * - * Do not confuse this with Authorization, where the credentials are believed but not allowed to perform the operation. - */ -case object Authentication extends ClientFailure { - override val failureName: String = "Authentication" -} - -/** - * The operation was forbidden (often, but not always, by a Strato access control policy). - * - * Do not confuse this with Authentication, where the given credentials were missing, not trusted, or expired. - */ -case object Unauthorized extends ClientFailure { - override val failureName: String = "Unauthorized" -} - -/** - * The operation returned a Not Found response. - */ -case object NotFound extends ClientFailure { - override val failureName: String = "NotFound" -} - -/** - * An invalid input is included in a cursor field. - */ -case object MalformedCursor extends ClientFailure { - override val failureName: String = "MalformedCursor" -} - -/** - * The operation did not succeed due to a closed gate - */ -case object ClosedGate extends ClientFailure { - override val failureName: String = "ClosedGate" -} - -/** - * Server Failures are failures for which the owner of the product is responsible. Typically this - * means the request was valid but an issue within Product Mixer or a dependent service prevented - * it from succeeding. - * - * Server Failures contribute to the success rate SLO for the product. - */ -trait ServerFailure extends PipelineFailureCategory { - override val categoryName: String = "ServerFailure" -} - -/** - * Unclassified failures occur when product code throws an exception that Product Mixer does not - * know how to classify. - * - * They can be used in failOpen policies, etc - but it's always preferred to instead add additional - * classification logic and to keep Unclassified failures at 0. - */ -case object UncategorizedServerFailure extends ServerFailure { - override val failureName: String = "UncategorizedServerFailure" -} - -/** - * A hydrator or transformer returned a misconfigured feature map, this indicates a customer - * configuration error. The owner of the component should make sure the hydrator always returns a - * [[FeatureMap]] with the all features defined in the hydrator also set in the map, it should not have - * any unregistered features nor should registered features be absent. - */ -case object MisconfiguredFeatureMapFailure extends ServerFailure { - override val failureName: String = "MisconfiguredFeatureMapFailure" -} - -/** - * A PipelineSelector returned an invalid ComponentIdentifier. - * - * A pipeline selector should choose the identifier of a pipeline that is contained by the 'pipelines' - * sequence of the ProductPipelineConfig. - */ -case object InvalidPipelineSelected extends ServerFailure { - override val failureName: String = "InvalidPipelineSelected" -} - -/** - * Failures that occur when product code reaches an unexpected or otherwise illegal state. - */ -case object IllegalStateFailure extends ServerFailure { - override val failureName: String = "IllegalStateFailure" -} - -/** - * An unexpected candidate was returned in a candidate source that was unable to be transformed. - */ -case object UnexpectedCandidateResult extends ServerFailure { - override val failureName: String = "UnexpectedCandidateResult" -} - -/** - * An unexpected Candidate was returned in a marshaller - */ -case object UnexpectedCandidateInMarshaller extends ServerFailure { - override val failureName: String = "UnexpectedCandidateInMarshaller" -} - -/** - * Pipeline execution failed due to an incorrectly configured quality factor (e.g, accessing - * quality factor state for a pipeline that does not have quality factor configured) - */ -case object MisconfiguredQualityFactor extends ServerFailure { - override val failureName: String = "MisconfiguredQualityFactor" -} - -/** - * Pipeline execution failed due to an incorrectly configured decorator (e.g, decorator - * returned the wrong type or tried to decorate an already decorated candidate) - */ -case object MisconfiguredDecorator extends ServerFailure { - override val failureName: String = "MisconfiguredDecorator" -} - -/** - * Candidate Source Pipeline execution failed due to a timeout. - */ -case object CandidateSourceTimeout extends ServerFailure { - override val failureName: String = "CandidateSourceTimeout" -} - -/** - * Platform Failures are issues in the core Product Mixer logic itself which prevent a pipeline from - * properly executing. These failures are the responsibility of the Product Mixer team. - */ -trait PlatformFailure extends PipelineFailureCategory { - override val categoryName: String = "PlatformFailure" -} - -/** - * Pipeline execution failed due to an unexpected error in Product Mixer. - * - * ExecutionFailed indicates a bug with the core Product Mixer execution logic rather than with a - * specific product. For example, a bug in PipelineBuilder leading to us returning a - * ProductPipelineResult that neither succeeded nor failed. - */ -case object ExecutionFailed extends PlatformFailure { - override val failureName: String = "ExecutionFailed" -} - -/** - * Pipeline execution failed due to a feature hydration failure. - * - * FeatureHydrationFailed indicates that the underlying hydration for a feature defined in a hydrator - * failed (e.g, typically from a RPC call failing). - */ -case object FeatureHydrationFailed extends PlatformFailure { - override val failureName: String = "FeatureHydrationFailed" -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/PipelineFailureClassifier.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/PipelineFailureClassifier.docx new file mode 100644 index 000000000..7f4261847 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/PipelineFailureClassifier.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/PipelineFailureClassifier.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/PipelineFailureClassifier.scala deleted file mode 100644 index 449542ba9..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/PipelineFailureClassifier.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.pipeline_failure - -/** Represents a way to classify a given [[Throwable]] to a [[PipelineFailure]] */ -case class PipelineFailureClassifier( - classifier: PartialFunction[Throwable, PipelineFailure]) - extends PartialFunction[Throwable, PipelineFailure] { - override def isDefinedAt(throwable: Throwable): Boolean = classifier.isDefinedAt(throwable) - override def apply(throwable: Throwable): PipelineFailure = classifier.apply(throwable) -} - -private[core] object PipelineFailureClassifier { - val Empty: PipelineFailureClassifier = PipelineFailureClassifier(PartialFunction.empty) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/PipelineFailureSerializer.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/PipelineFailureSerializer.docx new file mode 100644 index 000000000..3ff0d0dce Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/PipelineFailureSerializer.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/PipelineFailureSerializer.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/PipelineFailureSerializer.scala deleted file mode 100644 index 9e4607b4c..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure/PipelineFailureSerializer.scala +++ /dev/null @@ -1,67 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.pipeline_failure - -import com.fasterxml.jackson.core.JsonGenerator -import com.fasterxml.jackson.databind.JsonSerializer -import com.fasterxml.jackson.databind.SerializerProvider -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifierStack - -private[pipeline_failure] class PipelineFailureSerializer() - extends JsonSerializer[PipelineFailure] { - - private sealed trait BaseSerializableException - - private case class SerializableException( - `class`: String, - message: String, - stackTrace: Seq[String], - cause: Option[BaseSerializableException]) - extends BaseSerializableException - - private case class SerializablePipelineFailure( - category: String, - reason: String, - underlying: Option[BaseSerializableException], - componentStack: Option[ComponentIdentifierStack], - stackTrace: Seq[String]) - extends BaseSerializableException - - private def serializeStackTrace(stackTrace: Array[StackTraceElement]): Seq[String] = - stackTrace.map(stackTraceElement => "at " + stackTraceElement.toString) - - private def mkSerializableException( - t: Throwable, - recursionDepth: Int = 0 - ): Option[BaseSerializableException] = { - t match { - case _ if recursionDepth > 4 => - // in the unfortunate case of a super deep chain of exceptions, stop if we get too deep - None - case pipelineFailure: PipelineFailure => - Some( - SerializablePipelineFailure( - category = - pipelineFailure.category.categoryName + "/" + pipelineFailure.category.failureName, - reason = pipelineFailure.reason, - underlying = - pipelineFailure.underlying.flatMap(mkSerializableException(_, recursionDepth + 1)), - componentStack = pipelineFailure.componentStack, - stackTrace = serializeStackTrace(pipelineFailure.getStackTrace) - )) - case t => - Some( - SerializableException( - `class` = t.getClass.getName, - message = t.getMessage, - stackTrace = serializeStackTrace(t.getStackTrace), - cause = Option(t.getCause).flatMap(mkSerializableException(_, recursionDepth + 1)) - ) - ) - } - } - - override def serialize( - pipelineFailure: PipelineFailure, - gen: JsonGenerator, - serializers: SerializerProvider - ): Unit = serializers.defaultSerializeValue(mkSerializableException(pipelineFailure), gen) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/BUILD deleted file mode 100644 index 6a1c78097..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/BUILD +++ /dev/null @@ -1,51 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "configapi/configapi-core", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/gate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_execution_logger", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_selector_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/quality_factor_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "stitch/stitch-core", - "stringcenter/client/src/main/scala/com/twitter/stringcenter/client", - "stringcenter/client/src/main/scala/com/twitter/stringcenter/client/stitch", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "configapi/configapi-core", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/gate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_execution_logger", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_selector_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/quality_factor_executor", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/BUILD.docx new file mode 100644 index 000000000..afa5f4615 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipeline.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipeline.docx new file mode 100644 index 000000000..f105265e4 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipeline.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipeline.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipeline.scala deleted file mode 100644 index 407379234..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipeline.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.product - -import com.twitter.product_mixer.core.functional_component.common.access_policy.WithDebugAccessPolicies -import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.Request -import com.twitter.product_mixer.core.pipeline.Pipeline -import com.twitter.stitch.Arrow - -/** - * A Product Pipeline - * - * This is an abstract class, as we only construct these via the [[ProductPipelineBuilder]]. - * - * A [[ProductPipeline]] is capable of processing a [[Request]] and returning a response. - * - * @tparam RequestType the domain model for the query or request - * @tparam ResponseType the final marshalled result type - */ -abstract class ProductPipeline[RequestType <: Request, ResponseType] private[product] - extends Pipeline[ProductPipelineRequest[RequestType], ResponseType] - with WithDebugAccessPolicies { - override private[core] val config: ProductPipelineConfig[RequestType, _, ResponseType] - override val arrow: Arrow[ - ProductPipelineRequest[RequestType], - ProductPipelineResult[ResponseType] - ] - override val identifier: ProductPipelineIdentifier -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineBuilder.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineBuilder.docx new file mode 100644 index 000000000..d2867967f Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineBuilder.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineBuilder.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineBuilder.scala deleted file mode 100644 index 600a70eed..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineBuilder.scala +++ /dev/null @@ -1,385 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.product - -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.tracing.Trace -import com.twitter.finagle.transport.Transport -import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.gate.DenyLoggedOutUsersGate -import com.twitter.product_mixer.core.gate.ParamGate -import com.twitter.product_mixer.core.gate.ParamGate.EnabledGateSuffix -import com.twitter.product_mixer.core.gate.ParamGate.SupportedClientGateSuffix -import com.twitter.product_mixer.core.model.common.Component -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifierStack -import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.Request -import com.twitter.product_mixer.core.pipeline.InvalidStepStateException -import com.twitter.product_mixer.core.pipeline.Pipeline -import com.twitter.product_mixer.core.pipeline.PipelineBuilder -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineBuilderFactory -import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig -import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineResult -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailureClassifier -import com.twitter.product_mixer.core.pipeline.pipeline_failure.ProductDisabled -import com.twitter.product_mixer.core.pipeline.recommendation.RecommendationPipelineBuilderFactory -import com.twitter.product_mixer.core.pipeline.recommendation.RecommendationPipelineConfig -import com.twitter.product_mixer.core.pipeline.recommendation.RecommendationPipelineResult -import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus -import com.twitter.product_mixer.core.quality_factor.QualityFactorObserver -import com.twitter.product_mixer.core.quality_factor.QualityFactorStatus -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.gate_executor.GateExecutor -import com.twitter.product_mixer.core.service.gate_executor.GateExecutorResult -import com.twitter.product_mixer.core.service.gate_executor.StoppedGateException -import com.twitter.product_mixer.core.service.pipeline_execution_logger.PipelineExecutionLogger -import com.twitter.product_mixer.core.service.pipeline_executor.PipelineExecutor -import com.twitter.product_mixer.core.service.pipeline_executor.PipelineExecutorRequest -import com.twitter.product_mixer.core.service.pipeline_executor.PipelineExecutorResult -import com.twitter.product_mixer.core.service.pipeline_selector_executor.PipelineSelectorExecutor -import com.twitter.product_mixer.core.service.pipeline_selector_executor.PipelineSelectorExecutorResult -import com.twitter.product_mixer.core.service.quality_factor_executor.QualityFactorExecutorResult -import com.twitter.stitch.Arrow -import com.twitter.stringcenter.client.StringCenterRequestContext -import com.twitter.stringcenter.client.stitch.StringCenterRequestContextLetter -import com.twitter.timelines.configapi.Params -import com.twitter.util.logging.Logging -import org.slf4j.MDC - -class ProductPipelineBuilder[TRequest <: Request, Query <: PipelineQuery, Response]( - gateExecutor: GateExecutor, - pipelineSelectorExecutor: PipelineSelectorExecutor, - pipelineExecutor: PipelineExecutor, - mixerPipelineBuilderFactory: MixerPipelineBuilderFactory, - recommendationPipelineBuilderFactory: RecommendationPipelineBuilderFactory, - override val statsReceiver: StatsReceiver, - pipelineExecutionLogger: PipelineExecutionLogger) - extends PipelineBuilder[ProductPipelineRequest[TRequest]] - with Logging { builder => - - override type UnderlyingResultType = Response - override type PipelineResultType = ProductPipelineResult[Response] - - /** - * Query Transformer Step is implemented inline instead of using an executor. - * - * It's a simple, synchronous step that executes the query transformer. - * - * Since the output of the transformer is used in multiple other steps (Gate, Pipeline Execution), - * we've promoted the transformer to a step so that it's outputs can be reused easily. - */ - def pipelineQueryTransformerStep( - queryTransformer: (TRequest, Params) => Query, - context: Executor.Context - ): Step[ProductPipelineRequest[TRequest], Query] = - new Step[ProductPipelineRequest[TRequest], Query] { - - override def identifier: PipelineStepIdentifier = - ProductPipelineConfig.pipelineQueryTransformerStep - - override def executorArrow: Arrow[ProductPipelineRequest[TRequest], Query] = { - wrapWithErrorHandling(context, identifier)( - Arrow.map[ProductPipelineRequest[TRequest], Query] { - case ProductPipelineRequest(request, params) => queryTransformer(request, params) - } - ) - } - - override def inputAdaptor( - query: ProductPipelineRequest[TRequest], - previousResult: ProductPipelineResult[Response] - ): ProductPipelineRequest[TRequest] = query - - override def resultUpdater( - previousPipelineResult: ProductPipelineResult[Response], - executorResult: Query - ): ProductPipelineResult[Response] = - previousPipelineResult.copy(transformedQuery = Some(executorResult)) - } - - def qualityFactorStep( - qualityFactorStatus: QualityFactorStatus - ): Step[Query, QualityFactorExecutorResult] = { - new Step[Query, QualityFactorExecutorResult] { - override def identifier: PipelineStepIdentifier = ProductPipelineConfig.qualityFactorStep - - override def executorArrow: Arrow[Query, QualityFactorExecutorResult] = - Arrow - .map[Query, QualityFactorExecutorResult] { _ => - QualityFactorExecutorResult( - pipelineQualityFactors = - qualityFactorStatus.qualityFactorByPipeline.mapValues(_.currentValue) - ) - } - - override def inputAdaptor( - query: ProductPipelineRequest[TRequest], - previousResult: ProductPipelineResult[Response] - ): Query = previousResult.transformedQuery - .getOrElse { - throw InvalidStepStateException(identifier, "TransformedQuery") - }.asInstanceOf[Query] - - override def resultUpdater( - previousPipelineResult: ProductPipelineResult[Response], - executorResult: QualityFactorExecutorResult - ): ProductPipelineResult[Response] = { - previousPipelineResult.copy( - transformedQuery = previousPipelineResult.transformedQuery.map { - case queryWithQualityFactor: HasQualityFactorStatus => - queryWithQualityFactor - .withQualityFactorStatus(qualityFactorStatus).asInstanceOf[Query] - case query => - query - }, - qualityFactorResult = Some(executorResult) - ) - } - } - } - - def gatesStep( - gates: Seq[Gate[Query]], - context: Executor.Context - ): Step[Query, GateExecutorResult] = new Step[Query, GateExecutorResult] { - override def identifier: PipelineStepIdentifier = ProductPipelineConfig.gatesStep - - override def executorArrow: Arrow[Query, GateExecutorResult] = { - gateExecutor.arrow(gates, context) - } - - override def inputAdaptor( - query: ProductPipelineRequest[TRequest], - previousResult: ProductPipelineResult[Response] - ): Query = previousResult.transformedQuery - .getOrElse { - throw InvalidStepStateException(identifier, "TransformedQuery") - }.asInstanceOf[Query] - - override def resultUpdater( - previousPipelineResult: ProductPipelineResult[Response], - executorResult: GateExecutorResult - ): ProductPipelineResult[Response] = - previousPipelineResult.copy(gateResult = Some(executorResult)) - } - - def pipelineSelectorStep( - pipelineByIdentifer: Map[ComponentIdentifier, Pipeline[Query, Response]], - pipelineSelector: Query => ComponentIdentifier, - context: Executor.Context - ): Step[Query, PipelineSelectorExecutorResult] = - new Step[Query, PipelineSelectorExecutorResult] { - override def identifier: PipelineStepIdentifier = ProductPipelineConfig.pipelineSelectorStep - - override def executorArrow: Arrow[ - Query, - PipelineSelectorExecutorResult - ] = pipelineSelectorExecutor.arrow(pipelineByIdentifer, pipelineSelector, context) - - override def inputAdaptor( - query: ProductPipelineRequest[TRequest], - previousResult: ProductPipelineResult[Response] - ): Query = - previousResult.transformedQuery - .getOrElse(throw InvalidStepStateException(identifier, "TransformedQuery")).asInstanceOf[ - Query] - - override def resultUpdater( - previousPipelineResult: ProductPipelineResult[Response], - executorResult: PipelineSelectorExecutorResult - ): ProductPipelineResult[Response] = - previousPipelineResult.copy(pipelineSelectorResult = Some(executorResult)) - } - - def pipelineExecutionStep( - pipelineByIdentifier: Map[ComponentIdentifier, Pipeline[Query, Response]], - qualityFactorObserverByPipeline: Map[ComponentIdentifier, QualityFactorObserver], - context: Executor.Context - ): Step[PipelineExecutorRequest[Query], PipelineExecutorResult[Response]] = - new Step[PipelineExecutorRequest[Query], PipelineExecutorResult[Response]] { - override def identifier: PipelineStepIdentifier = ProductPipelineConfig.pipelineExecutionStep - - override def executorArrow: Arrow[ - PipelineExecutorRequest[Query], - PipelineExecutorResult[Response] - ] = { - pipelineExecutor.arrow(pipelineByIdentifier, qualityFactorObserverByPipeline, context) - } - - override def inputAdaptor( - request: ProductPipelineRequest[TRequest], - previousResult: ProductPipelineResult[Response] - ): PipelineExecutorRequest[Query] = { - val query = previousResult.transformedQuery - .getOrElse { - throw InvalidStepStateException(identifier, "TransformedQuery") - }.asInstanceOf[Query] - - val pipelineIdentifier = previousResult.pipelineSelectorResult - .map(_.pipelineIdentifier).getOrElse { - throw InvalidStepStateException(identifier, "PipelineSelectorResult") - } - - PipelineExecutorRequest(query, pipelineIdentifier) - } - - override def resultUpdater( - previousPipelineResult: ProductPipelineResult[Response], - executorResult: PipelineExecutorResult[Response] - ): ProductPipelineResult[Response] = { - - val mixerPipelineResult = executorResult.pipelineResult match { - case mixerPipelineResult: MixerPipelineResult[Response] @unchecked => - Some(mixerPipelineResult) - case _ => - None - } - - val recommendationPipelineResult = executorResult.pipelineResult match { - case recommendationPipelineResult: RecommendationPipelineResult[ - _, - Response - ] @unchecked => - Some(recommendationPipelineResult) - case _ => - None - } - - previousPipelineResult.copy( - mixerPipelineResult = mixerPipelineResult, - recommendationPipelineResult = recommendationPipelineResult, - traceId = Trace.idOption.map(_.traceId.toString()), - result = executorResult.pipelineResult.result - ) - } - } - - def build( - parentComponentIdentifierStack: ComponentIdentifierStack, - config: ProductPipelineConfig[TRequest, Query, Response] - ): ProductPipeline[TRequest, Response] = { - - val pipelineIdentifier = config.identifier - - val context = Executor.Context( - PipelineFailureClassifier( - config.failureClassifier.orElse(StoppedGateException.classifier(ProductDisabled))), - parentComponentIdentifierStack.push(pipelineIdentifier) - ) - - val denyLoggedOutUsersGate = if (config.denyLoggedOutUsers) { - Some(DenyLoggedOutUsersGate(pipelineIdentifier)) - } else { - None - } - val enabledGate: ParamGate = - ParamGate(pipelineIdentifier + EnabledGateSuffix, config.paramConfig.EnabledDeciderParam) - val supportedClientGate = - ParamGate( - pipelineIdentifier + SupportedClientGateSuffix, - config.paramConfig.SupportedClientParam) - - /** - * Evaluate enabled decider gate first since if it's off, there is no reason to proceed - * Next evaluate supported client feature switch gate, followed by customer configured gates - */ - val allGates = - denyLoggedOutUsersGate.toSeq ++: enabledGate +: supportedClientGate +: config.gates - - val childPipelines: Seq[Pipeline[Query, Response]] = - config.pipelines.map { - case mixerConfig: MixerPipelineConfig[Query, _, Response] => - mixerConfig.build(context.componentStack, mixerPipelineBuilderFactory) - case recommendationConfig: RecommendationPipelineConfig[Query, _, _, Response] => - recommendationConfig.build(context.componentStack, recommendationPipelineBuilderFactory) - case other => - throw new IllegalArgumentException( - s"Product Pipelines only support Mixer and Recommendation pipelines, not $other") - } - - val pipelineByIdentifier: Map[ComponentIdentifier, Pipeline[Query, Response]] = - childPipelines.map { pipeline => - (pipeline.identifier, pipeline) - }.toMap - - val qualityFactorStatus: QualityFactorStatus = - QualityFactorStatus.build(config.qualityFactorConfigs) - - val qualityFactorObserverByPipeline = qualityFactorStatus.qualityFactorByPipeline.mapValues { - qualityFactor => - qualityFactor.buildObserver() - } - - buildGaugesForQualityFactor(pipelineIdentifier, qualityFactorStatus, statsReceiver) - - /** - * Initialize MDC with access logging with everything we have at request time. We can put - * more stuff into MDC later down the pipeline, but at risk of exceptions/errors preventing - * them from being added - */ - val mdcInitArrow = - Arrow.map[ProductPipelineRequest[TRequest], ProductPipelineRequest[TRequest]] { request => - val serviceIdentifier = ServiceIdentifier.fromCertificate(Transport.peerCertificate) - MDC.put("product", config.product.identifier.name) - MDC.put("serviceIdentifier", ServiceIdentifier.asString(serviceIdentifier)) - request - } - - val builtSteps = Seq( - pipelineQueryTransformerStep(config.pipelineQueryTransformer, context), - qualityFactorStep(qualityFactorStatus), - gatesStep(allGates, context), - pipelineSelectorStep(pipelineByIdentifier, config.pipelineSelector, context), - pipelineExecutionStep(pipelineByIdentifier, qualityFactorObserverByPipeline, context) - ) - - val underlying: Arrow[ProductPipelineRequest[TRequest], ProductPipelineResult[Response]] = - buildCombinedArrowFromSteps( - steps = builtSteps, - context = context, - initialEmptyResult = ProductPipelineResult.empty, - stepsInOrderFromConfig = ProductPipelineConfig.stepsInOrder - ) - - /** - * Unlike other components and pipelines, [[ProductPipeline]] must be observed in the - * [[ProductPipelineBuilder]] directly because the resulting [[ProductPipeline.arrow]] - * is run directly without an executor so must contain all stats. - */ - val observed = - wrapProductPipelineWithExecutorBookkeeping[ - ProductPipelineRequest[TRequest], - ProductPipelineResult[Response] - ](context, pipelineIdentifier)(underlying) - - val finalArrow: Arrow[ProductPipelineRequest[TRequest], ProductPipelineResult[Response]] = - Arrow - .letWithArg[ - ProductPipelineRequest[TRequest], - ProductPipelineResult[Response], - StringCenterRequestContext](StringCenterRequestContextLetter)(request => - StringCenterRequestContext( - request.request.clientContext.languageCode, - request.request.clientContext.countryCode - ))( - mdcInitArrow - .andThen(observed) - .onSuccess(result => result.transformedQuery.map(pipelineExecutionLogger(_, result)))) - - val configFromBuilder = config - new ProductPipeline[TRequest, Response] { - override private[core] val config: ProductPipelineConfig[TRequest, _, Response] = - configFromBuilder - override val arrow: Arrow[ProductPipelineRequest[TRequest], ProductPipelineResult[Response]] = - finalArrow - override val identifier: ProductPipelineIdentifier = pipelineIdentifier - override val alerts: Seq[Alert] = config.alerts - override val debugAccessPolicies: Set[AccessPolicy] = config.debugAccessPolicies - override val children: Seq[Component] = allGates ++ childPipelines - } - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineBuilderFactory.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineBuilderFactory.docx new file mode 100644 index 000000000..ef64f1f74 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineBuilderFactory.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineBuilderFactory.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineBuilderFactory.scala deleted file mode 100644 index 9e609ed92..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineBuilderFactory.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.product - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.model.marshalling.request.Request -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineBuilderFactory -import com.twitter.product_mixer.core.pipeline.recommendation.RecommendationPipelineBuilderFactory -import com.twitter.product_mixer.core.service.gate_executor.GateExecutor -import com.twitter.product_mixer.core.service.pipeline_execution_logger.PipelineExecutionLogger -import com.twitter.product_mixer.core.service.pipeline_executor.PipelineExecutor -import com.twitter.product_mixer.core.service.pipeline_selector_executor.PipelineSelectorExecutor -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ProductPipelineBuilderFactory @Inject() ( - gateExecutor: GateExecutor, - pipelineSelectorExecutor: PipelineSelectorExecutor, - pipelineExecutor: PipelineExecutor, - mixerPipelineBuilderFactory: MixerPipelineBuilderFactory, - recommendationPipelineBuilderFactory: RecommendationPipelineBuilderFactory, - statsReceiver: StatsReceiver, - pipelineExecutionLogger: PipelineExecutionLogger) { - def get[ - TRequest <: Request, - Query <: PipelineQuery, - Response - ]: ProductPipelineBuilder[TRequest, Query, Response] = { - new ProductPipelineBuilder[TRequest, Query, Response]( - gateExecutor, - pipelineSelectorExecutor, - pipelineExecutor, - mixerPipelineBuilderFactory, - recommendationPipelineBuilderFactory, - statsReceiver, - pipelineExecutionLogger - ) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineConfig.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineConfig.docx new file mode 100644 index 000000000..d13d45b1e Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineConfig.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineConfig.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineConfig.scala deleted file mode 100644 index ade9b76a8..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineConfig.scala +++ /dev/null @@ -1,107 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.product - -import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.Product -import com.twitter.product_mixer.core.model.marshalling.request.Request -import com.twitter.product_mixer.core.pipeline.PipelineConfig -import com.twitter.product_mixer.core.pipeline.PipelineConfigCompanion -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.product.ProductParamConfig -import com.twitter.product_mixer.core.quality_factor.QualityFactorConfig -import com.twitter.timelines.configapi.Params - -trait ProductPipelineConfig[TRequest <: Request, Query <: PipelineQuery, Response] - extends PipelineConfig { - - override val identifier: ProductPipelineIdentifier - - val product: Product - val paramConfig: ProductParamConfig - - /** - * Product Pipeline Gates will be executed before any other step (including retrieval from mixer - * pipelines). They're executed sequentially, and any "Stop" result will prevent pipeline execution. - */ - def gates: Seq[Gate[Query]] = Seq.empty - - def pipelineQueryTransformer(request: TRequest, params: Params): Query - - /** - * A list of all pipelines that power this product directly (there is no need to include pipelines - * called by those pipelines). - * - * Only pipeline from this list should referenced from the pipelineSelector - */ - def pipelines: Seq[PipelineConfig] - - /** - * A pipeline selector selects a pipeline (from the list in `def pipelines`) to handle the - * current request. - */ - def pipelineSelector(query: Query): ComponentIdentifier - - /** - ** [[qualityFactorConfigs]] associates [[QualityFactorConfig]]s to specific pipelines - * using [[ComponentIdentifier]]. - */ - def qualityFactorConfigs: Map[ComponentIdentifier, QualityFactorConfig] = - Map.empty - - /** - * By default (for safety), product mixer pipelines do not allow logged out requests. - * A "DenyLoggedOutUsersGate" will be generated and added to the pipeline. - * - * You can disable this behavior by overriding `denyLoggedOutUsers` with False. - */ - val denyLoggedOutUsers: Boolean = true - - /** - * A pipeline can define a partial function to rescue failures here. They will be treated as failures - * from a monitoring standpoint, and cancellation exceptions will always be propagated (they cannot be caught here). - */ - def failureClassifier: PartialFunction[Throwable, PipelineFailure] = PartialFunction.empty - - /** - * Alerts can be used to indicate the pipeline's service level objectives. Alerts and - * dashboards will be automatically created based on this information. - */ - val alerts: Seq[Alert] = Seq.empty - - /** - * Access Policies can be used to gate who can query a product from Product Mixer's query tool - * (go/turntable). - * - * This will typically be gated by an LDAP group associated with your team. For example: - * - * {{{ - * override val debugAccessPolicies: Set[AccessPolicy] = Set(AllowedLdapGroups("NAME")) - * }}} - * - * You can disable all queries by using the [[com.twitter.product_mixer.core.functional_component.common.access_policy.BlockEverything]] policy. - */ - val debugAccessPolicies: Set[AccessPolicy] -} - -object ProductPipelineConfig extends PipelineConfigCompanion { - val pipelineQueryTransformerStep: PipelineStepIdentifier = PipelineStepIdentifier( - "PipelineQueryTransformer") - val qualityFactorStep: PipelineStepIdentifier = PipelineStepIdentifier("QualityFactor") - val gatesStep: PipelineStepIdentifier = PipelineStepIdentifier("Gates") - val pipelineSelectorStep: PipelineStepIdentifier = PipelineStepIdentifier("PipelineSelector") - val pipelineExecutionStep: PipelineStepIdentifier = PipelineStepIdentifier("PipelineExecution") - - /** All the Steps which are executed by a [[ProductPipeline]] in the order in which they are run */ - override val stepsInOrder: Seq[PipelineStepIdentifier] = Seq( - pipelineQueryTransformerStep, - qualityFactorStep, - gatesStep, - pipelineSelectorStep, - pipelineExecutionStep - ) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineRequest.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineRequest.docx new file mode 100644 index 000000000..962c0e042 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineRequest.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineRequest.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineRequest.scala deleted file mode 100644 index 7cefde3d7..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineRequest.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.product - -import com.twitter.timelines.configapi.Params - -case class ProductPipelineRequest[RequestType](request: RequestType, params: Params) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineResult.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineResult.docx new file mode 100644 index 000000000..094f93008 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineResult.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineResult.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineResult.scala deleted file mode 100644 index c0065992a..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product/ProductPipelineResult.scala +++ /dev/null @@ -1,62 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.product - -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.PipelineResult -import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineResult -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.recommendation.RecommendationPipelineResult -import com.twitter.product_mixer.core.service.gate_executor.GateExecutorResult -import com.twitter.product_mixer.core.service.pipeline_selector_executor.PipelineSelectorExecutorResult -import com.twitter.product_mixer.core.service.quality_factor_executor.QualityFactorExecutorResult - -case class ProductPipelineResult[Result]( - transformedQuery: Option[PipelineQuery], - qualityFactorResult: Option[QualityFactorExecutorResult], - gateResult: Option[GateExecutorResult], - pipelineSelectorResult: Option[PipelineSelectorExecutorResult], - mixerPipelineResult: Option[MixerPipelineResult[Result]], - recommendationPipelineResult: Option[RecommendationPipelineResult[_, Result]], - traceId: Option[String], - failure: Option[PipelineFailure], - result: Option[Result]) - extends PipelineResult[Result] { - - override val resultSize: Int = { - if (mixerPipelineResult.isDefined) { - mixerPipelineResult.map(_.resultSize).getOrElse(0) - } else { - recommendationPipelineResult.map(_.resultSize).getOrElse(0) - } - } - - override def withFailure(failure: PipelineFailure): PipelineResult[Result] = - copy(failure = Some(failure)) - - override def withResult(result: Result): PipelineResult[Result] = copy(result = Some(result)) -} - -object ProductPipelineResult { - def empty[A]: ProductPipelineResult[A] = ProductPipelineResult( - None, - None, - None, - None, - None, - None, - None, - None, - None - ) - - def fromResult[A](result: A): ProductPipelineResult[A] = ProductPipelineResult( - None, - None, - None, - None, - None, - None, - None, - None, - Some(result) - ) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/BUILD deleted file mode 100644 index 506e537c1..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/BUILD +++ /dev/null @@ -1,62 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector:insert_append_results", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/premarshaller", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/side_effect", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/async_feature_map_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_pipeline_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/filter_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/quality_factor_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/scoring_pipeline_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "stitch/stitch-core", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/premarshaller", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/side_effect", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/async_feature_map_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_pipeline_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/filter_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/quality_factor_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/scoring_pipeline_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/BUILD.docx new file mode 100644 index 000000000..41867cdda Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipeline.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipeline.docx new file mode 100644 index 000000000..92cc05092 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipeline.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipeline.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipeline.scala deleted file mode 100644 index fe379c9b9..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipeline.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.recommendation - -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.RecommendationPipelineIdentifier -import com.twitter.product_mixer.core.pipeline.Pipeline -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Arrow - -/** - * A Recommendation Pipeline - * - * This is an abstract class, as we only construct these via the [[RecommendationPipelineBuilder]]. - * - * A [[RecommendationPipeline]] is capable of processing requests (queries) and returning responses (results) - * in the correct format to directly send to users. - * - * @tparam Query the domain model for the query or request - * @tparam Candidate the type of the candidates - * @tparam Result the final marshalled result type - */ -abstract class RecommendationPipeline[ - Query <: PipelineQuery, - Candidate <: UniversalNoun[Any], - Result] - extends Pipeline[Query, Result] { - override private[core] val config: RecommendationPipelineConfig[Query, Candidate, _, Result] - override val arrow: Arrow[Query, RecommendationPipelineResult[Candidate, Result]] - override val identifier: RecommendationPipelineIdentifier -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipelineBuilder.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipelineBuilder.docx new file mode 100644 index 000000000..02f0f9dfb Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipelineBuilder.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipelineBuilder.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipelineBuilder.scala deleted file mode 100644 index 684ec3c6d..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipelineBuilder.scala +++ /dev/null @@ -1,1076 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.recommendation - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.util.logging.Logging -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.asyncfeaturemap.AsyncFeatureMap -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.Component -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifierStack -import com.twitter.product_mixer.core.model.common.identifier.RecommendationPipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ItemPresentation -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.FailOpenPolicy -import com.twitter.product_mixer.core.pipeline.InvalidStepStateException -import com.twitter.product_mixer.core.pipeline.PipelineBuilder -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipeline -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineBuilderFactory -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.pipeline_failure.IllegalStateFailure -import com.twitter.product_mixer.core.pipeline.pipeline_failure.MisconfiguredDecorator -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailureClassifier -import com.twitter.product_mixer.core.pipeline.pipeline_failure.ProductDisabled -import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipeline -import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipelineBuilderFactory -import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipelineConfig -import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus -import com.twitter.product_mixer.core.quality_factor.QualityFactorObserver -import com.twitter.product_mixer.core.quality_factor.QualityFactorStatus -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.async_feature_map_executor.AsyncFeatureMapExecutor -import com.twitter.product_mixer.core.service.async_feature_map_executor.AsyncFeatureMapExecutorResults -import com.twitter.product_mixer.core.service.candidate_decorator_executor.CandidateDecoratorExecutor -import com.twitter.product_mixer.core.service.candidate_decorator_executor.CandidateDecoratorExecutorResult -import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutor -import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutorResult -import com.twitter.product_mixer.core.service.candidate_pipeline_executor.CandidatePipelineExecutor -import com.twitter.product_mixer.core.service.candidate_pipeline_executor.CandidatePipelineExecutorResult -import com.twitter.product_mixer.core.service.domain_marshaller_executor.DomainMarshallerExecutor -import com.twitter.product_mixer.core.service.filter_executor.FilterExecutor -import com.twitter.product_mixer.core.service.filter_executor.FilterExecutorResult -import com.twitter.product_mixer.core.service.gate_executor.GateExecutor -import com.twitter.product_mixer.core.service.gate_executor.GateExecutorResult -import com.twitter.product_mixer.core.service.gate_executor.StoppedGateException -import com.twitter.product_mixer.core.service.pipeline_result_side_effect_executor.PipelineResultSideEffectExecutor -import com.twitter.product_mixer.core.service.quality_factor_executor.QualityFactorExecutorResult -import com.twitter.product_mixer.core.service.query_feature_hydrator_executor.QueryFeatureHydratorExecutor -import com.twitter.product_mixer.core.service.scoring_pipeline_executor.ScoringPipelineExecutor -import com.twitter.product_mixer.core.service.scoring_pipeline_executor.ScoringPipelineExecutorResult -import com.twitter.product_mixer.core.service.selector_executor.SelectorExecutor -import com.twitter.product_mixer.core.service.selector_executor.SelectorExecutorResult -import com.twitter.product_mixer.core.service.transport_marshaller_executor.TransportMarshallerExecutor -import com.twitter.stitch.Arrow - -/** - * RecommendationPipelineBuilder builds [[RecommendationPipeline]]s from [[RecommendationPipelineConfig]]s. - * - * You should inject a [[RecommendationPipelineBuilderFactory]] and call `.get` to build these. - * - * @see [[RecommendationPipelineConfig]] for the description of the type parameters. - * - * @note Almost a mirror of MixerPipelineBuilder - */ - -class RecommendationPipelineBuilder[ - Query <: PipelineQuery, - Candidate <: UniversalNoun[Any], - DomainResultType <: HasMarshalling, - Result -]( - candidatePipelineExecutor: CandidatePipelineExecutor, - gateExecutor: GateExecutor, - selectorExecutor: SelectorExecutor, - queryFeatureHydratorExecutor: QueryFeatureHydratorExecutor, - asyncFeatureMapExecutor: AsyncFeatureMapExecutor, - candidateFeatureHydratorExecutor: CandidateFeatureHydratorExecutor, - filterExecutor: FilterExecutor, - scoringPipelineExecutor: ScoringPipelineExecutor, - candidateDecoratorExecutor: CandidateDecoratorExecutor, - domainMarshallerExecutor: DomainMarshallerExecutor, - transportMarshallerExecutor: TransportMarshallerExecutor, - pipelineResultSideEffectExecutor: PipelineResultSideEffectExecutor, - candidatePipelineBuilderFactory: CandidatePipelineBuilderFactory, - scoringPipelineBuilderFactory: ScoringPipelineBuilderFactory, - override val statsReceiver: StatsReceiver) - extends PipelineBuilder[Query] - with Logging { - - override type UnderlyingResultType = Result - override type PipelineResultType = RecommendationPipelineResult[Candidate, Result] - - def qualityFactorStep( - qualityFactorStatus: QualityFactorStatus - ): Step[Query, QualityFactorExecutorResult] = - new Step[Query, QualityFactorExecutorResult] { - override def identifier: PipelineStepIdentifier = - RecommendationPipelineConfig.qualityFactorStep - - override def executorArrow: Arrow[Query, QualityFactorExecutorResult] = - Arrow - .map[Query, QualityFactorExecutorResult] { _ => - QualityFactorExecutorResult( - pipelineQualityFactors = - qualityFactorStatus.qualityFactorByPipeline.mapValues(_.currentValue) - ) - } - - override def inputAdaptor( - query: Query, - previousResult: RecommendationPipelineResult[Candidate, Result] - ): Query = query - - override def resultUpdater( - previousPipelineResult: RecommendationPipelineResult[Candidate, Result], - executorResult: QualityFactorExecutorResult - ): RecommendationPipelineResult[Candidate, Result] = - previousPipelineResult.copy(qualityFactorResult = Some(executorResult)) - - override def queryUpdater( - query: Query, - executorResult: QualityFactorExecutorResult - ): Query = { - query match { - case queryWithQualityFactor: HasQualityFactorStatus => - queryWithQualityFactor - .withQualityFactorStatus( - queryWithQualityFactor.qualityFactorStatus.getOrElse(QualityFactorStatus.empty) ++ - qualityFactorStatus - ).asInstanceOf[Query] - case _ => - query - } - } - } - - def gatesStep( - gates: Seq[Gate[Query]], - context: Executor.Context - ): Step[Query, GateExecutorResult] = new Step[Query, GateExecutorResult] { - override def identifier: PipelineStepIdentifier = RecommendationPipelineConfig.gatesStep - - override def executorArrow: Arrow[Query, GateExecutorResult] = - gateExecutor.arrow(gates, context) - - override def inputAdaptor( - query: Query, - previousResult: RecommendationPipelineResult[Candidate, Result] - ): Query = - query - - override def resultUpdater( - previousPipelineResult: RecommendationPipelineResult[Candidate, Result], - executorResult: GateExecutorResult - ): RecommendationPipelineResult[Candidate, Result] = - previousPipelineResult.copy(gateResult = Some(executorResult)) - } - - def fetchQueryFeaturesStep( - queryFeatureHydrators: Seq[BaseQueryFeatureHydrator[Query, _]], - stepIdentifier: PipelineStepIdentifier, - updater: ResultUpdater[ - RecommendationPipelineResult[Candidate, Result], - QueryFeatureHydratorExecutor.Result - ], - context: Executor.Context - ): Step[Query, QueryFeatureHydratorExecutor.Result] = - new Step[Query, QueryFeatureHydratorExecutor.Result] { - override def identifier: PipelineStepIdentifier = stepIdentifier - - override def executorArrow: Arrow[Query, QueryFeatureHydratorExecutor.Result] = - queryFeatureHydratorExecutor.arrow( - queryFeatureHydrators, - RecommendationPipelineConfig.stepsAsyncFeatureHydrationCanBeCompletedBy, - context) - - override def inputAdaptor( - query: Query, - previousResult: RecommendationPipelineResult[Candidate, Result] - ): Query = query - - override def resultUpdater( - previousPipelineResult: RecommendationPipelineResult[Candidate, Result], - executorResult: QueryFeatureHydratorExecutor.Result - ): RecommendationPipelineResult[Candidate, Result] = - updater(previousPipelineResult, executorResult) - - override def queryUpdater( - query: Query, - executorResult: QueryFeatureHydratorExecutor.Result - ): Query = - query - .withFeatureMap( - query.features - .getOrElse(FeatureMap.empty) ++ executorResult.featureMap).asInstanceOf[Query] - } - - def asyncFeaturesStep( - stepToHydrateFor: PipelineStepIdentifier, - context: Executor.Context - ): Step[AsyncFeatureMap, AsyncFeatureMapExecutorResults] = - new Step[AsyncFeatureMap, AsyncFeatureMapExecutorResults] { - override def identifier: PipelineStepIdentifier = - RecommendationPipelineConfig.asyncFeaturesStep(stepToHydrateFor) - - override def executorArrow: Arrow[AsyncFeatureMap, AsyncFeatureMapExecutorResults] = - asyncFeatureMapExecutor.arrow( - stepToHydrateFor, - identifier, - context - ) - - override def inputAdaptor( - query: Query, - previousResult: RecommendationPipelineResult[Candidate, Result] - ): AsyncFeatureMap = - previousResult.mergedAsyncQueryFeatures - .getOrElse( - throw InvalidStepStateException(identifier, "MergedAsyncQueryFeatures") - ) - - override def resultUpdater( - previousPipelineResult: RecommendationPipelineResult[Candidate, Result], - executorResult: AsyncFeatureMapExecutorResults - ): RecommendationPipelineResult[Candidate, Result] = - previousPipelineResult.copy( - asyncFeatureHydrationResults = previousPipelineResult.asyncFeatureHydrationResults match { - case Some(existingResults) => Some(existingResults ++ executorResult) - case None => Some(executorResult) - }) - - override def queryUpdater( - query: Query, - executorResult: AsyncFeatureMapExecutorResults - ): Query = - if (executorResult.featureMapsByStep - .getOrElse(stepToHydrateFor, FeatureMap.empty).isEmpty) { - query - } else { - query - .withFeatureMap( - query.features - .getOrElse(FeatureMap.empty) ++ executorResult.featureMapsByStep( - stepToHydrateFor)).asInstanceOf[Query] - } - } - - def candidatePipelinesStep( - candidatePipelines: Seq[CandidatePipeline[Query]], - defaultFailOpenPolicy: FailOpenPolicy, - failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy], - qualityFactorObserverByPipeline: Map[ComponentIdentifier, QualityFactorObserver], - context: Executor.Context - ): Step[CandidatePipeline.Inputs[Query], CandidatePipelineExecutorResult] = - new Step[CandidatePipeline.Inputs[Query], CandidatePipelineExecutorResult] { - override def identifier: PipelineStepIdentifier = - RecommendationPipelineConfig.candidatePipelinesStep - - override def executorArrow: Arrow[CandidatePipeline.Inputs[ - Query - ], CandidatePipelineExecutorResult] = - candidatePipelineExecutor - .arrow( - candidatePipelines, - defaultFailOpenPolicy, - failOpenPolicies, - qualityFactorObserverByPipeline, - context - ) - - override def inputAdaptor( - query: Query, - previousResult: RecommendationPipelineResult[Candidate, Result] - ): CandidatePipeline.Inputs[ - Query - ] = CandidatePipeline.Inputs(query, Seq.empty) - - override def resultUpdater( - previousPipelineResult: RecommendationPipelineResult[Candidate, Result], - executorResult: CandidatePipelineExecutorResult - ): RecommendationPipelineResult[Candidate, Result] = - previousPipelineResult.copy(candidatePipelineResults = Some(executorResult)) - - override def queryUpdater( - query: Query, - executorResult: CandidatePipelineExecutorResult - ): Query = { - val updatedFeatureMap = query.features - .getOrElse(FeatureMap.empty) ++ executorResult.queryFeatureMap - query - .withFeatureMap(updatedFeatureMap).asInstanceOf[Query] - } - } - - def dependentCandidatePipelinesStep( - candidatePipelines: Seq[CandidatePipeline[Query]], - defaultFailOpenPolicy: FailOpenPolicy, - failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy], - qualityFactorObserverByPipeline: Map[ComponentIdentifier, QualityFactorObserver], - context: Executor.Context - ): Step[CandidatePipeline.Inputs[Query], CandidatePipelineExecutorResult] = - new Step[CandidatePipeline.Inputs[Query], CandidatePipelineExecutorResult] { - override def identifier: PipelineStepIdentifier = - RecommendationPipelineConfig.dependentCandidatePipelinesStep - - override def executorArrow: Arrow[CandidatePipeline.Inputs[ - Query - ], CandidatePipelineExecutorResult] = - candidatePipelineExecutor - .arrow( - candidatePipelines, - defaultFailOpenPolicy, - failOpenPolicies, - qualityFactorObserverByPipeline, - context - ) - - override def inputAdaptor( - query: Query, - previousResult: RecommendationPipelineResult[Candidate, Result] - ): CandidatePipeline.Inputs[ - Query - ] = { - val previousCandidates = previousResult.candidatePipelineResults - .getOrElse { - throw InvalidStepStateException(identifier, "Candidates") - }.candidatePipelineResults.flatMap(_.result.getOrElse(Seq.empty)) - - CandidatePipeline.Inputs(query, previousCandidates) - } - - override def resultUpdater( - previousPipelineResult: RecommendationPipelineResult[Candidate, Result], - executorResult: CandidatePipelineExecutorResult - ): RecommendationPipelineResult[Candidate, Result] = - previousPipelineResult.copy(dependentCandidatePipelineResults = Some(executorResult)) - - override def queryUpdater( - query: Query, - executorResult: CandidatePipelineExecutorResult - ): Query = { - val updatedFeatureMap = query.features - .getOrElse(FeatureMap.empty) ++ executorResult.queryFeatureMap - query - .withFeatureMap(updatedFeatureMap).asInstanceOf[Query] - } - } - - abstract class FilterStep( - filters: Seq[Filter[Query, Candidate]], - context: Executor.Context, - override val identifier: PipelineStepIdentifier) - extends Step[ - (Query, Seq[CandidateWithFeatures[Candidate]]), - FilterExecutorResult[Candidate] - ] { - - def itemCandidates( - previousResult: RecommendationPipelineResult[Candidate, Result] - ): Seq[CandidateWithDetails] - - override def executorArrow: Arrow[ - (Query, Seq[CandidateWithFeatures[Candidate]]), - FilterExecutorResult[Candidate] - ] = - filterExecutor.arrow(filters, context) - - override def inputAdaptor( - query: Query, - previousResult: RecommendationPipelineResult[Candidate, Result] - ): (Query, Seq[CandidateWithFeatures[Candidate]]) = { - - val extractedItemCandidates = itemCandidates(previousResult).collect { - case itemCandidate: ItemCandidateWithDetails => itemCandidate - } - - (query, extractedItemCandidates.asInstanceOf[Seq[CandidateWithFeatures[Candidate]]]) - } - } - - def postCandidatePipelinesSelectorStep( - selectors: Seq[Selector[Query]], - context: Executor.Context - ): Step[SelectorExecutor.Inputs[Query], SelectorExecutorResult] = - new Step[SelectorExecutor.Inputs[Query], SelectorExecutorResult] { - override def identifier: PipelineStepIdentifier = - RecommendationPipelineConfig.postCandidatePipelinesSelectorsStep - - override def executorArrow: Arrow[SelectorExecutor.Inputs[ - Query - ], SelectorExecutorResult] = - selectorExecutor.arrow(selectors, context) - - override def inputAdaptor( - query: Query, - previousResult: RecommendationPipelineResult[Candidate, Result] - ): SelectorExecutor.Inputs[Query] = { - val candidatePipelineResults = previousResult.candidatePipelineResults - .getOrElse { - throw InvalidStepStateException(identifier, "CandidatePipelineResults") - }.candidatePipelineResults.flatMap(_.result.getOrElse(Seq.empty)) - val dependentCandidatePipelineResults = previousResult.dependentCandidatePipelineResults - .getOrElse { - throw InvalidStepStateException(identifier, "DependentCandidatePipelineResults") - }.candidatePipelineResults.flatMap(_.result.getOrElse(Seq.empty)) - - SelectorExecutor.Inputs( - query = query, - candidatesWithDetails = candidatePipelineResults ++ dependentCandidatePipelineResults - ) - } - - override def resultUpdater( - previousPipelineResult: RecommendationPipelineResult[Candidate, Result], - executorResult: SelectorExecutorResult - ): RecommendationPipelineResult[Candidate, Result] = - previousPipelineResult.copy(postCandidatePipelinesSelectorResults = Some(executorResult)) - } - - def postCandidatePipelinesFeatureHydrationStep( - hydrators: Seq[BaseCandidateFeatureHydrator[Query, Candidate, _]], - context: Executor.Context - ): Step[ - CandidateFeatureHydratorExecutor.Inputs[Query, Candidate], - CandidateFeatureHydratorExecutorResult[Candidate] - ] = new Step[ - CandidateFeatureHydratorExecutor.Inputs[Query, Candidate], - CandidateFeatureHydratorExecutorResult[Candidate] - ] { - override def identifier: PipelineStepIdentifier = - RecommendationPipelineConfig.postCandidatePipelinesFeatureHydrationStep - - override def executorArrow: Arrow[ - CandidateFeatureHydratorExecutor.Inputs[Query, Candidate], - CandidateFeatureHydratorExecutorResult[Candidate] - ] = - candidateFeatureHydratorExecutor.arrow(hydrators, context) - - override def inputAdaptor( - query: Query, - previousResult: RecommendationPipelineResult[Candidate, Result] - ): CandidateFeatureHydratorExecutor.Inputs[Query, Candidate] = { - val selectedCandidatesResult = - previousResult.postCandidatePipelinesSelectorResults.getOrElse { - throw InvalidStepStateException(identifier, "PostCandidatePipelinesSelectorResults") - }.selectedCandidates - - CandidateFeatureHydratorExecutor.Inputs( - query, - selectedCandidatesResult.asInstanceOf[Seq[CandidateWithFeatures[Candidate]]]) - } - - override def resultUpdater( - previousPipelineResult: RecommendationPipelineResult[Candidate, Result], - executorResult: CandidateFeatureHydratorExecutorResult[Candidate] - ): RecommendationPipelineResult[Candidate, Result] = previousPipelineResult.copy( - postCandidatePipelinesFeatureHydrationResults = Some(executorResult) - ) - } - - def globalFiltersStep( - filters: Seq[Filter[Query, Candidate]], - context: Executor.Context - ): Step[(Query, Seq[CandidateWithFeatures[Candidate]]), FilterExecutorResult[Candidate]] = - new FilterStep(filters, context, RecommendationPipelineConfig.globalFiltersStep) { - override def itemCandidates( - previousResult: RecommendationPipelineResult[Candidate, Result] - ): Seq[CandidateWithDetails] = { - val candidates = previousResult.postCandidatePipelinesSelectorResults - .getOrElse { - throw InvalidStepStateException(identifier, "PostCandidatePipelineSelectorResults") - }.selectedCandidates.collect { - case itemCandidate: ItemCandidateWithDetails => itemCandidate - } - - val featureMaps = previousResult.postCandidatePipelinesFeatureHydrationResults - .getOrElse { - throw InvalidStepStateException( - identifier, - "PostCandidatePipelineFeatureHydrationResults") - }.results.map(_.features) - // If no hydrators were run, this list would be empty. Otherwise, order and cardinality is - // always ensured to match. - if (featureMaps.isEmpty) { - candidates - } else { - candidates.zip(featureMaps).map { - case (candidate, featureMap) => - candidate.copy(features = candidate.features ++ featureMap) - } - } - } - - override def resultUpdater( - previousPipelineResult: RecommendationPipelineResult[Candidate, Result], - executorResult: FilterExecutorResult[Candidate] - ): RecommendationPipelineResult[Candidate, Result] = previousPipelineResult.copy( - globalFilterResults = Some(executorResult) - ) - } - - def scoringPipelinesStep( - scoringPipelines: Seq[ScoringPipeline[Query, Candidate]], - context: Executor.Context, - defaultFailOpenPolicy: FailOpenPolicy, - failOpenPolicies: Map[ScoringPipelineIdentifier, FailOpenPolicy], - qualityFactorObserverByPipeline: Map[ComponentIdentifier, QualityFactorObserver] - ): Step[ScoringPipelineExecutor.Inputs[Query], ScoringPipelineExecutorResult[ - Candidate - ]] = - new Step[ScoringPipelineExecutor.Inputs[Query], ScoringPipelineExecutorResult[ - Candidate - ]] { - override def identifier: PipelineStepIdentifier = - RecommendationPipelineConfig.scoringPipelinesStep - - override def executorArrow: Arrow[ - ScoringPipelineExecutor.Inputs[Query], - ScoringPipelineExecutorResult[Candidate] - ] = scoringPipelineExecutor.arrow( - scoringPipelines, - context, - defaultFailOpenPolicy, - failOpenPolicies, - qualityFactorObserverByPipeline - ) - - override def inputAdaptor( - query: Query, - previousResult: RecommendationPipelineResult[Candidate, Result] - ): ScoringPipelineExecutor.Inputs[Query] = { - val selectedCandidates = - previousResult.postCandidatePipelinesSelectorResults.getOrElse { - throw InvalidStepStateException(identifier, "PostCandidatePipelinesSelectorResults") - }.selectedCandidates - - val itemCandidates = selectedCandidates.collect { - case itemCandidate: ItemCandidateWithDetails => itemCandidate - } - - val featureMaps = previousResult.postCandidatePipelinesFeatureHydrationResults - .getOrElse { - throw InvalidStepStateException( - identifier, - "PostCandidatePipelineFeatureHydrationResults") - }.results.map(_.features) - // If no hydrators were run, this list would be empty. Otherwise, order and cardinality is - // always ensured to match. - val updatedCandidates = if (featureMaps.isEmpty) { - itemCandidates - } else { - itemCandidates.zip(featureMaps).map { - case (candidate, featureMap) => - candidate.copy(features = candidate.features ++ featureMap) - } - } - - // Filter the original list of candidates to keep only the ones that were kept from - // filtering - val filterResults: Set[Candidate] = previousResult.globalFilterResults - .getOrElse { - throw InvalidStepStateException(identifier, "FilterResults") - }.result.toSet - - val filteredItemCandidates = updatedCandidates.filter { itemCandidate => - filterResults.contains(itemCandidate.candidate.asInstanceOf[Candidate]) - } - - ScoringPipelineExecutor.Inputs( - query, - filteredItemCandidates - ) - } - - override def resultUpdater( - previousPipelineResult: RecommendationPipelineResult[Candidate, Result], - executorResult: ScoringPipelineExecutorResult[Candidate] - ): RecommendationPipelineResult[Candidate, Result] = previousPipelineResult - .copy(scoringPipelineResults = Some(executorResult)) - } - - def resultSelectorsStep( - selectors: Seq[Selector[Query]], - context: Executor.Context - ): Step[SelectorExecutor.Inputs[Query], SelectorExecutorResult] = - new Step[SelectorExecutor.Inputs[Query], SelectorExecutorResult] { - override def identifier: PipelineStepIdentifier = - RecommendationPipelineConfig.resultSelectorsStep - - override def executorArrow: Arrow[SelectorExecutor.Inputs[ - Query - ], SelectorExecutorResult] = - selectorExecutor.arrow(selectors, context) - - override def inputAdaptor( - query: Query, - previousResult: RecommendationPipelineResult[Candidate, Result] - ): SelectorExecutor.Inputs[Query] = { - - /** - * See [[ScoringPipelineExecutor]], scoringPipelineResults contains the fully re-merged - * and updated FeatureMap so there's no need to do any recomposition. Scoring Pipeline Results - * has only candidates that were kept in previous filtering, with their final merged feature - * map. - */ - val scorerResults = previousResult.scoringPipelineResults.getOrElse { - throw InvalidStepStateException(identifier, "Scores") - } - - SelectorExecutor.Inputs( - query = query, - candidatesWithDetails = scorerResults.result - ) - } - - override def resultUpdater( - previousPipelineResult: RecommendationPipelineResult[Candidate, Result], - executorResult: SelectorExecutorResult - ): RecommendationPipelineResult[Candidate, Result] = - previousPipelineResult.copy(resultSelectorResults = Some(executorResult)) - } - - def postSelectionFiltersStep( - filters: Seq[Filter[Query, Candidate]], - context: Executor.Context - ): Step[(Query, Seq[CandidateWithFeatures[Candidate]]), FilterExecutorResult[Candidate]] = - new FilterStep(filters, context, RecommendationPipelineConfig.postSelectionFiltersStep) { - - override def itemCandidates( - previousResult: RecommendationPipelineResult[Candidate, Result] - ): Seq[CandidateWithDetails] = { - previousResult.resultSelectorResults.getOrElse { - throw InvalidStepStateException(identifier, "Candidates") - }.selectedCandidates - } - - override def resultUpdater( - previousPipelineResult: RecommendationPipelineResult[Candidate, Result], - executorResult: FilterExecutorResult[Candidate] - ): RecommendationPipelineResult[Candidate, Result] = { - previousPipelineResult.copy(postSelectionFilterResults = Some(executorResult)) - } - } - - def decoratorStep( - decorator: Option[CandidateDecorator[Query, Candidate]], - context: Executor.Context - ): Step[(Query, Seq[CandidateWithFeatures[Candidate]]), CandidateDecoratorExecutorResult] = - new Step[(Query, Seq[CandidateWithFeatures[Candidate]]), CandidateDecoratorExecutorResult] { - override def identifier: PipelineStepIdentifier = RecommendationPipelineConfig.decoratorStep - - override lazy val executorArrow: Arrow[ - (Query, Seq[CandidateWithFeatures[Candidate]]), - CandidateDecoratorExecutorResult - ] = - candidateDecoratorExecutor.arrow(decorator, context) - - override def inputAdaptor( - query: Query, - previousResult: RecommendationPipelineResult[Candidate, Result] - ): (Query, Seq[CandidateWithFeatures[Candidate]]) = { - - val selectorResults = previousResult.resultSelectorResults - .getOrElse { - throw InvalidStepStateException(identifier, "SelectorResults") - }.selectedCandidates - .collect { case candidate: ItemCandidateWithDetails => candidate } - - val filterResults = previousResult.postSelectionFilterResults - .getOrElse { - throw InvalidStepStateException(identifier, "PostSelectionFilterResults") - }.result.toSet - - val itemCandidateWithDetailsPostFiltering = - selectorResults - .filter(candidateWithDetails => - filterResults.contains( - candidateWithDetails.candidate - .asInstanceOf[Candidate])) - .asInstanceOf[Seq[CandidateWithFeatures[Candidate]]] - - (query, itemCandidateWithDetailsPostFiltering) - } - - override def resultUpdater( - previousPipelineResult: RecommendationPipelineResult[Candidate, Result], - executorResult: CandidateDecoratorExecutorResult - ): RecommendationPipelineResult[Candidate, Result] = - previousPipelineResult.copy( - candidateDecoratorResult = Some(executorResult) - ) - } - - def domainMarshallingStep( - domainMarshaller: DomainMarshaller[Query, DomainResultType], - context: Executor.Context - ): Step[DomainMarshallerExecutor.Inputs[Query], DomainMarshallerExecutor.Result[ - DomainResultType - ]] = - new Step[DomainMarshallerExecutor.Inputs[Query], DomainMarshallerExecutor.Result[ - DomainResultType - ]] { - override def identifier: PipelineStepIdentifier = - RecommendationPipelineConfig.domainMarshallerStep - - override def executorArrow: Arrow[ - DomainMarshallerExecutor.Inputs[Query], - DomainMarshallerExecutor.Result[DomainResultType] - ] = - domainMarshallerExecutor.arrow(domainMarshaller, context) - - override def inputAdaptor( - query: Query, - previousResult: RecommendationPipelineResult[Candidate, Result] - ): DomainMarshallerExecutor.Inputs[Query] = { - val selectorResults = previousResult.resultSelectorResults.getOrElse { - throw InvalidStepStateException(identifier, "SelectorResults") - } - - val filterResults = previousResult.postSelectionFilterResults - .getOrElse { - throw InvalidStepStateException(identifier, "PostSelectionFilterResults") - }.result.toSet - - val filteredResults = selectorResults.selectedCandidates.collect { - case candidate: ItemCandidateWithDetails - if filterResults.contains(candidate.candidate.asInstanceOf[Candidate]) => - candidate - } - - val decoratorResults = previousResult.candidateDecoratorResult - .getOrElse(throw InvalidStepStateException(identifier, "DecoratorStep")).result.map { - decoration => - decoration.candidate -> decoration.presentation - }.toMap - - val finalResults = filteredResults.map { itemWithDetails => - decoratorResults.get(itemWithDetails.candidate) match { - case Some(presentation: ItemPresentation) => - if (itemWithDetails.presentation.isDefined) { - throw PipelineFailure( - category = MisconfiguredDecorator, - reason = "Item Candidate already decorated", - componentStack = Some(context.componentStack)) - } else { - itemWithDetails.copy(presentation = Some(presentation)) - } - case Some(_) => - throw PipelineFailure( - category = MisconfiguredDecorator, - reason = "Item Candidate got back a non ItemPresentation from decorator", - componentStack = Some(context.componentStack)) - case None => itemWithDetails - } - } - DomainMarshallerExecutor.Inputs( - query = query, - candidatesWithDetails = finalResults - ) - } - - override def resultUpdater( - previousPipelineResult: RecommendationPipelineResult[Candidate, Result], - executorResult: DomainMarshallerExecutor.Result[DomainResultType] - ): RecommendationPipelineResult[Candidate, Result] = previousPipelineResult.copy( - domainMarshallerResults = Some(executorResult) - ) - } - - def resultSideEffectsStep( - sideEffects: Seq[PipelineResultSideEffect[Query, DomainResultType]], - context: Executor.Context - ): Step[ - PipelineResultSideEffect.Inputs[Query, DomainResultType], - PipelineResultSideEffectExecutor.Result - ] = new Step[ - PipelineResultSideEffect.Inputs[Query, DomainResultType], - PipelineResultSideEffectExecutor.Result - ] { - override def identifier: PipelineStepIdentifier = - RecommendationPipelineConfig.resultSideEffectsStep - - override def executorArrow: Arrow[ - PipelineResultSideEffect.Inputs[Query, DomainResultType], - PipelineResultSideEffectExecutor.Result - ] = pipelineResultSideEffectExecutor.arrow(sideEffects, context) - - override def inputAdaptor( - query: Query, - previousResult: RecommendationPipelineResult[Candidate, Result] - ): PipelineResultSideEffect.Inputs[Query, DomainResultType] = { - - // Re-apply decorations to the selected results - val resultSelectorResults = { - val decoratorResults = previousResult.candidateDecoratorResult - .getOrElse(throw InvalidStepStateException(identifier, "DecoratorStep")).result.map { - decoration => - decoration.candidate -> decoration.presentation - }.toMap - - val previousSelectorResults = previousResult.resultSelectorResults.getOrElse { - throw InvalidStepStateException(identifier, "SelectorResults") - } - - val filterResults = previousResult.postSelectionFilterResults - .getOrElse { - throw InvalidStepStateException(identifier, "PostSelectionFilterResults") - }.result.toSet - - val filteredSelectorResults = previousSelectorResults.selectedCandidates.collect { - case candidate: ItemCandidateWithDetails - if filterResults.contains(candidate.candidate.asInstanceOf[Candidate]) => - candidate - } - - val decoratedSelectedResults = filteredSelectorResults.map { - case itemWithDetails: ItemCandidateWithDetails => - decoratorResults.get(itemWithDetails.candidate) match { - case Some(presentation: ItemPresentation) => - if (itemWithDetails.presentation.isDefined) { - throw PipelineFailure( - category = MisconfiguredDecorator, - reason = "Item Candidate already decorated", - componentStack = Some(context.componentStack)) - } else { - itemWithDetails.copy(presentation = Some(presentation)) - } - case Some(_) => - throw PipelineFailure( - category = MisconfiguredDecorator, - reason = "Item Candidate got back a non ItemPresentation from decorator", - componentStack = Some(context.componentStack)) - case None => itemWithDetails - } - case item => - // This branch should be impossible to hit since we do a .collect on ItemCandidateWithDetails - // as part of executing the candidate pipelines. - throw PipelineFailure( - category = IllegalStateFailure, - reason = - s"Only ItemCandidateWithDetails expected in pipeline, found: ${item.toString}", - componentStack = Some(context.componentStack) - ) - } - - previousSelectorResults.copy(selectedCandidates = decoratedSelectedResults) - } - - val domainMarshallerResults = previousResult.domainMarshallerResults.getOrElse { - throw InvalidStepStateException(identifier, "DomainMarshallerResults") - } - - PipelineResultSideEffect.Inputs[Query, DomainResultType]( - query = query, - selectedCandidates = resultSelectorResults.selectedCandidates, - remainingCandidates = resultSelectorResults.remainingCandidates, - droppedCandidates = resultSelectorResults.droppedCandidates, - response = domainMarshallerResults.result.asInstanceOf[DomainResultType] - ) - } - - override def resultUpdater( - previousPipelineResult: RecommendationPipelineResult[Candidate, Result], - executorResult: PipelineResultSideEffectExecutor.Result - ): RecommendationPipelineResult[Candidate, Result] = - previousPipelineResult.copy(resultSideEffectResults = Some(executorResult)) - } - - def transportMarshallingStep( - transportMarshaller: TransportMarshaller[DomainResultType, Result], - context: Executor.Context - ): Step[ - TransportMarshallerExecutor.Inputs[DomainResultType], - TransportMarshallerExecutor.Result[Result] - ] = new Step[TransportMarshallerExecutor.Inputs[ - DomainResultType - ], TransportMarshallerExecutor.Result[Result]] { - override def identifier: PipelineStepIdentifier = - RecommendationPipelineConfig.transportMarshallerStep - - override def executorArrow: Arrow[TransportMarshallerExecutor.Inputs[ - DomainResultType - ], TransportMarshallerExecutor.Result[Result]] = - transportMarshallerExecutor.arrow(transportMarshaller, context) - - override def inputAdaptor( - query: Query, - previousResult: RecommendationPipelineResult[Candidate, Result] - ): TransportMarshallerExecutor.Inputs[DomainResultType] = { - val domainMarshallingResults = previousResult.domainMarshallerResults.getOrElse { - throw InvalidStepStateException(identifier, "DomainMarshallerResults") - } - - // Since the PipelineResult just uses HasMarshalling - val domainResult = domainMarshallingResults.result.asInstanceOf[DomainResultType] - - TransportMarshallerExecutor.Inputs(domainResult) - } - - override def resultUpdater( - previousPipelineResult: RecommendationPipelineResult[Candidate, Result], - executorResult: TransportMarshallerExecutor.Result[Result] - ): RecommendationPipelineResult[Candidate, Result] = previousPipelineResult.copy( - transportMarshallerResults = Some(executorResult), - result = Some(executorResult.result) - ) - } - - def build( - parentComponentIdentifierStack: ComponentIdentifierStack, - config: RecommendationPipelineConfig[ - Query, - Candidate, - DomainResultType, - Result - ] - ): RecommendationPipeline[Query, Candidate, Result] = { - val pipelineIdentifier = config.identifier - - val context = Executor.Context( - PipelineFailureClassifier( - config.failureClassifier.orElse(StoppedGateException.classifier(ProductDisabled))), - parentComponentIdentifierStack.push(pipelineIdentifier) - ) - - val decorator = config.decorator.map(decorator => - CandidateDecorator.copyWithUpdatedIdentifier(decorator, pipelineIdentifier)) - - val qualityFactorStatus: QualityFactorStatus = - QualityFactorStatus.build(config.qualityFactorConfigs) - - val qualityFactorObserverByPipeline = - qualityFactorStatus.qualityFactorByPipeline.mapValues { qualityFactor => - qualityFactor.buildObserver() - } - - buildGaugesForQualityFactor(pipelineIdentifier, qualityFactorStatus, statsReceiver) - - val candidatePipelines: Seq[CandidatePipeline[Query]] = config.candidatePipelines.map { - pipelineConfig: CandidatePipelineConfig[Query, _, _, _] => - pipelineConfig.build(context.componentStack, candidatePipelineBuilderFactory) - } - - val dependentCandidatePipelines: Seq[CandidatePipeline[Query]] = - config.dependentCandidatePipelines.map { - pipelineConfig: DependentCandidatePipelineConfig[Query, _, _, _] => - pipelineConfig.build(context.componentStack, candidatePipelineBuilderFactory) - } - - val scoringPipelines: Seq[ScoringPipeline[Query, Candidate]] = config.scoringPipelines.map { - pipelineConfig: ScoringPipelineConfig[Query, Candidate] => - pipelineConfig.build(context.componentStack, scoringPipelineBuilderFactory) - } - - val builtSteps = Seq( - qualityFactorStep(qualityFactorStatus), - gatesStep(config.gates, context), - fetchQueryFeaturesStep( - config.fetchQueryFeatures, - RecommendationPipelineConfig.fetchQueryFeaturesStep, - (previousPipelineResult, executorResult) => - previousPipelineResult.copy(queryFeatures = Some(executorResult)), - context - ), - fetchQueryFeaturesStep( - config.fetchQueryFeaturesPhase2, - RecommendationPipelineConfig.fetchQueryFeaturesPhase2Step, - (previousPipelineResult, executorResult) => - previousPipelineResult.copy( - queryFeaturesPhase2 = Some(executorResult), - mergedAsyncQueryFeatures = Some( - previousPipelineResult.queryFeatures - .getOrElse(throw InvalidStepStateException( - RecommendationPipelineConfig.fetchQueryFeaturesPhase2Step, - "QueryFeatures")) - .asyncFeatureMap ++ executorResult.asyncFeatureMap) - ), - context - ), - asyncFeaturesStep(RecommendationPipelineConfig.candidatePipelinesStep, context), - candidatePipelinesStep( - candidatePipelines, - config.defaultFailOpenPolicy, - config.candidatePipelineFailOpenPolicies, - qualityFactorObserverByPipeline, - context), - asyncFeaturesStep(RecommendationPipelineConfig.dependentCandidatePipelinesStep, context), - dependentCandidatePipelinesStep( - dependentCandidatePipelines, - config.defaultFailOpenPolicy, - config.candidatePipelineFailOpenPolicies, - qualityFactorObserverByPipeline, - context), - asyncFeaturesStep(RecommendationPipelineConfig.postCandidatePipelinesSelectorsStep, context), - postCandidatePipelinesSelectorStep(config.postCandidatePipelinesSelectors, context), - asyncFeaturesStep( - RecommendationPipelineConfig.postCandidatePipelinesFeatureHydrationStep, - context), - postCandidatePipelinesFeatureHydrationStep( - config.postCandidatePipelinesFeatureHydration, - context), - asyncFeaturesStep(RecommendationPipelineConfig.globalFiltersStep, context), - globalFiltersStep(config.globalFilters, context), - asyncFeaturesStep(RecommendationPipelineConfig.scoringPipelinesStep, context), - scoringPipelinesStep( - scoringPipelines, - context, - config.defaultFailOpenPolicy, - config.scoringPipelineFailOpenPolicies, - qualityFactorObserverByPipeline - ), - asyncFeaturesStep(RecommendationPipelineConfig.resultSelectorsStep, context), - resultSelectorsStep(config.resultSelectors, context), - asyncFeaturesStep(RecommendationPipelineConfig.postSelectionFiltersStep, context), - postSelectionFiltersStep(config.postSelectionFilters, context), - asyncFeaturesStep(RecommendationPipelineConfig.decoratorStep, context), - decoratorStep(decorator, context), - domainMarshallingStep(config.domainMarshaller, context), - asyncFeaturesStep(RecommendationPipelineConfig.resultSideEffectsStep, context), - resultSideEffectsStep(config.resultSideEffects, context), - transportMarshallingStep(config.transportMarshaller, context) - ) - - val finalArrow = buildCombinedArrowFromSteps( - steps = builtSteps, - context = context, - initialEmptyResult = RecommendationPipelineResult.empty, - stepsInOrderFromConfig = RecommendationPipelineConfig.stepsInOrder - ) - - val configFromBuilder = config - new RecommendationPipeline[Query, Candidate, Result] { - override private[core] val config: RecommendationPipelineConfig[ - Query, - Candidate, - _, - Result - ] = - configFromBuilder - override val arrow: Arrow[Query, RecommendationPipelineResult[Candidate, Result]] = - finalArrow - override val identifier: RecommendationPipelineIdentifier = pipelineIdentifier - override val alerts: Seq[Alert] = config.alerts - override val children: Seq[Component] = - config.gates ++ - config.fetchQueryFeatures ++ - candidatePipelines ++ - dependentCandidatePipelines ++ - config.postCandidatePipelinesFeatureHydration ++ - config.globalFilters ++ - scoringPipelines ++ - config.postSelectionFilters ++ - config.resultSideEffects ++ - decorator.toSeq ++ - Seq(config.domainMarshaller, config.transportMarshaller) - } - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipelineBuilderFactory.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipelineBuilderFactory.docx new file mode 100644 index 000000000..004f1fadb Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipelineBuilderFactory.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipelineBuilderFactory.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipelineBuilderFactory.scala deleted file mode 100644 index 6df35985a..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipelineBuilderFactory.scala +++ /dev/null @@ -1,67 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.recommendation - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineBuilderFactory -import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipelineBuilderFactory -import com.twitter.product_mixer.core.service.candidate_decorator_executor.CandidateDecoratorExecutor -import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutor -import com.twitter.product_mixer.core.service.candidate_pipeline_executor.CandidatePipelineExecutor -import com.twitter.product_mixer.core.service.domain_marshaller_executor.DomainMarshallerExecutor -import com.twitter.product_mixer.core.service.filter_executor.FilterExecutor -import com.twitter.product_mixer.core.service.gate_executor.GateExecutor -import com.twitter.product_mixer.core.service.pipeline_result_side_effect_executor.PipelineResultSideEffectExecutor -import com.twitter.product_mixer.core.service.async_feature_map_executor.AsyncFeatureMapExecutor -import com.twitter.product_mixer.core.service.query_feature_hydrator_executor.QueryFeatureHydratorExecutor -import com.twitter.product_mixer.core.service.scoring_pipeline_executor.ScoringPipelineExecutor -import com.twitter.product_mixer.core.service.selector_executor.SelectorExecutor -import com.twitter.product_mixer.core.service.transport_marshaller_executor.TransportMarshallerExecutor - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class RecommendationPipelineBuilderFactory @Inject() ( - candidatePipelineExecutor: CandidatePipelineExecutor, - gateExecutor: GateExecutor, - selectorExecutor: SelectorExecutor, - queryFeatureHydratorExecutor: QueryFeatureHydratorExecutor, - asyncFeatureMapExecutor: AsyncFeatureMapExecutor, - candidateFeatureHydratorExecutor: CandidateFeatureHydratorExecutor, - filterExecutor: FilterExecutor, - scoringPipelineExecutor: ScoringPipelineExecutor, - candidateDecoratorExecutor: CandidateDecoratorExecutor, - domainMarshallerExecutor: DomainMarshallerExecutor, - transportMarshallerExecutor: TransportMarshallerExecutor, - pipelineResultSideEffectExecutor: PipelineResultSideEffectExecutor, - candidatePipelineBuilderFactory: CandidatePipelineBuilderFactory, - scoringPipelineBuilderFactory: ScoringPipelineBuilderFactory, - statsReceiver: StatsReceiver) { - - def get[ - Query <: PipelineQuery, - Candidate <: UniversalNoun[Any], - DomainResultType <: HasMarshalling, - Result - ]: RecommendationPipelineBuilder[Query, Candidate, DomainResultType, Result] = { - new RecommendationPipelineBuilder[Query, Candidate, DomainResultType, Result]( - candidatePipelineExecutor, - gateExecutor, - selectorExecutor, - queryFeatureHydratorExecutor, - asyncFeatureMapExecutor, - candidateFeatureHydratorExecutor, - filterExecutor, - scoringPipelineExecutor, - candidateDecoratorExecutor, - domainMarshallerExecutor, - transportMarshallerExecutor, - pipelineResultSideEffectExecutor, - candidatePipelineBuilderFactory, - scoringPipelineBuilderFactory, - statsReceiver - ) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipelineConfig.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipelineConfig.docx new file mode 100644 index 000000000..571f7e764 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipelineConfig.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipelineConfig.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipelineConfig.scala deleted file mode 100644 index 1e2fa59c2..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipelineConfig.scala +++ /dev/null @@ -1,262 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.recommendation - -import com.twitter.product_mixer.component_library.selector.InsertAppendResults -import com.twitter.product_mixer.core.functional_component.common.AllPipelines -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifierStack -import com.twitter.product_mixer.core.model.common.identifier.RecommendationPipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.FailOpenPolicy -import com.twitter.product_mixer.core.pipeline.PipelineConfig -import com.twitter.product_mixer.core.pipeline.PipelineConfigCompanion -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.pipeline_failure.ClosedGate -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipelineConfig -import com.twitter.product_mixer.core.quality_factor.QualityFactorConfig - -/** - * This is the configuration necessary to generate a Recommendation Pipeline. Product code should create a - * RecommendationPipelineConfig, and then use a RecommendationPipelineBuilder to get the final RecommendationPipeline which can - * process requests. - * - * @tparam Query - The domain model for the query or request - * @tparam Candidate - The type of the candidates that the Candidate Pipelines are generating - * @tparam UnmarshalledResultType - The result type of the pipeline, but before marshalling to a wire protocol like URT - * @tparam Result - The final result that will be served to users - */ -trait RecommendationPipelineConfig[ - Query <: PipelineQuery, - Candidate <: UniversalNoun[Any], - UnmarshalledResultType <: HasMarshalling, - Result] - extends PipelineConfig { - - override val identifier: RecommendationPipelineIdentifier - - /** - * Recommendation Pipeline Gates will be executed before any other step (including retrieval from candidate - * pipelines). They're executed sequentially, and any "Stop" result will prevent pipeline execution. - */ - def gates: Seq[Gate[Query]] = Seq.empty - - /** - * A recommendation pipeline can fetch query-level features before candidate pipelines are executed. - */ - def fetchQueryFeatures: Seq[BaseQueryFeatureHydrator[Query, _]] = Seq.empty - - /** - * Candidate pipelines retrieve candidates for possible inclusion in the result - */ - def fetchQueryFeaturesPhase2: Seq[BaseQueryFeatureHydrator[Query, _]] = Seq.empty - - /** - * What candidate pipelines should this Recommendations Pipeline get candidate from? - */ - def candidatePipelines: Seq[CandidatePipelineConfig[Query, _, _, _]] - - /** - * Dependent candidate pipelines to retrieve candidates that depend on the result of [[candidatePipelines]] - * [[DependentCandidatePipelineConfig]] have access to the list of previously retrieved & decorated - * candidates for use in constructing the query object. - */ - def dependentCandidatePipelines: Seq[DependentCandidatePipelineConfig[Query, _, _, _]] = Seq.empty - - /** - * Takes final ranked list of candidates & apply any business logic (e.g, deduplicating and merging - * candidates before scoring). - */ - def postCandidatePipelinesSelectors: Seq[Selector[Query]] = Seq(InsertAppendResults(AllPipelines)) - - /** - * After selectors are run, you can fetch features for each candidate. - * The existing features from previous hydrations are passed in as inputs. You are not expected to - * put them into the resulting feature map yourself - they will be merged for you by the platform. - */ - def postCandidatePipelinesFeatureHydration: Seq[ - BaseCandidateFeatureHydrator[Query, Candidate, _] - ] = - Seq.empty - - /** - * Global filters to run on all candidates. - */ - def globalFilters: Seq[Filter[Query, Candidate]] = Seq.empty - - /** - * By default, a Recommendation Pipeline will fail closed - if any candidate or scoring - * pipeline fails to return a result, then the Recommendation Pipeline will not return a result. - * You can adjust this default policy, or provide specific policies to specific pipelines. - * Those specific policies will take priority. - * - * FailOpenPolicy.All will always fail open (the RecommendationPipeline will continue without that pipeline) - * FailOpenPolicy.Never will always fail closed (the RecommendationPipeline will fail if that pipeline fails) - * - * There's a default policy, and a specific Map of policies that takes precedence. - */ - def defaultFailOpenPolicy: FailOpenPolicy = FailOpenPolicy(Set(ClosedGate)) - def candidatePipelineFailOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = - Map.empty - def scoringPipelineFailOpenPolicies: Map[ScoringPipelineIdentifier, FailOpenPolicy] = Map.empty - - /** - ** [[qualityFactorConfigs]] associates [[QualityFactorConfig]]s to specific candidate pipelines - * using [[ComponentIdentifier]]. - */ - def qualityFactorConfigs: Map[ComponentIdentifier, QualityFactorConfig] = - Map.empty - - /** - * Scoring pipelines for scoring candidates. - * @note These do not drop or re-order candidates, you should do those in the sub-sequent selectors - * step based off of the scores on candidates set in those [[ScoringPipeline]]s. - */ - def scoringPipelines: Seq[ScoringPipelineConfig[Query, Candidate]] - - /** - * Takes final ranked list of candidates & apply any business logic (e.g, capping number - * of ad accounts or pacing ad accounts). - */ - def resultSelectors: Seq[Selector[Query]] - - /** - * Takes the final selected list of candidates and applies a final list of filters. - * Useful for doing very expensive filtering at the end of your pipeline. - */ - def postSelectionFilters: Seq[Filter[Query, Candidate]] = Seq.empty - - /** - * Decorators allow for adding Presentations to candidates. While the Presentation can contain any - * arbitrary data, Decorators are often used to add a UrtItemPresentation for URT item support. Most - * customers will prefer to set a decorator in their respective candidate pipeline, however, a final - * global one is available for those that do global decoration as late possible to avoid unnecessary hydrations. - * @note This decorator can only return an ItemPresentation. - * @note This decorator cannot decorate an already decorated candidate from the prior decorator - * step in candidate pipelines. - */ - def decorator: Option[CandidateDecorator[Query, Candidate]] = None - - /** - * Domain marshaller transforms the selections into the model expected by the marshaller - */ - def domainMarshaller: DomainMarshaller[Query, UnmarshalledResultType] - - /** - * Mixer result side effects that are executed after selection and domain marshalling - */ - def resultSideEffects: Seq[PipelineResultSideEffect[Query, UnmarshalledResultType]] = Seq() - - /** - * Transport marshaller transforms the model into our line-level API like URT or JSON - */ - def transportMarshaller: TransportMarshaller[UnmarshalledResultType, Result] - - /** - * A pipeline can define a partial function to rescue failures here. They will be treated as failures - * from a monitoring standpoint, and cancellation exceptions will always be propagated (they cannot be caught here). - */ - def failureClassifier: PartialFunction[Throwable, PipelineFailure] = PartialFunction.empty - - /** - * Alerts can be used to indicate the pipeline's service level objectives. Alerts and - * dashboards will be automatically created based on this information. - */ - val alerts: Seq[Alert] = Seq.empty - - /** - * This method is used by the product mixer framework to build the pipeline. - */ - private[core] final def build( - parentComponentIdentifierStack: ComponentIdentifierStack, - builder: RecommendationPipelineBuilderFactory - ): RecommendationPipeline[Query, Candidate, Result] = - builder.get.build(parentComponentIdentifierStack, this) -} - -object RecommendationPipelineConfig extends PipelineConfigCompanion { - val qualityFactorStep: PipelineStepIdentifier = PipelineStepIdentifier("QualityFactor") - val gatesStep: PipelineStepIdentifier = PipelineStepIdentifier("Gates") - val fetchQueryFeaturesStep: PipelineStepIdentifier = PipelineStepIdentifier("FetchQueryFeatures") - val fetchQueryFeaturesPhase2Step: PipelineStepIdentifier = PipelineStepIdentifier( - "FetchQueryFeaturesPhase2") - val candidatePipelinesStep: PipelineStepIdentifier = PipelineStepIdentifier("CandidatePipelines") - val dependentCandidatePipelinesStep: PipelineStepIdentifier = - PipelineStepIdentifier("DependentCandidatePipelines") - val postCandidatePipelinesSelectorsStep: PipelineStepIdentifier = - PipelineStepIdentifier("PostCandidatePipelinesSelectors") - val postCandidatePipelinesFeatureHydrationStep: PipelineStepIdentifier = - PipelineStepIdentifier("PostCandidatePipelinesFeatureHydration") - val globalFiltersStep: PipelineStepIdentifier = PipelineStepIdentifier("GlobalFilters") - val scoringPipelinesStep: PipelineStepIdentifier = PipelineStepIdentifier("ScoringPipelines") - val resultSelectorsStep: PipelineStepIdentifier = PipelineStepIdentifier("ResultSelectors") - val postSelectionFiltersStep: PipelineStepIdentifier = PipelineStepIdentifier( - "PostSelectionFilters") - val decoratorStep: PipelineStepIdentifier = PipelineStepIdentifier("Decorator") - val domainMarshallerStep: PipelineStepIdentifier = PipelineStepIdentifier("DomainMarshaller") - val resultSideEffectsStep: PipelineStepIdentifier = PipelineStepIdentifier("ResultSideEffects") - val transportMarshallerStep: PipelineStepIdentifier = PipelineStepIdentifier( - "TransportMarshaller") - - /** All the Steps which are executed by a [[RecommendationPipeline]] in the order in which they are run */ - override val stepsInOrder: Seq[PipelineStepIdentifier] = Seq( - qualityFactorStep, - gatesStep, - fetchQueryFeaturesStep, - fetchQueryFeaturesPhase2Step, - asyncFeaturesStep(candidatePipelinesStep), - candidatePipelinesStep, - asyncFeaturesStep(dependentCandidatePipelinesStep), - dependentCandidatePipelinesStep, - asyncFeaturesStep(postCandidatePipelinesSelectorsStep), - postCandidatePipelinesSelectorsStep, - asyncFeaturesStep(postCandidatePipelinesFeatureHydrationStep), - postCandidatePipelinesFeatureHydrationStep, - asyncFeaturesStep(globalFiltersStep), - globalFiltersStep, - asyncFeaturesStep(scoringPipelinesStep), - scoringPipelinesStep, - asyncFeaturesStep(resultSelectorsStep), - resultSelectorsStep, - asyncFeaturesStep(postSelectionFiltersStep), - postSelectionFiltersStep, - asyncFeaturesStep(decoratorStep), - decoratorStep, - domainMarshallerStep, - asyncFeaturesStep(resultSideEffectsStep), - resultSideEffectsStep, - transportMarshallerStep - ) - - /** - * All the Steps which an [[com.twitter.product_mixer.core.functional_component.feature_hydrator.AsyncHydrator AsyncHydrator]] - * can be configured to [[com.twitter.product_mixer.core.functional_component.feature_hydrator.AsyncHydrator.hydrateBefore hydrateBefore]] - */ - override val stepsAsyncFeatureHydrationCanBeCompletedBy: Set[PipelineStepIdentifier] = Set( - candidatePipelinesStep, - dependentCandidatePipelinesStep, - postCandidatePipelinesSelectorsStep, - postCandidatePipelinesFeatureHydrationStep, - globalFiltersStep, - scoringPipelinesStep, - resultSelectorsStep, - postSelectionFiltersStep, - decoratorStep, - resultSideEffectsStep, - ) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipelineResult.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipelineResult.docx new file mode 100644 index 000000000..cbe7da59f Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipelineResult.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipelineResult.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipelineResult.scala deleted file mode 100644 index 9e75c8db3..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation/RecommendationPipelineResult.scala +++ /dev/null @@ -1,84 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.recommendation - -import com.twitter.product_mixer.core.feature.featuremap.asyncfeaturemap.AsyncFeatureMap -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.PipelineResult -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.service.async_feature_map_executor.AsyncFeatureMapExecutorResults -import com.twitter.product_mixer.core.service.candidate_decorator_executor.CandidateDecoratorExecutorResult -import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutorResult -import com.twitter.product_mixer.core.service.candidate_pipeline_executor.CandidatePipelineExecutorResult -import com.twitter.product_mixer.core.service.domain_marshaller_executor.DomainMarshallerExecutor -import com.twitter.product_mixer.core.service.filter_executor.FilterExecutorResult -import com.twitter.product_mixer.core.service.gate_executor.GateExecutorResult -import com.twitter.product_mixer.core.service.pipeline_result_side_effect_executor.PipelineResultSideEffectExecutor -import com.twitter.product_mixer.core.service.quality_factor_executor.QualityFactorExecutorResult -import com.twitter.product_mixer.core.service.query_feature_hydrator_executor.QueryFeatureHydratorExecutor -import com.twitter.product_mixer.core.service.scoring_pipeline_executor.ScoringPipelineExecutorResult -import com.twitter.product_mixer.core.service.selector_executor.SelectorExecutorResult -import com.twitter.product_mixer.core.service.transport_marshaller_executor.TransportMarshallerExecutor - -case class RecommendationPipelineResult[Candidate <: UniversalNoun[Any], ResultType]( - qualityFactorResult: Option[QualityFactorExecutorResult], - gateResult: Option[GateExecutorResult], - queryFeatures: Option[QueryFeatureHydratorExecutor.Result], - queryFeaturesPhase2: Option[QueryFeatureHydratorExecutor.Result], - mergedAsyncQueryFeatures: Option[AsyncFeatureMap], - candidatePipelineResults: Option[CandidatePipelineExecutorResult], - dependentCandidatePipelineResults: Option[CandidatePipelineExecutorResult], - postCandidatePipelinesSelectorResults: Option[SelectorExecutorResult], - postCandidatePipelinesFeatureHydrationResults: Option[ - CandidateFeatureHydratorExecutorResult[Candidate] - ], - globalFilterResults: Option[FilterExecutorResult[Candidate]], - scoringPipelineResults: Option[ScoringPipelineExecutorResult[Candidate]], - resultSelectorResults: Option[SelectorExecutorResult], - postSelectionFilterResults: Option[FilterExecutorResult[Candidate]], - candidateDecoratorResult: Option[CandidateDecoratorExecutorResult], - domainMarshallerResults: Option[DomainMarshallerExecutor.Result[HasMarshalling]], - resultSideEffectResults: Option[PipelineResultSideEffectExecutor.Result], - asyncFeatureHydrationResults: Option[AsyncFeatureMapExecutorResults], - transportMarshallerResults: Option[TransportMarshallerExecutor.Result[ResultType]], - failure: Option[PipelineFailure], - result: Option[ResultType]) - extends PipelineResult[ResultType] { - override val resultSize: Int = result match { - case Some(seqResult @ Seq(_)) => seqResult.length - case Some(_) => 1 - case None => 0 - } - - override def withFailure( - failure: PipelineFailure - ): RecommendationPipelineResult[Candidate, ResultType] = - copy(failure = Some(failure)) - override def withResult(result: ResultType): RecommendationPipelineResult[Candidate, ResultType] = - copy(result = Some(result)) -} - -object RecommendationPipelineResult { - def empty[A <: UniversalNoun[Any], B]: RecommendationPipelineResult[A, B] = - RecommendationPipelineResult( - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None - ) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/BUILD deleted file mode 100644 index 43c431a64..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/BUILD +++ /dev/null @@ -1,36 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector:insert_append_results", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/candidate_feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/gate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/scorer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/selector", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "stitch/stitch-core", - "util/util-core:scala", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/candidate_feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/gate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/scorer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/selector", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/BUILD.docx new file mode 100644 index 000000000..6a6af16b5 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/NewScoringPipelineBuilder.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/NewScoringPipelineBuilder.docx new file mode 100644 index 000000000..eb28ce1e1 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/NewScoringPipelineBuilder.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/NewScoringPipelineBuilder.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/NewScoringPipelineBuilder.scala deleted file mode 100644 index 9b3bf2a26..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/NewScoringPipelineBuilder.scala +++ /dev/null @@ -1,202 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.scoring - -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.decorator.Decoration -import com.twitter.product_mixer.core.functional_component.scorer.ScoredCandidateResult -import com.twitter.product_mixer.core.gate.ParamGate -import com.twitter.product_mixer.core.gate.ParamGate._ -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.Component -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifierStack -import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.pipeline.NewPipelineBuilder -import com.twitter.product_mixer.core.pipeline.NewPipelineArrowBuilder -import com.twitter.product_mixer.core.pipeline.NewPipelineResult -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.pipeline_failure.ClosedGate -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailureClassifier -import com.twitter.product_mixer.core.pipeline.state.HasCandidatesWithDetails -import com.twitter.product_mixer.core.pipeline.state.HasCandidatesWithFeatures -import com.twitter.product_mixer.core.pipeline.state.HasExecutorResults -import com.twitter.product_mixer.core.pipeline.state.HasQuery -import com.twitter.product_mixer.core.pipeline.state.HasResult -import com.twitter.product_mixer.core.pipeline.step.candidate_feature_hydrator.CandidateFeatureHydratorStep -import com.twitter.product_mixer.core.pipeline.step.gate.GateStep -import com.twitter.product_mixer.core.pipeline.step.scorer.ScorerStep -import com.twitter.product_mixer.core.pipeline.step.selector.SelectorStep -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.ExecutorResult -import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutorResult -import com.twitter.product_mixer.core.service.gate_executor.GateExecutorResult -import com.twitter.product_mixer.core.service.gate_executor.StoppedGateException -import com.twitter.product_mixer.core.service.selector_executor.SelectorExecutorResult -import com.twitter.stitch.Arrow -import javax.inject.Inject -import scala.collection.immutable.ListMap - -/** - * NewScoringPipelineBuilder builds [[ScoringPipeline]]s from [[ScoringPipelineConfig]]s. - * New because it's meant to eventually replace [[ScoringPipelineBuilder]] - * You should inject a [[ScoringPipelineBuilderFactory]] and call `.get` to build these. - * - * @see [[ScoringPipelineConfig]] for the description of the type parameters - * @tparam Query the type of query these accept. - * @tparam Candidate the domain model for the candidate being scored - */ -class NewScoringPipelineBuilder[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]] @Inject() ( - selectionStep: SelectorStep[Query, ScoringPipelineState[Query, Candidate]], - gateStep: GateStep[Query, ScoringPipelineState[Query, Candidate]], - candidateFeatureHydrationStep: CandidateFeatureHydratorStep[ - Query, - Candidate, - ScoringPipelineState[Query, Candidate] - ], - scorerStep: ScorerStep[Query, Candidate, ScoringPipelineState[Query, Candidate]]) - extends NewPipelineBuilder[ScoringPipelineConfig[Query, Candidate], Seq[ - CandidateWithFeatures[Candidate] - ], ScoringPipelineState[Query, Candidate], ScoringPipeline[Query, Candidate]] { - - override def build( - parentComponentIdentifierStack: ComponentIdentifierStack, - arrowBuilder: NewPipelineArrowBuilder[ArrowResult, ArrowState], - scoringPipelineConfig: ScoringPipelineConfig[Query, Candidate] - ): ScoringPipeline[Query, Candidate] = { - val pipelineIdentifier = scoringPipelineConfig.identifier - - val context = Executor.Context( - PipelineFailureClassifier( - scoringPipelineConfig.failureClassifier.orElse( - StoppedGateException.classifier(ClosedGate))), - parentComponentIdentifierStack.push(pipelineIdentifier) - ) - - val enabledGateOpt = scoringPipelineConfig.enabledDeciderParam.map { deciderParam => - ParamGate(pipelineIdentifier + EnabledGateSuffix, deciderParam) - } - val supportedClientGateOpt = scoringPipelineConfig.supportedClientParam.map { param => - ParamGate(pipelineIdentifier + SupportedClientGateSuffix, param) - } - - /** - * Evaluate enabled decider gate first since if it's off, there is no reason to proceed - * Next evaluate supported client feature switch gate, followed by customer configured gates - */ - val allGates = - enabledGateOpt.toSeq ++ supportedClientGateOpt.toSeq ++ scoringPipelineConfig.gates - - val underlyingArrow = arrowBuilder - .add(ScoringPipelineConfig.gatesStep, gateStep, allGates) - .add(ScoringPipelineConfig.selectorsStep, selectionStep, scoringPipelineConfig.selectors) - .add( - ScoringPipelineConfig.preScoringFeatureHydrationPhase1Step, - candidateFeatureHydrationStep, - scoringPipelineConfig.preScoringFeatureHydrationPhase1) - .add( - ScoringPipelineConfig.preScoringFeatureHydrationPhase2Step, - candidateFeatureHydrationStep, - scoringPipelineConfig.preScoringFeatureHydrationPhase2) - .add(ScoringPipelineConfig.scorersStep, scorerStep, scoringPipelineConfig.scorers).buildArrow( - context) - - val finalArrow = Arrow - .map { inputs: ScoringPipeline.Inputs[Query] => - ScoringPipelineState[Query, Candidate](inputs.query, inputs.candidates, ListMap.empty) - }.andThen(underlyingArrow).map { pipelineResult => - ScoringPipelineResult( - gateResults = pipelineResult.executorResultsByPipelineStep - .get(ScoringPipelineConfig.gatesStep) - .map(_.asInstanceOf[GateExecutorResult]), - selectorResults = pipelineResult.executorResultsByPipelineStep - .get(ScoringPipelineConfig.selectorsStep) - .map(_.asInstanceOf[SelectorExecutorResult]), - preScoringHydrationPhase1Result = pipelineResult.executorResultsByPipelineStep - .get(ScoringPipelineConfig.preScoringFeatureHydrationPhase1Step) - .map(_.asInstanceOf[CandidateFeatureHydratorExecutorResult[Candidate]]), - preScoringHydrationPhase2Result = pipelineResult.executorResultsByPipelineStep - .get(ScoringPipelineConfig.preScoringFeatureHydrationPhase2Step) - .map(_.asInstanceOf[CandidateFeatureHydratorExecutorResult[Candidate]]), - scorerResults = pipelineResult.executorResultsByPipelineStep - .get(ScoringPipelineConfig.scorersStep) - .map(_.asInstanceOf[CandidateFeatureHydratorExecutorResult[Candidate]]), - failure = pipelineResult match { - case failure: NewPipelineResult.Failure => - Some(failure.failure) - case _ => None - }, - result = pipelineResult match { - case result: NewPipelineResult.Success[Seq[CandidateWithFeatures[Candidate]]] => - Some(result.result.map { candidateWithFeatures => - ScoredCandidateResult( - candidateWithFeatures.candidate, - candidateWithFeatures.features) - }) - case _ => None - } - ) - } - - new ScoringPipeline[Query, Candidate] { - override val arrow: Arrow[ScoringPipeline.Inputs[Query], ScoringPipelineResult[Candidate]] = - finalArrow - - override val identifier: ScoringPipelineIdentifier = scoringPipelineConfig.identifier - - override val alerts: Seq[Alert] = scoringPipelineConfig.alerts - - override val children: Seq[Component] = - allGates ++ scoringPipelineConfig.preScoringFeatureHydrationPhase1 ++ scoringPipelineConfig.preScoringFeatureHydrationPhase2 ++ scoringPipelineConfig.scorers - - override private[core] val config = scoringPipelineConfig - } - } -} - -case class ScoringPipelineState[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]( - override val query: Query, - candidates: Seq[ItemCandidateWithDetails], - override val executorResultsByPipelineStep: ListMap[PipelineStepIdentifier, ExecutorResult]) - extends HasQuery[Query, ScoringPipelineState[Query, Candidate]] - with HasCandidatesWithDetails[ScoringPipelineState[Query, Candidate]] - with HasCandidatesWithFeatures[Candidate, ScoringPipelineState[Query, Candidate]] - with HasExecutorResults[ScoringPipelineState[Query, Candidate]] - with HasResult[Seq[CandidateWithFeatures[Candidate]]] { - - override val candidatesWithDetails: Seq[CandidateWithDetails] = candidates - - override val candidatesWithFeatures: Seq[CandidateWithFeatures[Candidate]] = - candidates.asInstanceOf[Seq[CandidateWithFeatures[Candidate]]] - - override val buildResult: Seq[CandidateWithFeatures[Candidate]] = candidatesWithFeatures - - override def updateCandidatesWithDetails( - newCandidates: Seq[CandidateWithDetails] - ): ScoringPipelineState[Query, Candidate] = { - this.copy(candidates = newCandidates.asInstanceOf[Seq[ItemCandidateWithDetails]]) - } - - override def updateQuery(newQuery: Query): ScoringPipelineState[Query, Candidate] = - this.copy(query = newQuery) - - override def updateDecorations( - decoration: Seq[Decoration] - ): ScoringPipelineState[Query, Candidate] = ??? - - override def updateCandidatesWithFeatures( - newCandidates: Seq[CandidateWithFeatures[Candidate]] - ): ScoringPipelineState[Query, Candidate] = { - val updatedCandidates = candidates.zip(newCandidates).map { - case (itemCandidateWithDetails, newCandidate) => - itemCandidateWithDetails.copy(features = - itemCandidateWithDetails.features ++ newCandidate.features) - } - this.copy(query, updatedCandidates) - } - - override private[pipeline] def setExecutorResults( - newMap: ListMap[PipelineStepIdentifier, ExecutorResult] - ) = this.copy(executorResultsByPipelineStep = newMap) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipeline.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipeline.docx new file mode 100644 index 000000000..e005d94c8 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipeline.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipeline.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipeline.scala deleted file mode 100644 index a02bbb8b4..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipeline.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.scoring - -import com.twitter.product_mixer.core.functional_component.scorer.ScoredCandidateResult -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.pipeline.Pipeline -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Arrow - -/** - * A Scoring Pipeline - * - * This is an abstract class, as we only construct these via the [[ScoringPipelineBuilder]]. - * - * A [[ScoringPipeline]] is capable of pre-filtering candidates for scoring, performing the scoring - * then running selection heuristics (ranking, dropping, etc) based off of the score. - * @tparam Query the domain model for the query or request - * @tparam Candidate the domain model for the candidate being scored - */ -abstract class ScoringPipeline[-Query <: PipelineQuery, Candidate <: UniversalNoun[Any]] - extends Pipeline[ScoringPipeline.Inputs[Query], Seq[ScoredCandidateResult[Candidate]]] { - override private[core] val config: ScoringPipelineConfig[Query, Candidate] - override val arrow: Arrow[ScoringPipeline.Inputs[Query], ScoringPipelineResult[Candidate]] - override val identifier: ScoringPipelineIdentifier -} - -object ScoringPipeline { - case class Inputs[+Query <: PipelineQuery]( - query: Query, - candidates: Seq[ItemCandidateWithDetails]) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipelineBuilder.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipelineBuilder.docx new file mode 100644 index 000000000..b3bb81694 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipelineBuilder.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipelineBuilder.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipelineBuilder.scala deleted file mode 100644 index d597fa9a5..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipelineBuilder.scala +++ /dev/null @@ -1,367 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.scoring - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.scorer.ScoredCandidateResult -import com.twitter.product_mixer.core.gate.ParamGate -import com.twitter.product_mixer.core.gate.ParamGate.EnabledGateSuffix -import com.twitter.product_mixer.core.gate.ParamGate.SupportedClientGateSuffix -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.Component -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifierStack -import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.pipeline.InvalidStepStateException -import com.twitter.product_mixer.core.pipeline.PipelineBuilder -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.pipeline_failure.ClosedGate -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailureClassifier -import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipeline.Inputs -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutor -import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutorResult -import com.twitter.product_mixer.core.service.gate_executor.GateExecutor -import com.twitter.product_mixer.core.service.gate_executor.GateExecutorResult -import com.twitter.product_mixer.core.service.gate_executor.StoppedGateException -import com.twitter.product_mixer.core.service.selector_executor.SelectorExecutor -import com.twitter.product_mixer.core.service.selector_executor.SelectorExecutorResult -import com.twitter.stitch.Arrow -import javax.inject.Inject - -/** - * ScoringPipelineBuilder builds [[ScoringPipeline]]s from [[ScoringPipelineConfig]]s. - * - * You should inject a [[ScoringPipelineBuilderFactory]] and call `.get` to build these. - * - * @see [[ScoringPipelineConfig]] for the description of the type parameters - * @tparam Query the type of query these accept. - * @tparam Candidate the domain model for the candidate being scored - */ -class ScoringPipelineBuilder[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]] @Inject() ( - gateExecutor: GateExecutor, - selectorExecutor: SelectorExecutor, - candidateFeatureHydratorExecutor: CandidateFeatureHydratorExecutor, - override val statsReceiver: StatsReceiver) - extends PipelineBuilder[Inputs[Query]] { - - override type UnderlyingResultType = Seq[ScoredCandidateResult[Candidate]] - override type PipelineResultType = ScoringPipelineResult[Candidate] - - def build( - parentComponentIdentifierStack: ComponentIdentifierStack, - config: ScoringPipelineConfig[Query, Candidate] - ): ScoringPipeline[Query, Candidate] = { - - val pipelineIdentifier = config.identifier - - val context = Executor.Context( - PipelineFailureClassifier( - config.failureClassifier.orElse(StoppedGateException.classifier(ClosedGate))), - parentComponentIdentifierStack.push(pipelineIdentifier) - ) - - val enabledGateOpt = config.enabledDeciderParam.map { deciderParam => - ParamGate(pipelineIdentifier + EnabledGateSuffix, deciderParam) - } - val supportedClientGateOpt = config.supportedClientParam.map { param => - ParamGate(pipelineIdentifier + SupportedClientGateSuffix, param) - } - - /** - * Evaluate enabled decider gate first since if it's off, there is no reason to proceed - * Next evaluate supported client feature switch gate, followed by customer configured gates - */ - val allGates = enabledGateOpt.toSeq ++ supportedClientGateOpt.toSeq ++ config.gates - - val GatesStep = new Step[Query, GateExecutorResult] { - override def identifier: PipelineStepIdentifier = ScoringPipelineConfig.gatesStep - - override lazy val executorArrow: Arrow[Query, GateExecutorResult] = - gateExecutor.arrow(allGates, context) - - override def inputAdaptor( - query: ScoringPipeline.Inputs[Query], - previousResult: ScoringPipelineResult[Candidate] - ): Query = { - query.query - } - - override def resultUpdater( - previousPipelineResult: ScoringPipelineResult[Candidate], - executorResult: GateExecutorResult - ): ScoringPipelineResult[Candidate] = - previousPipelineResult.copy(gateResults = Some(executorResult)) - } - - val SelectorsStep = new Step[SelectorExecutor.Inputs[Query], SelectorExecutorResult] { - override def identifier: PipelineStepIdentifier = ScoringPipelineConfig.selectorsStep - - override def executorArrow: Arrow[SelectorExecutor.Inputs[Query], SelectorExecutorResult] = - selectorExecutor.arrow(config.selectors, context) - - override def inputAdaptor( - query: ScoringPipeline.Inputs[Query], - previousResult: ScoringPipelineResult[Candidate] - ): SelectorExecutor.Inputs[Query] = SelectorExecutor.Inputs(query.query, query.candidates) - - override def resultUpdater( - previousPipelineResult: ScoringPipelineResult[Candidate], - executorResult: SelectorExecutorResult - ): ScoringPipelineResult[Candidate] = - previousPipelineResult.copy(selectorResults = Some(executorResult)) - } - - val PreScoringFeatureHydrationPhase1Step = - new Step[ - CandidateFeatureHydratorExecutor.Inputs[Query, Candidate], - CandidateFeatureHydratorExecutorResult[Candidate] - ] { - override def identifier: PipelineStepIdentifier = - ScoringPipelineConfig.preScoringFeatureHydrationPhase1Step - - override def executorArrow: Arrow[ - CandidateFeatureHydratorExecutor.Inputs[Query, Candidate], - CandidateFeatureHydratorExecutorResult[Candidate] - ] = - candidateFeatureHydratorExecutor.arrow(config.preScoringFeatureHydrationPhase1, context) - - override def inputAdaptor( - query: ScoringPipeline.Inputs[Query], - previousResult: ScoringPipelineResult[Candidate] - ): CandidateFeatureHydratorExecutor.Inputs[Query, Candidate] = { - val selectedCandidatesResult = previousResult.selectorResults.getOrElse { - throw InvalidStepStateException(identifier, "SelectorResults") - }.selectedCandidates - - CandidateFeatureHydratorExecutor.Inputs( - query.query, - selectedCandidatesResult.asInstanceOf[Seq[CandidateWithFeatures[Candidate]]]) - } - - override def resultUpdater( - previousPipelineResult: ScoringPipelineResult[Candidate], - executorResult: CandidateFeatureHydratorExecutorResult[Candidate] - ): ScoringPipelineResult[Candidate] = previousPipelineResult.copy( - preScoringHydrationPhase1Result = Some(executorResult) - ) - } - - val PreScoringFeatureHydrationPhase2Step = - new Step[ - CandidateFeatureHydratorExecutor.Inputs[Query, Candidate], - CandidateFeatureHydratorExecutorResult[Candidate] - ] { - override def identifier: PipelineStepIdentifier = - ScoringPipelineConfig.preScoringFeatureHydrationPhase2Step - - override def executorArrow: Arrow[ - CandidateFeatureHydratorExecutor.Inputs[Query, Candidate], - CandidateFeatureHydratorExecutorResult[Candidate] - ] = - candidateFeatureHydratorExecutor.arrow(config.preScoringFeatureHydrationPhase2, context) - - override def inputAdaptor( - query: ScoringPipeline.Inputs[Query], - previousResult: ScoringPipelineResult[Candidate] - ): CandidateFeatureHydratorExecutor.Inputs[Query, Candidate] = { - val preScoringHydrationPhase1FeatureMaps: Seq[FeatureMap] = - previousResult.preScoringHydrationPhase1Result - .getOrElse( - throw InvalidStepStateException(identifier, "PreScoringHydrationPhase1Result")) - .results.map(_.features) - - val itemCandidates = previousResult.selectorResults - .getOrElse(throw InvalidStepStateException(identifier, "SelectionResults")) - .selectedCandidates.collect { - case itemCandidate: ItemCandidateWithDetails => itemCandidate - } - // If there is no feature hydration (empty results), no need to attempt merging. - val candidates = if (preScoringHydrationPhase1FeatureMaps.isEmpty) { - itemCandidates - } else { - itemCandidates.zip(preScoringHydrationPhase1FeatureMaps).map { - case (itemCandidate, featureMap) => - itemCandidate.copy(features = itemCandidate.features ++ featureMap) - } - } - - CandidateFeatureHydratorExecutor.Inputs( - query.query, - candidates.asInstanceOf[Seq[CandidateWithFeatures[Candidate]]]) - } - - override def resultUpdater( - previousPipelineResult: ScoringPipelineResult[Candidate], - executorResult: CandidateFeatureHydratorExecutorResult[Candidate] - ): ScoringPipelineResult[Candidate] = previousPipelineResult.copy( - preScoringHydrationPhase2Result = Some(executorResult) - ) - } - - def getMergedPreScoringFeatureMap( - stepIdentifier: PipelineStepIdentifier, - previousResult: ScoringPipelineResult[Candidate] - ): Seq[FeatureMap] = { - val preScoringHydrationPhase1FeatureMaps: Seq[FeatureMap] = - previousResult.preScoringHydrationPhase1Result - .getOrElse( - throw InvalidStepStateException( - stepIdentifier, - "PreScoringHydrationPhase1Result")).results.map(_.features) - - val preScoringHydrationPhase2FeatureMaps: Seq[FeatureMap] = - previousResult.preScoringHydrationPhase2Result - .getOrElse( - throw InvalidStepStateException( - stepIdentifier, - "PreScoringHydrationPhase2Result")).results.map(_.features) - /* - * If either pre-scoring hydration phase feature map is empty, no need to merge them, - * we can just take all non-empty ones. - */ - if (preScoringHydrationPhase1FeatureMaps.isEmpty) { - preScoringHydrationPhase2FeatureMaps - } else if (preScoringHydrationPhase2FeatureMaps.isEmpty) { - preScoringHydrationPhase1FeatureMaps - } else { - // No need to check the size in both, since the inputs to both hydration phases are the - // same and each phase ensures the number of candidates and ordering matches the input. - preScoringHydrationPhase1FeatureMaps.zip(preScoringHydrationPhase2FeatureMaps).map { - case (preScoringHydrationPhase1FeatureMap, preScoringHydrationPhasesFeatureMap) => - preScoringHydrationPhase1FeatureMap ++ preScoringHydrationPhasesFeatureMap - } - } - } - - val ScorersStep = - new Step[ - CandidateFeatureHydratorExecutor.Inputs[Query, Candidate], - CandidateFeatureHydratorExecutorResult[Candidate] - ] { - override def identifier: PipelineStepIdentifier = ScoringPipelineConfig.scorersStep - - override def inputAdaptor( - query: ScoringPipeline.Inputs[Query], - previousResult: ScoringPipelineResult[Candidate] - ): CandidateFeatureHydratorExecutor.Inputs[Query, Candidate] = { - - val mergedPreScoringFeatureHydrationFeatures: Seq[FeatureMap] = - getMergedPreScoringFeatureMap(ScoringPipelineConfig.scorersStep, previousResult) - - val itemCandidates = previousResult.selectorResults - .getOrElse(throw InvalidStepStateException(identifier, "SelectionResults")) - .selectedCandidates.collect { - case itemCandidate: ItemCandidateWithDetails => itemCandidate - } - - // If there was no pre-scoring features hydration, no need to re-merge feature maps - // and construct a new item candidate - val updatedCandidates = if (mergedPreScoringFeatureHydrationFeatures.isEmpty) { - itemCandidates - } else { - itemCandidates.zip(mergedPreScoringFeatureHydrationFeatures).map { - case (itemCandidate, preScoringFeatureMap) => - itemCandidate.copy(features = itemCandidate.features ++ preScoringFeatureMap) - } - } - CandidateFeatureHydratorExecutor.Inputs( - query.query, - updatedCandidates.asInstanceOf[Seq[CandidateWithFeatures[Candidate]]]) - } - - override lazy val executorArrow: Arrow[ - CandidateFeatureHydratorExecutor.Inputs[Query, Candidate], - CandidateFeatureHydratorExecutorResult[ - Candidate - ] - ] = candidateFeatureHydratorExecutor.arrow(config.scorers.toSeq, context) - - override def resultUpdater( - previousPipelineResult: ScoringPipelineResult[Candidate], - executorResult: CandidateFeatureHydratorExecutorResult[Candidate] - ): ScoringPipelineResult[Candidate] = - previousPipelineResult.copy(scorerResults = Some(executorResult)) - } - - val ResultStep = - new Step[Seq[CandidateWithFeatures[UniversalNoun[Any]]], Seq[ - CandidateWithFeatures[UniversalNoun[Any]] - ]] { - override def identifier: PipelineStepIdentifier = ScoringPipelineConfig.resultStep - - override def executorArrow: Arrow[Seq[CandidateWithFeatures[UniversalNoun[Any]]], Seq[ - CandidateWithFeatures[UniversalNoun[Any]] - ]] = Arrow.identity - - override def inputAdaptor( - query: Inputs[Query], - previousResult: ScoringPipelineResult[Candidate] - ): Seq[CandidateWithFeatures[UniversalNoun[Any]]] = previousResult.selectorResults - .getOrElse(throw InvalidStepStateException(identifier, "SelectionResults")) - .selectedCandidates.collect { - case itemCandidate: ItemCandidateWithDetails => itemCandidate - } - - override def resultUpdater( - previousPipelineResult: ScoringPipelineResult[Candidate], - executorResult: Seq[CandidateWithFeatures[UniversalNoun[Any]]] - ): ScoringPipelineResult[Candidate] = { - val scorerResults: Seq[FeatureMap] = previousPipelineResult.scorerResults - .getOrElse(throw InvalidStepStateException(identifier, "ScorerResult")).results.map( - _.features) - - val mergedPreScoringFeatureHydrationFeatureMaps: Seq[FeatureMap] = - getMergedPreScoringFeatureMap(ScoringPipelineConfig.resultStep, previousPipelineResult) - - val itemCandidates = executorResult.asInstanceOf[Seq[ItemCandidateWithDetails]] - val finalFeatureMap = if (mergedPreScoringFeatureHydrationFeatureMaps.isEmpty) { - scorerResults - } else { - scorerResults - .zip(mergedPreScoringFeatureHydrationFeatureMaps).map { - case (preScoringFeatureMap, scoringFeatureMap) => - preScoringFeatureMap ++ scoringFeatureMap - } - } - - val finalResults = itemCandidates.zip(finalFeatureMap).map { - case (itemCandidate, featureMap) => - ScoredCandidateResult(itemCandidate.candidate.asInstanceOf[Candidate], featureMap) - } - previousPipelineResult.withResult(finalResults) - } - } - - val builtSteps = Seq( - GatesStep, - SelectorsStep, - PreScoringFeatureHydrationPhase1Step, - PreScoringFeatureHydrationPhase2Step, - ScorersStep, - ResultStep - ) - - /** The main execution logic for this Candidate Pipeline. */ - val finalArrow: Arrow[ScoringPipeline.Inputs[Query], ScoringPipelineResult[Candidate]] = - buildCombinedArrowFromSteps( - steps = builtSteps, - context = context, - initialEmptyResult = ScoringPipelineResult.empty, - stepsInOrderFromConfig = ScoringPipelineConfig.stepsInOrder - ) - - val configFromBuilder = config - new ScoringPipeline[Query, Candidate] { - override private[core] val config: ScoringPipelineConfig[Query, Candidate] = configFromBuilder - override val arrow: Arrow[ScoringPipeline.Inputs[Query], ScoringPipelineResult[Candidate]] = - finalArrow - override val identifier: ScoringPipelineIdentifier = pipelineIdentifier - override val alerts: Seq[Alert] = config.alerts - override val children: Seq[Component] = - allGates ++ config.preScoringFeatureHydrationPhase1 ++ config.preScoringFeatureHydrationPhase2 ++ config.scorers - } - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipelineBuilderFactory.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipelineBuilderFactory.docx new file mode 100644 index 000000000..d8409268e Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipelineBuilderFactory.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipelineBuilderFactory.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipelineBuilderFactory.scala deleted file mode 100644 index 5b65d0738..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipelineBuilderFactory.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.scoring - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutor -import com.twitter.product_mixer.core.service.gate_executor.GateExecutor -import com.twitter.product_mixer.core.service.selector_executor.SelectorExecutor -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ScoringPipelineBuilderFactory @Inject() ( - gateExecutor: GateExecutor, - selectorExecutor: SelectorExecutor, - candidateFeatureHydratorExecutor: CandidateFeatureHydratorExecutor, - statsReceiver: StatsReceiver) { - - def get[ - Query <: PipelineQuery, - Candidate <: UniversalNoun[Any] - ]: ScoringPipelineBuilder[Query, Candidate] = { - new ScoringPipelineBuilder[Query, Candidate]( - gateExecutor, - selectorExecutor, - candidateFeatureHydratorExecutor, - statsReceiver - ) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipelineConfig.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipelineConfig.docx new file mode 100644 index 000000000..3c2fbac5d Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipelineConfig.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipelineConfig.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipelineConfig.scala deleted file mode 100644 index d2b87e01b..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipelineConfig.scala +++ /dev/null @@ -1,131 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.scoring - -import com.twitter.product_mixer.component_library.selector.InsertAppendResults -import com.twitter.product_mixer.core.functional_component.common.AllPipelines -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.gate.BaseGate -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifierStack -import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineConfig -import com.twitter.product_mixer.core.pipeline.PipelineConfigCompanion -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.decider.DeciderParam - -/** - * This is the configuration necessary to generate a Scoring Pipeline. Product code should create a - * ScoringPipelineConfig, and then use a ScoringPipelineBuilder to get the final ScoringPipeline which can - * process requests. - * - * @tparam Query - The domain model for the query or request - * @tparam Candidate the domain model for the candidate being scored - */ -trait ScoringPipelineConfig[-Query <: PipelineQuery, Candidate <: UniversalNoun[Any]] - extends PipelineConfig { - - override val identifier: ScoringPipelineIdentifier - - /** - * When these Params are defined, they will automatically be added as Gates in the pipeline - * by the CandidatePipelineBuilder - * - * The enabled decider param can to be used to quickly disable a Candidate Pipeline via Decider - */ - val enabledDeciderParam: Option[DeciderParam[Boolean]] = None - - /** - * This supported client feature switch param can be used with a Feature Switch to control the - * rollout of a new Candidate Pipeline from dogfood to experiment to production - */ - val supportedClientParam: Option[FSParam[Boolean]] = None - - /** [[BaseGate]]s that are applied sequentially, the pipeline will only run if all the Gates are open */ - def gates: Seq[BaseGate[Query]] = Seq.empty - - /** - * Logic for selecting which candidates to score. Note, this doesn't drop the candidates from - * the final result, just whether to score it in this pipeline or not. - */ - def selectors: Seq[Selector[Query]] - - /** - * After selectors are run, you can fetch features for each candidate. - * The existing features from previous hydrations are passed in as inputs. You are not expected to - * put them into the resulting feature map yourself - they will be merged for you by the platform. - */ - def preScoringFeatureHydrationPhase1: Seq[BaseCandidateFeatureHydrator[Query, Candidate, _]] = - Seq.empty - - /** - * A second phase of feature hydration that can be run after selection and after the first phase - * of pre-scoring feature hydration. You are not expected to put them into the resulting - * feature map yourself - they will be merged for you by the platform. - */ - def preScoringFeatureHydrationPhase2: Seq[BaseCandidateFeatureHydrator[Query, Candidate, _]] = - Seq.empty - - /** - * Ranker Function for candidates. Scorers are executed in parallel. - * Note: Order does not matter, this could be a Set if Set was covariant over it's type. - */ - def scorers: Seq[Scorer[Query, Candidate]] - - /** - * A pipeline can define a partial function to rescue failures here. They will be treated as failures - * from a monitoring standpoint, and cancellation exceptions will always be propagated (they cannot be caught here). - */ - def failureClassifier: PartialFunction[Throwable, PipelineFailure] = PartialFunction.empty - - /** - * Alerts can be used to indicate the pipeline's service level objectives. Alerts and - * dashboards will be automatically created based on this information. - */ - val alerts: Seq[Alert] = Seq.empty - - /** - * This method is used by the product mixer framework to build the pipeline. - */ - private[core] final def build( - parentComponentIdentifierStack: ComponentIdentifierStack, - builder: ScoringPipelineBuilderFactory - ): ScoringPipeline[Query, Candidate] = - builder.get.build(parentComponentIdentifierStack, this) -} - -object ScoringPipelineConfig extends PipelineConfigCompanion { - def apply[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]( - scorer: Scorer[Query, Candidate] - ): ScoringPipelineConfig[Query, Candidate] = new ScoringPipelineConfig[Query, Candidate] { - override val identifier: ScoringPipelineIdentifier = ScoringPipelineIdentifier( - s"ScoreAll${scorer.identifier.name}") - - override val selectors: Seq[Selector[Query]] = Seq(InsertAppendResults(AllPipelines)) - - override val scorers: Seq[Scorer[Query, Candidate]] = Seq(scorer) - } - - val gatesStep: PipelineStepIdentifier = PipelineStepIdentifier("Gates") - val selectorsStep: PipelineStepIdentifier = PipelineStepIdentifier("Selectors") - val preScoringFeatureHydrationPhase1Step: PipelineStepIdentifier = - PipelineStepIdentifier("PreScoringFeatureHydrationPhase1") - val preScoringFeatureHydrationPhase2Step: PipelineStepIdentifier = - PipelineStepIdentifier("PreScoringFeatureHydrationPhase2") - val scorersStep: PipelineStepIdentifier = PipelineStepIdentifier("Scorers") - val resultStep: PipelineStepIdentifier = PipelineStepIdentifier("Result") - - /** All the Steps which are executed by a [[ScoringPipeline]] in the order in which they are run */ - override val stepsInOrder: Seq[PipelineStepIdentifier] = Seq( - gatesStep, - selectorsStep, - preScoringFeatureHydrationPhase1Step, - preScoringFeatureHydrationPhase2Step, - scorersStep, - resultStep - ) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipelineResult.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipelineResult.docx new file mode 100644 index 000000000..fd7a4b84d Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipelineResult.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipelineResult.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipelineResult.scala deleted file mode 100644 index ca1585b4c..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring/ScoringPipelineResult.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.scoring - -import com.twitter.product_mixer.core.functional_component.scorer.ScoredCandidateResult -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.pipeline.PipelineResult -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutorResult -import com.twitter.product_mixer.core.service.gate_executor.GateExecutorResult -import com.twitter.product_mixer.core.service.selector_executor.SelectorExecutorResult - -/** - * The Results of every step during the ScoringPipeline process. The end result contains - * only the candidates that were actually scored (e.g, not dropped by a filter) with an updated, - * combined feature map of all features that were passed in with the candidate plus all features - * returned as part of scoring. - */ -case class ScoringPipelineResult[Candidate <: UniversalNoun[Any]]( - gateResults: Option[GateExecutorResult], - selectorResults: Option[SelectorExecutorResult], - preScoringHydrationPhase1Result: Option[CandidateFeatureHydratorExecutorResult[Candidate]], - preScoringHydrationPhase2Result: Option[CandidateFeatureHydratorExecutorResult[Candidate]], - scorerResults: Option[CandidateFeatureHydratorExecutorResult[ - Candidate - ]], - failure: Option[PipelineFailure], - result: Option[Seq[ScoredCandidateResult[Candidate]]]) - extends PipelineResult[Seq[ScoredCandidateResult[Candidate]]] { - override val resultSize: Int = result.map(_.size).getOrElse(0) - - override def withFailure( - failure: PipelineFailure - ): ScoringPipelineResult[Candidate] = - copy(failure = Some(failure)) - override def withResult( - result: Seq[ScoredCandidateResult[Candidate]] - ): ScoringPipelineResult[Candidate] = - copy(result = Some(result)) -} - -object ScoringPipelineResult { - def empty[Candidate <: UniversalNoun[Any]]: ScoringPipelineResult[Candidate] = - ScoringPipelineResult( - None, - None, - None, - None, - None, - None, - None - ) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/BUILD deleted file mode 100644 index fe26c9f66..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/BUILD +++ /dev/null @@ -1,33 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator:decoration", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator:decoration", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/BUILD.docx new file mode 100644 index 000000000..484757df5 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasAsyncFeatureMap.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasAsyncFeatureMap.docx new file mode 100644 index 000000000..6d14aba88 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasAsyncFeatureMap.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasAsyncFeatureMap.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasAsyncFeatureMap.scala deleted file mode 100644 index 9ff0ed803..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasAsyncFeatureMap.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.state - -import com.twitter.product_mixer.core.feature.featuremap.asyncfeaturemap.AsyncFeatureMap - -trait HasAsyncFeatureMap[State] { - def asyncFeatureMap: AsyncFeatureMap - - private[core] def addAsyncFeatureMap(newFeatureMap: AsyncFeatureMap): State -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasCandidates.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasCandidates.docx new file mode 100644 index 000000000..239e3cfe5 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasCandidates.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasCandidates.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasCandidates.scala deleted file mode 100644 index 85f2937c7..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasCandidates.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.state - -import com.twitter.product_mixer.core.model.common.UniversalNoun - -trait HasCandidates[Candidate <: UniversalNoun[Any], T] { - def candidates: Seq[Candidate] - def updateCandidates(newCandidates: Seq[Candidate]): T -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasCandidatesWithDetails.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasCandidatesWithDetails.docx new file mode 100644 index 000000000..57229f2f2 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasCandidatesWithDetails.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasCandidatesWithDetails.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasCandidatesWithDetails.scala deleted file mode 100644 index f91d17bb3..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasCandidatesWithDetails.scala +++ /dev/null @@ -1,11 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.state - -import com.twitter.product_mixer.core.functional_component.decorator.Decoration -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails - -trait HasCandidatesWithDetails[T] { - def candidatesWithDetails: Seq[CandidateWithDetails] - def updateCandidatesWithDetails(newCandidates: Seq[CandidateWithDetails]): T - - def updateDecorations(decoration: Seq[Decoration]): T -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasCandidatesWithFeatures.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasCandidatesWithFeatures.docx new file mode 100644 index 000000000..6846cb67d Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasCandidatesWithFeatures.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasCandidatesWithFeatures.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasCandidatesWithFeatures.scala deleted file mode 100644 index 1fa327e6b..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasCandidatesWithFeatures.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.state - -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun - -trait HasCandidatesWithFeatures[Candidate <: UniversalNoun[Any], T] { - def candidatesWithFeatures: Seq[CandidateWithFeatures[Candidate]] - def updateCandidatesWithFeatures(newCandidates: Seq[CandidateWithFeatures[Candidate]]): T -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasExecutorResults.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasExecutorResults.docx new file mode 100644 index 000000000..c233b702a Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasExecutorResults.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasExecutorResults.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasExecutorResults.scala deleted file mode 100644 index c22dbe11c..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasExecutorResults.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.state - -import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier -import com.twitter.product_mixer.core.service.ExecutorResult -import scala.collection.immutable.ListMap - -trait HasExecutorResults[State] { - // We use a list map to maintain the insertion order - val executorResultsByPipelineStep: ListMap[PipelineStepIdentifier, ExecutorResult] - private[pipeline] def setExecutorResults( - newMap: ListMap[PipelineStepIdentifier, ExecutorResult] - ): State -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasParams.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasParams.docx new file mode 100644 index 000000000..e2fe9f612 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasParams.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasParams.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasParams.scala deleted file mode 100644 index 42f7f0062..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasParams.scala +++ /dev/null @@ -1,7 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.state - -import com.twitter.timelines.configapi.Params - -trait HasParams { - def params: Params -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasQuery.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasQuery.docx new file mode 100644 index 000000000..8dc28b7a6 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasQuery.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasQuery.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasQuery.scala deleted file mode 100644 index 89e9bd94e..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasQuery.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.state - -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -trait HasQuery[Query <: PipelineQuery, T] { - def query: Query - def updateQuery(query: Query): T -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasRequest.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasRequest.docx new file mode 100644 index 000000000..ccde225fd Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasRequest.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasRequest.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasRequest.scala deleted file mode 100644 index 9058f0561..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasRequest.scala +++ /dev/null @@ -1,7 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.state - -import com.twitter.product_mixer.core.model.marshalling.request.Request - -trait HasRequest[TRequest <: Request] { - def request: TRequest -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasResult.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasResult.docx new file mode 100644 index 000000000..b12f7de0e Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasResult.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasResult.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasResult.scala deleted file mode 100644 index 0c5f0dfef..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state/HasResult.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.state - -/** - * Defines how to build a result from a pipeline state. Pipeline States should extend this and - * implement [[buildResult]] which computes the final result from their current state. - * @tparam Result Type of result - */ -trait HasResult[+Result] { - def buildResult(): Result -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/BUILD deleted file mode 100644 index cbd8c036d..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/BUILD +++ /dev/null @@ -1,17 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "stitch/stitch-core", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/BUILD.docx new file mode 100644 index 000000000..97a20f9a3 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/Step.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/Step.docx new file mode 100644 index 000000000..ecbee8a84 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/Step.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/Step.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/Step.scala deleted file mode 100644 index 99aa9ae87..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/Step.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.step - -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.ExecutorResult -import com.twitter.stitch.Arrow - -/** - * A Step within a Pipeline, a Step is a unitary phase within an entire chain that makes up a pipeline. - * @tparam State The request domain model. - * @tparam ExecutorConfig The configs that should be passed into the executor at build time. - * @tparam ExecutorInput The input type that an executor takes at request time. - * @tparam ExResult The result that a step's executor will return. - * @tparam OutputState The final/updated state a step would output, this is typically taking the ExResult - * and mutating/transforming the State. - */ -trait Step[State, -Config, ExecutorInput, ExResult <: ExecutorResult] { - - /** - * Adapt the state into the expected input for the Step's Arrow. - * - * @param state State object passed into the Step. - * @param config The config object used to build the executor arrow or input. - * @return ExecutorInput that is used in the arrow of the underlying executor. - */ - def adaptInput(state: State, config: Config): ExecutorInput - - /** - * The actual arrow to be executed for the step, taking in the adapted input from [[adaptInput]] - * and returning the expected result. - * @param config Runtime configurations to configure the arrow with. - * @param context Context of Executor. - */ - def arrow( - config: Config, - context: Executor.Context - ): Arrow[ExecutorInput, ExResult] - - /** - * Whether the step is considered a noop/empty based off of input being passed in. Empty - * steps are skipped when being executed. - */ - def isEmpty(config: Config): Boolean - - /** - * Update the passed in state based off of the result from [[arrow]] - * @param state State object passed into the Step. - * @param executorResult Executor result returned from [[arrow]] - * @param config The config object used to build the executor arrow or input. - * @return Updated state object passed. - */ - def updateState(state: State, executorResult: ExResult, config: Config): State -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/async_feature_map/AsyncFeatureMapStep.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/async_feature_map/AsyncFeatureMapStep.docx new file mode 100644 index 000000000..8cbe487b6 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/async_feature_map/AsyncFeatureMapStep.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/async_feature_map/AsyncFeatureMapStep.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/async_feature_map/AsyncFeatureMapStep.scala deleted file mode 100644 index 9f0c787bb..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/async_feature_map/AsyncFeatureMapStep.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.step.async_feature_map - -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.asyncfeaturemap.AsyncFeatureMap -import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.state.HasAsyncFeatureMap -import com.twitter.product_mixer.core.pipeline.state.HasQuery -import com.twitter.product_mixer.core.pipeline.step.Step -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.async_feature_map_executor.AsyncFeatureMapExecutor -import com.twitter.product_mixer.core.service.async_feature_map_executor.AsyncFeatureMapExecutorResults -import com.twitter.stitch.Arrow -import javax.inject.Inject - -/** - * Async Feature Hydrator Step, it takes an existing asyn feature map and executes any hydration - * needed before the next step. The state object is responsible for keeping the updated query - * with the updated feature map. - * - * @param asyncFeatureMapExecutor Async feature map executor - * - * @tparam Query Type of PipelineQuery domain model - * @tparam State The pipeline state domain model. - */ -case class AsyncFeatureMapStep[ - Query <: PipelineQuery, - State <: HasQuery[Query, State] with HasAsyncFeatureMap[State]] @Inject() ( - asyncFeatureMapExecutor: AsyncFeatureMapExecutor) - extends Step[ - State, - AsyncFeatureMapStepConfig, - AsyncFeatureMap, - AsyncFeatureMapExecutorResults - ] { - override def isEmpty(config: AsyncFeatureMapStepConfig): Boolean = false - - override def adaptInput( - state: State, - config: AsyncFeatureMapStepConfig - ): AsyncFeatureMap = state.asyncFeatureMap - - override def arrow( - config: AsyncFeatureMapStepConfig, - context: Executor.Context - ): Arrow[AsyncFeatureMap, AsyncFeatureMapExecutorResults] = - asyncFeatureMapExecutor.arrow(config.stepToHydrateFor, config.currentStep, context) - - override def updateState( - state: State, - executorResult: AsyncFeatureMapExecutorResults, - config: AsyncFeatureMapStepConfig - ): State = { - val hydratedFeatureMap = - executorResult.featureMapsByStep.getOrElse(config.stepToHydrateFor, FeatureMap.empty) - if (hydratedFeatureMap.isEmpty) { - state - } else { - val updatedFeatureMap = state.query.features - .getOrElse(FeatureMap.empty) ++ hydratedFeatureMap - state.updateQuery( - state.query - .withFeatureMap(updatedFeatureMap).asInstanceOf[Query]) - } - } -} - -case class AsyncFeatureMapStepConfig( - stepToHydrateFor: PipelineStepIdentifier, - currentStep: PipelineStepIdentifier) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/async_feature_map/BUILD.bazel b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/async_feature_map/BUILD.bazel deleted file mode 100644 index 144fc73d8..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/async_feature_map/BUILD.bazel +++ /dev/null @@ -1,36 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/async_feature_map_executor", - "stitch/stitch-core", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/async_feature_map_executor", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/async_feature_map/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/async_feature_map/BUILD.docx new file mode 100644 index 000000000..4b80d3985 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/async_feature_map/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/candidate_feature_hydrator/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/candidate_feature_hydrator/BUILD deleted file mode 100644 index a15f0c954..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/candidate_feature_hydrator/BUILD +++ /dev/null @@ -1,36 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_hydrator_executor", - "stitch/stitch-core", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_hydrator_executor", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/candidate_feature_hydrator/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/candidate_feature_hydrator/BUILD.docx new file mode 100644 index 000000000..50a95a39d Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/candidate_feature_hydrator/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/candidate_feature_hydrator/CandidateFeatureHydratorStep.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/candidate_feature_hydrator/CandidateFeatureHydratorStep.docx new file mode 100644 index 000000000..446587734 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/candidate_feature_hydrator/CandidateFeatureHydratorStep.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/candidate_feature_hydrator/CandidateFeatureHydratorStep.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/candidate_feature_hydrator/CandidateFeatureHydratorStep.scala deleted file mode 100644 index 94c11256d..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/candidate_feature_hydrator/CandidateFeatureHydratorStep.scala +++ /dev/null @@ -1,71 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.step.candidate_feature_hydrator - -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.state.HasCandidatesWithFeatures -import com.twitter.product_mixer.core.pipeline.state.HasQuery -import com.twitter.product_mixer.core.pipeline.step.Step -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutor -import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutorResult -import com.twitter.stitch.Arrow -import javax.inject.Inject - -/** - * A candidate level feature hydration step, it takes the input list of candidates and the given - * hydrators and executes them. The [[State]] object is responsible for merging the resulting - * feature maps with the hydrated ones in its updateCandidatesWithFeatures. - * - * @param candidateFeatureHydratorExecutor Hydrator Executor - * @tparam Query Type of PipelineQuery domain model - * @tparam Candidate Type of Candidates to hydrate features for. - * @tparam State The pipeline state domain model. - */ -case class CandidateFeatureHydratorStep[ - Query <: PipelineQuery, - Candidate <: UniversalNoun[Any], - State <: HasQuery[Query, State] with HasCandidatesWithFeatures[ - Candidate, - State - ]] @Inject() ( - candidateFeatureHydratorExecutor: CandidateFeatureHydratorExecutor) - extends Step[State, Seq[ - BaseCandidateFeatureHydrator[Query, Candidate, _] - ], CandidateFeatureHydratorExecutor.Inputs[ - Query, - Candidate - ], CandidateFeatureHydratorExecutorResult[Candidate]] { - - override def adaptInput( - state: State, - config: Seq[BaseCandidateFeatureHydrator[Query, Candidate, _]] - ): CandidateFeatureHydratorExecutor.Inputs[Query, Candidate] = - CandidateFeatureHydratorExecutor.Inputs(state.query, state.candidatesWithFeatures) - - override def arrow( - config: Seq[BaseCandidateFeatureHydrator[Query, Candidate, _]], - context: Executor.Context - ): Arrow[ - CandidateFeatureHydratorExecutor.Inputs[Query, Candidate], - CandidateFeatureHydratorExecutorResult[Candidate] - ] = candidateFeatureHydratorExecutor.arrow(config, context) - - override def updateState( - input: State, - executorResult: CandidateFeatureHydratorExecutorResult[Candidate], - config: Seq[BaseCandidateFeatureHydrator[Query, Candidate, _]] - ): State = { - val candidatesWithHydratedFeatures = executorResult.results - if (candidatesWithHydratedFeatures.isEmpty) { - input - } else { - input.updateCandidatesWithFeatures(candidatesWithHydratedFeatures) - } - } - - override def isEmpty( - config: Seq[BaseCandidateFeatureHydrator[Query, Candidate, _]] - ): Boolean = - config.isEmpty -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/candidate_source/BUILD.bazel b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/candidate_source/BUILD.bazel deleted file mode 100644 index 933fd47a3..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/candidate_source/BUILD.bazel +++ /dev/null @@ -1,36 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_source_executor", - "stitch/stitch-core", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_source_executor", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/candidate_source/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/candidate_source/BUILD.docx new file mode 100644 index 000000000..b29b1eb82 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/candidate_source/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/candidate_source/CandidateSourceStep.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/candidate_source/CandidateSourceStep.docx new file mode 100644 index 000000000..bfc0ab177 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/candidate_source/CandidateSourceStep.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/candidate_source/CandidateSourceStep.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/candidate_source/CandidateSourceStep.scala deleted file mode 100644 index 359ff64cd..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/candidate_source/CandidateSourceStep.scala +++ /dev/null @@ -1,84 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.step.candidate_source - -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.transformer.BaseCandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.state.HasCandidatesWithFeatures -import com.twitter.product_mixer.core.pipeline.state.HasQuery -import com.twitter.product_mixer.core.pipeline.step.Step -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.candidate_source_executor.CandidateSourceExecutor -import com.twitter.product_mixer.core.service.candidate_source_executor.CandidateSourceExecutorResult -import com.twitter.stitch.Arrow -import javax.inject.Inject - -/** - * A candidate source step, which takes the query and gets csandidates from the candidate source. - * - * @param candidateSourceExecutor Candidate Source Executor - * @tparam Query Type of PipelineQuery domain model - * @tparam Candidate Type of Candidates to filter - * @tparam State The pipeline state domain model. - */ -case class CandidateSourceStep[ - Query <: PipelineQuery, - CandidateSourceQuery, - CandidateSourceResult, - Candidate <: UniversalNoun[Any], - State <: HasQuery[Query, State] with HasCandidatesWithFeatures[Candidate, State]] @Inject() ( - candidateSourceExecutor: CandidateSourceExecutor) - extends Step[ - State, - CandidateSourceConfig[Query, CandidateSourceQuery, CandidateSourceResult, Candidate], - Query, - CandidateSourceExecutorResult[ - Candidate - ] - ] { - override def isEmpty( - config: CandidateSourceConfig[Query, CandidateSourceQuery, CandidateSourceResult, Candidate] - ): Boolean = false - - override def adaptInput( - state: State, - config: CandidateSourceConfig[Query, CandidateSourceQuery, CandidateSourceResult, Candidate] - ): Query = state.query - - override def arrow( - config: CandidateSourceConfig[Query, CandidateSourceQuery, CandidateSourceResult, Candidate], - context: Executor.Context - ): Arrow[Query, CandidateSourceExecutorResult[Candidate]] = candidateSourceExecutor.arrow( - config.candidateSource, - config.queryTransformer, - config.resultTransformer, - config.resultFeaturesTransformers, - context - ) - - override def updateState( - state: State, - executorResult: CandidateSourceExecutorResult[Candidate], - config: CandidateSourceConfig[Query, CandidateSourceQuery, CandidateSourceResult, Candidate] - ): State = state - .updateQuery( - state.query - .withFeatureMap(executorResult.candidateSourceFeatureMap).asInstanceOf[ - Query]).updateCandidatesWithFeatures(executorResult.candidates) -} - -case class CandidateSourceConfig[ - Query <: PipelineQuery, - CandidateSourceQuery, - CandidateSourceResult, - Candidate <: UniversalNoun[Any] -]( - candidateSource: BaseCandidateSource[CandidateSourceQuery, CandidateSourceResult], - queryTransformer: BaseCandidatePipelineQueryTransformer[ - Query, - CandidateSourceQuery - ], - resultTransformer: CandidatePipelineResultsTransformer[CandidateSourceResult, Candidate], - resultFeaturesTransformers: Seq[CandidateFeatureTransformer[CandidateSourceResult]]) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/decorator/BUILD.bazel b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/decorator/BUILD.bazel deleted file mode 100644 index 88d9b3be9..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/decorator/BUILD.bazel +++ /dev/null @@ -1,36 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_decorator_executor", - "stitch/stitch-core", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_decorator_executor", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/decorator/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/decorator/BUILD.docx new file mode 100644 index 000000000..36872cc9f Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/decorator/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/decorator/DecoratorStep.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/decorator/DecoratorStep.docx new file mode 100644 index 000000000..e25c3204d Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/decorator/DecoratorStep.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/decorator/DecoratorStep.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/decorator/DecoratorStep.scala deleted file mode 100644 index 9f77bd110..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/decorator/DecoratorStep.scala +++ /dev/null @@ -1,63 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.step.decorator - -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.state.HasCandidatesWithDetails -import com.twitter.product_mixer.core.pipeline.state.HasCandidatesWithFeatures -import com.twitter.product_mixer.core.pipeline.state.HasQuery -import com.twitter.product_mixer.core.pipeline.step.Step -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.candidate_decorator_executor.CandidateDecoratorExecutor -import com.twitter.product_mixer.core.service.candidate_decorator_executor.CandidateDecoratorExecutorResult -import com.twitter.stitch.Arrow -import javax.inject.Inject - -/** - * A candidate decoration step, which takes the query and candidates and outputs decorations for them - * - * @param candidateDecoratorExecutor Candidate Source Executor - * @tparam Query Type of PipelineQuery domain model - * @tparam Candidate Type of Candidates to filter - * @tparam State The pipeline state domain model. - */ -case class DecoratorStep[ - Query <: PipelineQuery, - Candidate <: UniversalNoun[Any], - State <: HasQuery[Query, State] with HasCandidatesWithDetails[ - State - ] with HasCandidatesWithFeatures[ - Candidate, - State - ]] @Inject() (candidateDecoratorExecutor: CandidateDecoratorExecutor) - extends Step[ - State, - Option[CandidateDecorator[Query, Candidate]], - (Query, Seq[CandidateWithFeatures[Candidate]]), - CandidateDecoratorExecutorResult - ] { - - override def isEmpty(config: Option[CandidateDecorator[Query, Candidate]]): Boolean = - config.isEmpty - - override def adaptInput( - state: State, - config: Option[CandidateDecorator[Query, Candidate]] - ): (Query, Seq[CandidateWithFeatures[Candidate]]) = - (state.query, state.candidatesWithFeatures) - - override def arrow( - config: Option[CandidateDecorator[Query, Candidate]], - context: Executor.Context - ): Arrow[(Query, Seq[CandidateWithFeatures[Candidate]]), CandidateDecoratorExecutorResult] = - candidateDecoratorExecutor.arrow(config, context) - - override def updateState( - state: State, - executorResult: CandidateDecoratorExecutorResult, - config: Option[CandidateDecorator[Query, Candidate]] - ): State = { - state.updateDecorations(executorResult.result) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/domain_marshaller/BUILD.bazel b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/domain_marshaller/BUILD.bazel deleted file mode 100644 index 35b2606fe..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/domain_marshaller/BUILD.bazel +++ /dev/null @@ -1,30 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/premarshaller", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/domain_marshaller_executor", - "stitch/stitch-core", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/premarshaller", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/domain_marshaller_executor", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/domain_marshaller/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/domain_marshaller/BUILD.docx new file mode 100644 index 000000000..bd07268b8 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/domain_marshaller/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/domain_marshaller/DomainMarshallerStep.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/domain_marshaller/DomainMarshallerStep.docx new file mode 100644 index 000000000..5624325c7 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/domain_marshaller/DomainMarshallerStep.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/domain_marshaller/DomainMarshallerStep.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/domain_marshaller/DomainMarshallerStep.scala deleted file mode 100644 index 18fee29a8..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/domain_marshaller/DomainMarshallerStep.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.step.domain_marshaller - -import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.state.HasCandidatesWithDetails -import com.twitter.product_mixer.core.pipeline.state.HasQuery -import com.twitter.product_mixer.core.pipeline.step.Step -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.domain_marshaller_executor.DomainMarshallerExecutor -import com.twitter.stitch.Arrow -import javax.inject.Inject - -/** - * A domain marshaller step, it takes the input list of candidates and the given - * domain marshaller and executes its to return a marshalled result. The [[State]] object is - * responsible for keeping a reference of the built Response. - * - * @param domainMarshallerExecutor Domain Marshaller executor. - * @tparam Query Type of PipelineQuery domain model - * @tparam ResponseType the domain marshalling type expected to be returned. - * @tparam State The pipeline state domain model. - */ -case class DomainMarshallerStep[ - Query <: PipelineQuery, - ResponseType <: HasMarshalling, - State <: HasQuery[Query, State] with HasCandidatesWithDetails[State]] @Inject() ( - domainMarshallerExecutor: DomainMarshallerExecutor) - extends Step[State, DomainMarshaller[Query, ResponseType], DomainMarshallerExecutor.Inputs[ - Query - ], DomainMarshallerExecutor.Result[ResponseType]] { - - override def isEmpty(config: DomainMarshaller[Query, ResponseType]): Boolean = false - - override def adaptInput( - state: State, - config: DomainMarshaller[Query, ResponseType] - ): DomainMarshallerExecutor.Inputs[Query] = - DomainMarshallerExecutor.Inputs(state.query, state.candidatesWithDetails) - - override def arrow( - config: DomainMarshaller[Query, ResponseType], - context: Executor.Context - ): Arrow[DomainMarshallerExecutor.Inputs[Query], DomainMarshallerExecutor.Result[ResponseType]] = - domainMarshallerExecutor.arrow(config, context) - - // Noop since the pipeline updates the executor results for us - override def updateState( - state: State, - executorResult: DomainMarshallerExecutor.Result[ResponseType], - config: DomainMarshaller[Query, ResponseType] - ): State = state - -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/filter/BUILD.bazel b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/filter/BUILD.bazel deleted file mode 100644 index c5c363f7b..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/filter/BUILD.bazel +++ /dev/null @@ -1,36 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/filter_executor", - "stitch/stitch-core", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/filter_executor", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/filter/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/filter/BUILD.docx new file mode 100644 index 000000000..ebf602370 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/filter/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/filter/FilterStep.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/filter/FilterStep.docx new file mode 100644 index 000000000..c1e2b9073 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/filter/FilterStep.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/filter/FilterStep.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/filter/FilterStep.scala deleted file mode 100644 index f8d9770e1..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/filter/FilterStep.scala +++ /dev/null @@ -1,64 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.step.filter - -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.state.HasCandidatesWithFeatures -import com.twitter.product_mixer.core.pipeline.state.HasQuery -import com.twitter.product_mixer.core.pipeline.step.Step -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.filter_executor.FilterExecutor -import com.twitter.product_mixer.core.service.filter_executor.FilterExecutorResult -import com.twitter.stitch.Arrow -import javax.inject.Inject - -/** - * A candidate filter step, it takes the input list of candidates and the given filter and applies - * the filters on the candidates in sequence, returning the final kept candidates list to State. - * - * @param filterExecutor Filter Executor - * @tparam Query Type of PipelineQuery domain model - * @tparam Candidate Type of Candidates to filter - * @tparam State The pipeline state domain model. - */ -case class FilterStep[ - Query <: PipelineQuery, - Candidate <: UniversalNoun[Any], - State <: HasQuery[Query, State] with HasCandidatesWithFeatures[ - Candidate, - State - ]] @Inject() (filterExecutor: FilterExecutor) - extends Step[State, Seq[ - Filter[Query, Candidate] - ], (Query, Seq[CandidateWithFeatures[Candidate]]), FilterExecutorResult[Candidate]] { - - override def isEmpty(config: Seq[Filter[Query, Candidate]]): Boolean = config.isEmpty - - override def adaptInput( - state: State, - config: Seq[Filter[Query, Candidate]] - ): (Query, Seq[CandidateWithFeatures[Candidate]]) = - (state.query, state.candidatesWithFeatures) - - override def arrow( - config: Seq[Filter[Query, Candidate]], - context: Executor.Context - ): Arrow[(Query, Seq[CandidateWithFeatures[Candidate]]), FilterExecutorResult[Candidate]] = - filterExecutor.arrow(config, context) - - override def updateState( - state: State, - executorResult: FilterExecutorResult[Candidate], - config: Seq[Filter[Query, Candidate]] - ): State = { - val keptCandidates = executorResult.result - val candidatesMap = state.candidatesWithFeatures.map { candidatesWithFeatures => - candidatesWithFeatures.candidate -> candidatesWithFeatures - }.toMap - val newCandidates = keptCandidates.flatMap { candidate => - candidatesMap.get(candidate) - } - state.updateCandidatesWithFeatures(newCandidates) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/gate/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/gate/BUILD deleted file mode 100644 index 5ea9ee1bb..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/gate/BUILD +++ /dev/null @@ -1,36 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor", - "stitch/stitch-core", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/gate/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/gate/BUILD.docx new file mode 100644 index 000000000..84d1ee9a3 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/gate/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/gate/GateStep.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/gate/GateStep.docx new file mode 100644 index 000000000..1014786a0 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/gate/GateStep.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/gate/GateStep.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/gate/GateStep.scala deleted file mode 100644 index c100fb1ed..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/gate/GateStep.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.step.gate - -import com.twitter.product_mixer.core.functional_component.gate.BaseGate -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.state.HasQuery -import com.twitter.product_mixer.core.pipeline.step.Step -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.gate_executor.GateExecutor -import com.twitter.product_mixer.core.service.gate_executor.GateExecutorResult -import com.twitter.stitch.Arrow -import javax.inject.Inject - -/** - * A gate step, it takes the query and the given gates and executes them. Gates do not update state - * if they return continue, and throw an exception if any gate says stopped, thus no state changes - * are expected in this step. The [[NewPipelineArrowBuilder]] and [[PipelineStep]] handle short - * circuiting the pipeline's execution if this throws. - * - * @param gateExecutor Gate Executor for executing the gates - * @tparam Query Type of PipelineQuery domain model - * @tparam State The pipeline state domain model. - */ -case class GateStep[Query <: PipelineQuery, State <: HasQuery[Query, State]] @Inject() ( - gateExecutor: GateExecutor) - extends Step[State, Seq[BaseGate[Query]], Query, GateExecutorResult] { - - override def adaptInput(state: State, config: Seq[BaseGate[Query]]): Query = state.query - - override def arrow( - config: Seq[BaseGate[Query]], - context: Executor.Context - ): Arrow[Query, GateExecutorResult] = gateExecutor.arrow(config, context) - - // Gate Executor is a noop, if it continues, the state isn't changed. If it stops the world, - // an exception gets thrown. - override def updateState( - input: State, - executorResult: GateExecutorResult, - config: Seq[BaseGate[Query]] - ): State = input - - override def isEmpty(config: Seq[BaseGate[Query]]): Boolean = config.isEmpty -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/group_results/BUILD.bazel b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/group_results/BUILD.bazel deleted file mode 100644 index 1a727b523..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/group_results/BUILD.bazel +++ /dev/null @@ -1,29 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/group_results_executor", - "stitch/stitch-core", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/group_results_executor", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/group_results/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/group_results/BUILD.docx new file mode 100644 index 000000000..bd2ae47c9 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/group_results/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/group_results/GroupResultsStep.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/group_results/GroupResultsStep.docx new file mode 100644 index 000000000..21a4dc6ad Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/group_results/GroupResultsStep.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/group_results/GroupResultsStep.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/group_results/GroupResultsStep.scala deleted file mode 100644 index bb152be5d..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/group_results/GroupResultsStep.scala +++ /dev/null @@ -1,67 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.step.group_results - -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.pipeline.state.HasCandidatesWithDetails -import com.twitter.product_mixer.core.pipeline.state.HasCandidatesWithFeatures -import com.twitter.product_mixer.core.pipeline.step.Step -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.group_results_executor.GroupResultsExecutor -import com.twitter.product_mixer.core.service.group_results_executor.GroupResultsExecutorInput -import com.twitter.product_mixer.core.service.group_results_executor.GroupResultsExecutorResult -import com.twitter.stitch.Arrow -import javax.inject.Inject - -/** - * A group results step, it takes the input list of candidates and decorations, and assembles - * properly decorated candidates with details. - * - * @param groupResultsExecutor Group results executor - * @tparam Candidate Type of candidates - * @tparam State The pipeline state domain model. - */ -case class GroupResultsStep[ - Candidate <: UniversalNoun[Any], - State <: HasCandidatesWithDetails[State] with HasCandidatesWithFeatures[ - Candidate, - State - ]] @Inject() ( - groupResultsExecutor: GroupResultsExecutor) - extends Step[State, CandidatePipelineContext, GroupResultsExecutorInput[ - Candidate - ], GroupResultsExecutorResult] { - - override def isEmpty(config: CandidatePipelineContext): Boolean = false - override def adaptInput( - state: State, - config: CandidatePipelineContext - ): GroupResultsExecutorInput[Candidate] = { - val presentationMap = state.candidatesWithDetails.flatMap { candidateWithDetails => - candidateWithDetails.presentation - .map { presentation => - candidateWithDetails.getCandidate[UniversalNoun[Any]] -> presentation - } - }.toMap - GroupResultsExecutorInput(state.candidatesWithFeatures, presentationMap) - } - - override def arrow( - config: CandidatePipelineContext, - context: Executor.Context - ): Arrow[GroupResultsExecutorInput[Candidate], GroupResultsExecutorResult] = - groupResultsExecutor.arrow( - config.candidatePipelineIdentifier, - config.candidateSourceIdentifier, - context) - - override def updateState( - state: State, - executorResult: GroupResultsExecutorResult, - config: CandidatePipelineContext - ): State = state.updateCandidatesWithDetails(executorResult.candidatesWithDetails) -} - -case class CandidatePipelineContext( - candidatePipelineIdentifier: CandidatePipelineIdentifier, - candidateSourceIdentifier: CandidateSourceIdentifier) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/pipeline_executor/BUILD.bazel b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/pipeline_executor/BUILD.bazel deleted file mode 100644 index 94b17381a..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/pipeline_executor/BUILD.bazel +++ /dev/null @@ -1,32 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = [ - "bazel-compatible", - "bazel-only", - ], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/pipeline_selector", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_executor", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_executor", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/pipeline_executor/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/pipeline_executor/BUILD.docx new file mode 100644 index 000000000..52017b797 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/pipeline_executor/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/pipeline_executor/PipelineExecutorStep.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/pipeline_executor/PipelineExecutorStep.docx new file mode 100644 index 000000000..5c3eedad6 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/pipeline_executor/PipelineExecutorStep.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/pipeline_executor/PipelineExecutorStep.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/pipeline_executor/PipelineExecutorStep.scala deleted file mode 100644 index c2bdb6ea1..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/pipeline_executor/PipelineExecutorStep.scala +++ /dev/null @@ -1,81 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.step.pipeline_executor - -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier -import com.twitter.product_mixer.core.pipeline.Pipeline -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.pipeline_failure.IllegalStateFailure -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.state.HasExecutorResults -import com.twitter.product_mixer.core.pipeline.state.HasQuery -import com.twitter.product_mixer.core.pipeline.state.HasResult -import com.twitter.product_mixer.core.pipeline.step.Step -import com.twitter.product_mixer.core.pipeline.step.pipeline_selector.PipelineSelectorResult -import com.twitter.product_mixer.core.quality_factor.QualityFactorObserver -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.pipeline_executor.PipelineExecutor -import com.twitter.product_mixer.core.service.pipeline_executor.PipelineExecutorRequest -import com.twitter.product_mixer.core.service.pipeline_executor.PipelineExecutorResult -import com.twitter.stitch.Arrow -import javax.inject.Inject - -/** - * Pipeline Execution step that takes a selected pipeline and executes it. - * - * @param pipelineExecutor Pipeline executor that executes the selected pipeline - * - * @tparam Query Pipeline query model with quality factor status - * @tparam Result The expected result type - * @tparam State The pipeline state domain model. - */ -case class PipelineExecutorStep[ - Query <: PipelineQuery, - Result, - State <: HasQuery[Query, State] with HasExecutorResults[State] with HasResult[Result]] @Inject() ( - pipelineExecutor: PipelineExecutor) - extends Step[ - State, - PipelineExecutorStepConfig[Query, Result], - PipelineExecutorRequest[Query], - PipelineExecutorResult[Result] - ] { - - override def isEmpty(config: PipelineExecutorStepConfig[Query, Result]): Boolean = - false - - override def adaptInput( - state: State, - config: PipelineExecutorStepConfig[Query, Result] - ): PipelineExecutorRequest[Query] = { - val pipelineSelectorResult = state.executorResultsByPipelineStep - .getOrElse( - config.selectedPipelineResultIdentifier, - throw PipelineFailure( - IllegalStateFailure, - "Missing Selected Pipeline in Pipeline Executor Step")).asInstanceOf[ - PipelineSelectorResult] - PipelineExecutorRequest(state.query, pipelineSelectorResult.pipelineIdentifier) - } - - override def arrow( - config: PipelineExecutorStepConfig[Query, Result], - context: Executor.Context - ): Arrow[PipelineExecutorRequest[Query], PipelineExecutorResult[Result]] = pipelineExecutor.arrow( - config.pipelinesByIdentifier, - config.qualityFactorObserversByIdentifier, - context - ) - - // Noop since the platform will add the final result to the executor result map then state - // is responsible for reading it in [[WithResult]] - override def updateState( - state: State, - executorResult: PipelineExecutorResult[Result], - config: PipelineExecutorStepConfig[Query, Result] - ): State = state -} - -case class PipelineExecutorStepConfig[Query <: PipelineQuery, Result]( - pipelinesByIdentifier: Map[ComponentIdentifier, Pipeline[Query, Result]], - selectedPipelineResultIdentifier: PipelineStepIdentifier, - qualityFactorObserversByIdentifier: Map[ComponentIdentifier, QualityFactorObserver]) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/pipeline_selector/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/pipeline_selector/BUILD deleted file mode 100644 index e0babba8b..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/pipeline_selector/BUILD +++ /dev/null @@ -1,26 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/pipeline_selector/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/pipeline_selector/BUILD.docx new file mode 100644 index 000000000..087f609cc Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/pipeline_selector/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/pipeline_selector/PipelineSelectorStep.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/pipeline_selector/PipelineSelectorStep.docx new file mode 100644 index 000000000..30b6a0893 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/pipeline_selector/PipelineSelectorStep.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/pipeline_selector/PipelineSelectorStep.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/pipeline_selector/PipelineSelectorStep.scala deleted file mode 100644 index aa40d4829..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/pipeline_selector/PipelineSelectorStep.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.step.pipeline_selector - -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.state.HasQuery -import com.twitter.product_mixer.core.pipeline.step.Step -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.ExecutorResult -import com.twitter.stitch.Arrow -import javax.inject.Inject - -/** - * Pipeline Selection step to decide which pipeline to execute. This step doesn't update state, as - * the selected pipeline identifier is added to the executor results list map for later retrieval - * - * @tparam Query Pipeline query model - * @tparam State The pipeline state domain model. - */ -case class PipelineSelectorStep[Query <: PipelineQuery, State <: HasQuery[Query, State]] @Inject() ( -) extends Step[State, Query => ComponentIdentifier, Query, PipelineSelectorResult] { - override def isEmpty(config: Query => ComponentIdentifier): Boolean = false - - override def adaptInput( - state: State, - config: Query => ComponentIdentifier - ): Query = state.query - - override def arrow( - config: Query => ComponentIdentifier, - context: Executor.Context - ): Arrow[Query, PipelineSelectorResult] = Arrow.map { query: Query => - PipelineSelectorResult(config(query)) - } - - // Noop since we keep the identifier in the executor results - override def updateState( - state: State, - executorResult: PipelineSelectorResult, - config: Query => ComponentIdentifier - ): State = state -} - -case class PipelineSelectorResult(pipelineIdentifier: ComponentIdentifier) extends ExecutorResult diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/quality_factor/BUILD.bazel b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/quality_factor/BUILD.bazel deleted file mode 100644 index a220a9550..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/quality_factor/BUILD.bazel +++ /dev/null @@ -1,31 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = [ - "bazel-compatible", - "bazel-only", - ], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/quality_factor/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/quality_factor/BUILD.docx new file mode 100644 index 000000000..f1d4852e5 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/quality_factor/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/quality_factor/QualityFactorStep.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/quality_factor/QualityFactorStep.docx new file mode 100644 index 000000000..7d15bc77e Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/quality_factor/QualityFactorStep.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/quality_factor/QualityFactorStep.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/quality_factor/QualityFactorStep.scala deleted file mode 100644 index 0d66a4b57..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/quality_factor/QualityFactorStep.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.step.quality_factor - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.state.HasQuery -import com.twitter.product_mixer.core.pipeline.step.Step -import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus -import com.twitter.product_mixer.core.quality_factor.QualityFactorStatus -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.ExecutorResult -import com.twitter.stitch.Arrow -import javax.inject.Inject - -/** - * Quality Factor building step that builds up the state snapshot for a map of configs. - * - * @param statsReceiver Stats Receiver used to build finagle gauges for QF State - * - * @tparam Query Pipeline query model with quality factor status - * @tparam State The pipeline state domain model. - */ -case class QualityFactorStep[ - Query <: PipelineQuery with HasQualityFactorStatus, - State <: HasQuery[Query, State]] @Inject() ( - statsReceiver: StatsReceiver) - extends Step[ - State, - QualityFactorStepConfig, - Any, - QualityFactorStepResult - ] { - override def isEmpty(config: QualityFactorStepConfig): Boolean = - config.qualityFactorStatus.qualityFactorByPipeline.isEmpty - - override def adaptInput( - state: State, - config: QualityFactorStepConfig - ): Any = () - - override def arrow( - config: QualityFactorStepConfig, - context: Executor.Context - ): Arrow[Any, QualityFactorStepResult] = { - // We use provideGauge so these gauges live forever even without a reference. - val currentValues = config.qualityFactorStatus.qualityFactorByPipeline.map { - case (identifier, qualityFactor) => - // QF is a relative stat (since the parent pipeline is monitoring a child pipeline) - val scopes = config.pipelineIdentifier.toScopes ++ identifier.toScopes :+ "QualityFactor" - val currentValue = qualityFactor.currentValue.toFloat - statsReceiver.provideGauge(scopes: _*) { - currentValue - } - identifier -> currentValue - } - Arrow.value(QualityFactorStepResult(currentValues)) - } - - override def updateState( - state: State, - executorResult: QualityFactorStepResult, - config: QualityFactorStepConfig - ): State = state.updateQuery( - state.query.withQualityFactorStatus(config.qualityFactorStatus).asInstanceOf[Query]) -} - -case class QualityFactorStepConfig( - pipelineIdentifier: ComponentIdentifier, - qualityFactorStatus: QualityFactorStatus) - -case class QualityFactorStepResult(currentValues: Map[ComponentIdentifier, Float]) - extends ExecutorResult diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/query_feature_hydrator/BUILD.bazel b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/query_feature_hydrator/BUILD.bazel deleted file mode 100644 index 2ed3762c4..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/query_feature_hydrator/BUILD.bazel +++ /dev/null @@ -1,36 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor", - "stitch/stitch-core", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/query_feature_hydrator/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/query_feature_hydrator/BUILD.docx new file mode 100644 index 000000000..482db932d Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/query_feature_hydrator/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/query_feature_hydrator/QueryFeatureHydratorStep.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/query_feature_hydrator/QueryFeatureHydratorStep.docx new file mode 100644 index 000000000..0e6c6b952 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/query_feature_hydrator/QueryFeatureHydratorStep.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/query_feature_hydrator/QueryFeatureHydratorStep.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/query_feature_hydrator/QueryFeatureHydratorStep.scala deleted file mode 100644 index 1b119fc2f..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/query_feature_hydrator/QueryFeatureHydratorStep.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.step.query_feature_hydrator - -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.state.HasAsyncFeatureMap -import com.twitter.product_mixer.core.pipeline.state.HasQuery -import com.twitter.product_mixer.core.pipeline.step.Step -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.query_feature_hydrator_executor.QueryFeatureHydratorExecutor -import com.twitter.stitch.Arrow -import javax.inject.Inject - -/** - * A query level feature hydration step, it takes the input list of candidates and the given - * hydrators and executes them. The [[State]] object is responsible for merging the resulting - * feature maps with the hydrated ones in its updateCandidatesWithFeatures. - * - * @param queryFeatureHydratorExecutor Hydrator Executor - * @tparam Query Type of PipelineQuery domain model - * @tparam State The pipeline state domain model. - */ -case class QueryFeatureHydratorStep[ - Query <: PipelineQuery, - State <: HasQuery[Query, State] with HasAsyncFeatureMap[State]] @Inject() ( - queryFeatureHydratorExecutor: QueryFeatureHydratorExecutor) - extends Step[State, QueryFeatureHydratorStepConfig[ - Query - ], Query, QueryFeatureHydratorExecutor.Result] { - override def isEmpty(config: QueryFeatureHydratorStepConfig[Query]): Boolean = - config.hydrators.isEmpty - - override def adaptInput(state: State, config: QueryFeatureHydratorStepConfig[Query]): Query = - state.query - - override def arrow( - config: QueryFeatureHydratorStepConfig[Query], - context: Executor.Context - ): Arrow[Query, QueryFeatureHydratorExecutor.Result] = - queryFeatureHydratorExecutor.arrow( - config.hydrators, - config.validPipelineStepIdentifiers, - context) - - override def updateState( - state: State, - executorResult: QueryFeatureHydratorExecutor.Result, - config: QueryFeatureHydratorStepConfig[Query] - ): State = { - val updatedQuery = state.query - .withFeatureMap(executorResult.featureMap).asInstanceOf[Query] - state - .updateQuery(updatedQuery).addAsyncFeatureMap(executorResult.asyncFeatureMap) - } -} - -case class QueryFeatureHydratorStepConfig[Query <: PipelineQuery]( - hydrators: Seq[BaseQueryFeatureHydrator[Query, _]], - validPipelineStepIdentifiers: Set[PipelineStepIdentifier]) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/query_transformer/BUILD.bazel b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/query_transformer/BUILD.bazel deleted file mode 100644 index dd63b24f6..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/query_transformer/BUILD.bazel +++ /dev/null @@ -1,29 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = [ - "bazel-compatible", - "bazel-only", - ], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/query_transformer/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/query_transformer/BUILD.docx new file mode 100644 index 000000000..69c3a168d Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/query_transformer/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/query_transformer/QueryTransformerStep.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/query_transformer/QueryTransformerStep.docx new file mode 100644 index 000000000..732dc8b84 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/query_transformer/QueryTransformerStep.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/query_transformer/QueryTransformerStep.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/query_transformer/QueryTransformerStep.scala deleted file mode 100644 index 37814714b..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/query_transformer/QueryTransformerStep.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.step.query_transformer - -import com.twitter.product_mixer.core.model.marshalling.request.Request -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.state.HasParams -import com.twitter.product_mixer.core.pipeline.state.HasQuery -import com.twitter.product_mixer.core.pipeline.state.HasRequest -import com.twitter.product_mixer.core.pipeline.step.Step -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.ExecutorResult -import com.twitter.stitch.Arrow -import com.twitter.timelines.configapi.Params - -/** - * Query Transformation Step that takes an incoming thrift request model object and returns a - * pipeline query. The pipeline state is responsible for keeping the updated query. - * - * @tparam TRequest Thrift request domain model - * @tparam Query PipelineQuery type to transform to h - * @tparam State The request domain model - */ -case class QueryTransformerStep[ - TRequest <: Request, - Query <: PipelineQuery, - State <: HasQuery[Query, State] with HasRequest[TRequest] with HasParams -]() extends Step[State, (TRequest, Params) => Query, (TRequest, Params), QueryTransformerResult[ - Query - ]] { - - override def isEmpty(config: (TRequest, Params) => Query): Boolean = false - - override def arrow( - config: (TRequest, Params) => Query, - context: Executor.Context - ): Arrow[(TRequest, Params), QueryTransformerResult[Query]] = Arrow.map { - case (request: TRequest @unchecked, params: Params) => - QueryTransformerResult(config(request, params)) - } - - override def updateState( - state: State, - executorResult: QueryTransformerResult[Query], - config: (TRequest, Params) => Query - ): State = state.updateQuery(executorResult.query) - - override def adaptInput( - state: State, - config: (TRequest, Params) => Query - ): (TRequest, Params) = (state.request, state.params) -} - -case class QueryTransformerResult[Query <: PipelineQuery](query: Query) extends ExecutorResult diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/scorer/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/scorer/BUILD deleted file mode 100644 index 105cd49a1..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/scorer/BUILD +++ /dev/null @@ -1,36 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/scorer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_hydrator_executor", - "stitch/stitch-core", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/scorer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_hydrator_executor", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/scorer/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/scorer/BUILD.docx new file mode 100644 index 000000000..972638a62 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/scorer/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/scorer/ScorerStep.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/scorer/ScorerStep.docx new file mode 100644 index 000000000..b5e52f4c1 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/scorer/ScorerStep.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/scorer/ScorerStep.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/scorer/ScorerStep.scala deleted file mode 100644 index 938b18d62..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/scorer/ScorerStep.scala +++ /dev/null @@ -1,69 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.step.scorer - -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.state.HasCandidatesWithFeatures -import com.twitter.product_mixer.core.pipeline.state.HasQuery -import com.twitter.product_mixer.core.pipeline.step.Step -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutor -import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutorResult -import com.twitter.stitch.Arrow -import javax.inject.Inject - -/** - * A scoring step, it takes the input list of candidates and the given - * scorers and executes them. The [[State]] object is responsible for merging the resulting - * feature maps with the scored ones in its updateCandidatesWithFeatures. - * - * @param candidateFeatureHydratorExecutor Hydrator Executor - * @tparam Query Type of PipelineQuery domain model - * @tparam Candidate Type of Candidates to hydrate features for. - * @tparam State The pipeline state domain model. - */ -case class ScorerStep[ - Query <: PipelineQuery, - Candidate <: UniversalNoun[Any], - State <: HasQuery[Query, State] with HasCandidatesWithFeatures[ - Candidate, - State - ]] @Inject() ( - candidateFeatureHydratorExecutor: CandidateFeatureHydratorExecutor) - extends Step[State, Seq[ - Scorer[Query, Candidate] - ], CandidateFeatureHydratorExecutor.Inputs[ - Query, - Candidate - ], CandidateFeatureHydratorExecutorResult[Candidate]] { - - override def adaptInput( - state: State, - config: Seq[Scorer[Query, Candidate]] - ): CandidateFeatureHydratorExecutor.Inputs[Query, Candidate] = - CandidateFeatureHydratorExecutor.Inputs(state.query, state.candidatesWithFeatures) - - override def arrow( - config: Seq[Scorer[Query, Candidate]], - context: Executor.Context - ): Arrow[ - CandidateFeatureHydratorExecutor.Inputs[Query, Candidate], - CandidateFeatureHydratorExecutorResult[Candidate] - ] = candidateFeatureHydratorExecutor.arrow(config, context) - - override def updateState( - input: State, - executorResult: CandidateFeatureHydratorExecutorResult[Candidate], - config: Seq[Scorer[Query, Candidate]] - ): State = { - val resultCandidates = executorResult.results - if (resultCandidates.isEmpty) { - input - } else { - input.updateCandidatesWithFeatures(resultCandidates) - } - } - - override def isEmpty(config: Seq[Scorer[Query, Candidate]]): Boolean = - config.isEmpty -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/selector/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/selector/BUILD deleted file mode 100644 index 4479122cd..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/selector/BUILD +++ /dev/null @@ -1,36 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor", - "stitch/stitch-core", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/selector/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/selector/BUILD.docx new file mode 100644 index 000000000..e112e71fb Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/selector/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/selector/SelectorStep.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/selector/SelectorStep.docx new file mode 100644 index 000000000..1c0bf65d7 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/selector/SelectorStep.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/selector/SelectorStep.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/selector/SelectorStep.scala deleted file mode 100644 index bc74d2d2b..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/selector/SelectorStep.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.step.selector - -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.state.HasCandidatesWithDetails -import com.twitter.product_mixer.core.pipeline.state.HasQuery -import com.twitter.product_mixer.core.pipeline.step.Step -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.selector_executor.SelectorExecutor -import com.twitter.product_mixer.core.service.selector_executor.SelectorExecutorResult -import com.twitter.stitch.Arrow -import javax.inject.Inject - -/** - * A selection step, it takes the input list of candidates with details and the given - * selectors and executes them to decide which candidates should be selected. - * - * @param selectorExecutor Selector Executor - * @tparam Query Type of PipelineQuery domain model - * @tparam State The pipeline state domain model. - */ -case class SelectorStep[ - Query <: PipelineQuery, - State <: HasQuery[Query, State] with HasCandidatesWithDetails[State]] @Inject() ( - selectorExecutor: SelectorExecutor) - extends Step[State, Seq[ - Selector[Query] - ], SelectorExecutor.Inputs[ - Query - ], SelectorExecutorResult] { - - override def adaptInput( - state: State, - config: Seq[Selector[Query]] - ): SelectorExecutor.Inputs[Query] = - SelectorExecutor.Inputs(state.query, state.candidatesWithDetails) - - override def arrow( - config: Seq[Selector[Query]], - context: Executor.Context - ): Arrow[SelectorExecutor.Inputs[Query], SelectorExecutorResult] = - selectorExecutor.arrow(config, context) - - override def updateState( - input: State, - executorResult: SelectorExecutorResult, - config: Seq[Selector[Query]] - ): State = input.updateCandidatesWithDetails(executorResult.selectedCandidates) - - // Selection is a bit different to other steps (i.e, other steps, empty means don't change anything) - // where an empty selection list drops all candidates. - override def isEmpty(config: Seq[Selector[Query]]): Boolean = false -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/side_effect/BUILD.bazel b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/side_effect/BUILD.bazel deleted file mode 100644 index fb9d33c98..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/side_effect/BUILD.bazel +++ /dev/null @@ -1,32 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/side_effect", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/domain_marshaller_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_result_side_effect_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor", - "stitch/stitch-core", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/side_effect", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_result_side_effect_executor", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/side_effect/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/side_effect/BUILD.docx new file mode 100644 index 000000000..80ded0404 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/side_effect/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/side_effect/SideEffectStep.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/side_effect/SideEffectStep.docx new file mode 100644 index 000000000..915b31527 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/side_effect/SideEffectStep.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/side_effect/SideEffectStep.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/side_effect/SideEffectStep.scala deleted file mode 100644 index 36e72cff6..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/side_effect/SideEffectStep.scala +++ /dev/null @@ -1,102 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.step.side_effect - -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.pipeline_failure.IllegalStateFailure -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.state.HasExecutorResults -import com.twitter.product_mixer.core.pipeline.state.HasQuery -import com.twitter.product_mixer.core.pipeline.step.Step -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.domain_marshaller_executor.DomainMarshallerExecutor -import com.twitter.product_mixer.core.service.pipeline_result_side_effect_executor.PipelineResultSideEffectExecutor -import com.twitter.product_mixer.core.service.selector_executor.SelectorExecutorResult -import com.twitter.stitch.Arrow -import javax.inject.Inject - -/** - * A side effect step, it takes the input list of side effects and and executes them. - * - * @param sideEffectExecutor Side Effect Executor - * - * @tparam Query Type of PipelineQuery domain model - * @tparam DomainResultType Domain Marshaller result type - * @tparam State The pipeline state domain model. - */ -case class SideEffectStep[ - Query <: PipelineQuery, - DomainResultType <: HasMarshalling, - State <: HasQuery[Query, State] with HasExecutorResults[State]] @Inject() ( - sideEffectExecutor: PipelineResultSideEffectExecutor) - extends Step[ - State, - PipelineStepConfig[Query, DomainResultType], - PipelineResultSideEffect.Inputs[ - Query, - DomainResultType - ], - PipelineResultSideEffectExecutor.Result - ] { - override def isEmpty(config: PipelineStepConfig[Query, DomainResultType]): Boolean = - config.sideEffects.isEmpty - - override def adaptInput( - state: State, - config: PipelineStepConfig[Query, DomainResultType] - ): PipelineResultSideEffect.Inputs[Query, DomainResultType] = { - val selectorResults = state.executorResultsByPipelineStep - .getOrElse( - config.selectorStepIdentifier, - throw PipelineFailure( - IllegalStateFailure, - "Missing Selector Result in Side Effect Step")).asInstanceOf[SelectorExecutorResult] - - val domainMarshallerResult = state.executorResultsByPipelineStep - .getOrElse( - config.domainMarshallerStepIdentifier, - throw PipelineFailure( - IllegalStateFailure, - "Missing Domain Marshaller Result in Side Effect Step")).asInstanceOf[ - DomainMarshallerExecutor.Result[DomainResultType]] - - PipelineResultSideEffect.Inputs( - query = state.query, - selectedCandidates = selectorResults.selectedCandidates, - remainingCandidates = selectorResults.remainingCandidates, - droppedCandidates = selectorResults.droppedCandidates, - response = domainMarshallerResult.result - ) - } - - override def arrow( - config: PipelineStepConfig[Query, DomainResultType], - context: Executor.Context - ): Arrow[ - PipelineResultSideEffect.Inputs[Query, DomainResultType], - PipelineResultSideEffectExecutor.Result - ] = sideEffectExecutor.arrow(config.sideEffects, context) - - override def updateState( - state: State, - executorResult: PipelineResultSideEffectExecutor.Result, - config: PipelineStepConfig[Query, DomainResultType] - ): State = state -} - -/** - * Wrapper case class containing side effects to be executed and other information needed to execute - * @param sideEffects The side effects to execute. - * @param selectorStepIdentifier The identifier of the selector step in the parent - * pipeline to get selection results from. - * @param domainMarshallerStepIdentifier The identifier of the domain marshaller step in the parent - * pipeline to get domain marshalled results from. - * - * @tparam Query Type of PipelineQuery domain model - * @tparam DomainResultType Domain Marshaller result type - */ -case class PipelineStepConfig[Query <: PipelineQuery, DomainResultType <: HasMarshalling]( - sideEffects: Seq[PipelineResultSideEffect[Query, DomainResultType]], - selectorStepIdentifier: PipelineStepIdentifier, - domainMarshallerStepIdentifier: PipelineStepIdentifier) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/transport_marshaller/BUILD.bazel b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/transport_marshaller/BUILD.bazel deleted file mode 100644 index 7b097b33f..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/transport_marshaller/BUILD.bazel +++ /dev/null @@ -1,31 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/domain_marshaller_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transport_marshaller_executor", - "stitch/stitch-core", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/state", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transport_marshaller_executor", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/transport_marshaller/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/transport_marshaller/BUILD.docx new file mode 100644 index 000000000..ce5c6556b Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/transport_marshaller/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/transport_marshaller/TransportMarshallerStep.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/transport_marshaller/TransportMarshallerStep.docx new file mode 100644 index 000000000..33f815727 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/transport_marshaller/TransportMarshallerStep.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/transport_marshaller/TransportMarshallerStep.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/transport_marshaller/TransportMarshallerStep.scala deleted file mode 100644 index 577df3fdd..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/step/transport_marshaller/TransportMarshallerStep.scala +++ /dev/null @@ -1,76 +0,0 @@ -package com.twitter.product_mixer.core.pipeline.step.transport_marshaller - -import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller -import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.pipeline_failure.IllegalStateFailure -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.state.HasExecutorResults -import com.twitter.product_mixer.core.pipeline.step.Step -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.domain_marshaller_executor.DomainMarshallerExecutor -import com.twitter.product_mixer.core.service.transport_marshaller_executor.TransportMarshallerExecutor -import com.twitter.stitch.Arrow -import javax.inject.Inject - -/** - * A transport marshaller step, it takes domain marshalled result as input and returns trasnport - * ready marshalled object. - * The [[State]] object is responsible for keeping a reference of the built marshalled response. - * - * @param transportMarshallerExecutor Domain Marshaller executor. - * @tparam Query Type of PipelineQuery domain model - * @tparam DomainResponseType the domain marshalling type used as input - * @tparam TransportResponseType the expected returned transport type - * @tparam State The pipeline state domain model. - */ -case class TransportMarshallerStep[ - DomainResponseType <: HasMarshalling, - TransportResponseType, - State <: HasExecutorResults[State]] @Inject() ( - transportMarshallerExecutor: TransportMarshallerExecutor) - extends Step[ - State, - TransportMarshallerConfig[DomainResponseType, TransportResponseType], - TransportMarshallerExecutor.Inputs[DomainResponseType], - TransportMarshallerExecutor.Result[TransportResponseType] - ] { - - override def isEmpty( - config: TransportMarshallerConfig[DomainResponseType, TransportResponseType] - ): Boolean = false - - override def adaptInput( - state: State, - config: TransportMarshallerConfig[DomainResponseType, TransportResponseType] - ): TransportMarshallerExecutor.Inputs[DomainResponseType] = { - val domainMarshallerResult = state.executorResultsByPipelineStep - .getOrElse( - config.domainMarshallerStepIdentifier, - throw PipelineFailure( - IllegalStateFailure, - "Missing Domain Marshaller in Transport Marshaller Step")).asInstanceOf[ - DomainMarshallerExecutor.Result[DomainResponseType]] - TransportMarshallerExecutor.Inputs(domainMarshallerResult.result) - } - - // Noop as platform updates executor result - override def updateState( - state: State, - executorResult: TransportMarshallerExecutor.Result[TransportResponseType], - config: TransportMarshallerConfig[DomainResponseType, TransportResponseType] - ): State = state - - override def arrow( - config: TransportMarshallerConfig[DomainResponseType, TransportResponseType], - context: Executor.Context - ): Arrow[TransportMarshallerExecutor.Inputs[ - DomainResponseType - ], TransportMarshallerExecutor.Result[TransportResponseType]] = - transportMarshallerExecutor.arrow(config.transportMarshaller, context) - -} - -case class TransportMarshallerConfig[DomainResponseType <: HasMarshalling, TransportResponseType]( - transportMarshaller: TransportMarshaller[DomainResponseType, TransportResponseType], - domainMarshallerStepIdentifier: PipelineStepIdentifier) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/BUILD deleted file mode 100644 index cc258ffe8..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi/registry", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer", - "stitch/stitch-core", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi/registry", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/BUILD.docx new file mode 100644 index 000000000..3def815e6 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/ProductParamConfig.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/ProductParamConfig.docx new file mode 100644 index 000000000..a45c0c23c Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/ProductParamConfig.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/ProductParamConfig.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/ProductParamConfig.scala deleted file mode 100644 index c390b599e..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/ProductParamConfig.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.product_mixer.core.product - -import com.twitter.product_mixer.core.functional_component.configapi.registry.ParamConfig -import com.twitter.servo.decider.DeciderKeyName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.decider.BooleanDeciderParam - -trait ProductParamConfig extends ParamConfig with ProductParamConfigBuilder { - - /** - * This enabled decider param can to be used to quickly disable a Product via Decider - * - * This value must correspond to the deciders configured in the `resources/config/decider.yml` file - */ - val enabledDeciderKey: DeciderKeyName - - /** - * This supported client feature switch param can be used with a Feature Switch to control the - * rollout of a new Product from dogfood to experiment to production - * - * FeatureSwitches are configured by defining both a [[com.twitter.timelines.configapi.Param]] in code - * and in an associated `.yml` file in the __config repo__. - * - * The `.yml` file path is determined by the `feature_switches_path` in your aurora file and tge Product name - * so the resulting path in the __config repo__ is essentially `s"{feature_switches_path}/{snakeCase(Product.identifier)}"` - */ - val supportedClientFSName: String - - object EnabledDeciderParam extends BooleanDeciderParam(enabledDeciderKey) - - object SupportedClientParam - extends FSParam( - name = supportedClientFSName, - default = false - ) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/ProductParamConfigBuilder.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/ProductParamConfigBuilder.docx new file mode 100644 index 000000000..1ef0d1995 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/ProductParamConfigBuilder.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/ProductParamConfigBuilder.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/ProductParamConfigBuilder.scala deleted file mode 100644 index 0d3490ebe..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/ProductParamConfigBuilder.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.product_mixer.core.product - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.functional_component.configapi.registry.ParamConfigBuilder -import com.twitter.servo.decider.DeciderGateBuilder -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.OptionalOverride -import com.twitter.timelines.configapi.decider.DeciderUtils - -trait ProductParamConfigBuilder extends ParamConfigBuilder { - productParamConfig: ProductParamConfig => - - override def build( - deciderGateBuilder: DeciderGateBuilder, - statsReceiver: StatsReceiver - ): Seq[OptionalOverride[_]] = { - DeciderUtils.getBooleanDeciderOverrides(deciderGateBuilder, EnabledDeciderParam) ++ - FeatureSwitchOverrideUtil.getBooleanFSOverrides(SupportedClientParam) ++ - super.build(deciderGateBuilder, statsReceiver) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice/BUILD deleted file mode 100644 index 998a18e9d..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - ], - exports = [ - "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice/BUILD.docx new file mode 100644 index 000000000..baceb900e Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice/ProductScope.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice/ProductScope.docx new file mode 100644 index 000000000..bacb50275 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice/ProductScope.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice/ProductScope.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice/ProductScope.scala deleted file mode 100644 index a4ae726dd..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice/ProductScope.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.product_mixer.core.product.guice -import com.twitter.product_mixer.core.model.marshalling.request.Product -import com.google.inject.Key - -/** - * A specialization of SimpleScope - a simple Guice Scope that takes an initial Product Mixer Product as a key - */ -class ProductScope extends SimpleScope { - def let[T](product: Product)(f: => T): T = super.let(Map(Key.get(classOf[Product]) -> product))(f) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice/ProductScopeModule.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice/ProductScopeModule.docx new file mode 100644 index 000000000..674aec74c Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice/ProductScopeModule.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice/ProductScopeModule.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice/ProductScopeModule.scala deleted file mode 100644 index 8345f166e..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice/ProductScopeModule.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.product_mixer.core.product.guice - -import com.google.inject.Provides -import com.twitter.inject.TwitterModule -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.product_mixer.core.model.marshalling.request.Product -import javax.inject.Singleton - -/** - * Registers the @ProductScoped scope. - * - * See https://github.com/google/guice/wiki/CustomScopes#registering-the-scope - */ -@Singleton -class ProductScopeModule extends TwitterModule { - - val productScope: ProductScope = new ProductScope - - override def configure(): Unit = { - bindScope(classOf[ProductScoped], productScope) - - bind[Product].toProvider(SimpleScope.SEEDED_KEY_PROVIDER).in(classOf[ProductScoped]) - } - - @Provides - def providesProductScope(): ProductScope = productScope -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice/SimpleScope.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice/SimpleScope.docx new file mode 100644 index 000000000..60ffcb947 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice/SimpleScope.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice/SimpleScope.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice/SimpleScope.scala deleted file mode 100644 index ccbf77121..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice/SimpleScope.scala +++ /dev/null @@ -1,68 +0,0 @@ -package com.twitter.product_mixer.core.product.guice - -import com.google.inject.Key -import com.google.inject.OutOfScopeException -import com.google.inject.Provider -import com.google.inject.Scope -import com.google.inject.Scopes -import com.twitter.util.Local -import scala.collection.concurrent -import scala.collection.mutable - -/** - * A scala-esque implementation of SimpleScope: https://github.com/google/guice/wiki/CustomScopes#implementing-scope - * - * Scopes the execution of a single block of code via `let` - */ -class SimpleScope extends Scope { - - private val values = new Local[concurrent.Map[Key[_], Any]]() - - /** - * Execute a block with a fresh scope. - * - * You can optionally supply a map of initialObjects to 'seed' the new scope. - */ - def let[T](initialObjects: Map[Key[_], Any] = Map.empty)(f: => T): T = { - val newMap: concurrent.Map[Key[_], Any] = concurrent.TrieMap.empty - - initialObjects.foreach { case (key, value) => newMap.put(key, value) } - - values.let(newMap)(f) - } - - override def scope[T]( - key: Key[T], - unscoped: Provider[T] - ): Provider[T] = () => { - val scopedObjects: mutable.Map[Key[T], Any] = getScopedObjectMap(key) - - scopedObjects - .get(key).map(_.asInstanceOf[T]).getOrElse { - val objectFromUnscoped: T = unscoped.get() - - if (Scopes.isCircularProxy(objectFromUnscoped)) { - objectFromUnscoped // Don't remember proxies - } else { - scopedObjects.put(key, objectFromUnscoped) - objectFromUnscoped - } - } - } - - def getScopedObjectMap[T](key: Key[T]): concurrent.Map[Key[T], Any] = { - values() - .getOrElse( - throw new OutOfScopeException(s"Cannot access $key outside of a scoping block") - ).asInstanceOf[concurrent.Map[Key[T], Any]] - } -} - -object SimpleScope { - - val SEEDED_KEY_PROVIDER: Provider[Nothing] = () => - throw new IllegalStateException( - """If you got here then it means that your code asked for scoped object which should have - | been explicitly seeded in this scope by calling SimpleScope.seed(), - | but was not.""".stripMargin) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry/BUILD deleted file mode 100644 index 9ecf4adca..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry/BUILD +++ /dev/null @@ -1,26 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi/registry", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/component_registry", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi/registry", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/component_registry", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry/BUILD.docx new file mode 100644 index 000000000..1beaf795a Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry/ProductParamRegistry.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry/ProductParamRegistry.docx new file mode 100644 index 000000000..baa71baa2 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry/ProductParamRegistry.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry/ProductParamRegistry.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry/ProductParamRegistry.scala deleted file mode 100644 index 089aaea5e..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry/ProductParamRegistry.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.product_mixer.core.product.registry - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.servo.decider.DeciderGateBuilder -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.Config -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ProductParamRegistry @Inject() ( - productPipelineRegistryConfig: ProductPipelineRegistryConfig, - deciderGateBuilder: DeciderGateBuilder, - statsReceiver: StatsReceiver) { - - def build(): Seq[Config] = { - val productConfigs = productPipelineRegistryConfig.productPipelineConfigs.map { - productPipelineConfig => - BaseConfigBuilder( - productPipelineConfig.paramConfig.build(deciderGateBuilder, statsReceiver)) - .build(productPipelineConfig.paramConfig.getClass.getSimpleName) - } - - productConfigs - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry/ProductPipelineRegistry.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry/ProductPipelineRegistry.docx new file mode 100644 index 000000000..099e0e847 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry/ProductPipelineRegistry.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry/ProductPipelineRegistry.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry/ProductPipelineRegistry.scala deleted file mode 100644 index e04322f6a..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry/ProductPipelineRegistry.scala +++ /dev/null @@ -1,189 +0,0 @@ -package com.twitter.product_mixer.core.product.registry - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifierStack -import com.twitter.product_mixer.core.model.common.identifier.ProductIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.RootIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.Product -import com.twitter.product_mixer.core.model.marshalling.request.Request -import com.twitter.product_mixer.core.pipeline.Pipeline -import com.twitter.product_mixer.core.pipeline.product.ProductPipeline -import com.twitter.product_mixer.core.pipeline.product.ProductPipelineBuilderFactory -import com.twitter.product_mixer.core.service.component_registry.ComponentRegistry -import com.twitter.product_mixer.core.service.component_registry.ComponentRegistrySnapshot -import com.twitter.product_mixer.shared_library.observer.Observer -import com.twitter.util.Try -import com.twitter.util.Var -import com.twitter.util.logging.Logging -import javax.inject.Inject -import javax.inject.Singleton -import scala.reflect.runtime.universe._ - -@Singleton -class ProductPipelineRegistry @Inject() ( - componentRegistry: ComponentRegistry, - productPipelineRegistryConfig: ProductPipelineRegistryConfig, - productPipelineBuilderFactory: ProductPipelineBuilderFactory, - statsReceiver: StatsReceiver) - extends Logging { - - private val rootIdentifierStack = ComponentIdentifierStack(RootIdentifier()) - - private val rebuildObserver = - Observer.function[Unit](statsReceiver, "ProductPipelineRegistry", "rebuild") - - /** - * Internal state of ProductPipelineRegistry. - * - * Build once on startup, and later whenever `rebuild()` is called. - */ - private[this] val productPipelineByProduct = - Var[Map[Product, ProductPipeline[_ <: Request, _]]](buildProductPipelineByProduct()) - - /** - * Triggers a rebuild of the ProductPipelineRegistry and also the ComponentRegistry - * - * Failed rebuilds will throw an exception - likely one of the listed ones - and the product - * registry and component registry will not be modified. - * - * @throws MultipleProductPipelinesForAProductException - * @throws ComponentIdentifierCollisionException - * @throws ChildComponentCollisionException - */ - private[core] def rebuild(): Unit = { - Try { - rebuildObserver { - productPipelineByProduct.update(buildProductPipelineByProduct()) - } - }.onFailure { ex => - error("Failed to rebuild ProductPipelineRegistry", ex) - }.get() - } - - /** - * register the provided pipeline recursively register all of it's children components - * that are added to the [[Pipeline]]'s [[Pipeline.children]] - */ - private def registerPipelineAndChildren( - componentRegistrySnapshot: ComponentRegistrySnapshot, - pipeline: Pipeline[_, _], - parentIdentifierStack: ComponentIdentifierStack - ): Unit = { - val identifierStackString = - s"${parentIdentifierStack.componentIdentifiers.reverse.mkString("\t->\t")}\t->\t${pipeline.identifier}" - info(identifierStackString) - - componentRegistrySnapshot.register( - component = pipeline, - parentIdentifierStack = parentIdentifierStack) - - val identifierStackWithCurrentPipeline = parentIdentifierStack.push(pipeline.identifier) - pipeline.children.foreach { - case childPipeline: Pipeline[_, _] => - info(s"$identifierStackString\t->\t${childPipeline.identifier}") - registerPipelineAndChildren( - componentRegistrySnapshot, - childPipeline, - identifierStackWithCurrentPipeline) - case component => - info(s"$identifierStackString\t->\t${component.identifier}") - componentRegistrySnapshot.register( - component = component, - parentIdentifierStack = identifierStackWithCurrentPipeline) - } - } - - /* - * Internal method (not for callers outside of this class, see rebuild() for those) - * - * Produces an updated Map[Product, ProductPipeline] and also refreshes the global component registry - */ - private[this] def buildProductPipelineByProduct( - ): Map[Product, ProductPipeline[_ <: Request, _]] = { - - // Build a new component registry snapshot. - val newComponentRegistry = new ComponentRegistrySnapshot() - - info( - "Registering all products, pipelines, and components (this may be helpful if you encounter dependency injection errors)") - info("debug details are in the form of `parent -> child`") - - // handle the case of multiple ProductPipelines having the same product - checkForAndThrowMultipleProductPipelinesForAProduct() - - // Build a Map[Product, ProductPipeline], registering everything in the new component registry recursively - val pipelinesByProduct: Map[Product, ProductPipeline[_ <: Request, _]] = - productPipelineRegistryConfig.productPipelineConfigs.map { productPipelineConfig => - val product = productPipelineConfig.product - info(s"Recursively registering ${product.identifier}") - - // gets the ComponentIdentifierStack without the RootIdentifier since - // we don't want RootIdentifier to show up in stats or errors - val productPipeline = - productPipelineBuilderFactory.get.build( - ComponentIdentifierStack(product.identifier), - productPipelineConfig) - - // gets RootIdentifier so we can register Products under the correct hierarchy - newComponentRegistry.register(product, rootIdentifierStack) - registerPipelineAndChildren( - newComponentRegistry, - productPipeline, - rootIdentifierStack.push(product.identifier)) - - // In addition to registering the component in the main registry, we want to maintain a map of - // product to the product pipeline to allow for O(1) lookup by product on the request hot path - product -> productPipeline - }.toMap - - info( - s"Successfully registered ${newComponentRegistry.getAllRegisteredComponents - .count(_.identifier.isInstanceOf[ProductIdentifier])} products and " + - s"${newComponentRegistry.getAllRegisteredComponents.length} " + - s"components total, query the component registry endpoint for details") - - componentRegistry.set(newComponentRegistry) - - pipelinesByProduct - } - - // handle the case of multiple ProductPipelines having the same product - private def checkForAndThrowMultipleProductPipelinesForAProduct(): Unit = { - productPipelineRegistryConfig.productPipelineConfigs.groupBy(_.product.identifier).foreach { - case (product, productPipelines) if productPipelines.length != 1 => - throw new MultipleProductPipelinesForAProductException( - product, - productPipelines.map(_.identifier)) - case _ => - } - } - - def getProductPipeline[MixerRequest <: Request: TypeTag, ResponseType: TypeTag]( - product: Product - ): ProductPipeline[MixerRequest, ResponseType] = { - // Check and cast the bounded existential types to the concrete types - (typeOf[MixerRequest], typeOf[ResponseType]) match { - case (req, res) if req =:= typeOf[MixerRequest] && res =:= typeOf[ResponseType] => - productPipelineByProduct.sample - .getOrElse(product, throw new ProductNotFoundException(product)) - .asInstanceOf[ProductPipeline[MixerRequest, ResponseType]] - case _ => - throw new UnknownPipelineResponseException(product) - } - } -} - -class ProductNotFoundException(product: Product) - extends RuntimeException(s"No Product found for $product") - -class UnknownPipelineResponseException(product: Product) - extends RuntimeException(s"Unknown pipeline response for $product") - -class MultipleProductPipelinesForAProductException( - product: ProductIdentifier, - pipelineIdentifiers: Seq[ProductPipelineIdentifier]) - extends IllegalStateException(s"Multiple ProductPipelines found for $product, found " + - s"${pipelineIdentifiers - .map(productPipelineIdentifier => s"$productPipelineIdentifier from ${productPipelineIdentifier.file}") - .mkString(", ")} ") diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry/ProductPipelineRegistryConfig.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry/ProductPipelineRegistryConfig.docx new file mode 100644 index 000000000..7230c214d Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry/ProductPipelineRegistryConfig.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry/ProductPipelineRegistryConfig.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry/ProductPipelineRegistryConfig.scala deleted file mode 100644 index bfa13c1c5..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry/ProductPipelineRegistryConfig.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.product_mixer.core.product.registry - -import com.twitter.product_mixer.core.model.marshalling.request.Request -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.product.ProductPipelineConfig - -trait ProductPipelineRegistryConfig { - def productPipelineConfigs: Seq[ProductPipelineConfig[_ <: Request, _ <: PipelineQuery, _]] -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/BUILD deleted file mode 100644 index 758b66385..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finagle/finagle-mux/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "servo/util", - "stitch/stitch-core", - "util/util-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/BUILD.docx new file mode 100644 index 000000000..36579e6ed Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/Bounds.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/Bounds.docx new file mode 100644 index 000000000..541605bb2 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/Bounds.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/Bounds.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/Bounds.scala deleted file mode 100644 index d94accb8f..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/Bounds.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.product_mixer.core.quality_factor - -/** - * Provides a way to apply inclusive min/max bounds to a given value. - */ -case class Bounds[T](minInclusive: T, maxInclusive: T)(implicit ordering: Ordering[T]) { - - def apply(value: T): T = ordering.min(maxInclusive, ordering.max(minInclusive, value)) - - def isWithin(value: T): Boolean = - ordering.gteq(value, minInclusive) && ordering.lteq(value, maxInclusive) - - def throwIfOutOfBounds(value: T, messagePrefix: String): Unit = - require(isWithin(value), s"$messagePrefix: value must be within $toString") - - override def toString: String = s"[$minInclusive, $maxInclusive]" -} - -object BoundsWithDefault { - def apply[T]( - minInclusive: T, - maxInclusive: T, - default: T - )( - implicit ordering: Ordering[T] - ): BoundsWithDefault[T] = BoundsWithDefault(Bounds(minInclusive, maxInclusive), default) -} - -case class BoundsWithDefault[T](bounds: Bounds[T], default: T)(implicit ordering: Ordering[T]) { - bounds.throwIfOutOfBounds(default, "default") - - def apply(valueOpt: Option[T]): T = valueOpt.map(bounds.apply).getOrElse(default) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/LinearLatencyQualityFactor.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/LinearLatencyQualityFactor.docx new file mode 100644 index 000000000..ebd22f0bf Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/LinearLatencyQualityFactor.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/LinearLatencyQualityFactor.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/LinearLatencyQualityFactor.scala deleted file mode 100644 index 9750f21f6..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/LinearLatencyQualityFactor.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.product_mixer.core.quality_factor - -import com.twitter.util.Duration -import com.twitter.util.Stopwatch - -case class LinearLatencyQualityFactor( - override val config: LinearLatencyQualityFactorConfig) - extends QualityFactor[Duration] { - - private val delayedUntilInMillis = Stopwatch.timeMillis() + config.initialDelay.inMillis - - private var state: Double = config.qualityFactorBounds.default - - override def currentValue: Double = state - - override def update(latency: Duration): Unit = { - if (Stopwatch.timeMillis() >= delayedUntilInMillis) { - if (latency > config.targetLatency) { - adjustState(getNegativeDelta) - } else { - adjustState(config.delta) - } - } - } - - override def buildObserver(): QualityFactorObserver = LinearLatencyQualityFactorObserver(this) - - private def getNegativeDelta: Double = - -config.delta * config.targetLatencyPercentile / (100.0 - config.targetLatencyPercentile) - - private def adjustState(delta: Double): Unit = { - state = config.qualityFactorBounds.bounds(state + delta) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/LinearLatencyQualityFactorObserver.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/LinearLatencyQualityFactorObserver.docx new file mode 100644 index 000000000..f4685fcfb Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/LinearLatencyQualityFactorObserver.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/LinearLatencyQualityFactorObserver.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/LinearLatencyQualityFactorObserver.scala deleted file mode 100644 index bd5931d80..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/LinearLatencyQualityFactorObserver.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.product_mixer.core.quality_factor - -import com.twitter.util.Duration -import com.twitter.util.Try - -case class LinearLatencyQualityFactorObserver( - override val qualityFactor: LinearLatencyQualityFactor) - extends QualityFactorObserver { - - override def apply(result: Try[_], latency: Duration): Unit = { - result - .onSuccess(_ => qualityFactor.update(latency)) - .onFailure { - case t if qualityFactor.config.ignorableFailures.isDefinedAt(t) => () - case _ => qualityFactor.update(Duration.Top) - } - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QualityFactor.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QualityFactor.docx new file mode 100644 index 000000000..073f2da13 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QualityFactor.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QualityFactor.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QualityFactor.scala deleted file mode 100644 index 7c65656f7..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QualityFactor.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.product_mixer.core.quality_factor - -/** - * [[QualityFactor]] is an abstract number that enables a feedback loop to control operation costs and ultimately - * maintain the operation success rate. Abstractly, if operations/calls are too expensive (such as high - * latencies), the quality factor should go down, which helps future calls to ease their demand/load (such as - * reducing request width); if ops/calls are fast, the quality factor should go up, so we can incur more load. - * - * @note to avoid overhead the underlying state may sometimes not be synchronized. - * If a part of an application is unhealthy, it will likely be unhealthy for all threads, - * it will eventually result in a close-enough quality factor value for all thread's view of the state. - * - * In extremely low volume scenarios such as manual testing in a development environment, - * it's possible that different threads will have vastly different views of the underling state, - * but in practice, in production systems, they will be close-enough. - */ -trait QualityFactor[Input] { self => - - /** get the current [[QualityFactor]]'s value */ - def currentValue: Double - - def config: QualityFactorConfig - - /** update of the current `factor` value */ - def update(input: Input): Unit - - /** a [[QualityFactorObserver]] for this [[QualityFactor]] */ - def buildObserver(): QualityFactorObserver - - override def toString: String = { - self.getClass.getSimpleName.stripSuffix("$") - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QualityFactorConfig.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QualityFactorConfig.docx new file mode 100644 index 000000000..3622eadab Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QualityFactorConfig.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QualityFactorConfig.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QualityFactorConfig.scala deleted file mode 100644 index e0ba6f465..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QualityFactorConfig.scala +++ /dev/null @@ -1,120 +0,0 @@ -package com.twitter.product_mixer.core.quality_factor - -import com.twitter.product_mixer.core.pipeline.pipeline_failure.ClientFailure -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.quality_factor.QualityFactorConfig.defaultIgnorableFailures -import com.twitter.servo.util.CancelledExceptionExtractor -import com.twitter.util.Duration -import com.twitter.conversions.DurationOps.RichDuration - -/** - * Quality factor is an abstract number that enables a feedback loop to control operation costs and ultimately - * maintain the operation success rate. Abstractly, if operations/calls are too expensive (such as high - * latencies), the quality factor should go down, which helps future calls to ease their demand/load (such as - * reducing request width); if ops/calls are fast, the quality factor should go up, so we can incur more load. - */ -sealed trait QualityFactorConfig { - - /** - * specifies the quality factor min and max bounds and default value. - */ - def qualityFactorBounds: BoundsWithDefault[Double] - - /** - * initialDelay Specifies how much delay we should have before the quality factor calculation start to kick in. This is - * mostly to ease the load during the initial warmup/startup. - */ - def initialDelay: Duration - - /** - * [[Throwable]]s that should be ignored when calculating - * the [[QualityFactor]] if this is [[PartialFunction.isDefinedAt]] - */ - def ignorableFailures: PartialFunction[Throwable, Unit] = defaultIgnorableFailures -} - -object QualityFactorConfig { - - /** - * Default value for [[QualityFactorConfig.ignorableFailures]] that ignores any - * Cancelled requests and [[ClientFailure]] - */ - val defaultIgnorableFailures: PartialFunction[Throwable, Unit] = { - case PipelineFailure(_: ClientFailure, _, _, _) => () - case CancelledExceptionExtractor(_) => () - } -} - -/** - * This is a linear quality factor implementation, aimed to achieve and maintain a percentile latency target. - * - * If we call quality factor q, target latency t and target percentile p, - * then the q (quality factor) formula should be: - * q += delta for each request with latency <= t - * q -= delta * p / (100 - p) for each request with latency > t ms or a timeout. - * - * When percentile p latency stays at target latency t, then based on the formula above, q will - * stay constant (fluctuates around a constant value). - * - * For example, assume t = 100ms, p = p99, and q = 0.5 - * let's say, p99 latency stays at 100ms when q = 0.5. p99 means that out of every 100 latencies, - * 99 times the latency is below 100ms and 1 time it is above 100ms. So based on the formula above, - * q will increase by "delta" 99 times and it will decrease by delta * p / (100 - p) = delta * 99 once, - * which results in the same q = 0.5. - * - * @param targetLatency This is the latency target, calls with latencies above which will cause quality - * factor to go down, and vice versa. e.g. 500ms. - * @param targetLatencyPercentile This the percentile where the target latency is aimed at. e.g. 95.0. - * @param delta the step for adjusting quality factor. It should be a positive double. If delta is - * too large, then quality factor will fluctuate more, and if it is too small, the - * responsiveness will be reduced. - */ -case class LinearLatencyQualityFactorConfig( - override val qualityFactorBounds: BoundsWithDefault[Double], - override val initialDelay: Duration, - targetLatency: Duration, - targetLatencyPercentile: Double, - delta: Double, - override val ignorableFailures: PartialFunction[Throwable, Unit] = - QualityFactorConfig.defaultIgnorableFailures) - extends QualityFactorConfig { - require( - targetLatencyPercentile >= 50.0 && targetLatencyPercentile < 100.0, - s"Invalid targetLatencyPercentile value: ${targetLatencyPercentile}.\n" + - s"Correct sample values: 95.0, 99.9. Incorrect sample values: 0.95, 0.999." - ) -} - -/** - * A quality factor provides component capacity state based on sampling component - * Queries Per Second (qps) at local host level. - * - * If we call quality factor q, max qps R: - * then the q (quality factor) formula should be: - * q = Math.min([[qualityFactorBounds.bounds.maxInclusive]], q + delta) for each request that observed qps <= R on local host - * q -= delta for each request that observed qps > R on local host - * - * When qps r stays below R, q will stay as constant (value at [[qualityFactorBounds.bounds.maxInclusive]]). - * When qps r starts to increase above R, q will decrease by delta per request, - * with delta being an additive factor that controls how sensitive q is when max qps R is exceeded. - * - * @param initialDelay Specifies an initial delay time to allow query rate counter warm up to start reflecting actual traffic load. - * Qf value would only start to update after this initial delay. - * @param maxQueriesPerSecond The max qps the underlying component can take. Requests go above this qps threshold will cause quality factor to go down. - * @param queriesPerSecondSampleWindow The window of underlying query rate counter counting with and calculate an average qps over the window, - * default to count with 10 seconds time window (i.e. qps = total requests over last 10 secs / 10). - * Note: underlying query rate counter has a sliding window with 10 fixed slices. Therefore a larger - * window would lead to a coarser qps calculation. (e.g. with 60 secs time window, it sliding over 6 seconds slice (60 / 10 = 6 secs)). - * A larger time window also lead to a slower reaction to sudden qps burst, but more robust to flaky qps pattern. - * @param delta The step for adjusting quality factor. It should be a positive double. If the delta is large, the quality factor - * will fluctuate more and be more responsive to exceeding max qps, and if it is small, the quality factor will be less responsive. - */ -case class QueriesPerSecondBasedQualityFactorConfig( - override val qualityFactorBounds: BoundsWithDefault[Double], - override val initialDelay: Duration, - maxQueriesPerSecond: Int, - queriesPerSecondSampleWindow: Duration = 10.seconds, - delta: Double = 0.001, - override val ignorableFailures: PartialFunction[Throwable, Unit] = - QualityFactorConfig.defaultIgnorableFailures) - extends QualityFactorConfig diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QualityFactorObserver.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QualityFactorObserver.docx new file mode 100644 index 000000000..a6b398506 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QualityFactorObserver.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QualityFactorObserver.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QualityFactorObserver.scala deleted file mode 100644 index e15fb5491..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QualityFactorObserver.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.product_mixer.core.quality_factor - -import com.twitter.util.Duration -import com.twitter.util.Try - -/** Updates the [[QualityFactor]] */ -trait QualityFactorObserver { - - /** The [[QualityFactor]] to update when observing */ - def qualityFactor: QualityFactor[_] - - /** - * updates the [[qualityFactor]] given the result [[Try]] and the latency - * @note implementations must be sure to correctly ignore - * [[QualityFactor.config]]'s [[QualityFactorConfig.ignorableFailures]] - */ - def apply(result: Try[_], latency: Duration): Unit -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QualityFactorStatus.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QualityFactorStatus.docx new file mode 100644 index 000000000..74a9d1a1f Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QualityFactorStatus.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QualityFactorStatus.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QualityFactorStatus.scala deleted file mode 100644 index 01a327f86..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QualityFactorStatus.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.product_mixer.core.quality_factor - -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.pipeline_failure.MisconfiguredQualityFactor -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure - -case class QualityFactorStatus( - qualityFactorByPipeline: Map[ComponentIdentifier, QualityFactor[_]]) { - - /** - * returns a new [[QualityFactorStatus]] with all the elements of current QualityFactorStatus and `other`. - * If a [[ComponentIdentifier]] exists in both maps, the Value from `other` takes precedence - */ - def ++(other: QualityFactorStatus): QualityFactorStatus = { - if (other.qualityFactorByPipeline.isEmpty) { - this - } else if (qualityFactorByPipeline.isEmpty) { - other - } else { - QualityFactorStatus(qualityFactorByPipeline ++ other.qualityFactorByPipeline) - } - } -} - -object QualityFactorStatus { - def build[Identifier <: ComponentIdentifier]( - qualityFactorConfigs: Map[Identifier, QualityFactorConfig] - ): QualityFactorStatus = { - QualityFactorStatus( - qualityFactorConfigs.map { - case (key, config: LinearLatencyQualityFactorConfig) => - key -> LinearLatencyQualityFactor(config) - case (key, config: QueriesPerSecondBasedQualityFactorConfig) => - key -> QueriesPerSecondBasedQualityFactor(config) - } - ) - } - - val empty: QualityFactorStatus = QualityFactorStatus(Map.empty) -} - -trait HasQualityFactorStatus { - def qualityFactorStatus: Option[QualityFactorStatus] = None - def withQualityFactorStatus(qualityFactorStatus: QualityFactorStatus): PipelineQuery - - def getQualityFactorCurrentValue( - identifier: ComponentIdentifier - ): Double = getQualityFactor(identifier).currentValue - - def getQualityFactor( - identifier: ComponentIdentifier - ): QualityFactor[_] = qualityFactorStatus - .flatMap(_.qualityFactorByPipeline.get(identifier)) - .getOrElse { - throw PipelineFailure( - MisconfiguredQualityFactor, - s"Quality factor not configured for $identifier") - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QueriesPerSecondBasedQualityFactor.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QueriesPerSecondBasedQualityFactor.docx new file mode 100644 index 000000000..6bca74426 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QueriesPerSecondBasedQualityFactor.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QueriesPerSecondBasedQualityFactor.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QueriesPerSecondBasedQualityFactor.scala deleted file mode 100644 index a1ba3f373..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QueriesPerSecondBasedQualityFactor.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.product_mixer.core.quality_factor - -import com.google.common.annotations.VisibleForTesting -import com.twitter.util.Stopwatch - -case class QueriesPerSecondBasedQualityFactor( - override val config: QueriesPerSecondBasedQualityFactorConfig) - extends QualityFactor[Int] { - - @VisibleForTesting - private[quality_factor] val queryRateCounter: QueryRateCounter = QueryRateCounter( - config.queriesPerSecondSampleWindow) - - private val delayedUntilInMillis = Stopwatch.timeMillis() + config.initialDelay.inMillis - - private var state: Double = config.qualityFactorBounds.default - - override def currentValue: Double = state - - override def update(count: Int = 1): Unit = { - val queryRate = incrementAndGetQueryRateCount(count) - - // Only update quality factor until the initial delay past. - // This allows query rate counter get warm up to reflect - // actual traffic load by the time initial delay expires. - if (Stopwatch.timeMillis() >= delayedUntilInMillis) { - if (queryRate > config.maxQueriesPerSecond) { - state = config.qualityFactorBounds.bounds(state - config.delta) - } else { - state = config.qualityFactorBounds.bounds(state + config.delta) - } - } - } - - private def incrementAndGetQueryRateCount(count: Int): Double = { - // Int.MaxValue is used as a special signal from [[QueriesPerSecondBasedQualityFactorObserver]] - // to indicate a component failure is observed. - // In this case, we do not update queryRateCounter and instead return Int.MaxValue. - // As the largest Int value, this should result in the threshold qps for quality factor being - // exceeded and directly decrementing quality factor. - if (count == Int.MaxValue) { - Int.MaxValue.toDouble - } else { - queryRateCounter.increment(count) - queryRateCounter.getRate() - } - } - - override def buildObserver(): QualityFactorObserver = - QueriesPerSecondBasedQualityFactorObserver(this) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QueriesPerSecondBasedQualityFactorObserver.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QueriesPerSecondBasedQualityFactorObserver.docx new file mode 100644 index 000000000..3979e17c6 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QueriesPerSecondBasedQualityFactorObserver.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QueriesPerSecondBasedQualityFactorObserver.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QueriesPerSecondBasedQualityFactorObserver.scala deleted file mode 100644 index 27f13c8a3..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QueriesPerSecondBasedQualityFactorObserver.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.product_mixer.core.quality_factor - -import com.twitter.util.Duration -import com.twitter.util.Try - -case class QueriesPerSecondBasedQualityFactorObserver( - override val qualityFactor: QueriesPerSecondBasedQualityFactor) - extends QualityFactorObserver { - override def apply( - result: Try[_], - latency: Duration - ): Unit = { - result - .onSuccess(_ => qualityFactor.update()) - .onFailure { - case t if qualityFactor.config.ignorableFailures.isDefinedAt(t) => () - // Degrade qf as a proactive mitigation for any non ignorable failures. - case _ => qualityFactor.update(Int.MaxValue) - } - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QueryRateCounter.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QueryRateCounter.docx new file mode 100644 index 000000000..85132932b Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QueryRateCounter.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QueryRateCounter.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QueryRateCounter.scala deleted file mode 100644 index 33d4a5375..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor/QueryRateCounter.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.product_mixer.core.quality_factor - -import com.twitter.util.Duration -import com.twitter.util.Stopwatch -import com.twitter.util.TokenBucket - -/** - * Query rate counter based on a leaky bucket. For more, see [[com.twitter.util.TokenBucket]]. - */ -case class QueryRateCounter private[quality_factor] ( - queryRateWindow: Duration) { - - private val queryRateWindowInSeconds = queryRateWindow.inSeconds - - private val leakyBucket: TokenBucket = - TokenBucket.newLeakyBucket(ttl = queryRateWindow, reserve = 0, nowMs = Stopwatch.timeMillis) - - def increment(count: Int): Unit = leakyBucket.put(count) - - def getRate(): Double = leakyBucket.count / queryRateWindowInSeconds -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/BUILD deleted file mode 100644 index 72565730e..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/BUILD +++ /dev/null @@ -1,32 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "finagle/finagle-core", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor", - "product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/observer", - "servo/util", - "stitch/stitch-core", - "util/util-core", - "util/util-stats", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline:executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor", - "product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/observer", - "stitch/stitch-core", - "util/util-stats", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/BUILD.docx new file mode 100644 index 000000000..70328882e Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/Executor.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/Executor.docx new file mode 100644 index 000000000..9032ae422 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/Executor.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/Executor.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/Executor.scala deleted file mode 100644 index d56c1e0fc..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/Executor.scala +++ /dev/null @@ -1,700 +0,0 @@ -package com.twitter.product_mixer.core.service - -import com.twitter.finagle.stats.BroadcastStatsReceiver -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.DefaultStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.tracing.Annotation -import com.twitter.finagle.tracing.Record -import com.twitter.finagle.tracing.Trace -import com.twitter.finagle.tracing.TraceId -import com.twitter.finagle.tracing.TraceServiceName -import com.twitter.finagle.tracing.Tracing.LocalBeginAnnotation -import com.twitter.finagle.tracing.Tracing.LocalEndAnnotation -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifierStack -import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier -import com.twitter.product_mixer.core.pipeline.FailOpenPolicy -import com.twitter.product_mixer.core.pipeline.PipelineResult -import com.twitter.product_mixer.core.pipeline.pipeline_failure.FeatureHydrationFailed -import com.twitter.product_mixer.core.pipeline.pipeline_failure.MisconfiguredFeatureMapFailure -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailureClassifier -import com.twitter.product_mixer.core.pipeline.pipeline_failure.UncategorizedServerFailure -import com.twitter.product_mixer.core.quality_factor.QualityFactorObserver -import com.twitter.product_mixer.core.service.Executor.AlwaysFailOpenIncludingProgrammerErrors -import com.twitter.product_mixer.core.service.Executor.Context -import com.twitter.product_mixer.core.service.Executor.TracingConfig -import com.twitter.product_mixer.core.service.Executor.toPipelineFailureWithComponentIdentifierStack -import com.twitter.servo.util.CancelledExceptionExtractor -import com.twitter.stitch.Arrow -import com.twitter.stitch.Stitch -import com.twitter.stitch.Stitch.Letter -import com.twitter.util.Duration -import com.twitter.util.Return -import com.twitter.util.Throw -import com.twitter.util.Time -import com.twitter.util.Try - -/** - * Base trait that all executors implement - * - * All executors should: - * - implement a `def arrow` or `def apply` with the relevant types for their use case - * and take in an implicit [[PipelineFailureClassifier]] and [[ComponentIdentifierStack]]. - * - add a `@singleton` annotation to the class and `@inject` annotation to the argument list - * - take in a [[StatsReceiver]] - * - * @example {{{ - * @Singleton class MyExecutor @Inject() ( - * override val statsReceiver: StatsReceiver - * ) extends Executor { - * def arrow( - * arg: MyArg, - * ..., - * context: Context - * ): Arrow[In,Out] = ??? - * } - * }}} - */ -private[core] trait Executor { - val statsReceiver: StatsReceiver - - /** - * Applies the `pipelineFailureClassifier` to the output of the `arrow` - * and adds the `componentStack` to the [[PipelineFailure]] - */ - def wrapWithErrorHandling[In, Out]( - context: Context, - currentComponentIdentifier: ComponentIdentifier - )( - arrow: Arrow[In, Out] - ): Arrow[In, Out] = { - arrow.mapFailure( - toPipelineFailureWithComponentIdentifierStack(context, currentComponentIdentifier)) - } - - /** - * Chain a `Seq` of [[Arrow.Iso]], only passing successful results to the next [[Arrow.Iso]] - * - * @note the resulting [[Arrow]] runs the passed in [[Arrow]]s one after the other, - * as an ordered execution, this means that each [[Arrow]] is dependent - * on all previous [[Arrow]]s in the `Seq` so no `Stitch` batching can occur - * between them. - */ - def isoArrowsSequentially[T](arrows: Seq[Arrow.Iso[T]]): Arrow.Iso[T] = { - // avoid excess Arrow complexity when there is only a single Arrow - arrows match { - case Seq() => Arrow.identity - case Seq(onlyOneArrow) => onlyOneArrow - case Seq(head, tail @ _*) => - tail.foldLeft(head) { - case (combinedArrow, nextArrow) => combinedArrow.flatMapArrow(nextArrow) - } - } - } - - /** - * Start running the [[Arrow]] in the background returning a [[Stitch.Ref]] which will complete - * when the background task is finished - */ - def startArrowAsync[In, Out](arrow: Arrow[In, Out]): Arrow[In, Stitch[Out]] = { - Arrow - .map { arg: In => - // wrap in a `ref` so we only compute it's value once - Stitch.ref(arrow(arg)) - } - .andThen( - Arrow.zipWithArg( - // satisfy the `ref` async - Arrow.async(Arrow.flatMap[Stitch[Out], Out](identity)))) - .map { case (ref, _) => ref } - } - - /** - * for [[com.twitter.product_mixer.core.model.common.Component]]s which - * are executed per-candidate or which we don't want to record stats for. - * This performs Tracing but does not record Stats - * - * @note This should be used around the computation that includes the execution of the - * underlying Component over all the Candidates, not around each execution - * of the component around each candidate for per-candidate Components. - * - * @note when using this you should only use [[wrapPerCandidateComponentWithExecutorBookkeepingWithoutTracing]] - * for handling Stats. - */ - def wrapComponentsWithTracingOnly[In, Out]( - context: Context, - currentComponentIdentifier: ComponentIdentifier - )( - arrow: Arrow[In, Out] - ): Arrow[In, Out] = { - Executor.wrapArrowWithLocalTracingSpan( - Arrow - .time(arrow) - .map { - case (result, latency) => - Executor.recordTraceData( - componentStack = context.componentStack, - componentIdentifier = currentComponentIdentifier, - result = result, - latency = latency, - size = None) - result - }.lowerFromTry) - } - - /** - * for [[com.twitter.product_mixer.core.model.common.Component]]s which - * are executed per-candidate. Records Stats but does not do Tracing. - * - * @note when using this you should only use [[wrapPerCandidateComponentsWithTracingOnly]] - * for handling Tracing - */ - def wrapPerCandidateComponentWithExecutorBookkeepingWithoutTracing[In, Out]( - context: Context, - currentComponentIdentifier: ComponentIdentifier - )( - arrow: Arrow[In, Out] - ): Arrow[In, Out] = { - val observerSideEffect = - ExecutorObserver.executorObserver[Out](context, currentComponentIdentifier, statsReceiver) - - Executor.wrapWithExecutorBookkeeping[In, Out, Out]( - context = context, - currentComponentIdentifier = currentComponentIdentifier, - executorResultSideEffect = observerSideEffect, - transformer = Return(_), - tracingConfig = TracingConfig.NoTracing - )(arrow) - } - - /** for [[com.twitter.product_mixer.core.model.common.Component]]s */ - def wrapComponentWithExecutorBookkeeping[In, Out]( - context: Context, - currentComponentIdentifier: ComponentIdentifier - )( - arrow: Arrow[In, Out] - ): Arrow[In, Out] = { - val observerSideEffect = - ExecutorObserver.executorObserver[Out](context, currentComponentIdentifier, statsReceiver) - - Executor.wrapWithExecutorBookkeeping[In, Out, Out]( - context = context, - currentComponentIdentifier = currentComponentIdentifier, - executorResultSideEffect = observerSideEffect, - transformer = Return(_) - )(arrow) - } - - /** - * for [[com.twitter.product_mixer.core.model.common.Component]]s which an `onSuccess` - * to add custom stats or logging of results - */ - def wrapComponentWithExecutorBookkeeping[In, Out]( - context: Context, - currentComponentIdentifier: ComponentIdentifier, - onSuccess: Out => Unit - )( - arrow: Arrow[In, Out] - ): Arrow[In, Out] = { - val observerSideEffect = - ExecutorObserver.executorObserver[Out](context, currentComponentIdentifier, statsReceiver) - - Executor.wrapWithExecutorBookkeeping[In, Out, Out]( - context = context, - currentComponentIdentifier = currentComponentIdentifier, - executorResultSideEffect = observerSideEffect, - transformer = Return(_), - onComplete = (transformed: Try[Out]) => transformed.onSuccess(onSuccess) - )(arrow) - } - - /** for [[com.twitter.product_mixer.core.pipeline.Pipeline]]s */ - def wrapPipelineWithExecutorBookkeeping[In, Out <: PipelineResult[_]]( - context: Context, - currentComponentIdentifier: ComponentIdentifier, - qualityFactorObserver: Option[QualityFactorObserver], - failOpenPolicy: FailOpenPolicy = FailOpenPolicy.Never - )( - arrow: Arrow[In, Out] - ): Arrow[In, Out] = { - val observerSideEffect = - ExecutorObserver - .pipelineExecutorObserver[Out](context, currentComponentIdentifier, statsReceiver) - - Executor.wrapWithExecutorBookkeeping[In, Out, Out]( - context = context, - currentComponentIdentifier = currentComponentIdentifier, - executorResultSideEffect = observerSideEffect, - transformer = (result: Out) => result.toTry, - size = Some(_.resultSize()), - failOpenPolicy = failOpenPolicy, - qualityFactorObserver = qualityFactorObserver - )(arrow) - } - - /** for [[com.twitter.product_mixer.core.pipeline.product.ProductPipeline]]s */ - def wrapProductPipelineWithExecutorBookkeeping[In, Out <: PipelineResult[_]]( - context: Context, - currentComponentIdentifier: ProductPipelineIdentifier - )( - arrow: Arrow[In, Out] - ): Arrow[In, Out] = { - - val observerSideEffect = - ExecutorObserver - .productPipelineExecutorObserver[Out](currentComponentIdentifier, statsReceiver) - - Executor.wrapWithExecutorBookkeeping[In, Out, Out]( - context = context, - currentComponentIdentifier = currentComponentIdentifier, - executorResultSideEffect = observerSideEffect, - transformer = _.toTry, - size = Some(_.resultSize()), - failOpenPolicy = - // always save Failures in the Result object instead of failing the request - AlwaysFailOpenIncludingProgrammerErrors - )(arrow) - } - - /** for [[com.twitter.product_mixer.core.model.common.Component]]s which need a result size stat */ - def wrapComponentWithExecutorBookkeepingWithSize[In, Out]( - context: Context, - currentComponentIdentifier: CandidateSourceIdentifier, - size: Out => Int - )( - arrow: Arrow[In, Out] - ): Arrow[In, Out] = { - val observerSideEffect = - ExecutorObserver.executorObserverWithSize(context, currentComponentIdentifier, statsReceiver) - - Executor.wrapWithExecutorBookkeeping[In, Out, Int]( - context = context, - currentComponentIdentifier = currentComponentIdentifier, - executorResultSideEffect = observerSideEffect, - transformer = (out: Out) => Try(size(out)), - size = Some(identity) - )(arrow) - } - - /** for [[com.twitter.product_mixer.core.pipeline.PipelineBuilder.Step]]s */ - def wrapStepWithExecutorBookkeeping[In, Out]( - context: Context, - identifier: PipelineStepIdentifier, - arrow: Arrow[In, Out], - transformer: Out => Try[Unit] - ): Arrow[In, Out] = { - val observerSideEffect = - ExecutorObserver.stepExecutorObserver(context, identifier, statsReceiver) - - Executor.wrapWithExecutorBookkeeping[In, Out, Unit]( - context = context, - currentComponentIdentifier = identifier, - executorResultSideEffect = observerSideEffect, - transformer = transformer, - failOpenPolicy = AlwaysFailOpenIncludingProgrammerErrors - )(arrow) - } -} - -private[core] object Executor { - - private[service] object TracingConfig { - - /** Used to specify whether a wrapped Arrow should be Traced in [[wrapWithExecutorBookkeeping]] */ - sealed trait TracingConfig - case object NoTracing extends TracingConfig - case object WrapWithSpanAndTracingData extends TracingConfig - } - - /** - * Always fail-open and return the [[com.twitter.product_mixer.core.pipeline.product.ProductPipelineResult]] - * containing the exception, this differs from [[FailOpenPolicy.Always]] in that this will still - * fail-open and return the overall result object even if the underlying failure is the result - * of programmer error. - */ - private val AlwaysFailOpenIncludingProgrammerErrors: FailOpenPolicy = _ => true - - /** - * Wraps an [[Arrow]] so that bookkeeping around the execution occurs uniformly. - * - * @note should __never__ be called directly! - * - * - For successful results, apply the `transformer` - * - convert any exceptions to PipelineFailures - * - record stats and update [[QualityFactorObserver]] - * - wraps the execution in a Trace span and record Trace data (can be turned off by [[TracingConfig]]) - * - applies a trace span and records metadata to the provided `arrow` - * - determine whether to fail-open based on `result.flatMap(transformer)` - * - if failing-open, always return the original result - * - if failing-closed and successful, return the original result - * - otherwise, return the failure (from `result.flatMap(transformer)`) - * - * @param context the [[Executor.Context]] - * @param currentComponentIdentifier the current component's [[ComponentIdentifier]] - * @param executorResultSideEffect the [[ExecutorObserver]] used to record stats - * @param transformer function to convert a successful result into possibly a failed result - * @param failOpenPolicy [[FailOpenPolicy]] to apply to the results of `result.flatMap(transformer)` - * @param qualityFactorObserver [[QualityFactorObserver]] to update based on the results of `result.flatMap(transformer)` - * @param tracingConfig indicates whether the [[Arrow]] should be wrapped with Tracing - * @param onComplete runs the function for its side effects with the result of `result.flatMap(transformer)` - * @param arrow an input [[Arrow]] to wrap so that after it's execution, we perform all these operations - * - * @return the wrapped [[Arrow]] - */ - private[service] def wrapWithExecutorBookkeeping[In, Out, Transformed]( - context: Context, - currentComponentIdentifier: ComponentIdentifier, - executorResultSideEffect: ExecutorObserver[Transformed], - transformer: Out => Try[Transformed], - size: Option[Transformed => Int] = None, - failOpenPolicy: FailOpenPolicy = FailOpenPolicy.Never, - qualityFactorObserver: Option[QualityFactorObserver] = None, - tracingConfig: TracingConfig.TracingConfig = TracingConfig.WrapWithSpanAndTracingData, - onComplete: Try[Transformed] => Unit = { _: Try[Transformed] => () } - )( - arrow: Arrow[In, Out] - ): Arrow[In, Out] = { - - val failureClassifier = - toPipelineFailureWithComponentIdentifierStack(context, currentComponentIdentifier) - - /** transform the results, mapping all exceptions to [[PipelineFailure]]s, and tuple with original result */ - val transformResultAndClassifyFailures: Arrow[Out, (Out, Try[Transformed])] = - Arrow.join( - Arrow.mapFailure(failureClassifier), - Arrow - .transformTry[Out, Transformed](result => - result - .flatMap(transformer) - .rescue { case t => Throw(failureClassifier(t)) }) - .liftToTry - ) - - /** Only record tracing data if [[TracingConfig.WrapWithSpanAndTracingData]] */ - val maybeRecordTracingData: (Try[Transformed], Duration) => Unit = tracingConfig match { - case TracingConfig.NoTracing => (_, _) => () - case TracingConfig.WrapWithSpanAndTracingData => - (transformedAndClassifiedResult, latency) => - recordTraceData( - context.componentStack, - currentComponentIdentifier, - transformedAndClassifiedResult, - latency, - transformedAndClassifiedResult.toOption.flatMap(result => size.map(_.apply(result))) - ) - } - - /** Will never be in a failed state so we can do a simple [[Arrow.map]] */ - val recordStatsAndUpdateQualityFactor = - Arrow - .map[(Try[(Out, Try[Transformed])], Duration), Unit] { - case (tryResultAndTryTransformed, latency) => - val transformedAndClassifiedResult = tryResultAndTryTransformed.flatMap { - case (_, transformed) => transformed - } - executorResultSideEffect(transformedAndClassifiedResult, latency) - qualityFactorObserver.foreach(_.apply(transformedAndClassifiedResult, latency)) - onComplete(transformedAndClassifiedResult) - maybeRecordTracingData(transformedAndClassifiedResult, latency) - }.unit - - /** - * Applies the provided [[FailOpenPolicy]] based on the [[transformer]]'s results, - * returning the original result or an exception - */ - val applyFailOpenPolicyBasedOnTransformedResult: Arrow[ - (Try[(Out, Try[Transformed])], Duration), - Out - ] = - Arrow - .map[(Try[(Out, Try[Transformed])], Duration), Try[(Out, Try[Transformed])]] { - case (tryResultAndTryTransformed, _) => tryResultAndTryTransformed - } - .lowerFromTry - .map { - case (result, Throw(pipelineFailure: PipelineFailure)) - if failOpenPolicy(pipelineFailure.category) => - Return(result) - case (_, t: Throw[_]) => t.asInstanceOf[Throw[Out]] - case (result, _) => Return(result) - }.lowerFromTry - - /** The complete Arrow minus a Local span wrapping */ - val arrowWithTimingExecutorSideEffects = Arrow - .time(arrow.andThen(transformResultAndClassifyFailures)) - .applyEffect(recordStatsAndUpdateQualityFactor) - .andThen(applyFailOpenPolicyBasedOnTransformedResult) - - /** Dont wrap with a span if we are not tracing */ - tracingConfig match { - case TracingConfig.WrapWithSpanAndTracingData => - wrapArrowWithLocalTracingSpan(arrowWithTimingExecutorSideEffects) - case TracingConfig.NoTracing => - arrowWithTimingExecutorSideEffects - } - } - - /** Let-scopes a [[TraceId]] around a computation */ - private[this] object TracingLetter extends Letter[TraceId] { - override def let[S](traceId: TraceId)(s: => S): S = Trace.letId(traceId)(s) - } - - /** - * Wraps the Arrow's execution in a new trace span as a child of the current parent span - * - * @note Should __never__ be called directly! - * - * It's expected that the contained `arrow` will invoke [[recordTraceData]] exactly ONCE - * during it's execution. - * - * @note this does not record any data about the trace, it only sets the [[Trace]] Span - * for the execution of `arrow` - */ - private[service] def wrapArrowWithLocalTracingSpan[In, Out]( - arrow: Arrow[In, Out] - ): Arrow[In, Out] = - Arrow.ifelse( - _ => Trace.isActivelyTracing, - Arrow.let(TracingLetter)(Trace.nextId)(arrow), - arrow - ) - - private[this] object Tracing { - - /** - * Duplicate of [[com.twitter.finagle.tracing.Tracing]]'s `localSpans` which - * uses an un-scoped [[StatsReceiver]] - * - * Since we needed to roll-our-own latency measurement we are unable to increment the - * `local_spans` metric automatically, this is important in the event a service is - * unexpectedly not recording spans or unexpectedly recording too many, so we manually - * increment it - */ - val localSpans: Counter = DefaultStatsReceiver.counter("tracing", "local_spans") - - /** Local Component field of a span in the UI */ - val localComponentTag = "lc" - val sizeTag = "product_mixer.result.size" - val successTag = "product_mixer.result.success" - val successValue = "success" - val cancelledTag = "product_mixer.result.cancelled" - val failureTag = "product_mixer.result.failure" - } - - /** - * Records metadata onto the current [[Trace]] Span - * - * @note Should __never__ be called directly! - * - * This should be called exactly ONCE in the Arrow passed into [[wrapArrowWithLocalTracingSpan]] - * to record data for the Span. - */ - private[service] def recordTraceData[T]( - componentStack: ComponentIdentifierStack, - componentIdentifier: ComponentIdentifier, - result: Try[T], - latency: Duration, - size: Option[Int] = None - ): Unit = { - if (Trace.isActivelyTracing) { - Tracing.localSpans.incr() - val traceId = Trace.id - val endTime = Time.nowNanoPrecision - - // These annotations are needed for the Zipkin UI to display the span properly - TraceServiceName().foreach(Trace.recordServiceName) - Trace.recordRpc(componentIdentifier.snakeCase) // name of the span in the UI - Trace.recordBinary( - Tracing.localComponentTag, - componentStack.peek.toString + "/" + componentIdentifier.toString) - Trace.record(Record(traceId, endTime - latency, Annotation.Message(LocalBeginAnnotation))) - Trace.record(Record(traceId, endTime, Annotation.Message(LocalEndAnnotation))) - - // product mixer specific zipkin data - size.foreach(size => Trace.recordBinary(Tracing.sizeTag, size)) - result match { - case Return(_) => - Trace.recordBinary(Tracing.successTag, Tracing.successValue) - case Throw(CancelledExceptionExtractor(e)) => - Trace.recordBinary(Tracing.cancelledTag, e) - case Throw(e) => - Trace.recordBinary(Tracing.failureTag, e) - } - } - } - - /** - * Returns a tuple of the stats scopes for the current component and the relative scope for - * the parent component and the current component together - * - * This is useful when recording stats for a component by itself as well as stats for calls to that component from it's parent. - * - * @example if the current component has a scope of "currentComponent" and the parent component has a scope of "parentComponent" - * then this will return `(Seq("currentComponent"), Seq("parentComponent", "currentComponent"))` - */ - def buildScopes( - context: Context, - currentComponentIdentifier: ComponentIdentifier - ): Executor.Scopes = { - val parentScopes = context.componentStack.peek.toScopes - val componentScopes = currentComponentIdentifier.toScopes - val relativeScopes = parentScopes ++ componentScopes - Executor.Scopes(componentScopes, relativeScopes) - } - - /** - * Makes a [[BroadcastStatsReceiver]] that will broadcast stats to the correct - * current component's scope and to the scope relative to the parent. - */ - def broadcastStatsReceiver( - context: Context, - currentComponentIdentifier: ComponentIdentifier, - statsReceiver: StatsReceiver - ): StatsReceiver = { - val Executor.Scopes(componentScopes, relativeScopes) = - Executor.buildScopes(context, currentComponentIdentifier) - - BroadcastStatsReceiver( - Seq(statsReceiver.scope(relativeScopes: _*), statsReceiver.scope(componentScopes: _*))) - } - - /** - * Returns a feature map containing all the [[com.twitter.product_mixer.core.feature.Feature]]s - * stored as failures using the exception provided with as the reason wrapped in a PipelineFailure. - * e.g, for features A & B that threw an ExampleException b, this will return: - * { A -> Throw(PipelineFailure(...)), B -> Throw(PipelineFailure(...)) } - */ - def featureMapWithFailuresForFeatures( - features: Set[Feature[_, _]], - error: Throwable, - context: Executor.Context - ): FeatureMap = { - val builder = FeatureMapBuilder() - features.foreach { feature => - val pipelineFailure = PipelineFailure( - FeatureHydrationFailed, - s"Feature hydration failed for ${feature.toString}", - Some(error), - Some(context.componentStack)) - builder.addFailure(feature, pipelineFailure) - } - builder.build() - } - - /** - * Validates and returns back the passed feature map if it passes validation. A feature map - * is considered valid if it contains only the passed `registeredFeatures` features in it, - * nothing else and nothing missing. - */ - @throws(classOf[PipelineFailure]) - def validateFeatureMap( - registeredFeatures: Set[Feature[_, _]], - featureMap: FeatureMap, - context: Executor.Context - ): FeatureMap = { - val hydratedFeatures = featureMap.getFeatures - if (hydratedFeatures == registeredFeatures) { - featureMap - } else { - val missingFeatures = registeredFeatures -- hydratedFeatures - val unregisteredFeatures = hydratedFeatures -- registeredFeatures - throw PipelineFailure( - MisconfiguredFeatureMapFailure, - s"Unregistered features $unregisteredFeatures and missing features $missingFeatures", - None, - Some(context.componentStack) - ) - } - } - - object NotAMisconfiguredFeatureMapFailure { - - /** - * Will return any exception that isn't a [[MisconfiguredFeatureMapFailure]] [[PipelineFailure]] - * Allows for easy [[Arrow.handle]]ing all exceptions that aren't [[MisconfiguredFeatureMapFailure]]s - */ - def unapply(e: Throwable): Option[Throwable] = e match { - case pipelineFailure: PipelineFailure - if pipelineFailure.category == MisconfiguredFeatureMapFailure => - None - case e => Some(e) - } - } - - /** - * contains the scopes for recording metrics for the component by itself and - * the relative scope of that component within it's parent component scope - * - * @see [[Executor.buildScopes]] - */ - case class Scopes(componentScopes: Seq[String], relativeScope: Seq[String]) - - /** - * Wrap the [[Throwable]] in a [[UncategorizedServerFailure]] [[PipelineFailure]] with the original - * [[Throwable]] as the cause, even if it's already a [[PipelineFailure]]. - * - * This ensures that any access to the stored feature will result in a meaningful [[UncategorizedServerFailure]] - * [[com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailureCategory]] in stats which is more useful - * for customers components which access a failed [[Feature]] than the original [[com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailureCategory]]. - */ - def uncategorizedServerFailure( - componentStack: ComponentIdentifierStack, - throwable: Throwable - ): PipelineFailure = { - PipelineFailure( - UncategorizedServerFailure, - reason = "Unclassified Failure in Pipeline", - Some(throwable), - Some(componentStack) - ) - } - - /** - * [[PartialFunction]] that converts any [[Throwable]] into a - * [[PipelineFailure]] based on the provided `failureClassifier` - */ - def toPipelineFailureWithComponentIdentifierStack( - context: Context, - currentComponentIdentifier: ComponentIdentifier - ): PipelineFailureClassifier = { - // if given a `currentComponentIdentifier` then ensure we correctly handle `BasedOnParentComponent` identifier types - val contextWithCurrentComponentIdentifier = - context.pushToComponentStack(currentComponentIdentifier) - PipelineFailureClassifier( - contextWithCurrentComponentIdentifier.pipelineFailureClassifier - .orElse[Throwable, PipelineFailure] { - case CancelledExceptionExtractor(throwable) => throw throwable - case pipelineFailure: PipelineFailure => pipelineFailure - case throwable => - uncategorizedServerFailure( - contextWithCurrentComponentIdentifier.componentStack, - throwable) - }.andThen { pipelineFailure => - pipelineFailure.componentStack match { - case _: Some[_] => pipelineFailure - case None => - pipelineFailure.copy(componentStack = - Some(contextWithCurrentComponentIdentifier.componentStack)) - } - } - ) - } - - /** - * information used by an [[Executor]] that provides context around execution - */ - case class Context( - pipelineFailureClassifier: PipelineFailureClassifier, - componentStack: ComponentIdentifierStack) { - - def pushToComponentStack(newComponentIdentifier: ComponentIdentifier): Context = - copy(componentStack = componentStack.push(newComponentIdentifier)) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/ExecutorObserver.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/ExecutorObserver.docx new file mode 100644 index 000000000..e7a770a1d Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/ExecutorObserver.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/ExecutorObserver.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/ExecutorObserver.scala deleted file mode 100644 index 16df331fb..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/ExecutorObserver.scala +++ /dev/null @@ -1,146 +0,0 @@ -package com.twitter.product_mixer.core.service - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineResult -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.service.Executor.Context -import com.twitter.product_mixer.shared_library.observer.Observer -import com.twitter.product_mixer.shared_library.observer.Observer.Observer -import com.twitter.product_mixer.shared_library.observer.ResultsStatsObserver.ResultsStatsObserver -import com.twitter.util.Duration -import com.twitter.util.Return -import com.twitter.util.Throw -import com.twitter.util.Try - -private[core] object ExecutorObserver { - - /** Make a [[ExecutorObserver]] with stats for the [[ComponentIdentifier]] and relative to the parent in the [[Context.componentStack]] */ - def executorObserver[T]( - context: Context, - currentComponentIdentifier: ComponentIdentifier, - statsReceiver: StatsReceiver - ): ExecutorObserver[T] = new ExecutorObserver[T]( - Executor.broadcastStatsReceiver(context, currentComponentIdentifier, statsReceiver)) - - /** Make a [[ExecutorObserverWithSize]] with stats for the [[ComponentIdentifier]] and relative to the parent in the [[Context.componentStack]] */ - def executorObserverWithSize( - context: Context, - currentComponentIdentifier: ComponentIdentifier, - statsReceiver: StatsReceiver - ): ExecutorObserverWithSize = new ExecutorObserverWithSize( - Executor.broadcastStatsReceiver(context, currentComponentIdentifier, statsReceiver)) - - /** Make a [[PipelineExecutorObserver]] with stats for the [[ComponentIdentifier]] and relative to the parent in the [[Context.componentStack]] */ - def pipelineExecutorObserver[T <: PipelineResult[_]]( - context: Context, - currentComponentIdentifier: ComponentIdentifier, - statsReceiver: StatsReceiver - ): PipelineExecutorObserver[T] = new PipelineExecutorObserver[T]( - Executor.broadcastStatsReceiver(context, currentComponentIdentifier, statsReceiver)) - - /** - * Make a [[PipelineExecutorObserver]] specifically for a [[com.twitter.product_mixer.core.pipeline.product.ProductPipeline]] - * with no relative stats - */ - def productPipelineExecutorObserver[T <: PipelineResult[_]]( - currentComponentIdentifier: ProductPipelineIdentifier, - statsReceiver: StatsReceiver - ): PipelineExecutorObserver[T] = - new PipelineExecutorObserver[T](statsReceiver.scope(currentComponentIdentifier.toScopes: _*)) - - /** - * Make a [[PipelineExecutorObserver]] with only stats relative to the parent pipeline - * for [[com.twitter.product_mixer.core.pipeline.PipelineBuilder.Step]]s - */ - def stepExecutorObserver( - context: Context, - currentComponentIdentifier: PipelineStepIdentifier, - statsReceiver: StatsReceiver - ): ExecutorObserver[Unit] = { - new ExecutorObserver[Unit]( - statsReceiver.scope( - Executor.buildScopes(context, currentComponentIdentifier).relativeScope: _*)) - } -} - -/** - * An [[Observer]] which is called as a side effect. Unlike the other observers which wrap a computation, - * this [[Observer]] expects the caller to provide the latency value and wire it in - */ -private[core] sealed class ExecutorObserver[T]( - override val statsReceiver: StatsReceiver) - extends { - - /** - * always empty because we expect an already scoped [[com.twitter.finagle.stats.BroadcastStatsReceiver]] to be passed in - * @note uses early definitions [[https://docs.scala-lang.org/tutorials/FAQ/initialization-order.html]] to avoid null values for `scopes` in [[Observer]] - */ - override val scopes: Seq[String] = Seq.empty -} with Observer[T] { - - /** - * Serialize the provided [[Throwable]], prefixing [[PipelineFailure]]s with their - * [[com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailureCategory.categoryName]] and - * [[com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailureCategory.failureName]] - */ - override def serializeThrowable(throwable: Throwable): Seq[String] = { - throwable match { - case PipelineFailure(category, _, None, _) => - Seq(category.categoryName, category.failureName) - case PipelineFailure(category, _, Some(underlying), _) => - Seq(category.categoryName, category.failureName) ++ serializeThrowable(underlying) - case throwable: Throwable => super.serializeThrowable(throwable) - } - } - - /** record success, failure, and latency stats based on `t` and `latency` */ - def apply(t: Try[T], latency: Duration): Unit = observe(t, latency) -} - -/** - * Same as [[ExecutorObserver]] but records a size stat for [[PipelineResult]]s and - * records a failure counter for the cause of the failure under `failures/$pipelineFailureCategory/$componentType/$componentName`. - * - * @example if `GateIdentifier("MyGate")` is at the top of the [[PipelineFailure.componentStack]] then - * the resulting metric `failures/ClientFailure/Gate/MyGate` will be incremented. - */ -private[core] final class PipelineExecutorObserver[T <: PipelineResult[_]]( - override val statsReceiver: StatsReceiver) - extends ExecutorObserver[T](statsReceiver) - with ResultsStatsObserver[T] { - override val size: T => Int = _.resultSize() - - override def apply(t: Try[T], latency: Duration): Unit = { - super.apply(t, latency) - t match { - case Return(result) => observeResults(result) - case Throw(PipelineFailure(category, _, _, Some(componentIdentifierStack))) => - statsReceiver - .counter( - Seq( - Observer.Failures, - category.categoryName, - category.failureName) ++ componentIdentifierStack.peek.toScopes: _*).incr() - case _ => - } - } -} - -/** Same as [[ExecutorObserver]] but records a size stat */ -private[core] final class ExecutorObserverWithSize( - override val statsReceiver: StatsReceiver) - extends ExecutorObserver[Int](statsReceiver) - with ResultsStatsObserver[Int] { - override val size: Int => Int = identity - - override def apply(t: Try[Int], latency: Duration): Unit = { - super.apply(t, latency) - t match { - case Return(result) => observeResults(result) - case _ => - } - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/ExecutorResult.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/ExecutorResult.docx new file mode 100644 index 000000000..6b9f8b5ea Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/ExecutorResult.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/ExecutorResult.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/ExecutorResult.scala deleted file mode 100644 index 3a37870bd..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/ExecutorResult.scala +++ /dev/null @@ -1,3 +0,0 @@ -package com.twitter.product_mixer.core.service - -trait ExecutorResult diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/async_feature_map_executor/AsyncFeatureMapExecutor.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/async_feature_map_executor/AsyncFeatureMapExecutor.docx new file mode 100644 index 000000000..0f40cb5ae Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/async_feature_map_executor/AsyncFeatureMapExecutor.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/async_feature_map_executor/AsyncFeatureMapExecutor.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/async_feature_map_executor/AsyncFeatureMapExecutor.scala deleted file mode 100644 index 799112be2..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/async_feature_map_executor/AsyncFeatureMapExecutor.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.product_mixer.core.service.async_feature_map_executor - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.asyncfeaturemap.AsyncFeatureMap -import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.Executor._ -import com.twitter.product_mixer.core.service.ExecutorResult -import com.twitter.stitch.Arrow -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class AsyncFeatureMapExecutor @Inject() ( - override val statsReceiver: StatsReceiver) - extends Executor { - - /** - * Forces an [[AsyncFeatureMap]] to hydrate and resolve into a [[FeatureMap]] - * containing all [[com.twitter.product_mixer.core.feature.Feature]]s that are - * supposed to be hydrated before `stepToHydrateBefore`. - */ - def arrow( - stepToHydrateFor: PipelineStepIdentifier, - currentStep: PipelineStepIdentifier, - context: Context - ): Arrow[AsyncFeatureMap, AsyncFeatureMapExecutorResults] = { - Arrow - .map[AsyncFeatureMap, Option[Stitch[FeatureMap]]](_.hydrate(stepToHydrateFor)) - .andThen( - Arrow.choose( - Arrow.Choice.ifDefinedAt( - { case Some(stitchOfFeatureMap) => stitchOfFeatureMap }, - // only stat if there's something to hydrate - wrapComponentWithExecutorBookkeeping(context, currentStep)( - Arrow - .flatMap[Stitch[FeatureMap], FeatureMap](identity) - .map(featureMap => - AsyncFeatureMapExecutorResults(Map(stepToHydrateFor -> featureMap))) - ) - ), - Arrow.Choice.otherwise(Arrow.value(AsyncFeatureMapExecutorResults(Map.empty))) - ) - ) - } -} - -case class AsyncFeatureMapExecutorResults( - featureMapsByStep: Map[PipelineStepIdentifier, FeatureMap]) - extends ExecutorResult { - def ++( - asyncFeatureMapExecutorResults: AsyncFeatureMapExecutorResults - ): AsyncFeatureMapExecutorResults = - AsyncFeatureMapExecutorResults( - featureMapsByStep ++ asyncFeatureMapExecutorResults.featureMapsByStep) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/async_feature_map_executor/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/async_feature_map_executor/BUILD deleted file mode 100644 index 7b70b50e6..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/async_feature_map_executor/BUILD +++ /dev/null @@ -1,19 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/async_feature_map_executor/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/async_feature_map_executor/BUILD.docx new file mode 100644 index 000000000..9c5822b55 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/async_feature_map_executor/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_decorator_executor/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_decorator_executor/BUILD deleted file mode 100644 index f1b84bbc7..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_decorator_executor/BUILD +++ /dev/null @@ -1,22 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_decorator_executor/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_decorator_executor/BUILD.docx new file mode 100644 index 000000000..c39ad1a13 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_decorator_executor/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_decorator_executor/CandidateDecoratorExecutor.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_decorator_executor/CandidateDecoratorExecutor.docx new file mode 100644 index 000000000..13cd19c28 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_decorator_executor/CandidateDecoratorExecutor.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_decorator_executor/CandidateDecoratorExecutor.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_decorator_executor/CandidateDecoratorExecutor.scala deleted file mode 100644 index 141d9bce1..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_decorator_executor/CandidateDecoratorExecutor.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.product_mixer.core.service.candidate_decorator_executor - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.decorator.Decoration -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.service.Executor -import com.twitter.stitch.Arrow - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class CandidateDecoratorExecutor @Inject() (override val statsReceiver: StatsReceiver) - extends Executor { - def arrow[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]( - decoratorOpt: Option[CandidateDecorator[Query, Candidate]], - context: Executor.Context - ): Arrow[(Query, Seq[CandidateWithFeatures[Candidate]]), CandidateDecoratorExecutorResult] = { - val decoratorArrow = - decoratorOpt match { - case Some(decorator) => - val candidateDecoratorArrow = - Arrow.flatMap[(Query, Seq[CandidateWithFeatures[Candidate]]), Seq[Decoration]] { - case (query, candidatesWithFeatures) => decorator.apply(query, candidatesWithFeatures) - } - - wrapComponentWithExecutorBookkeeping(context, decorator.identifier)( - candidateDecoratorArrow) - - case _ => Arrow.value(Seq.empty[Decoration]) - } - - decoratorArrow.map(CandidateDecoratorExecutorResult) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_decorator_executor/CandidateDecoratorExecutorResult.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_decorator_executor/CandidateDecoratorExecutorResult.docx new file mode 100644 index 000000000..d852bb55b Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_decorator_executor/CandidateDecoratorExecutorResult.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_decorator_executor/CandidateDecoratorExecutorResult.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_decorator_executor/CandidateDecoratorExecutorResult.scala deleted file mode 100644 index 3f882c963..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_decorator_executor/CandidateDecoratorExecutorResult.scala +++ /dev/null @@ -1,6 +0,0 @@ -package com.twitter.product_mixer.core.service.candidate_decorator_executor - -import com.twitter.product_mixer.core.functional_component.decorator.Decoration -import com.twitter.product_mixer.core.service.ExecutorResult - -case class CandidateDecoratorExecutorResult(result: Seq[Decoration]) extends ExecutorResult diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_hydrator_executor/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_hydrator_executor/BUILD deleted file mode 100644 index 8395e2754..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_hydrator_executor/BUILD +++ /dev/null @@ -1,25 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator/featurestorev1", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/feature_hydrator_observer", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator/featurestorev1", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_hydrator_executor/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_hydrator_executor/BUILD.docx new file mode 100644 index 000000000..80f8af20e Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_hydrator_executor/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_hydrator_executor/CandidateFeatureHydratorExecutor.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_hydrator_executor/CandidateFeatureHydratorExecutor.docx new file mode 100644 index 000000000..72bc3d8f5 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_hydrator_executor/CandidateFeatureHydratorExecutor.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_hydrator_executor/CandidateFeatureHydratorExecutor.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_hydrator_executor/CandidateFeatureHydratorExecutor.scala deleted file mode 100644 index f25e7bedf..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_hydrator_executor/CandidateFeatureHydratorExecutor.scala +++ /dev/null @@ -1,277 +0,0 @@ -package com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseBulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.HydratorCandidateResult -import com.twitter.product_mixer.core.functional_component.feature_hydrator.featurestorev1.FeatureStoreV1CandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.Conditionally -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.pipeline_failure.MisconfiguredFeatureMapFailure -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.Executor._ -import com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor.CandidateFeatureHydratorExecutor.Inputs -import com.twitter.product_mixer.core.service.feature_hydrator_observer.FeatureHydratorObserver -import com.twitter.stitch.Arrow -import com.twitter.util.Try -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class CandidateFeatureHydratorExecutor @Inject() (override val statsReceiver: StatsReceiver) - extends Executor { - def arrow[Query <: PipelineQuery, Result <: UniversalNoun[Any]]( - hydrators: Seq[BaseCandidateFeatureHydrator[Query, Result, _]], - context: Executor.Context - ): Arrow[ - Inputs[Query, Result], - CandidateFeatureHydratorExecutorResult[ - Result - ] - ] = { - - val observer = new FeatureHydratorObserver(statsReceiver, hydrators, context) - - val candidateFeatureHydratorExecutorResults: Seq[Arrow[ - Inputs[Query, Result], - CandidateFeatureHydratorExecutorResult[Result] - ]] = hydrators.map(getCandidateHydratorArrow(_, context, observer)) - - val runHydrators = Arrow.collect(candidateFeatureHydratorExecutorResults).map { - candidateFeatureHydratorExecutorResult: Seq[CandidateFeatureHydratorExecutorResult[Result]] => - candidateFeatureHydratorExecutorResult.foldLeft( - CandidateFeatureHydratorExecutorResult[Result]( - Seq.empty, - Map.empty - ) - ) { (accumulator, additionalResult) => - // accumulator.results and additionalResults.results are either the same length or one may be empty - // checks in each Hydrator's Arrow implementation ensure the ordering and length are correct - val mergedFeatureMaps = - if (accumulator.results.length == additionalResult.results.length) { - // merge if there are results for both and they are the same size - // also handles both being empty - accumulator.results.zip(additionalResult.results).map { - case (accumulatedScoredCandidate, resultScoredCandidate) => - val updatedFeatureMap = - accumulatedScoredCandidate.features ++ resultScoredCandidate.features - HydratorCandidateResult(resultScoredCandidate.candidate, updatedFeatureMap) - } - } else if (accumulator.results.isEmpty) { - // accumulator is empty (the initial case) so use additional results - additionalResult.results - } else { - // empty results but non-empty accumulator due to Hydrator being turned off so use accumulator results - accumulator.results - } - - CandidateFeatureHydratorExecutorResult( - mergedFeatureMaps, - accumulator.individualFeatureHydratorResults ++ additionalResult.individualFeatureHydratorResults - ) - } - } - - Arrow.ifelse[Inputs[Query, Result], CandidateFeatureHydratorExecutorResult[Result]]( - _.candidates.nonEmpty, - runHydrators, - Arrow.value(CandidateFeatureHydratorExecutorResult(Seq.empty, Map.empty))) - } - - /** @note the returned [[Arrow]] must have a result for every candidate passed into it in the same order OR a completely empty result */ - private def getCandidateHydratorArrow[Query <: PipelineQuery, Result <: UniversalNoun[Any]]( - hydrator: BaseCandidateFeatureHydrator[Query, Result, _], - context: Executor.Context, - candidateFeatureHydratorObserver: FeatureHydratorObserver - ): Arrow[ - Inputs[Query, Result], - CandidateFeatureHydratorExecutorResult[Result] - ] = { - val componentExecutorContext = context.pushToComponentStack(hydrator.identifier) - - val validateFeatureMapFn: FeatureMap => FeatureMap = - hydrator match { - // Feature store candidate hydrators store the resulting PredictionRecords and - // not the features, so we cannot validate the same way - case _: FeatureStoreV1CandidateFeatureHydrator[Query, Result] => - identity - case _ => - validateFeatureMap( - hydrator.features.asInstanceOf[Set[Feature[_, _]]], - _, - componentExecutorContext) - } - - val hydratorBaseArrow = hydrator match { - case hydrator: CandidateFeatureHydrator[Query, Result] => - singleCandidateHydratorArrow( - hydrator, - validateFeatureMapFn, - componentExecutorContext, - parentContext = context) - - case hydrator: BaseBulkCandidateFeatureHydrator[Query, Result, _] => - bulkCandidateHydratorArrow( - hydrator, - validateFeatureMapFn, - componentExecutorContext, - parentContext = context) - } - - val candidateFeatureHydratorArrow = - Arrow - .zipWithArg(hydratorBaseArrow) - .map { - case ( - arg: CandidateFeatureHydratorExecutor.Inputs[Query, Result], - featureMapSeq: Seq[FeatureMap]) => - val candidates = arg.candidates.map(_.candidate) - - candidateFeatureHydratorObserver.observeFeatureSuccessAndFailures( - hydrator, - featureMapSeq) - - // Build a map from candidate to FeatureMap - val candidateAndFeatureMaps = if (candidates.size == featureMapSeq.size) { - candidates.zip(featureMapSeq).map { - case (candidate, featureMap) => HydratorCandidateResult(candidate, featureMap) - } - } else { - throw PipelineFailure( - MisconfiguredFeatureMapFailure, - s"Unexpected response length from ${hydrator.identifier}, ensure hydrator returns feature map for all candidates") - } - val individualFeatureHydratorFeatureMaps = - Map(hydrator.identifier -> IndividualFeatureHydratorResult(candidateAndFeatureMaps)) - CandidateFeatureHydratorExecutorResult( - candidateAndFeatureMaps, - individualFeatureHydratorFeatureMaps) - } - - val conditionallyRunArrow = hydrator match { - case hydrator: BaseCandidateFeatureHydrator[Query, Result, _] with Conditionally[ - Query @unchecked - ] => - Arrow.ifelse[Inputs[Query, Result], CandidateFeatureHydratorExecutorResult[Result]]( - { case Inputs(query: Query @unchecked, _) => hydrator.onlyIf(query) }, - candidateFeatureHydratorArrow, - Arrow.value( - CandidateFeatureHydratorExecutorResult( - Seq.empty, - Map(hydrator.identifier -> FeatureHydratorDisabled[Result]()) - )) - ) - case _ => candidateFeatureHydratorArrow - } - - wrapWithErrorHandling(context, hydrator.identifier)(conditionallyRunArrow) - } - - private def singleCandidateHydratorArrow[Query <: PipelineQuery, Result <: UniversalNoun[Any]]( - hydrator: CandidateFeatureHydrator[Query, Result], - validateFeatureMap: FeatureMap => FeatureMap, - componentContext: Context, - parentContext: Context - ): Arrow[Inputs[Query, Result], Seq[FeatureMap]] = { - val inputTransformer = Arrow - .map { inputs: Inputs[Query, Result] => - inputs.candidates.map { candidate => - (inputs.query, candidate.candidate, candidate.features) - } - } - - val hydratorArrow = Arrow - .flatMap[(Query, Result, FeatureMap), FeatureMap] { - case (query, candidate, featureMap) => - hydrator.apply(query, candidate, featureMap) - } - - // validate before observing so validation failures are caught in the metrics - val hydratorArrowWithValidation = hydratorArrow.map(validateFeatureMap) - - // no tracing here since per-Component spans is overkill - val observedArrow = - wrapPerCandidateComponentWithExecutorBookkeepingWithoutTracing( - parentContext, - hydrator.identifier - )(hydratorArrowWithValidation) - - // only handle non-validation failures - val liftNonValidationFailuresToFailedFeatures = Arrow.handle[FeatureMap, FeatureMap] { - case NotAMisconfiguredFeatureMapFailure(e) => - featureMapWithFailuresForFeatures(hydrator.features, e, componentContext) - } - - wrapComponentsWithTracingOnly(parentContext, hydrator.identifier)( - inputTransformer.andThen( - Arrow.sequence(observedArrow.andThen(liftNonValidationFailuresToFailedFeatures)) - ) - ) - } - - private def bulkCandidateHydratorArrow[Query <: PipelineQuery, Result <: UniversalNoun[Any]]( - hydrator: BaseBulkCandidateFeatureHydrator[Query, Result, _], - validateFeatureMap: FeatureMap => FeatureMap, - componentContext: Context, - parentContext: Context - ): Arrow[Inputs[Query, Result], Seq[FeatureMap]] = { - val hydratorArrow: Arrow[Inputs[Query, Result], Seq[FeatureMap]] = - Arrow.flatMap { inputs => - hydrator.apply(inputs.query, inputs.candidates) - } - - val validationArrow: Arrow[(Inputs[Query, Result], Seq[FeatureMap]), Seq[FeatureMap]] = Arrow - .map[(Inputs[Query, Result], Seq[FeatureMap]), Seq[FeatureMap]] { - case (inputs, results) => - // For bulk APIs, this ensures no candidates are omitted and also ensures the order is preserved. - if (inputs.candidates.length != results.length) { - throw PipelineFailure( - MisconfiguredFeatureMapFailure, - s"Unexpected response from ${hydrator.identifier}, ensure hydrator returns features for all candidates. Missing results for ${inputs.candidates.length - results.length} candidates" - ) - } - - results.map(validateFeatureMap) - } - - // validate before observing so validation failures are caught in the metrics - val hydratorArrowWithValidation: Arrow[Inputs[Query, Result], Seq[FeatureMap]] = - Arrow.zipWithArg(hydratorArrow).andThen(validationArrow) - - val observedArrow = - wrapComponentWithExecutorBookkeeping(parentContext, hydrator.identifier)( - hydratorArrowWithValidation) - - // only handle non-validation failures - val liftNonValidationFailuresToFailedFeatures = - Arrow.map[(Inputs[Query, Result], Try[Seq[FeatureMap]]), Try[Seq[FeatureMap]]] { - case (inputs, resultTry) => - resultTry.handle { - case NotAMisconfiguredFeatureMapFailure(e) => - val errorFeatureMap = - featureMapWithFailuresForFeatures( - hydrator.features.asInstanceOf[Set[Feature[_, _]]], - e, - componentContext) - inputs.candidates.map(_ => errorFeatureMap) - } - } - - Arrow - .zipWithArg(observedArrow.liftToTry) - .andThen(liftNonValidationFailuresToFailedFeatures) - .lowerFromTry - } -} - -object CandidateFeatureHydratorExecutor { - case class Inputs[+Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]( - query: Query, - candidates: Seq[CandidateWithFeatures[Candidate]]) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_hydrator_executor/CandidateFeatureHydratorExecutorResult.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_hydrator_executor/CandidateFeatureHydratorExecutorResult.docx new file mode 100644 index 000000000..66166748e Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_hydrator_executor/CandidateFeatureHydratorExecutorResult.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_hydrator_executor/CandidateFeatureHydratorExecutorResult.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_hydrator_executor/CandidateFeatureHydratorExecutorResult.scala deleted file mode 100644 index 9a865a7b0..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_hydrator_executor/CandidateFeatureHydratorExecutorResult.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.product_mixer.core.service.candidate_feature_hydrator_executor - -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.service.ExecutorResult - -case class CandidateFeatureHydratorExecutorResult[+Result <: UniversalNoun[Any]]( - results: Seq[CandidateWithFeatures[Result]], - individualFeatureHydratorResults: Map[ - _ <: ComponentIdentifier, - BaseIndividualFeatureHydratorResult[Result] - ]) extends ExecutorResult - -sealed trait BaseIndividualFeatureHydratorResult[+Result <: UniversalNoun[Any]] -case class FeatureHydratorDisabled[+Result <: UniversalNoun[Any]]() - extends BaseIndividualFeatureHydratorResult[Result] -case class IndividualFeatureHydratorResult[+Result <: UniversalNoun[Any]]( - result: Seq[CandidateWithFeatures[Result]]) - extends BaseIndividualFeatureHydratorResult[Result] diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_transformer_executor/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_transformer_executor/BUILD deleted file mode 100644 index 40702d280..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_transformer_executor/BUILD +++ /dev/null @@ -1,22 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_transformer_executor/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_transformer_executor/BUILD.docx new file mode 100644 index 000000000..75dd394d7 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_transformer_executor/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_transformer_executor/CandidateFeatureTransformerExecutor.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_transformer_executor/CandidateFeatureTransformerExecutor.docx new file mode 100644 index 000000000..864a06f51 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_transformer_executor/CandidateFeatureTransformerExecutor.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_transformer_executor/CandidateFeatureTransformerExecutor.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_transformer_executor/CandidateFeatureTransformerExecutor.scala deleted file mode 100644 index 4c4ee849b..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_transformer_executor/CandidateFeatureTransformerExecutor.scala +++ /dev/null @@ -1,93 +0,0 @@ -package com.twitter.product_mixer.core.service.candidate_feature_transformer_executor - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.Executor._ -import com.twitter.stitch.Arrow -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class CandidateFeatureTransformerExecutor @Inject() (override val statsReceiver: StatsReceiver) - extends Executor { - def arrow[Result]( - transformers: Seq[CandidateFeatureTransformer[Result]], - context: Executor.Context - ): Arrow[Seq[Result], CandidateFeatureTransformerExecutorResult] = { - if (transformers.isEmpty) { - // must always return a Seq of FeatureMaps, even if there are no Transformers - Arrow.map[Seq[Result], CandidateFeatureTransformerExecutorResult] { candidates => - CandidateFeatureTransformerExecutorResult(candidates.map(_ => FeatureMap.empty), Seq.empty) - } - } else { - val transformerArrows: Seq[Arrow[Seq[Result], Seq[(TransformerIdentifier, FeatureMap)]]] = - transformers.map { transformer => - val transformerContext = context.pushToComponentStack(transformer.identifier) - - val liftNonValidationFailuresToFailedFeatures = - Arrow.handle[FeatureMap, FeatureMap] { - case NotAMisconfiguredFeatureMapFailure(e) => - featureMapWithFailuresForFeatures(transformer.features, e, transformerContext) - } - - val underlyingArrow = Arrow - .map(transformer.transform) - .map(validateFeatureMap(transformer.features, _, transformerContext)) - - val observedArrowWithoutTracing = - wrapPerCandidateComponentWithExecutorBookkeepingWithoutTracing( - context, - transformer.identifier)(underlyingArrow) - - val seqArrow = - Arrow.sequence( - observedArrowWithoutTracing - .andThen(liftNonValidationFailuresToFailedFeatures) - .map(transformer.identifier -> _) - ) - - wrapComponentsWithTracingOnly(context, transformer.identifier)(seqArrow) - } - - Arrow.collect(transformerArrows).map { results => - /** - * Inner Seqs are a given Transformer applied to all the candidates - * - * We want to merge the FeatureMaps for each candidate - * from all the Transformers. We do this by merging all the FeatureMaps at - * each index `i` of each Seq in `results` by `transpose`-ing the `results` - * so the inner Seq becomes all the FeatureMaps for Candidate - * at index `i` in the input Seq. - * - * {{{ - * Seq( - * Seq(transformer1FeatureMapCandidate1, ..., transformer1FeatureMapCandidateN), - * ..., - * Seq(transformerMFeatureMapCandidate1, ..., transformerMFeatureMapCandidateN) - * ).transpose == Seq( - * Seq(transformer1FeatureMapCandidate1, ..., transformerMFeatureMapCandidate1), - * ..., - * Seq(transformer1FeatureMapCandidateN, ..., transformerMFeatureMapCandidateN) - * ) - * }}} - * - * we could avoid the transpose if we ran each candidate through all the transformers - * one-after-the-other, but then we couldn't have a single tracing span for all applications - * of a Transformer, so instead we apply each transformer to all candidates together, then - * move onto the next transformer. - * - * It's worth noting that the outer Seq is bounded by the number of Transformers that are - * applied which will typically be small. - */ - val transposed = results.transpose - val combinedMaps = transposed.map(featureMapsForSingleCandidate => - FeatureMap.merge(featureMapsForSingleCandidate.map { case (_, maps) => maps })) - - CandidateFeatureTransformerExecutorResult(combinedMaps, transposed.map(_.toMap)) - } - } - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_transformer_executor/CandidateFeatureTransformerExecutorResult.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_transformer_executor/CandidateFeatureTransformerExecutorResult.docx new file mode 100644 index 000000000..342b6828e Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_transformer_executor/CandidateFeatureTransformerExecutorResult.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_transformer_executor/CandidateFeatureTransformerExecutorResult.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_transformer_executor/CandidateFeatureTransformerExecutorResult.scala deleted file mode 100644 index 8947897a6..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_transformer_executor/CandidateFeatureTransformerExecutorResult.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.product_mixer.core.service.candidate_feature_transformer_executor - -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier - -case class CandidateFeatureTransformerExecutorResult( - featureMaps: Seq[FeatureMap], - individualFeatureMaps: Seq[Map[TransformerIdentifier, FeatureMap]]) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_pipeline_executor/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_pipeline_executor/BUILD deleted file mode 100644 index f89ba2858..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_pipeline_executor/BUILD +++ /dev/null @@ -1,27 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-slf4j/src/main/scala/com/twitter/inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "stitch/stitch-core", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-slf4j/src/main/scala/com/twitter/inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_pipeline_executor/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_pipeline_executor/BUILD.docx new file mode 100644 index 000000000..eee28b394 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_pipeline_executor/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_pipeline_executor/CandidatePipelineExecutor.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_pipeline_executor/CandidatePipelineExecutor.docx new file mode 100644 index 000000000..4c7674167 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_pipeline_executor/CandidatePipelineExecutor.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_pipeline_executor/CandidatePipelineExecutor.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_pipeline_executor/CandidatePipelineExecutor.scala deleted file mode 100644 index 6031d6e34..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_pipeline_executor/CandidatePipelineExecutor.scala +++ /dev/null @@ -1,82 +0,0 @@ -package com.twitter.product_mixer.core.service.candidate_pipeline_executor - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.pipeline.CandidatePipelineResults -import com.twitter.product_mixer.core.pipeline.FailOpenPolicy -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipeline -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineResult -import com.twitter.product_mixer.core.quality_factor.QualityFactorObserver -import com.twitter.product_mixer.core.service.Executor -import com.twitter.stitch.Arrow -import com.twitter.util.logging.Logging - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class CandidatePipelineExecutor @Inject() (override val statsReceiver: StatsReceiver) - extends Executor - with Logging { - - def arrow[Query <: PipelineQuery]( - candidatePipelines: Seq[CandidatePipeline[Query]], - defaultFailOpenPolicy: FailOpenPolicy, - failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy], - qualityFactorObserverByPipeline: Map[ComponentIdentifier, QualityFactorObserver], - context: Executor.Context - ): Arrow[CandidatePipeline.Inputs[Query], CandidatePipelineExecutorResult] = { - - // Get the `.arrow` of each Candidate Pipeline, and wrap it in a ResultObserver - val observedArrows: Seq[Arrow[CandidatePipeline.Inputs[Query], CandidatePipelineResult]] = - candidatePipelines.map { pipeline => - wrapPipelineWithExecutorBookkeeping( - context = context, - currentComponentIdentifier = pipeline.identifier, - qualityFactorObserver = qualityFactorObserverByPipeline.get(pipeline.identifier), - failOpenPolicy = failOpenPolicies.getOrElse(pipeline.identifier, defaultFailOpenPolicy) - )(pipeline.arrow) - } - - // Collect the results from all the candidate pipelines together - Arrow.zipWithArg(Arrow.collect(observedArrows)).map { - case (input: CandidatePipeline.Inputs[Query], results: Seq[CandidatePipelineResult]) => - val candidateWithDetails = results.flatMap(_.result.getOrElse(Seq.empty)) - val previousCandidateWithDetails = input.query.features - .map(_.getOrElse(CandidatePipelineResults, Seq.empty)) - .getOrElse(Seq.empty) - - val featureMapWithCandidates = FeatureMapBuilder() - .add(CandidatePipelineResults, previousCandidateWithDetails ++ candidateWithDetails) - .build() - - // Merge the query feature hydrator and candidate source query features back in. While this - // is done internally in the pipeline, we have to pass it back since we don't expose the - // updated pipeline query today. - val queryFeatureHydratorFeatureMaps = - results - .flatMap(result => Seq(result.queryFeatures, result.queryFeaturesPhase2)) - .collect { case Some(result) => result.featureMap } - val asyncFeatureHydratorFeatureMaps = - results - .flatMap(_.asyncFeatureHydrationResults) - .flatMap(_.featureMapsByStep.values) - - val candidateSourceFeatureMaps = - results - .flatMap(_.candidateSourceResult) - .map(_.candidateSourceFeatureMap) - - val featureMaps = - (featureMapWithCandidates +: queryFeatureHydratorFeatureMaps) ++ asyncFeatureHydratorFeatureMaps ++ candidateSourceFeatureMaps - val mergedFeatureMap = FeatureMap.merge(featureMaps) - CandidatePipelineExecutorResult( - candidatePipelineResults = results, - queryFeatureMap = mergedFeatureMap) - } - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_pipeline_executor/CandidatePipelineExecutorResult.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_pipeline_executor/CandidatePipelineExecutorResult.docx new file mode 100644 index 000000000..89cdfac4b Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_pipeline_executor/CandidatePipelineExecutorResult.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_pipeline_executor/CandidatePipelineExecutorResult.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_pipeline_executor/CandidatePipelineExecutorResult.scala deleted file mode 100644 index 8f3805f2f..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_pipeline_executor/CandidatePipelineExecutorResult.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.product_mixer.core.service.candidate_pipeline_executor - -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineResult - -case class CandidatePipelineExecutorResult( - candidatePipelineResults: Seq[CandidatePipelineResult], - queryFeatureMap: FeatureMap) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_source_executor/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_source_executor/BUILD deleted file mode 100644 index c34688181..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_source_executor/BUILD +++ /dev/null @@ -1,35 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_transformer_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transformer_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "stitch/stitch-core", - "util/util-core:scala", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_transformer_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transformer_executor", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_source_executor/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_source_executor/BUILD.docx new file mode 100644 index 000000000..d8ae4923c Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_source_executor/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_source_executor/CandidateSourceExecutor.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_source_executor/CandidateSourceExecutor.docx new file mode 100644 index 000000000..ae323bfaa Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_source_executor/CandidateSourceExecutor.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_source_executor/CandidateSourceExecutor.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_source_executor/CandidateSourceExecutor.scala deleted file mode 100644 index 5d6df2678..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_source_executor/CandidateSourceExecutor.scala +++ /dev/null @@ -1,173 +0,0 @@ -package com.twitter.product_mixer.core.service.candidate_source_executor - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSourceWithExtractedFeatures -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidatesWithSourceFeatures -import com.twitter.product_mixer.core.functional_component.transformer.BaseCandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.presentation.CandidateSourcePosition -import com.twitter.product_mixer.core.model.common.presentation.CandidateSources -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.pipeline_failure.ExecutionFailed -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.pipeline_failure.UnexpectedCandidateResult -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.candidate_feature_transformer_executor.CandidateFeatureTransformerExecutor -import com.twitter.product_mixer.core.service.transformer_executor.PerCandidateTransformerExecutor -import com.twitter.product_mixer.core.service.transformer_executor.TransformerExecutor -import com.twitter.stitch.Arrow -import com.twitter.util.Return -import com.twitter.util.Throw -import com.twitter.util.Try -import com.twitter.util.logging.Logging -import javax.inject.Inject -import javax.inject.Singleton -import scala.collection.immutable.ListSet - -/** - * [[CandidateSourceExecutor]]: - * - Executes a [[BaseCandidateSource]], using a [[BaseCandidatePipelineQueryTransformer]] and a [[CandidatePipelineResultsTransformer]] - * - in parallel, uses a [[CandidateFeatureTransformer]] to optionally extract [[com.twitter.product_mixer.core.feature.Feature]]s from the result - * - Handles [[UnexpectedCandidateResult]] [[PipelineFailure]]s returned from [[CandidatePipelineResultsTransformer]] failures by removing those candidates from the result - */ -@Singleton -class CandidateSourceExecutor @Inject() ( - override val statsReceiver: StatsReceiver, - candidateFeatureTransformerExecutor: CandidateFeatureTransformerExecutor, - transformerExecutor: TransformerExecutor, - perCandidateTransformerExecutor: PerCandidateTransformerExecutor) - extends Executor - with Logging { - - def arrow[ - Query <: PipelineQuery, - CandidateSourceQuery, - CandidateSourceResult, - Candidate <: UniversalNoun[Any] - ]( - candidateSource: BaseCandidateSource[CandidateSourceQuery, CandidateSourceResult], - queryTransformer: BaseCandidatePipelineQueryTransformer[ - Query, - CandidateSourceQuery - ], - resultTransformer: CandidatePipelineResultsTransformer[CandidateSourceResult, Candidate], - resultFeaturesTransformers: Seq[CandidateFeatureTransformer[CandidateSourceResult]], - context: Executor.Context - ): Arrow[Query, CandidateSourceExecutorResult[Candidate]] = { - - val candidateSourceArrow: Arrow[CandidateSourceQuery, CandidatesWithSourceFeatures[ - CandidateSourceResult - ]] = - candidateSource match { - case regularCandidateSource: CandidateSource[CandidateSourceQuery, CandidateSourceResult] => - Arrow.flatMap(regularCandidateSource.apply).map { candidates => - CandidatesWithSourceFeatures(candidates, FeatureMap.empty) - } - case candidateSourceWithExtractedFeatures: CandidateSourceWithExtractedFeatures[ - CandidateSourceQuery, - CandidateSourceResult - ] => - Arrow.flatMap(candidateSourceWithExtractedFeatures.apply) - } - - val resultsTransformerArrow: Arrow[Seq[CandidateSourceResult], Seq[Try[Candidate]]] = - perCandidateTransformerExecutor.arrow(resultTransformer, context) - - val featureMapTransformersArrow: Arrow[ - Seq[CandidateSourceResult], - Seq[FeatureMap] - ] = - candidateFeatureTransformerExecutor - .arrow(resultFeaturesTransformers, context).map(_.featureMaps) - - val candidatesResultArrow: Arrow[CandidatesWithSourceFeatures[CandidateSourceResult], Seq[ - (Candidate, FeatureMap) - ]] = Arrow - .map[CandidatesWithSourceFeatures[CandidateSourceResult], Seq[CandidateSourceResult]]( - _.candidates) - .andThen(Arrow - .joinMap(resultsTransformerArrow, featureMapTransformersArrow) { - case (transformed, features) => - if (transformed.length != features.length) - throw PipelineFailure( - ExecutionFailed, - s"Found ${transformed.length} candidates and ${features.length} FeatureMaps, expected their lengths to be equal") - transformed.iterator - .zip(features.iterator) - .collect { case ErrorHandling(result) => result } - .toSeq - }) - - // Build the final CandidateSourceExecutorResult - val executorResultArrow: Arrow[ - (FeatureMap, Seq[(Candidate, FeatureMap)]), - CandidateSourceExecutorResult[ - Candidate - ] - ] = Arrow.map { - case (queryFeatures: FeatureMap, results: Seq[(Candidate, FeatureMap)]) => - val candidatesWithFeatures: Seq[FetchedCandidateWithFeatures[Candidate]] = - results.zipWithIndex.map { - case ((candidate, featureMap), index) => - FetchedCandidateWithFeatures( - candidate, - featureMap + (CandidateSourcePosition, index) + (CandidateSources, ListSet( - candidateSource.identifier)) - ) - } - CandidateSourceExecutorResult( - candidates = candidatesWithFeatures, - candidateSourceFeatureMap = queryFeatures - ) - } - - val queryTransformerArrow = - transformerExecutor.arrow[Query, CandidateSourceQuery](queryTransformer, context) - - val combinedArrow = - queryTransformerArrow - .andThen(candidateSourceArrow) - .andThen( - Arrow - .join( - Arrow.map[CandidatesWithSourceFeatures[CandidateSourceResult], FeatureMap]( - _.features), - candidatesResultArrow - )) - .andThen(executorResultArrow) - - wrapComponentWithExecutorBookkeepingWithSize[Query, CandidateSourceExecutorResult[Candidate]]( - context, - candidateSource.identifier, - result => result.candidates.size - )(combinedArrow) - } - - object ErrorHandling { - - /** Silently drop [[UnexpectedCandidateResult]] */ - def unapply[Candidate]( - candidateTryAndFeatureMap: (Try[Candidate], FeatureMap) - ): Option[(Candidate, FeatureMap)] = { - val (candidateTry, featureMap) = candidateTryAndFeatureMap - val candidateOpt = candidateTry match { - case Throw(PipelineFailure(UnexpectedCandidateResult, _, _, _)) => None - case Throw(ex) => throw ex - case Return(r) => Some(r) - } - - candidateOpt.map { candidate => (candidate, featureMap) } - } - } -} - -case class FetchedCandidateWithFeatures[Candidate <: UniversalNoun[Any]]( - candidate: Candidate, - features: FeatureMap) - extends CandidateWithFeatures[Candidate] diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_source_executor/CandidateSourceExecutorResult.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_source_executor/CandidateSourceExecutorResult.docx new file mode 100644 index 000000000..b3755f045 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_source_executor/CandidateSourceExecutorResult.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_source_executor/CandidateSourceExecutorResult.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_source_executor/CandidateSourceExecutorResult.scala deleted file mode 100644 index 451bae1ae..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_source_executor/CandidateSourceExecutorResult.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.product_mixer.core.service.candidate_source_executor - -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.service.ExecutorResult - -case class CandidateSourceExecutorResult[Candidate <: UniversalNoun[Any]]( - candidates: Seq[FetchedCandidateWithFeatures[Candidate]], - candidateSourceFeatureMap: FeatureMap) - extends ExecutorResult diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/component_registry/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/component_registry/BUILD deleted file mode 100644 index 34d6b7fda..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/component_registry/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "util/util-slf4j-api", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/component_registry/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/component_registry/BUILD.docx new file mode 100644 index 000000000..3d90d9b05 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/component_registry/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/component_registry/ComponentRegistry.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/component_registry/ComponentRegistry.docx new file mode 100644 index 000000000..1708b12ba Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/component_registry/ComponentRegistry.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/component_registry/ComponentRegistry.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/component_registry/ComponentRegistry.scala deleted file mode 100644 index 14b70aa84..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/component_registry/ComponentRegistry.scala +++ /dev/null @@ -1,182 +0,0 @@ -package com.twitter.product_mixer.core.service.component_registry - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.model.common.Component -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifierStack -import com.twitter.util.Activity -import com.twitter.util.Future -import com.twitter.util.Try -import com.twitter.util.logging.Logging -import java.util.concurrent.ConcurrentHashMap -import javax.inject.Inject -import javax.inject.Singleton -import scala.collection.JavaConverters._ - -/** - * The [[ComponentRegistry]] works closely with [[ComponentIdentifier]]s and the [[com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry]] - * to provide the Product Mixer framework information about the [[com.twitter.product_mixer.core.pipeline.Pipeline]]s and [[Component]]s - * that make up an application. - * - * This registration allows us to configure alerts and dashboards, - * to query your application structure letting us display the graph of the execution and the results of queries, - * and to garner insight into usages. - * - * The registry is a snapshot of the state of the world when pipelines were last built successfully. - * For most services, this only happens once on startup. However, some services may rebuild their - * pipelines dynamically later on. - */ - -@Singleton -class ComponentRegistry @Inject() (statsReceiver: StatsReceiver) { - // Initially pending until the first snapshot is built by [[ProductPipelineRegistry]] - private val (snapshotActivity, snapshotWitness) = Activity[ComponentRegistrySnapshot]() - private val snapshotCount = statsReceiver.counter("ComponentRegistry", "SnapshotCount") - - def get: Future[ComponentRegistrySnapshot] = snapshotActivity.values.toFuture.lowerFromTry - private[core] def set(snapshot: ComponentRegistrySnapshot): Unit = { - snapshotCount.incr() - snapshotWitness.notify(Try(snapshot)) - } -} - -class ComponentRegistrySnapshot() extends Logging { - - /** for storing the [[RegisteredComponent]]s */ - private[this] val componentRegistry = - new ConcurrentHashMap[ComponentIdentifier, RegisteredComponent] - - /** for determining the children of a [[ComponentIdentifier]] */ - private[this] val componentChildren = - new ConcurrentHashMap[ComponentIdentifier, Set[ComponentIdentifier]] - - /** for determining [[ComponentIdentifier]] uniqueness within a given [[ComponentIdentifierStack]] */ - private[this] val componentHierarchy = - new ConcurrentHashMap[ComponentIdentifierStack, Set[ComponentIdentifier]] - - /** - * Register the given [[Component]] at the end of path provided by `parentIdentifierStack` - * or throws an exception if adding the component results in an invalid configuration. - * - * @throws ChildComponentCollisionException if a [[Component]] with the same [[ComponentIdentifier]] is registered - * more than once under the same parent. - * e.g. if you register `ComponentA` under `ProductA -> PipelineA` twice, - * this exception will be thrown when registering `ComponentA` the second - * time. This is pretty much always a configuration error due to copy-pasting - * and forgetting to update the identifier, or accidentally using the same - * component twice under the same parent. If this didn't throw, stats from - * these 2 components would be indistinguishable. - * - * @throws ComponentIdentifierCollisionException if a [[Component]] with the same [[ComponentIdentifier]] is registered - * but it's type is not the same as a previously registered [[Component]] - * with the same [[ComponentIdentifier]] - * e.g. if you register 2 [[Component]]s with the same [[ComponentIdentifier]] - * such as `new Component` and an instance of - * `class MyComponent extends Component` the `new Component` will have a - * type of `Component` and the other one will have a type of `MyComponent` - * which will throw. This is usually due to copy-pasting a component as - * a starting point and forgetting to update the identifier. If this - * didn't throw, absolute stats from these 2 components would be - * indistinguishable. - * - * - * @note this will log details of component identifier reuse if the underling components are not equal, but otherwise are of the same class. - * Their stats will be merged and indistinguishable but since they are the same name and same class, we assume the differences are - * minor enough that this is okay, but make a note in the log at startup in case someone sees unexpected metrics, we can look - * back at the logs and see the details. - * - * @param component the component to register - * @param parentIdentifierStack the complete [[ComponentIdentifierStack]] excluding the current [[Component]]'s [[ComponentIdentifier]] - */ - def register( - component: Component, - parentIdentifierStack: ComponentIdentifierStack - ): Unit = synchronized { - val identifier = component.identifier - val parentIdentifier = parentIdentifierStack.peek - - val registeredComponent = - RegisteredComponent(identifier, component, component.identifier.file.value) - - componentRegistry.asScala - .get(identifier) - .filter(_.component != component) // only do the foreach if the components aren't equal - .foreach { - case existingComponent if existingComponent.component.getClass != component.getClass => - /** - * The same component may be registered under different parent components. - * However, different component types cannot use the same component identifier. - * - * This catches some copy-pasting of a config or component and forgetting to update the identifier. - */ - throw new ComponentIdentifierCollisionException( - componentIdentifier = identifier, - component = registeredComponent, - existingComponent = componentRegistry.get(identifier), - parentIdentifierStack = parentIdentifierStack, - existingIdentifierStack = componentHierarchy.search[ComponentIdentifierStack]( - 1, - (stack, identifiers) => if (identifiers.contains(identifier)) stack else null) - ) - case existingComponent => - /** - * The same component may be registered under different parent components. - * However, if the components are not equal it __may be__ a configuration error - * so we log a detailed description of the issue in case they need to debug. - * - * This warns customers of some copy-pasting of a config or component and forgetting to update the - * identifier and of reusing components with hard-coded values which are configured differently. - */ - val existingIdentifierStack = componentHierarchy.search[ComponentIdentifierStack]( - 1, - (stack, identifiers) => if (identifiers.contains(identifier)) stack else null) - logger.info( - s"Found duplicate identifiers for non-equal components, $identifier from ${registeredComponent.sourceFile} " + - s"under ${parentIdentifierStack.componentIdentifiers.reverse.mkString(" -> ")} " + - s"was already defined and is unequal to ${existingComponent.sourceFile} " + - s"under ${existingIdentifierStack.componentIdentifiers.reverse.mkString(" -> ")}. " + - s"Merging these components in the registry, this will result in their metrics being merged. " + - s"If these components should have separate metrics, consider providing unique identifiers for them instead." - ) - } - - /** The same component may not be registered multiple times under the same parent */ - if (componentHierarchy.getOrDefault(parentIdentifierStack, Set.empty).contains(identifier)) - throw new ChildComponentCollisionException(identifier, parentIdentifierStack) - - // add component to registry - componentRegistry.putIfAbsent(identifier, registeredComponent) - // add component to parent's `children` set for easy lookup - componentChildren.merge(parentIdentifier, Set(identifier), _ ++ _) - // add the component to the hierarchy under it's parent's identifier stack - componentHierarchy.merge(parentIdentifierStack, Set(identifier), _ ++ _) - } - - def getAllRegisteredComponents: Seq[RegisteredComponent] = - componentRegistry.values.asScala.toSeq.sorted - - def getChildComponents(component: ComponentIdentifier): Seq[ComponentIdentifier] = - Option(componentChildren.get(component)) match { - case Some(components) => components.toSeq.sorted(ComponentIdentifier.ordering) - case None => Seq.empty - } -} - -class ComponentIdentifierCollisionException( - componentIdentifier: ComponentIdentifier, - component: RegisteredComponent, - existingComponent: RegisteredComponent, - parentIdentifierStack: ComponentIdentifierStack, - existingIdentifierStack: ComponentIdentifierStack) - extends IllegalArgumentException( - s"Tried to register component $componentIdentifier: of type ${component.component.getClass} from ${component.sourceFile} " + - s"under ${parentIdentifierStack.componentIdentifiers.reverse.mkString(" -> ")} " + - s"but it was already defined with a different type ${existingComponent.component.getClass} from ${existingComponent.sourceFile} " + - s"under ${existingIdentifierStack.componentIdentifiers.reverse.mkString(" -> ")}. " + - s"Ensure you aren't reusing a component identifier which can happen when copy-pasting existing component code by accident") - -class ChildComponentCollisionException( - componentIdentifier: ComponentIdentifier, - parentIdentifierStack: ComponentIdentifierStack) - extends IllegalArgumentException( - s"Component $componentIdentifier already defined under parent component $parentIdentifierStack") diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/component_registry/RegisteredComponent.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/component_registry/RegisteredComponent.docx new file mode 100644 index 000000000..2c47e309c Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/component_registry/RegisteredComponent.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/component_registry/RegisteredComponent.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/component_registry/RegisteredComponent.scala deleted file mode 100644 index 6ddadc033..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/component_registry/RegisteredComponent.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.product_mixer.core.service.component_registry - -import com.twitter.product_mixer.core.model.common.Component -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier - -object RegisteredComponent { - // Sort by ComponentIdentifier which has its own implicit ordering defined - implicit val ordering: Ordering[RegisteredComponent] = - Ordering.by[RegisteredComponent, ComponentIdentifier](_.component.identifier) -} - -case class RegisteredComponent( - identifier: ComponentIdentifier, - component: Component, - sourceFile: String) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/AuthorizationService.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/AuthorizationService.docx new file mode 100644 index 000000000..cd507fcf4 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/AuthorizationService.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/AuthorizationService.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/AuthorizationService.scala deleted file mode 100644 index 4c0aa8b88..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/AuthorizationService.scala +++ /dev/null @@ -1,82 +0,0 @@ -package com.twitter.product_mixer.core.service.debug_query - -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.inject.annotations.Flag -import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy -import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicyEvaluator -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifierStack -import com.twitter.product_mixer.core.module.product_mixer_flags.ProductMixerFlagModule.ServiceLocal -import com.twitter.product_mixer.core.pipeline.pipeline_failure.Authentication -import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.turntable.{thriftscala => t} -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Basic class that provides a verification method for checking if a call to our debugging - * features is allowed/authorized to make said call. - * @param isServiceLocal Whether the service is being run locally. - */ -@Singleton -class AuthorizationService @Inject() (@Flag(ServiceLocal) isServiceLocal: Boolean) { - import AuthorizationService._ - - /** - * Check whether a call to a given product is authorized. Throws an [[UnauthorizedServiceCallException]] - * if not. - * @param requestingServiceIdentifier The Service Identifier of the calling service - * @param productAccessPolicies The access policies of the product being called. - * @param requestContext The request context of the caller. - */ - def verifyRequestAuthorization( - componentIdentifierStack: ComponentIdentifierStack, - requestingServiceIdentifier: ServiceIdentifier, - productAccessPolicies: Set[AccessPolicy], - requestContext: t.TurntableRequestContext - ): Unit = { - val isServiceCallAuthorized = - requestingServiceIdentifier.role == AllowedServiceIdentifierRole && requestingServiceIdentifier.service == AllowedServiceIdentifierName - val userLdapGroups = requestContext.ldapGroups.map(_.toSet) - - val accessPolicyAllowed = AccessPolicyEvaluator.evaluate( - productAccessPolicies = productAccessPolicies, - userLdapGroups = userLdapGroups.getOrElse(Set.empty) - ) - - if (!isServiceLocal && !isServiceCallAuthorized) { - throw new UnauthorizedServiceCallException( - requestingServiceIdentifier, - componentIdentifierStack) - } - - if (!isServiceLocal && !accessPolicyAllowed) { - throw new InsufficientAccessException( - userLdapGroups, - productAccessPolicies, - componentIdentifierStack) - } - } -} - -object AuthorizationService { - final val AllowedServiceIdentifierRole = "turntable" - final val AllowedServiceIdentifierName = "turntable" -} - -class UnauthorizedServiceCallException( - serviceIdentifier: ServiceIdentifier, - componentIdentifierStack: ComponentIdentifierStack) - extends PipelineFailure( - BadRequest, - s"Unexpected Service tried to call Turntable Debug endpoint: ${ServiceIdentifier.asString(serviceIdentifier)}", - componentStack = Some(componentIdentifierStack)) - -class InsufficientAccessException( - ldapGroups: Option[Set[String]], - desiredAccessPolicies: Set[AccessPolicy], - componentIdentifierStack: ComponentIdentifierStack) - extends PipelineFailure( - Authentication, - s"Request did not satisfy access policies: $desiredAccessPolicies with ldapGroups = $ldapGroups", - componentStack = Some(componentIdentifierStack)) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/BUILD deleted file mode 100644 index 251996247..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/BUILD +++ /dev/null @@ -1,20 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "configapi/configapi-core", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry", - "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", - "product-mixer/turntable/service/src/main/scala/com/twitter/turntable/context:key", - "stitch/stitch-core", - "util/util-core:scala", - "util/util-jackson/src/main/scala/com/twitter/util/jackson", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/BUILD.docx new file mode 100644 index 000000000..03722994f Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/DebugQueryNotSupportedService.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/DebugQueryNotSupportedService.docx new file mode 100644 index 000000000..a48528088 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/DebugQueryNotSupportedService.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/DebugQueryNotSupportedService.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/DebugQueryNotSupportedService.scala deleted file mode 100644 index 22870d3d5..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/DebugQueryNotSupportedService.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.twitter.product_mixer.core.service.debug_query - -import com.twitter.finagle.Service -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.pipeline_failure.ProductDisabled -import com.twitter.product_mixer.core.pipeline.product.ProductPipelineResult -import com.twitter.scrooge.{Request => ScroogeRequest} -import com.twitter.scrooge.{Response => ScroogeResponse} -import com.twitter.util.Future -import com.twitter.product_mixer.core.{thriftscala => t} -import com.twitter.util.jackson.ScalaObjectMapper - -/** - * All Mixers must implement a debug query interface. This can be a problem for in-place migrations - * where a service may only partially implement Product Mixer patterns. This service can be used as - * a noop implementation of [[DebugQueryService]] until the Mixer service is fully migrated. - */ -object DebugQueryNotSupportedService - extends Service[ScroogeRequest[_], ScroogeResponse[t.PipelineExecutionResult]] { - - val failureJson: String = { - val message = "This service does not support debug queries, this is usually due to an active " + - "in-place migration to Product Mixer. Please reach out in #product-mixer if you have any questions." - - ScalaObjectMapper().writeValueAsString( - ProductPipelineResult( - transformedQuery = None, - qualityFactorResult = None, - gateResult = None, - pipelineSelectorResult = None, - mixerPipelineResult = None, - recommendationPipelineResult = None, - traceId = None, - failure = Some(PipelineFailure(ProductDisabled, message)), - result = None, - )) - } - - override def apply( - thriftRequest: ScroogeRequest[_] - ): Future[ScroogeResponse[t.PipelineExecutionResult]] = - Future.value(ScroogeResponse(t.PipelineExecutionResult(failureJson))) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/DebugQueryService.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/DebugQueryService.docx new file mode 100644 index 000000000..c898808a7 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/DebugQueryService.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/DebugQueryService.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/DebugQueryService.scala deleted file mode 100644 index 9a8f5f413..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/DebugQueryService.scala +++ /dev/null @@ -1,109 +0,0 @@ -package com.twitter.product_mixer.core.service.debug_query - -import com.fasterxml.jackson.databind.SerializationFeature -import com.twitter.finagle.Service -import com.twitter.finagle.context.Contexts -import com.twitter.finagle.tracing.Trace.traceLocal -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.transport.Transport -import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifierStack -import com.twitter.product_mixer.core.model.marshalling.request.Product -import com.twitter.product_mixer.core.model.marshalling.request.Request -import com.twitter.product_mixer.core.pipeline.product.ProductPipeline -import com.twitter.product_mixer.core.pipeline.product.ProductPipelineRequest -import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry -import com.twitter.product_mixer.core.{thriftscala => t} -import com.twitter.scrooge.ThriftStruct -import com.twitter.scrooge.{Request => ScroogeRequest} -import com.twitter.scrooge.{Response => ScroogeResponse} -import com.twitter.stitch.Stitch -import com.twitter.turntable.context.TurntableRequestContextKey -import com.twitter.util.jackson.ScalaObjectMapper -import javax.inject.Inject -import javax.inject.Singleton -import scala.reflect.runtime.universe.TypeTag - -/** - * Returns the complete execution log for a pipeline query. These endpoints are intended for - * debugging (primarily through Turntable). - */ -@Singleton -class DebugQueryService @Inject() ( - productPipelineRegistry: ProductPipelineRegistry, - paramsBuilder: ParamsBuilder, - authorizationService: AuthorizationService) { - - private val mapper = - ScalaObjectMapper.builder - .withAdditionalJacksonModules(Seq(ParamsSerializerModule)) - .withSerializationConfig( - Map( - // These are copied from the default serialization config. - SerializationFeature.WRITE_DATES_AS_TIMESTAMPS -> false, - SerializationFeature.WRITE_ENUMS_USING_TO_STRING -> true, - // Generally we want to be defensive when serializing since we don't control everything that's - // serialized. This issue also came up when trying to serialize Unit as part of sync side effects. - SerializationFeature.FAIL_ON_EMPTY_BEANS -> false, - )) - // The default implementation represents numbers as JSON Numbers (i.e. Double with 53 bit precision - // which leads to Snowflake IDs being cropped in the case of tweets. - .withNumbersAsStrings(true) - .objectMapper - - def apply[ - ThriftRequest <: ThriftStruct with Product1[MixerServiceRequest], - MixerServiceRequest <: ThriftStruct, - MixerRequest <: Request - ]( - unmarshaller: MixerServiceRequest => MixerRequest - )( - implicit requestTypeTag: TypeTag[MixerRequest] - ): Service[ScroogeRequest[ThriftRequest], ScroogeResponse[t.PipelineExecutionResult]] = { - (thriftRequest: ScroogeRequest[ThriftRequest]) => - { - - val request = unmarshaller(thriftRequest.args._1) - val params = paramsBuilder.build( - clientContext = request.clientContext, - product = request.product, - featureOverrides = request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty) - ) - - val productPipeline = productPipelineRegistry - .getProductPipeline[MixerRequest, Any](request.product) - verifyRequestAuthorization(request.product, productPipeline) - Contexts.broadcast.letClear(TurntableRequestContextKey) { - Stitch - .run(productPipeline - .arrow(ProductPipelineRequest(request, params)).map { detailedResult => - // Serialization can be slow so a trace is useful both for optimization by the Promix - // team and to give visibility to customers. - val serializedJSON = - traceLocal("serialize_debug_response")(mapper.writeValueAsString(detailedResult)) - t.PipelineExecutionResult(serializedJSON) - }) - .map(ScroogeResponse(_)) - } - } - } - - private def verifyRequestAuthorization( - product: Product, - productPipeline: ProductPipeline[_, _] - ): Unit = { - val serviceIdentifier = ServiceIdentifier.fromCertificate(Transport.peerCertificate) - val requestContext = Contexts.broadcast - .get(TurntableRequestContextKey).getOrElse(throw MissingTurntableRequestContextException) - - val componentStack = ComponentIdentifierStack(productPipeline.identifier, product.identifier) - authorizationService.verifyRequestAuthorization( - componentStack, - serviceIdentifier, - productPipeline.debugAccessPolicies, - requestContext) - } -} - -object MissingTurntableRequestContextException - extends Exception("Request is missing turntable request context") diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/ParamsSerializerModule.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/ParamsSerializerModule.docx new file mode 100644 index 000000000..64e3f01a5 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/ParamsSerializerModule.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/ParamsSerializerModule.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/ParamsSerializerModule.scala deleted file mode 100644 index e7265378f..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query/ParamsSerializerModule.scala +++ /dev/null @@ -1,35 +0,0 @@ -package com.twitter.product_mixer.core.service.debug_query - -import com.fasterxml.jackson.core.JsonGenerator -import com.fasterxml.jackson.databind.SerializerProvider -import com.fasterxml.jackson.databind.ser.std.StdSerializer -import com.twitter.timelines.configapi.Params -import com.fasterxml.jackson.databind.module.SimpleModule -import com.twitter.timelines.configapi.Config - -object ParamsSerializerModule extends SimpleModule { - addSerializer(ParamsConfigSerializer) - addSerializer(ParamsStdSerializer) -} - -object ParamsStdSerializer extends StdSerializer[Params](classOf[Params]) { - override def serialize( - value: Params, - gen: JsonGenerator, - provider: SerializerProvider - ): Unit = { - gen.writeStartObject() - gen.writeObjectField("applied_params", value.allAppliedValues) - gen.writeEndObject() - } -} - -object ParamsConfigSerializer extends StdSerializer[Config](classOf[Config]) { - override def serialize( - value: Config, - gen: JsonGenerator, - provider: SerializerProvider - ): Unit = { - gen.writeString(value.simpleName) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/domain_marshaller_executor/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/domain_marshaller_executor/BUILD deleted file mode 100644 index a56fdf2f4..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/domain_marshaller_executor/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/premarshaller", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "stitch/stitch-core", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/premarshaller", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/domain_marshaller_executor/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/domain_marshaller_executor/BUILD.docx new file mode 100644 index 000000000..5932bf4ba Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/domain_marshaller_executor/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/domain_marshaller_executor/DomainMarshallerExecutor.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/domain_marshaller_executor/DomainMarshallerExecutor.docx new file mode 100644 index 000000000..3d87eb7b6 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/domain_marshaller_executor/DomainMarshallerExecutor.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/domain_marshaller_executor/DomainMarshallerExecutor.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/domain_marshaller_executor/DomainMarshallerExecutor.scala deleted file mode 100644 index f0dc0e263..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/domain_marshaller_executor/DomainMarshallerExecutor.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.twitter.product_mixer.core.service.domain_marshaller_executor - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.ExecutorResult -import com.twitter.product_mixer.core.service.domain_marshaller_executor.DomainMarshallerExecutor.Inputs -import com.twitter.product_mixer.core.service.domain_marshaller_executor.DomainMarshallerExecutor.Result -import com.twitter.stitch.Arrow -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Executes a [[DomainMarshaller]]. - * - * @note This is a synchronous transform, so we don't observe it directly. Failures and such - * can be observed at the parent pipeline. - */ -@Singleton -class DomainMarshallerExecutor @Inject() (override val statsReceiver: StatsReceiver) - extends Executor { - def arrow[Query <: PipelineQuery, DomainResponseType <: HasMarshalling]( - marshaller: DomainMarshaller[Query, DomainResponseType], - context: Executor.Context - ): Arrow[Inputs[Query], Result[DomainResponseType]] = { - val arrow = Arrow - .map[Inputs[Query], DomainMarshallerExecutor.Result[DomainResponseType]] { - case Inputs(query, candidates) => - DomainMarshallerExecutor.Result(marshaller(query, candidates)) - } - - wrapComponentWithExecutorBookkeeping(context, marshaller.identifier)(arrow) - } -} - -object DomainMarshallerExecutor { - case class Inputs[Query <: PipelineQuery]( - query: Query, - candidatesWithDetails: Seq[CandidateWithDetails]) - case class Result[+DomainResponseType](result: DomainResponseType) extends ExecutorResult -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/feature_hydrator_observer/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/feature_hydrator_observer/BUILD deleted file mode 100644 index f3991968f..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/feature_hydrator_observer/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator/featurestorev1", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator/featurestorev1", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/feature_hydrator_observer/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/feature_hydrator_observer/BUILD.docx new file mode 100644 index 000000000..bda2258a1 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/feature_hydrator_observer/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/feature_hydrator_observer/FeatureHydratorObserver.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/feature_hydrator_observer/FeatureHydratorObserver.docx new file mode 100644 index 000000000..15e6b11ed Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/feature_hydrator_observer/FeatureHydratorObserver.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/feature_hydrator_observer/FeatureHydratorObserver.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/feature_hydrator_observer/FeatureHydratorObserver.scala deleted file mode 100644 index b37134907..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/feature_hydrator_observer/FeatureHydratorObserver.scala +++ /dev/null @@ -1,136 +0,0 @@ -package com.twitter.product_mixer.core.service.feature_hydrator_observer - -import com.twitter.finagle.stats.BroadcastStatsReceiver -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.RollupStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.ml.featurestore.lib.data.HydrationError -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featurestorev1.featurevalue.FeatureStoreV1ResponseFeature -import com.twitter.product_mixer.core.functional_component.feature_hydrator.FeatureHydrator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.featurestorev1.FeatureStoreV1CandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.featurestorev1.FeatureStoreV1QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.shared_library.observer.Observer -import com.twitter.servo.util.CancelledExceptionExtractor -import com.twitter.util.Throw -import com.twitter.util.Throwables - -class FeatureHydratorObserver( - statsReceiver: StatsReceiver, - hydrators: Seq[FeatureHydrator[_]], - context: Executor.Context) { - - private val hydratorAndFeatureToStats: Map[ - ComponentIdentifier, - Map[Feature[_, _], FeatureCounters] - ] = - hydrators.map { hydrator => - val hydratorScope = Executor.buildScopes(context, hydrator.identifier) - val featureToCounterMap: Map[Feature[_, _], FeatureCounters] = hydrator.features - .asInstanceOf[Set[Feature[_, _]]].map { feature => - val scopedStats = scopedBroadcastStats(hydratorScope, feature) - // Initialize so we have them registered - val requestsCounter = scopedStats.counter(Observer.Requests) - val successCounter = scopedStats.counter(Observer.Success) - // These are dynamic so we can't really cache them - scopedStats.counter(Observer.Failures) - scopedStats.counter(Observer.Cancelled) - feature -> FeatureCounters(requestsCounter, successCounter, scopedStats) - }.toMap - hydrator.identifier -> featureToCounterMap - }.toMap - - def observeFeatureSuccessAndFailures( - hydrator: FeatureHydrator[_], - featureMaps: Seq[FeatureMap] - ): Unit = { - - val features = hydrator.features.asInstanceOf[Set[Feature[_, _]]] - - val failedFeaturesWithErrorNames: Map[Feature[_, _], Seq[Seq[String]]] = hydrator match { - case _: FeatureStoreV1QueryFeatureHydrator[_] | - _: FeatureStoreV1CandidateFeatureHydrator[_, _] => - featureMaps.toIterator - .flatMap(_.getTry(FeatureStoreV1ResponseFeature).toOption.map(_.failedFeatures)).flatMap { - failureMap: Map[_ <: Feature[_, _], Set[HydrationError]] => - failureMap.flatMap { - case (feature, errors: Set[HydrationError]) => - errors.headOption.map { error => - feature -> Seq(Observer.Failures, error.errorType) - } - }.toIterator - }.toSeq.groupBy { case (feature, _) => feature }.mapValues { seqOfTuples => - seqOfTuples.map { case (_, error) => error } - } - - case _: FeatureHydrator[_] => - features.toIterator - .flatMap { feature => - featureMaps - .flatMap(_.underlyingMap - .get(feature).collect { - case Throw(CancelledExceptionExtractor(throwable)) => - (feature, Observer.Cancelled +: Throwables.mkString(throwable)) - case Throw(throwable) => - (feature, Observer.Failures +: Throwables.mkString(throwable)) - }) - }.toSeq.groupBy { case (feature, _) => feature }.mapValues { seqOfTuples => - seqOfTuples.map { case (_, error) => error } - } - } - - val failedFeaturesWithErrorCountsMap: Map[Feature[_, _], Map[Seq[String], Int]] = - failedFeaturesWithErrorNames.mapValues(_.groupBy { statKey => statKey }.mapValues(_.size)) - - val featuresToCounterMap = hydratorAndFeatureToStats.getOrElse( - hydrator.identifier, - throw new MissingHydratorException(hydrator.identifier)) - features.foreach { feature => - val hydratorFeatureCounters: FeatureCounters = featuresToCounterMap.getOrElse( - feature, - throw new MissingFeatureException(hydrator.identifier, feature)) - val failedMapsCount = failedFeaturesWithErrorNames.getOrElse(feature, Seq.empty).size - val failedFeatureErrorCounts = failedFeaturesWithErrorCountsMap.getOrElse(feature, Map.empty) - - hydratorFeatureCounters.requestsCounter.incr(featureMaps.size) - hydratorFeatureCounters.successCounter.incr(featureMaps.size - failedMapsCount) - failedFeatureErrorCounts.foreach { - case (failure, count) => - hydratorFeatureCounters.scopedStats.counter(failure: _*).incr(count) - } - } - } - - private def scopedBroadcastStats( - hydratorScope: Executor.Scopes, - feature: Feature[_, _], - ): StatsReceiver = { - val suffix = Seq("Feature", feature.toString) - val localScope = hydratorScope.componentScopes ++ suffix - val relativeScope = hydratorScope.relativeScope ++ suffix - new RollupStatsReceiver( - BroadcastStatsReceiver( - Seq( - statsReceiver.scope(localScope: _*), - statsReceiver.scope(relativeScope: _*), - ) - )) - } -} - -case class FeatureCounters( - requestsCounter: Counter, - successCounter: Counter, - scopedStats: StatsReceiver) - -class MissingHydratorException(featureHydratorIdentifier: ComponentIdentifier) - extends Exception(s"Missing Feature Hydrator in Stats Map: ${featureHydratorIdentifier.name}") - -class MissingFeatureException( - featureHydratorIdentifier: ComponentIdentifier, - feature: Feature[_, _]) - extends Exception( - s"Missing Feature in Stats Map: ${feature.toString} for ${featureHydratorIdentifier.name}") diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/filter_executor/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/filter_executor/BUILD deleted file mode 100644 index 313fde03c..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/filter_executor/BUILD +++ /dev/null @@ -1,24 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "stitch/stitch-core", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/filter_executor/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/filter_executor/BUILD.docx new file mode 100644 index 000000000..c1c5c54b5 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/filter_executor/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/filter_executor/FilterExecutor.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/filter_executor/FilterExecutor.docx new file mode 100644 index 000000000..0523a75f3 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/filter_executor/FilterExecutor.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/filter_executor/FilterExecutor.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/filter_executor/FilterExecutor.scala deleted file mode 100644 index 2d463e90b..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/filter_executor/FilterExecutor.scala +++ /dev/null @@ -1,172 +0,0 @@ -package com.twitter.product_mixer.core.service.filter_executor - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.Conditionally -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.filter_executor.FilterExecutor.FilterState -import com.twitter.stitch.Arrow -import com.twitter.stitch.Arrow.Iso -import javax.inject.Inject -import javax.inject.Singleton -import scala.collection.immutable.Queue - -/** - * Applies a `Seq[Filter]` in sequential order. - * Returns the results and a detailed Seq of each filter's results (for debugging / coherence). - * - * Note that each successive filter is only passed the 'kept' Seq from the previous filter, not the full - * set of candidates. - */ -@Singleton -class FilterExecutor @Inject() (override val statsReceiver: StatsReceiver) extends Executor { - - private val Kept = "kept" - private val Removed = "removed" - - def arrow[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]( - filters: Seq[Filter[Query, Candidate]], - context: Executor.Context - ): Arrow[(Query, Seq[CandidateWithFeatures[Candidate]]), FilterExecutorResult[Candidate]] = { - - val filterArrows = filters.map(getIsoArrowForFilter(_, context)) - val combinedArrow = isoArrowsSequentially(filterArrows) - - Arrow - .map[(Query, Seq[CandidateWithFeatures[Candidate]]), FilterState[Query, Candidate]] { - case (query, filterCandidates) => - // transform the input to the initial state of a `FilterExecutorResult` - val initialFilterExecutorResult = - FilterExecutorResult(filterCandidates.map(_.candidate), Queue.empty) - val allCandidates: Map[Candidate, CandidateWithFeatures[Candidate]] = - filterCandidates.map { fc => - (fc.candidate, fc) - }.toMap - - FilterState(query, allCandidates, initialFilterExecutorResult) - } - .flatMapArrow(combinedArrow) - .map { - case FilterState(_, _, filterExecutorResult) => - filterExecutorResult.copy(individualFilterResults = - // materialize the Queue into a List for faster future iterations - filterExecutorResult.individualFilterResults.toList) - } - - } - - /** - * Adds filter specific stats, generic [[wrapComponentWithExecutorBookkeeping]] stats, and wraps with failure handling - * - * If the filter is a [[Conditionally]] ensures that we dont record stats if its turned off - * - * @note For performance, the [[FilterExecutorResult.individualFilterResults]] is build backwards - the head being the most recent result. - * @param filter the filter to make an [[Arrow]] out of - * @param context the [[Executor.Context]] for the pipeline this is a part of - */ - private def getIsoArrowForFilter[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]( - filter: Filter[Query, Candidate], - context: Executor.Context - ): Iso[FilterState[Query, Candidate]] = { - val broadcastStatsReceiver = - Executor.broadcastStatsReceiver(context, filter.identifier, statsReceiver) - - val keptCounter = broadcastStatsReceiver.counter(Kept) - val removedCounter = broadcastStatsReceiver.counter(Removed) - - val filterArrow = Arrow.flatMap[ - (Query, Seq[CandidateWithFeatures[Candidate]]), - FilterExecutorIndividualResult[Candidate] - ] { - case (query, candidates) => - filter.apply(query, candidates).map { result => - FilterExecutorIndividualResult( - identifier = filter.identifier, - kept = result.kept, - removed = result.removed) - } - } - - val observedArrow: Arrow[ - (Query, Seq[CandidateWithFeatures[Candidate]]), - FilterExecutorIndividualResult[ - Candidate - ] - ] = wrapComponentWithExecutorBookkeeping( - context = context, - currentComponentIdentifier = filter.identifier, - onSuccess = { result: FilterExecutorIndividualResult[Candidate] => - keptCounter.incr(result.kept.size) - removedCounter.incr(result.removed.size) - } - )( - filterArrow - ) - - val conditionallyRunArrow: Arrow[ - (Query, Seq[CandidateWithFeatures[Candidate]]), - IndividualFilterResults[ - Candidate - ] - ] = - filter match { - case filter: Filter[Query, Candidate] with Conditionally[ - Filter.Input[Query, Candidate] @unchecked - ] => - Arrow.ifelse( - { - case (query, candidates) => - filter.onlyIf(Filter.Input(query, candidates)) - }, - observedArrow, - Arrow.value(ConditionalFilterDisabled(filter.identifier)) - ) - case _ => observedArrow - } - - // return an `Iso` arrow for easier composition later - Arrow - .zipWithArg( - Arrow - .map[FilterState[Query, Candidate], (Query, Seq[CandidateWithFeatures[Candidate]])] { - case FilterState(query, candidateToFeaturesMap, FilterExecutorResult(result, _)) => - (query, result.flatMap(candidateToFeaturesMap.get)) - }.andThen(conditionallyRunArrow)) - .map { - case ( - FilterState(query, allCandidates, filterExecutorResult), - filterResult: FilterExecutorIndividualResult[Candidate]) => - val resultWithCurrentPrepended = - filterExecutorResult.individualFilterResults :+ filterResult - val newFilterExecutorResult = FilterExecutorResult( - result = filterResult.kept, - individualFilterResults = resultWithCurrentPrepended) - FilterState(query, allCandidates, newFilterExecutorResult) - - case (filterState, filterDisabledResult: ConditionalFilterDisabled) => - filterState.copy( - executorResult = filterState.executorResult.copy( - individualFilterResults = - filterState.executorResult.individualFilterResults :+ filterDisabledResult - )) - } - } -} - -object FilterExecutor { - - /** - * FilterState is an internal representation of the state that is passed between each individual filter arrow. - * - * @param query: The query - * @param candidateToFeaturesMap: A lookup mapping from Candidate -> FilterCandidate, to rebuild the inputs quickly for the next filter - * @param executorResult: The in-progress executor result - */ - private case class FilterState[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]( - query: Query, - candidateToFeaturesMap: Map[Candidate, CandidateWithFeatures[Candidate]], - executorResult: FilterExecutorResult[Candidate]) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/filter_executor/FilterExecutorResult.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/filter_executor/FilterExecutorResult.docx new file mode 100644 index 000000000..2de4e61ba Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/filter_executor/FilterExecutorResult.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/filter_executor/FilterExecutorResult.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/filter_executor/FilterExecutorResult.scala deleted file mode 100644 index bc0336620..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/filter_executor/FilterExecutorResult.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.product_mixer.core.service.filter_executor - -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.service.ExecutorResult - -case class FilterExecutorResult[Candidate]( - result: Seq[Candidate], - individualFilterResults: Seq[IndividualFilterResults[Candidate]]) - extends ExecutorResult - -sealed trait IndividualFilterResults[+Candidate] -case class ConditionalFilterDisabled(identifier: FilterIdentifier) - extends IndividualFilterResults[Nothing] -case class FilterExecutorIndividualResult[+Candidate]( - identifier: FilterIdentifier, - kept: Seq[Candidate], - removed: Seq[Candidate]) - extends IndividualFilterResults[Candidate] diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/BUILD deleted file mode 100644 index 8b4ea4292..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/BUILD +++ /dev/null @@ -1,25 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "stitch/stitch-core", - "util/util-core:scala", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/BUILD.docx new file mode 100644 index 000000000..69b789b1f Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/ExecutedGateResult.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/ExecutedGateResult.docx new file mode 100644 index 000000000..ec2dc25e7 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/ExecutedGateResult.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/ExecutedGateResult.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/ExecutedGateResult.scala deleted file mode 100644 index 32d4e4410..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/ExecutedGateResult.scala +++ /dev/null @@ -1,6 +0,0 @@ -package com.twitter.product_mixer.core.service.gate_executor - -import com.twitter.product_mixer.core.functional_component.gate.GateResult -import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier - -case class ExecutedGateResult(identifier: GateIdentifier, result: GateResult) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/GateExecutor.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/GateExecutor.docx new file mode 100644 index 000000000..20fb59547 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/GateExecutor.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/GateExecutor.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/GateExecutor.scala deleted file mode 100644 index c1de292e8..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/GateExecutor.scala +++ /dev/null @@ -1,107 +0,0 @@ -package com.twitter.product_mixer.core.service.gate_executor - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.functional_component.gate.BaseGate -import com.twitter.product_mixer.core.functional_component.gate.GateResult -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.service.Executor -import com.twitter.stitch.Arrow -import com.twitter.stitch.Arrow.Iso -import com.twitter.util.Return -import com.twitter.util.Throw - -import javax.inject.Inject -import javax.inject.Singleton -import scala.collection.immutable.Queue - -/** - * A GateExecutor takes a Seq[Gate], executes them all sequentially, and - * determines a final Continue or Stop decision. - */ -@Singleton -class GateExecutor @Inject() (override val statsReceiver: StatsReceiver) extends Executor { - - private val Continue = "continue" - private val Skipped = "skipped" - private val Stop = "stop" - - def arrow[Query <: PipelineQuery]( - gates: Seq[BaseGate[Query]], - context: Executor.Context - ): Arrow[Query, GateExecutorResult] = { - - val gateArrows = gates.map(getIsoArrowForGate(_, context)) - val combinedArrow = isoArrowsSequentially(gateArrows) - - Arrow - .map { query: Query => (query, GateExecutorResult(Queue.empty)) } - .andThen(combinedArrow) - .map { - case (_, gateExecutorResult) => - // materialize the Queue into a List for faster future iterations - GateExecutorResult(gateExecutorResult.individualGateResults.toList) - } - } - - /** - * Each gate is transformed into a Iso Arrow over (Quest, List[GatewayResult]). - * - * This arrow: - * - Adapts the input and output types of the underlying Gate arrow (an [[Iso[(Query, QueryResult)]]) - * - throws a [[StoppedGateException]] if [[GateResult.continue]] is false - * - if its not false, prepends the current results to the [[GateExecutorResult.individualGateResults]] list - */ - private def getIsoArrowForGate[Query <: PipelineQuery]( - gate: BaseGate[Query], - context: Executor.Context - ): Iso[(Query, GateExecutorResult)] = { - val broadcastStatsReceiver = - Executor.broadcastStatsReceiver(context, gate.identifier, statsReceiver) - - val continueCounter = broadcastStatsReceiver.counter(Continue) - val skippedCounter = broadcastStatsReceiver.counter(Skipped) - val stopCounter = broadcastStatsReceiver.counter(Stop) - - val observedArrow = wrapComponentWithExecutorBookkeeping( - context, - gate.identifier, - onSuccess = { gateResult: GateResult => - gateResult match { - case GateResult.Continue => continueCounter.incr() - case GateResult.Skipped => skippedCounter.incr() - case GateResult.Stop => stopCounter.incr() - } - } - )(gate.arrow) - - val inputAdapted: Arrow[(Query, GateExecutorResult), GateResult] = - Arrow - .map[(Query, GateExecutorResult), Query] { case (query, _) => query } - .andThen(observedArrow) - - val zipped = Arrow.zipWithArg(inputAdapted) - - // at each step, the current `GateExecutorResult.continue` value is correct for all already run gates - val withStoppedGatesAsExceptions = zipped.map { - case ((query, previousResults), currentResult) if currentResult.continue => - Return( - ( - query, - GateExecutorResult( - previousResults.individualGateResults :+ ExecutedGateResult( - gate.identifier, - currentResult)) - )) - case _ => Throw(StoppedGateException(gate.identifier)) - }.lowerFromTry - - /** - * we gather stats before converting closed gates to exceptions because a closed gate - * isn't a failure for the gate, its a normal behavior - * but we do want to remap the the [[StoppedGateException]] created because the [[BaseGate]] is closed - * to the correct [[com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure]], - * so we remap with [[wrapWithErrorHandling]] - */ - wrapWithErrorHandling(context, gate.identifier)(withStoppedGatesAsExceptions) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/GateExecutorResult.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/GateExecutorResult.docx new file mode 100644 index 000000000..6367b2feb Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/GateExecutorResult.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/GateExecutorResult.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/GateExecutorResult.scala deleted file mode 100644 index 2a38cd59b..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/GateExecutorResult.scala +++ /dev/null @@ -1,7 +0,0 @@ -package com.twitter.product_mixer.core.service.gate_executor - -import com.twitter.product_mixer.core.service.ExecutorResult - -case class GateExecutorResult( - individualGateResults: Seq[ExecutedGateResult]) - extends ExecutorResult diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/StoppedGateException.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/StoppedGateException.docx new file mode 100644 index 000000000..ee5e12414 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/StoppedGateException.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/StoppedGateException.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/StoppedGateException.scala deleted file mode 100644 index 7bd7bae29..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/gate_executor/StoppedGateException.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.product_mixer.core.service.gate_executor - -import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailureCategory -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailureClassifier - -import scala.util.control.NoStackTrace - -case class StoppedGateException(identifier: GateIdentifier) - extends Exception("Closed gate stopped execution of the pipeline") - with NoStackTrace { - override def toString: String = s"StoppedGateException($identifier)" -} - -object StoppedGateException { - - /** - * Creates a [[PipelineFailureClassifier]] that is used as the default for classifying failures - * in a pipeline by mapping [[StoppedGateException]] to a [[PipelineFailure]] with the provided - * [[PipelineFailureCategory]] - */ - def classifier( - category: PipelineFailureCategory - ): PipelineFailureClassifier = PipelineFailureClassifier { - case stoppedGateException: StoppedGateException => - PipelineFailure(category, stoppedGateException.getMessage, Some(stoppedGateException)) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/group_results_executor/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/group_results_executor/BUILD deleted file mode 100644 index c34688181..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/group_results_executor/BUILD +++ /dev/null @@ -1,35 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_transformer_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transformer_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "stitch/stitch-core", - "util/util-core:scala", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/candidate_feature_transformer_executor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transformer_executor", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/group_results_executor/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/group_results_executor/BUILD.docx new file mode 100644 index 000000000..9208590a4 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/group_results_executor/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/group_results_executor/GroupResultsExecutor.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/group_results_executor/GroupResultsExecutor.docx new file mode 100644 index 000000000..7df09702b Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/group_results_executor/GroupResultsExecutor.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/group_results_executor/GroupResultsExecutor.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/group_results_executor/GroupResultsExecutor.scala deleted file mode 100644 index beb665c64..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/group_results_executor/GroupResultsExecutor.scala +++ /dev/null @@ -1,122 +0,0 @@ -package com.twitter.product_mixer.core.service.group_results_executor - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.model.common.identifier.PlatformIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidatePipelines -import com.twitter.product_mixer.core.model.common.presentation.CandidateSourcePosition -import com.twitter.product_mixer.core.model.common.presentation.CandidateSources -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ItemPresentation -import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ModulePresentation -import com.twitter.product_mixer.core.model.common.presentation.UniversalPresentation -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.ExecutorResult -import com.twitter.stitch.Arrow -import javax.inject.Inject -import javax.inject.Singleton -import scala.collection.immutable.ListSet - -// Most executors are in the core.service package, but this one is pipeline specific -@Singleton -class GroupResultsExecutor @Inject() (override val statsReceiver: StatsReceiver) extends Executor { - - val identifier: ComponentIdentifier = PlatformIdentifier("GroupResults") - - def arrow[Candidate <: UniversalNoun[Any]]( - pipelineIdentifier: CandidatePipelineIdentifier, - sourceIdentifier: CandidateSourceIdentifier, - context: Executor.Context - ): Arrow[GroupResultsExecutorInput[Candidate], GroupResultsExecutorResult] = { - - val groupArrow = Arrow.map { input: GroupResultsExecutorInput[Candidate] => - val modules: Map[Option[ModulePresentation], Seq[CandidateWithFeatures[Candidate]]] = - input.candidates - .map { candidate: CandidateWithFeatures[Candidate] => - val modulePresentationOpt: Option[ModulePresentation] = - input.decorations.get(candidate.candidate).collect { - case itemPresentation: ItemPresentation - if itemPresentation.modulePresentation.isDefined => - itemPresentation.modulePresentation.get - } - - (candidate, modulePresentationOpt) - }.groupBy { - case (_, modulePresentationOpt) => modulePresentationOpt - }.mapValues { - resultModuleOptTuples: Seq[ - (CandidateWithFeatures[Candidate], Option[ModulePresentation]) - ] => - resultModuleOptTuples.map { - case (result, _) => result - } - } - - // Modules should be in their original order, but the groupBy above isn't stable. - // Sort them by the sourcePosition, using the sourcePosition of their first contained - // candidate. - val sortedModules: Seq[(Option[ModulePresentation], Seq[CandidateWithFeatures[Candidate]])] = - modules.toSeq - .sortBy { - case (_, results) => - results.headOption.map(_.features.get(CandidateSourcePosition)) - } - - val candidatesWithDetails: Seq[CandidateWithDetails] = sortedModules.flatMap { - case (modulePresentationOpt, resultsSeq) => - val itemsWithDetails = resultsSeq.map { result => - val presentationOpt = input.decorations.get(result.candidate) match { - case itemPresentation @ Some(_: ItemPresentation) => itemPresentation - case _ => None - } - - val baseFeatureMap = FeatureMapBuilder() - .add(CandidatePipelines, ListSet.empty + pipelineIdentifier) - .build() - - ItemCandidateWithDetails( - candidate = result.candidate, - presentation = presentationOpt, - features = baseFeatureMap ++ result.features - ) - } - - modulePresentationOpt - .map { modulePresentation => - val moduleSourcePosition = - resultsSeq.head.features.get(CandidateSourcePosition) - val baseFeatureMap = FeatureMapBuilder() - .add(CandidatePipelines, ListSet.empty + pipelineIdentifier) - .add(CandidateSourcePosition, moduleSourcePosition) - .add(CandidateSources, ListSet.empty + sourceIdentifier) - .build() - - Seq( - ModuleCandidateWithDetails( - candidates = itemsWithDetails, - presentation = Some(modulePresentation), - features = baseFeatureMap - )) - }.getOrElse(itemsWithDetails) - } - - GroupResultsExecutorResult(candidatesWithDetails) - } - - wrapWithErrorHandling(context, identifier)(groupArrow) - } -} - -case class GroupResultsExecutorInput[Candidate <: UniversalNoun[Any]]( - candidates: Seq[CandidateWithFeatures[Candidate]], - decorations: Map[UniversalNoun[Any], UniversalPresentation]) - -case class GroupResultsExecutorResult(candidatesWithDetails: Seq[CandidateWithDetails]) - extends ExecutorResult diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_execution_logger/AllowListedPipelineExecutionLogger.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_execution_logger/AllowListedPipelineExecutionLogger.docx new file mode 100644 index 000000000..140b7c09f Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_execution_logger/AllowListedPipelineExecutionLogger.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_execution_logger/AllowListedPipelineExecutionLogger.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_execution_logger/AllowListedPipelineExecutionLogger.scala deleted file mode 100644 index af5ed42b9..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_execution_logger/AllowListedPipelineExecutionLogger.scala +++ /dev/null @@ -1,183 +0,0 @@ -package com.twitter.product_mixer.core.service.pipeline_execution_logger - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.annotations.Flag -import com.twitter.product_mixer.core.module.product_mixer_flags.ProductMixerFlagModule.PipelineExecutionLoggerAllowList -import com.twitter.product_mixer.core.module.product_mixer_flags.ProductMixerFlagModule.ServiceLocal -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.FuturePools -import com.twitter.product_mixer.shared_library.observer.Observer.FutureObserver -import com.twitter.util.Try -import com.twitter.util.logging.Logging -import pprint.PPrinter -import pprint.Tree -import pprint.Util -import pprint.tuplePrefix -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Initial implementation from: - * https://stackoverflow.com/questions/15718506/scala-how-to-print-case-classes-like-pretty-printed-tree/57080463#57080463 - */ -object AllowListedPipelineExecutionLogger { - - /** - * Given a case class who's arguments are all declared fields on the class, - * returns an iterator of the field name and values - */ - private[pipeline_execution_logger] def caseClassFields( - caseClass: Product - ): Iterator[(String, Any)] = { - val fieldValues = caseClass.productIterator.toSet - val fields = caseClass.getClass.getDeclaredFields.toSeq - .filterNot(f => f.isSynthetic || java.lang.reflect.Modifier.isStatic(f.getModifiers)) - fields.iterator - .map { f => - f.setAccessible(true) - f.getName -> f.get(caseClass) - }.filter { case (_, v) => fieldValues.contains(v) } - } - - /** - * Returns whether a given [[Product]] is a case class which we can render nicely which: - * - has a [[Product.productArity]] <= the number of declared fields - * - isn't a built in binary operator - * - isn't a tuple - * - who's arguments are fields (not methods) - * - every [[Product.productElement]] has a corresponding field - * - * This will return false for some case classes where we can not reliably determine which field names correspond to - * each value in the case class (this can happen if a case class implements an abstract class resulting in val fields - * becoming methods. - */ - private[pipeline_execution_logger] def isRenderableCaseClass(caseClass: Product): Boolean = { - val possibleToBeRenderableCaseClass = - caseClass.getClass.getDeclaredFields.length >= caseClass.productArity - val isntBuiltInOperator = - !(caseClass.productArity == 2 && Util.isOperator(caseClass.productPrefix)) - val isntTuple = !caseClass.getClass.getName.startsWith(tuplePrefix) - val declaredFieldsMatchesCaseClassFields = { - val caseClassFields = caseClass.productIterator.toSet - caseClass.getClass.getDeclaredFields.iterator - .filterNot(f => f.isSynthetic || java.lang.reflect.Modifier.isStatic(f.getModifiers)) - .count { f => - f.setAccessible(true) - caseClassFields.contains(f.get(caseClass)) - } >= caseClass.productArity - } - - possibleToBeRenderableCaseClass && isntBuiltInOperator && isntTuple && declaredFieldsMatchesCaseClassFields - } - - /** Makes a [[Tree]] which will render as `key = value` */ - private def keyValuePair(key: String, value: Tree): Tree = { - Tree.Infix(Tree.Literal(key), "=", value) - } - - /** - * Special handling for case classes who's field names can be easily paired with their values. - * This will make the [[PPrinter]] render them as - * {{{ - * CaseClassName( - * field1 = value1, - * field2 = value2 - * ) - * }}} - * instead of - * {{{ - * CaseClassName( - * value1, - * value2 - * ) - * }}} - * - * For case classes who's fields end up being compiled as methods, this will fall back - * to the built in handling of case classes without their field names. - */ - private[pipeline_execution_logger] def additionalHandlers: PartialFunction[Any, Tree] = { - case caseClass: Product if isRenderableCaseClass(caseClass) => - Tree.Apply( - caseClass.productPrefix, - caseClassFields(caseClass).flatMap { - case (key, value) => - val valueTree = printer.treeify(value, false, true) - Seq(keyValuePair(key, valueTree)) - } - ) - } - - /** - * [[PPrinter]] instance to use when rendering scala objects - * uses BlackAndWhite because colors mangle the output when looking at the logs in plain text - */ - private val printer: PPrinter = PPrinter.BlackWhite.copy( - // arbitrary high value to turn off truncation - defaultHeight = Int.MaxValue, - // the relatively high width will cause some wrapping but many of the pretty printed objects - // will be sparse (e.g. None,\n None,\n, None,\n) - defaultWidth = 300, - // use reflection to print field names (can be deleted in Scala 2.13) - additionalHandlers = additionalHandlers - ) - - /** Given any scala object, return a String representation of it */ - private[pipeline_execution_logger] def objectAsString(o: Any): String = - printer.tokenize(o).mkString -} - -@Singleton -class AllowListedPipelineExecutionLogger @Inject() ( - @Flag(ServiceLocal) isServiceLocal: Boolean, - @Flag(PipelineExecutionLoggerAllowList) allowList: Seq[String], - statsReceiver: StatsReceiver) - extends PipelineExecutionLogger - with Logging { - - private val scopedStats = statsReceiver.scope("AllowListedPipelineExecutionLogger") - - val allowListRoles: Set[String] = allowList.toSet - - private val futurePool = - FuturePools.boundedFixedThreadPool( - "AllowListedPipelineExecutionLogger", - // single thread, may need to be adjusted higher if it cant keep up with the work queue - fixedThreadCount = 1, - // arbitrarily large enough to handle spikes without causing large allocations or retaining past multiple GC cycles - workQueueSize = 100, - scopedStats - ) - - private val futureObserver = new FutureObserver[Unit](scopedStats, Seq.empty) - - private val loggerOutputPath = Try(System.getProperty("log.allow_listed_execution_logger.output")) - - override def apply(pipelineQuery: PipelineQuery, message: Any): Unit = { - - val userRoles: Set[String] = pipelineQuery.clientContext.userRoles.getOrElse(Set.empty) - - // Check if this request is in the allowlist via a cleverly optimized set intersection - val allowListed = - if (allowListRoles.size > userRoles.size) - userRoles.exists(allowListRoles.contains) - else - allowListRoles.exists(userRoles.contains) - - if (isServiceLocal || allowListed) { - futureObserver( - /** - * failure to enqueue the work will result with a failed [[com.twitter.util.Future]] - * containing a [[java.util.concurrent.RejectedExecutionException]] which the wrapping [[futureObserver]] - * will record metrics for. - */ - futurePool { - logger.info(AllowListedPipelineExecutionLogger.objectAsString(message)) - - if (isServiceLocal && loggerOutputPath.isReturn) { - println(s"Logged request to: ${loggerOutputPath.get()}") - } - } - ) - } - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_execution_logger/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_execution_logger/BUILD deleted file mode 100644 index dffbcc694..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_execution_logger/BUILD +++ /dev/null @@ -1,33 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/com/lihaoyi:pprint", - "3rdparty/jvm/net/codingwell:scala-guice", - "finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "finatra/inject/inject-slf4j/src/main/scala/com/twitter/inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/product_mixer_flags", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - ], - exports = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/com/lihaoyi:pprint", - "3rdparty/jvm/net/codingwell:scala-guice", - "finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-slf4j/src/main/scala/com/twitter/inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/product_mixer_flags", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_execution_logger/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_execution_logger/BUILD.docx new file mode 100644 index 000000000..56c375e96 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_execution_logger/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_execution_logger/PipelineExecutionLogger.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_execution_logger/PipelineExecutionLogger.docx new file mode 100644 index 000000000..5dc2a16c3 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_execution_logger/PipelineExecutionLogger.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_execution_logger/PipelineExecutionLogger.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_execution_logger/PipelineExecutionLogger.scala deleted file mode 100644 index e6c7535c7..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_execution_logger/PipelineExecutionLogger.scala +++ /dev/null @@ -1,7 +0,0 @@ -package com.twitter.product_mixer.core.service.pipeline_execution_logger - -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -trait PipelineExecutionLogger { - def apply(pipelineQuery: PipelineQuery, message: Any): Unit -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_executor/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_executor/BUILD deleted file mode 100644 index fbc1b6b8c..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_executor/BUILD +++ /dev/null @@ -1,25 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "stitch/stitch-core", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_executor/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_executor/BUILD.docx new file mode 100644 index 000000000..6dbb61b2e Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_executor/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_executor/PipelineExecutor.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_executor/PipelineExecutor.docx new file mode 100644 index 000000000..46cfcd107 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_executor/PipelineExecutor.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_executor/PipelineExecutor.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_executor/PipelineExecutor.scala deleted file mode 100644 index 511973cb1..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_executor/PipelineExecutor.scala +++ /dev/null @@ -1,66 +0,0 @@ -package com.twitter.product_mixer.core.service.pipeline_executor - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.pipeline.Pipeline -import com.twitter.product_mixer.core.pipeline.pipeline_failure.InvalidPipelineSelected -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.quality_factor.QualityFactorObserver -import com.twitter.product_mixer.core.service.Executor -import com.twitter.stitch.Arrow -import com.twitter.util.logging.Logging - -import javax.inject.Inject -import javax.inject.Singleton - -/** - * PipelineExecutor executes a single pipeline (of any type) - * It does not currently support fail open/closed policies like CandidatePipelineExecutor does - * In the future, maybe they can be merged. - */ - -case class PipelineExecutorRequest[Query](query: Query, pipelineIdentifier: ComponentIdentifier) - -@Singleton -class PipelineExecutor @Inject() (override val statsReceiver: StatsReceiver) - extends Executor - with Logging { - - def arrow[Query, ResultType]( - pipelineByIdentifier: Map[ComponentIdentifier, Pipeline[Query, ResultType]], - qualityFactorObserverByPipeline: Map[ComponentIdentifier, QualityFactorObserver], - context: Executor.Context - ): Arrow[PipelineExecutorRequest[Query], PipelineExecutorResult[ResultType]] = { - - val wrappedPipelineArrowsByIdentifier = pipelineByIdentifier.mapValues { pipeline => - wrapPipelineWithExecutorBookkeeping( - context, - pipeline.identifier, - qualityFactorObserverByPipeline.get(pipeline.identifier))(pipeline.arrow) - } - - val appliedPipelineArrow = Arrow - .identity[PipelineExecutorRequest[Query]] - .map { - case PipelineExecutorRequest(query, pipelineIdentifier) => - val pipeline = wrappedPipelineArrowsByIdentifier.getOrElse( - pipelineIdentifier, - // throwing instead of returning a `Throw(_)` and then `.lowerFromTry` because this is an exceptional case and we want to emphasize that by explicitly throwing - // this case should never happen since this is checked in the `PipelineSelectorExecutor` but we check it anyway - throw PipelineFailure( - InvalidPipelineSelected, - s"${context.componentStack.peek} attempted to execute $pipelineIdentifier", - // the `componentStack` includes the missing pipeline so it can show up in metrics easier - componentStack = Some(context.componentStack.push(pipelineIdentifier)) - ) - ) - (pipeline, query) - } - // less efficient than an `andThen` but since we dispatch this dynamically we need to use either `applyArrow` or `flatMap` and this is the better of those options - .applyArrow - .map(PipelineExecutorResult(_)) - - // no additional error handling needed since we populate the component stack above already - appliedPipelineArrow - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_executor/PipelineExecutorResult.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_executor/PipelineExecutorResult.docx new file mode 100644 index 000000000..58d98d51b Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_executor/PipelineExecutorResult.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_executor/PipelineExecutorResult.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_executor/PipelineExecutorResult.scala deleted file mode 100644 index 9953b4666..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_executor/PipelineExecutorResult.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.product_mixer.core.service.pipeline_executor - -import com.twitter.product_mixer.core.pipeline.PipelineResult -import com.twitter.product_mixer.core.service.ExecutorResult - -case class PipelineExecutorResult[ResultType]( - pipelineResult: PipelineResult[ResultType]) - extends ExecutorResult diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_result_side_effect_executor/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_result_side_effect_executor/BUILD deleted file mode 100644 index 1aecf5960..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_result_side_effect_executor/BUILD +++ /dev/null @@ -1,24 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/side_effect", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "stitch/stitch-core", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/side_effect", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_result_side_effect_executor/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_result_side_effect_executor/BUILD.docx new file mode 100644 index 000000000..4a0ba9d1c Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_result_side_effect_executor/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_result_side_effect_executor/PipelineResultSideEffectExecutor.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_result_side_effect_executor/PipelineResultSideEffectExecutor.docx new file mode 100644 index 000000000..7c70eb306 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_result_side_effect_executor/PipelineResultSideEffectExecutor.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_result_side_effect_executor/PipelineResultSideEffectExecutor.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_result_side_effect_executor/PipelineResultSideEffectExecutor.scala deleted file mode 100644 index 2506db301..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_result_side_effect_executor/PipelineResultSideEffectExecutor.scala +++ /dev/null @@ -1,91 +0,0 @@ -package com.twitter.product_mixer.core.service.pipeline_result_side_effect_executor - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.functional_component.side_effect.ExecuteSynchronously -import com.twitter.product_mixer.core.functional_component.side_effect.FailOpen -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect.Inputs -import com.twitter.product_mixer.core.model.common.Conditionally -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.ExecutorResult -import com.twitter.product_mixer.core.service.pipeline_result_side_effect_executor.PipelineResultSideEffectExecutor._ -import com.twitter.stitch.Arrow -import com.twitter.util.Return -import com.twitter.util.Try -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class PipelineResultSideEffectExecutor @Inject() (override val statsReceiver: StatsReceiver) - extends Executor { - def arrow[Query <: PipelineQuery, MixerDomainResultType <: HasMarshalling]( - sideEffects: Seq[PipelineResultSideEffect[Query, MixerDomainResultType]], - context: Executor.Context - ): Arrow[Inputs[Query, MixerDomainResultType], PipelineResultSideEffectExecutor.Result] = { - - val individualArrows: Seq[ - Arrow[Inputs[Query, MixerDomainResultType], (SideEffectIdentifier, SideEffectResultType)] - ] = sideEffects.map { - case synchronousSideEffect: ExecuteSynchronously => - val failsRequestIfThrows = { - wrapComponentWithExecutorBookkeeping(context, synchronousSideEffect.identifier)( - Arrow.flatMap(synchronousSideEffect.apply)) - } - synchronousSideEffect match { - case failOpen: FailOpen => - // lift the failure - failsRequestIfThrows.liftToTry.map(t => - (failOpen.identifier, SynchronousSideEffectResult(t))) - case _ => - // don't encapsulate the failure - failsRequestIfThrows.map(_ => - (synchronousSideEffect.identifier, SynchronousSideEffectResult(Return.Unit))) - } - - case sideEffect => - Arrow - .async( - wrapComponentWithExecutorBookkeeping(context, sideEffect.identifier)( - Arrow.flatMap(sideEffect.apply))) - .andThen(Arrow.value((sideEffect.identifier, SideEffectResult))) - } - - val conditionallyRunArrows = sideEffects.zip(individualArrows).map { - case ( - sideEffect: Conditionally[ - PipelineResultSideEffect.Inputs[Query, MixerDomainResultType] @unchecked - ], - arrow) => - Arrow.ifelse[ - Inputs[Query, MixerDomainResultType], - (SideEffectIdentifier, SideEffectResultType)]( - input => sideEffect.onlyIf(input), - arrow, - Arrow.value((sideEffect.identifier, TurnedOffByConditionally))) - case (_, arrow) => arrow - } - - Arrow - .collect(conditionallyRunArrows) - .map(results => Result(results)) - } -} - -object PipelineResultSideEffectExecutor { - case class Result(sideEffects: Seq[(SideEffectIdentifier, SideEffectResultType)]) - extends ExecutorResult - - sealed trait SideEffectResultType - - /** The [[PipelineResultSideEffect]] was executed asynchronously in a fire-and-forget way so no result will be available */ - case object SideEffectResult extends SideEffectResultType - - /** The result of the [[PipelineResultSideEffect]] that was executed with [[ExecuteSynchronously]] */ - case class SynchronousSideEffectResult(result: Try[Unit]) extends SideEffectResultType - - /** The result for when a [[PipelineResultSideEffect]] is turned off by [[Conditionally]]'s [[Conditionally.onlyIf]] */ - case object TurnedOffByConditionally extends SideEffectResultType -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_selector_executor/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_selector_executor/BUILD deleted file mode 100644 index 7c5725005..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_selector_executor/BUILD +++ /dev/null @@ -1,26 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-slf4j/src/main/scala/com/twitter/inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "stitch/stitch-core", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_selector_executor/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_selector_executor/BUILD.docx new file mode 100644 index 000000000..ba06ba15d Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_selector_executor/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_selector_executor/PipelineSelectorExecutor.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_selector_executor/PipelineSelectorExecutor.docx new file mode 100644 index 000000000..561aeaa66 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_selector_executor/PipelineSelectorExecutor.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_selector_executor/PipelineSelectorExecutor.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_selector_executor/PipelineSelectorExecutor.scala deleted file mode 100644 index bf211b63d..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_selector_executor/PipelineSelectorExecutor.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.twitter.product_mixer.core.service.pipeline_selector_executor - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.util.logging.Logging -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.model.common.identifier.PlatformIdentifier -import com.twitter.product_mixer.core.pipeline.Pipeline -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.pipeline_failure.InvalidPipelineSelected -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.service.Executor -import com.twitter.stitch.Arrow - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class PipelineSelectorExecutor @Inject() (override val statsReceiver: StatsReceiver) - extends Executor - with Logging { - - val identifier: ComponentIdentifier = PlatformIdentifier("PipelineSelector") - - def arrow[Query <: PipelineQuery, Response]( - pipelineByIdentifier: Map[ComponentIdentifier, Pipeline[Query, Response]], - pipelineSelector: Query => ComponentIdentifier, - context: Executor.Context - ): Arrow[Query, PipelineSelectorExecutorResult] = { - - val validateSelectedPipelineExists = Arrow - .map(pipelineSelector) - .map { chosenIdentifier => - if (pipelineByIdentifier.contains(chosenIdentifier)) { - PipelineSelectorExecutorResult(chosenIdentifier) - } else { - // throwing instead of returning a `Throw(_)` and then `.lowerFromTry` because this is an exceptional case and we want to emphasize that by explicitly throwing - throw PipelineFailure( - InvalidPipelineSelected, - s"${context.componentStack.peek} attempted to select $chosenIdentifier", - // the `componentStack` includes the missing pipeline so it can show up in metrics easier - componentStack = Some(context.componentStack.push(chosenIdentifier)) - ) - } - } - - wrapWithErrorHandling(context, identifier)(validateSelectedPipelineExists) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_selector_executor/PipelineSelectorExecutorResult.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_selector_executor/PipelineSelectorExecutorResult.docx new file mode 100644 index 000000000..c766a545a Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_selector_executor/PipelineSelectorExecutorResult.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_selector_executor/PipelineSelectorExecutorResult.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_selector_executor/PipelineSelectorExecutorResult.scala deleted file mode 100644 index 3ffc8297f..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/pipeline_selector_executor/PipelineSelectorExecutorResult.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.product_mixer.core.service.pipeline_selector_executor - -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier - -case class PipelineSelectorExecutorResult(pipelineIdentifier: ComponentIdentifier) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/quality_factor_executor/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/quality_factor_executor/BUILD deleted file mode 100644 index ae1c9b4fd..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/quality_factor_executor/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "stitch/stitch-core", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/quality_factor_executor/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/quality_factor_executor/BUILD.docx new file mode 100644 index 000000000..2e52490d8 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/quality_factor_executor/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/quality_factor_executor/QualityFactorExecutorResult.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/quality_factor_executor/QualityFactorExecutorResult.docx new file mode 100644 index 000000000..6c0e226e2 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/quality_factor_executor/QualityFactorExecutorResult.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/quality_factor_executor/QualityFactorExecutorResult.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/quality_factor_executor/QualityFactorExecutorResult.scala deleted file mode 100644 index 43acb992f..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/quality_factor_executor/QualityFactorExecutorResult.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.product_mixer.core.service.quality_factor_executor - -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier - -case class QualityFactorExecutorResult( - pipelineQualityFactors: Map[ComponentIdentifier, Double]) - -object QualityFactorExecutorResult { - val empty: QualityFactorExecutorResult = QualityFactorExecutorResult(Map.empty) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor/AsyncIndividualFeatureHydratorResultSerializer.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor/AsyncIndividualFeatureHydratorResultSerializer.docx new file mode 100644 index 000000000..9d49d7784 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor/AsyncIndividualFeatureHydratorResultSerializer.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor/AsyncIndividualFeatureHydratorResultSerializer.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor/AsyncIndividualFeatureHydratorResultSerializer.scala deleted file mode 100644 index b7a3e5819..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor/AsyncIndividualFeatureHydratorResultSerializer.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.product_mixer.core.service.query_feature_hydrator_executor - -import com.fasterxml.jackson.core.JsonGenerator -import com.fasterxml.jackson.databind.JsonSerializer -import com.fasterxml.jackson.databind.SerializerProvider -import com.twitter.product_mixer.core.service.query_feature_hydrator_executor.QueryFeatureHydratorExecutor.AsyncIndividualFeatureHydratorResult - -/** A [[JsonSerializer]] that skips the `Stitch` values */ -private[query_feature_hydrator_executor] class AsyncIndividualFeatureHydratorResultSerializer() - extends JsonSerializer[AsyncIndividualFeatureHydratorResult] { - - override def serialize( - asyncIndividualFeatureHydratorResult: AsyncIndividualFeatureHydratorResult, - gen: JsonGenerator, - serializers: SerializerProvider - ): Unit = - serializers.defaultSerializeValue( - // implicitly calls `toString` on the identifier because they are keys in the Map - Map( - asyncIndividualFeatureHydratorResult.hydrateBefore -> - asyncIndividualFeatureHydratorResult.features.map(_.toString)), - gen - ) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor/BUILD deleted file mode 100644 index ffceb6261..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor/BUILD +++ /dev/null @@ -1,28 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator/featurestorev1", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/feature_hydrator_observer", - "util/util-jackson/src/main/scala/com/twitter/util/jackson", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator/featurestorev1", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor/BUILD.docx new file mode 100644 index 000000000..d78a44271 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor/QueryFeatureHydratorExecutor.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor/QueryFeatureHydratorExecutor.docx new file mode 100644 index 000000000..df62123f7 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor/QueryFeatureHydratorExecutor.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor/QueryFeatureHydratorExecutor.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor/QueryFeatureHydratorExecutor.scala deleted file mode 100644 index 8d6e3c4c6..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/query_feature_hydrator_executor/QueryFeatureHydratorExecutor.scala +++ /dev/null @@ -1,217 +0,0 @@ -package com.twitter.product_mixer.core.service.query_feature_hydrator_executor - -import com.fasterxml.jackson.databind.annotation.JsonSerialize -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.asyncfeaturemap.AsyncFeatureMap -import com.twitter.product_mixer.core.functional_component.feature_hydrator.AsyncHydrator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.featurestorev1.FeatureStoreV1QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.Conditionally -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.Executor._ -import com.twitter.product_mixer.core.service.ExecutorResult -import com.twitter.product_mixer.core.service.feature_hydrator_observer.FeatureHydratorObserver -import com.twitter.product_mixer.core.service.query_feature_hydrator_executor.QueryFeatureHydratorExecutor.AsyncIndividualFeatureHydratorResult -import com.twitter.product_mixer.core.service.query_feature_hydrator_executor.QueryFeatureHydratorExecutor.BaseIndividualFeatureHydratorResult -import com.twitter.product_mixer.core.service.query_feature_hydrator_executor.QueryFeatureHydratorExecutor.FeatureHydratorDisabled -import com.twitter.product_mixer.core.service.query_feature_hydrator_executor.QueryFeatureHydratorExecutor.IndividualFeatureHydratorResult -import com.twitter.product_mixer.core.service.query_feature_hydrator_executor.QueryFeatureHydratorExecutor.validateAsyncQueryFeatureHydrator -import com.twitter.stitch.Arrow -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class QueryFeatureHydratorExecutor @Inject() (override val statsReceiver: StatsReceiver) - extends Executor { - - def arrow[Query <: PipelineQuery]( - hydrators: Seq[BaseQueryFeatureHydrator[Query, _]], - validPipelineSteps: Set[PipelineStepIdentifier], - context: Executor.Context - ): Arrow[Query, QueryFeatureHydratorExecutor.Result] = { - - val observer = new FeatureHydratorObserver(statsReceiver, hydrators, context) - val hydratorsWithErrorHandling = - hydrators.map { hydrator => - val queryFeatureHydratorArrow = - getQueryHydratorArrow(hydrator, context, observer) - val wrappedWithAsyncHandling = - handleAsyncHydrator(hydrator, validPipelineSteps, queryFeatureHydratorArrow) - handleConditionally(hydrator, wrappedWithAsyncHandling) - } - - Arrow - .collect(hydratorsWithErrorHandling) - .map { - results: Seq[ - (FeatureHydratorIdentifier, BaseIndividualFeatureHydratorResult) - ] => - val combinedFeatureMap = FeatureMap.merge(results.collect { - case (_, IndividualFeatureHydratorResult(featureMap)) => featureMap - }) - - val asyncFeatureMaps = results.collect { - case ( - hydratorIdentifier, - AsyncIndividualFeatureHydratorResult(hydrateBefore, featuresToHydrate, ref)) => - (hydratorIdentifier, hydrateBefore, featuresToHydrate, ref) - } - - QueryFeatureHydratorExecutor.Result( - individualFeatureMaps = results.toMap, - featureMap = combinedFeatureMap, - asyncFeatureMap = AsyncFeatureMap.fromFeatureMaps(asyncFeatureMaps) - ) - } - } - - def handleConditionally[Query <: PipelineQuery]( - hydrator: BaseQueryFeatureHydrator[Query, _], - arrow: Arrow[ - Query, - BaseIndividualFeatureHydratorResult - ] - ): Arrow[ - Query, - (FeatureHydratorIdentifier, BaseIndividualFeatureHydratorResult) - ] = { - val conditionallyRunArrow = hydrator match { - case hydrator: BaseQueryFeatureHydrator[Query, _] with Conditionally[Query @unchecked] => - Arrow.ifelse[Query, BaseIndividualFeatureHydratorResult]( - hydrator.onlyIf, - arrow, - Arrow.value(FeatureHydratorDisabled) - ) - case _ => arrow - } - - Arrow.join( - Arrow.value(hydrator.identifier), - conditionallyRunArrow - ) - } - - def handleAsyncHydrator[Query <: PipelineQuery]( - hydrator: BaseQueryFeatureHydrator[Query, _], - validPipelineSteps: Set[PipelineStepIdentifier], - arrow: Arrow[ - Query, - IndividualFeatureHydratorResult - ] - ): Arrow[Query, BaseIndividualFeatureHydratorResult] = { - hydrator match { - case hydrator: BaseQueryFeatureHydrator[ - Query, - _ - ] with AsyncHydrator => - validateAsyncQueryFeatureHydrator(hydrator, validPipelineSteps) - - startArrowAsync(arrow.map(_.featureMap)) - .map { ref => - AsyncIndividualFeatureHydratorResult( - hydrator.hydrateBefore, - hydrator.features.asInstanceOf[Set[Feature[_, _]]], - ref - ) - } - - case _ => arrow - } - } - - def getQueryHydratorArrow[Query <: PipelineQuery]( - hydrator: BaseQueryFeatureHydrator[Query, _], - context: Executor.Context, - queryFeatureHydratorObserver: FeatureHydratorObserver - ): Arrow[Query, IndividualFeatureHydratorResult] = { - - val componentExecutorContext = context.pushToComponentStack(hydrator.identifier) - val hydratorArrow: Arrow[Query, FeatureMap] = - Arrow.flatMap { query: Query => hydrator.hydrate(query) } - - val validationFn: FeatureMap => FeatureMap = hydrator match { - // Feature store query hydrators store the resulting PredictionRecord and not - // the features, so we cannot validate the same way - case _: FeatureStoreV1QueryFeatureHydrator[Query] => - identity - case _ => - validateFeatureMap( - hydrator.features.asInstanceOf[Set[Feature[_, _]]], - _, - componentExecutorContext) - } - - // record the component-level stats - val observedArrow = - wrapComponentWithExecutorBookkeeping[Query, FeatureMap]( - context, - hydrator.identifier - )(hydratorArrow.map(validationFn)) - - // store non-configuration errors in the FeatureMap - val liftNonValidationFailuresToFailedFeatures = Arrow.handle[FeatureMap, FeatureMap] { - case NotAMisconfiguredFeatureMapFailure(e) => - featureMapWithFailuresForFeatures( - hydrator.features.asInstanceOf[Set[Feature[_, _]]], - e, - componentExecutorContext) - } - - val observedLiftedAndWrapped = observedArrow - .andThen(liftNonValidationFailuresToFailedFeatures) - .applyEffect(Arrow.map[FeatureMap, Unit](featureMap => - // record per-feature stats, this is separate from the component stats handled by `wrapWithExecutorBookkeeping` - queryFeatureHydratorObserver.observeFeatureSuccessAndFailures(hydrator, Seq(featureMap)))) - .map(IndividualFeatureHydratorResult) - - observedLiftedAndWrapped - } -} - -object QueryFeatureHydratorExecutor { - case class Result( - individualFeatureMaps: Map[ - FeatureHydratorIdentifier, - BaseIndividualFeatureHydratorResult - ], - featureMap: FeatureMap, - asyncFeatureMap: AsyncFeatureMap) - extends ExecutorResult - - sealed trait BaseIndividualFeatureHydratorResult - - case object FeatureHydratorDisabled extends BaseIndividualFeatureHydratorResult - case class IndividualFeatureHydratorResult(featureMap: FeatureMap) - extends BaseIndividualFeatureHydratorResult - - /** Async result, serializes without the [[Stitch]] field since it's not serializable */ - @JsonSerialize(using = classOf[AsyncIndividualFeatureHydratorResultSerializer]) - case class AsyncIndividualFeatureHydratorResult( - hydrateBefore: PipelineStepIdentifier, - features: Set[Feature[_, _]], - ref: Stitch[FeatureMap]) - extends BaseIndividualFeatureHydratorResult - - /** - * Validates whether the [[AsyncHydrator.hydrateBefore]] [[PipelineStepIdentifier]] is valid - * - * @param asyncQueryFeatureHydrator the hydrator to validate - * @param validPipelineSteps a Set of [[PipelineStepIdentifier]]s which are valid places to populate async - * [[Feature]]s in a [[com.twitter.product_mixer.core.pipeline.Pipeline]] - */ - def validateAsyncQueryFeatureHydrator( - asyncQueryFeatureHydrator: AsyncHydrator, - validPipelineSteps: Set[PipelineStepIdentifier] - ): Unit = - require( - validPipelineSteps.contains(asyncQueryFeatureHydrator.hydrateBefore), - s"`AsyncHydrator.hydrateBefore` contained ${asyncQueryFeatureHydrator.hydrateBefore} which was not in the parent pipeline's " + - s"`PipelineConfig` Companion object field `stepsAsyncFeatureHydrationCanBeCompletedBy = $validPipelineSteps`." - ) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/scoring_pipeline_executor/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/scoring_pipeline_executor/BUILD deleted file mode 100644 index 2b03f2775..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/scoring_pipeline_executor/BUILD +++ /dev/null @@ -1,24 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "stitch/stitch-core", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/scoring_pipeline_executor/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/scoring_pipeline_executor/BUILD.docx new file mode 100644 index 000000000..47dfcf5ce Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/scoring_pipeline_executor/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/scoring_pipeline_executor/ScoringPipelineExecutor.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/scoring_pipeline_executor/ScoringPipelineExecutor.docx new file mode 100644 index 000000000..99076d306 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/scoring_pipeline_executor/ScoringPipelineExecutor.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/scoring_pipeline_executor/ScoringPipelineExecutor.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/scoring_pipeline_executor/ScoringPipelineExecutor.scala deleted file mode 100644 index 47b3a762a..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/scoring_pipeline_executor/ScoringPipelineExecutor.scala +++ /dev/null @@ -1,172 +0,0 @@ -package com.twitter.product_mixer.core.service.scoring_pipeline_executor - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.pipeline.FailOpenPolicy -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.pipeline_failure.IllegalStateFailure -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipeline -import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipelineResult -import com.twitter.product_mixer.core.quality_factor.QualityFactorObserver -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.scoring_pipeline_executor.ScoringPipelineExecutor.ScoringPipelineState -import com.twitter.stitch.Arrow -import com.twitter.stitch.Arrow.Iso -import com.twitter.util.logging.Logging - -import javax.inject.Inject -import javax.inject.Singleton -import scala.collection.immutable.Queue - -@Singleton -class ScoringPipelineExecutor @Inject() (override val statsReceiver: StatsReceiver) - extends Executor - with Logging { - def arrow[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]( - pipelines: Seq[ScoringPipeline[Query, Candidate]], - context: Executor.Context, - defaultFailOpenPolicy: FailOpenPolicy, - failOpenPolicies: Map[ScoringPipelineIdentifier, FailOpenPolicy], - qualityFactorObserverByPipeline: Map[ComponentIdentifier, QualityFactorObserver], - ): Arrow[ScoringPipelineExecutor.Inputs[Query], ScoringPipelineExecutorResult[Candidate]] = { - val scoringPipelineArrows = pipelines.map { pipeline => - val failOpenPolicy = failOpenPolicies.getOrElse(pipeline.identifier, defaultFailOpenPolicy) - val qualityFactorObserver = qualityFactorObserverByPipeline.get(pipeline.identifier) - - getIsoArrowForScoringPipeline( - pipeline, - context, - failOpenPolicy, - qualityFactorObserver - ) - } - val combinedArrow = isoArrowsSequentially(scoringPipelineArrows) - Arrow - .map[ScoringPipelineExecutor.Inputs[Query], ScoringPipelineState[Query, Candidate]] { - case input => - ScoringPipelineState( - input.query, - input.itemCandidatesWithDetails, - ScoringPipelineExecutorResult(input.itemCandidatesWithDetails, Queue.empty)) - }.flatMapArrow(combinedArrow).map { state => - state.executorResult.copy(individualPipelineResults = - // materialize the Queue into a List for faster future iterations - state.executorResult.individualPipelineResults.toList) - } - } - - private def getIsoArrowForScoringPipeline[ - Query <: PipelineQuery, - Candidate <: UniversalNoun[Any] - ]( - pipeline: ScoringPipeline[Query, Candidate], - context: Executor.Context, - failOpenPolicy: FailOpenPolicy, - qualityFactorObserver: Option[QualityFactorObserver] - ): Iso[ScoringPipelineState[Query, Candidate]] = { - val pipelineArrow = Arrow - .map[ScoringPipelineState[Query, Candidate], ScoringPipeline.Inputs[Query]] { state => - ScoringPipeline.Inputs(state.query, state.allCandidates) - }.flatMapArrow(pipeline.arrow) - - val observedArrow = wrapPipelineWithExecutorBookkeeping( - context, - pipeline.identifier, - qualityFactorObserver, - failOpenPolicy)(pipelineArrow) - - Arrow - .zipWithArg( - observedArrow - ).map { - case ( - scoringPipelinesState: ScoringPipelineState[Query, Candidate], - scoringPipelineResult: ScoringPipelineResult[Candidate]) => - val updatedCandidates: Seq[ItemCandidateWithDetails] = - mkUpdatedCandidates(pipeline.identifier, scoringPipelinesState, scoringPipelineResult) - ScoringPipelineState( - scoringPipelinesState.query, - updatedCandidates, - scoringPipelinesState.executorResult - .copy( - updatedCandidates, - scoringPipelinesState.executorResult.individualPipelineResults :+ scoringPipelineResult) - ) - } - } - - private def mkUpdatedCandidates[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]( - scoringPipelineIdentifier: ScoringPipelineIdentifier, - scoringPipelinesState: ScoringPipelineState[Query, Candidate], - scoringPipelineResult: ScoringPipelineResult[Candidate] - ): Seq[ItemCandidateWithDetails] = { - if (scoringPipelineResult.failure.isEmpty) { - - /** - * It's important that we map back from which actual item candidate was scored by looking - * at the selector results. This is to defend against the same candidate being selected - * from two different candidate pipelines. If one is selected and the other isn't, we - * should only score the selected one. If both are selected and each is scored differently - * we should get the right score for each. - */ - val selectedItemCandidates: Seq[ItemCandidateWithDetails] = - scoringPipelineResult.selectorResults - .getOrElse(throw PipelineFailure( - IllegalStateFailure, - s"Missing Selector Results in Scoring Pipeline $scoringPipelineIdentifier")).selectedCandidates.collect { - case itemCandidateWithDetails: ItemCandidateWithDetails => - itemCandidateWithDetails - } - val scoredFeatureMaps: Seq[FeatureMap] = scoringPipelineResult.result - .getOrElse(Seq.empty).map(_.features) - - if (scoredFeatureMaps.isEmpty) { - // It's possible that all Scorers are [[Conditionally]] off. In this case, we return empty - // and don't validate the list size since this is done in the hydrator/scorer executor. - scoringPipelinesState.allCandidates - } else if (selectedItemCandidates.length != scoredFeatureMaps.length) { - // The length of the inputted candidates should always match the returned feature map, unless - throw PipelineFailure( - IllegalStateFailure, - s"Missing configured scorer result, length of scorer results does not match the length of selected candidates") - } else { - /* Zip the selected item candidate seq back to the scored feature maps, this works - * because the scored results will always have the same number of elements returned - * and it should match the same order. We then loop through all candidates because the - * expectation is to always keep the result since a subsequent scoring pipeline can score a - * candidate that the current one did not. We only update the feature map of the candidate - * if it was selected and scored. - */ - val selectedItemCandidateToScorerMap: Map[ItemCandidateWithDetails, FeatureMap] = - selectedItemCandidates.zip(scoredFeatureMaps).toMap - scoringPipelinesState.allCandidates.map { itemCandidateWithDetails => - selectedItemCandidateToScorerMap.get(itemCandidateWithDetails) match { - case Some(scorerResult) => - itemCandidateWithDetails.copy(features = - itemCandidateWithDetails.features ++ scorerResult) - case None => itemCandidateWithDetails - } - } - } - } else { - // If the underlying scoring pipeline has failed open, just keep the existing candidates - scoringPipelinesState.allCandidates - } - } -} - -object ScoringPipelineExecutor { - private case class ScoringPipelineState[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]( - query: Query, - allCandidates: Seq[ItemCandidateWithDetails], - executorResult: ScoringPipelineExecutorResult[Candidate]) - - case class Inputs[Query <: PipelineQuery]( - query: Query, - itemCandidatesWithDetails: Seq[ItemCandidateWithDetails]) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/scoring_pipeline_executor/ScoringPipelineExecutorResult.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/scoring_pipeline_executor/ScoringPipelineExecutorResult.docx new file mode 100644 index 000000000..9edc40132 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/scoring_pipeline_executor/ScoringPipelineExecutorResult.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/scoring_pipeline_executor/ScoringPipelineExecutorResult.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/scoring_pipeline_executor/ScoringPipelineExecutorResult.scala deleted file mode 100644 index a018be70c..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/scoring_pipeline_executor/ScoringPipelineExecutorResult.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.product_mixer.core.service.scoring_pipeline_executor - -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipelineResult - -case class ScoringPipelineExecutorResult[Candidate <: UniversalNoun[Any]]( - result: Seq[ItemCandidateWithDetails], - individualPipelineResults: Seq[ScoringPipelineResult[Candidate]]) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor/BUILD deleted file mode 100644 index 34a1ddc21..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor/BUILD +++ /dev/null @@ -1,26 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "stitch/stitch-core", - "util/util-core:scala", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor/BUILD.docx new file mode 100644 index 000000000..dfe05df9a Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor/SelectorExecutor.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor/SelectorExecutor.docx new file mode 100644 index 000000000..149a4f386 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor/SelectorExecutor.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor/SelectorExecutor.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor/SelectorExecutor.scala deleted file mode 100644 index ba9b946a0..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor/SelectorExecutor.scala +++ /dev/null @@ -1,105 +0,0 @@ -package com.twitter.product_mixer.core.service.selector_executor - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.identifier.SelectorIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.pipeline_failure.IllegalStateFailure -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.service.Executor -import com.twitter.stitch.Arrow - -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Applies a `Seq[Selector]` in sequential order. - * Returns the results, and also a detailed list each selector's results (for debugging / understandability). - */ -@Singleton -class SelectorExecutor @Inject() (override val statsReceiver: StatsReceiver) extends Executor { - def arrow[Query <: PipelineQuery]( - selectors: Seq[Selector[Query]], - context: Executor.Context - ): Arrow[SelectorExecutor.Inputs[Query], SelectorExecutorResult] = { - - if (selectors.isEmpty) { - throw PipelineFailure( - IllegalStateFailure, - "Must provide a non-empty Seq of Selectors. Check the config indicated by the componentStack and ensure that a non-empty Selector Seq is provided.", - componentStack = Some(context.componentStack) - ) - } - - val selectorArrows = - selectors.zipWithIndex.foldLeft(Arrow.identity[(Query, IndexedSeq[SelectorResult])]) { - case (previousSelectorArrows, (selector, index)) => - val selectorResult = getIndividualSelectorIsoArrow(selector, index, context) - previousSelectorArrows.andThen(selectorResult) - } - - Arrow - .zipWithArg( - Arrow - .map[SelectorExecutor.Inputs[Query], (Query, IndexedSeq[SelectorResult])] { - case SelectorExecutor.Inputs(query, candidates) => - (query, IndexedSeq(SelectorResult(candidates, Seq.empty))) - }.andThen(selectorArrows)).map { - case (inputs, (_, selectorResults)) => - // the last results, safe because it's always non-empty since it starts with 1 element in it - val SelectorResult(remainingCandidates, result) = selectorResults.last - - val resultsAndRemainingCandidates = - (result.iterator ++ remainingCandidates.iterator).toSet - - // the droppedCandidates are all the candidates which are in neither the result or remainingCandidates - val droppedCandidates = inputs.candidatesWithDetails.iterator - .filterNot(resultsAndRemainingCandidates.contains) - .toIndexedSeq - - SelectorExecutorResult( - selectedCandidates = result, - remainingCandidates = remainingCandidates, - droppedCandidates = droppedCandidates, - individualSelectorResults = - selectorResults.tail // `.tail` to remove the initial state we had - ) - } - } - - private def getIndividualSelectorIsoArrow[Query <: PipelineQuery]( - selector: Selector[Query], - index: Int, - context: Executor.Context - ): Arrow.Iso[(Query, IndexedSeq[SelectorResult])] = { - val identifier = SelectorIdentifier(selector.getClass.getSimpleName, index) - - val arrow = Arrow - .identity[(Query, IndexedSeq[SelectorResult])] - .map { - case (query, previousResults) => - // last is safe here because we pass in a non-empty IndexedSeq - val previousResult = previousResults.last - val currentResult = selector.apply( - query, - previousResult.remainingCandidates, - previousResult.result - ) - (query, previousResults :+ currentResult) - } - - wrapComponentsWithTracingOnly(context, identifier)( - wrapWithErrorHandling(context, identifier)( - arrow - ) - ) - } -} - -object SelectorExecutor { - case class Inputs[Query <: PipelineQuery]( - query: Query, - candidatesWithDetails: Seq[CandidateWithDetails]) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor/SelectorExecutorResult.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor/SelectorExecutorResult.docx new file mode 100644 index 000000000..517c47da1 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor/SelectorExecutorResult.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor/SelectorExecutorResult.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor/SelectorExecutorResult.scala deleted file mode 100644 index 8b6038feb..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/selector_executor/SelectorExecutorResult.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.product_mixer.core.service.selector_executor - -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.service.ExecutorResult - -case class SelectorExecutorResult( - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - individualSelectorResults: Seq[SelectorResult]) - extends ExecutorResult diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/slice/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/slice/BUILD deleted file mode 100644 index 4cb8369e2..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/slice/BUILD +++ /dev/null @@ -1,22 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "configapi/configapi-core", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry", - "stitch/stitch-core", - "strato/config/src/thrift/com/twitter/strato/graphql:api-media-graphql-scala", - "strato/config/src/thrift/com/twitter/strato/graphql:graphql-scala", - "strato/config/src/thrift/com/twitter/strato/graphql:topics-graphql-scala", - "util/util-core:scala", - ], - exports = [ - "strato/config/src/thrift/com/twitter/strato/graphql:graphql-scala", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/slice/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/slice/BUILD.docx new file mode 100644 index 000000000..322c2e03b Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/slice/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/slice/SliceService.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/slice/SliceService.docx new file mode 100644 index 000000000..66f9bb207 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/slice/SliceService.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/slice/SliceService.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/slice/SliceService.scala deleted file mode 100644 index 0dbda7321..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/slice/SliceService.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.product_mixer.core.service.slice - -import com.twitter.product_mixer.core.model.marshalling.request.Request -import com.twitter.product_mixer.core.pipeline.product.ProductPipelineRequest -import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry -import com.twitter.stitch.Stitch -import com.twitter.strato.graphql.thriftscala.SliceResult -import com.twitter.timelines.configapi.Params - -import javax.inject.Inject -import javax.inject.Singleton -import scala.reflect.runtime.universe.TypeTag - -/** - * Look up and execute Slice products in the [[ProductPipelineRegistry]] - */ -@Singleton -class SliceService @Inject() (productPipelineRegistry: ProductPipelineRegistry) { - - def getSliceResponse[RequestType <: Request]( - request: RequestType, - params: Params - )( - implicit requestTypeTag: TypeTag[RequestType] - ): Stitch[SliceResult] = - productPipelineRegistry - .getProductPipeline[RequestType, SliceResult](request.product) - .process(ProductPipelineRequest(request, params)) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transformer_executor/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transformer_executor/BUILD deleted file mode 100644 index 2e7568b01..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transformer_executor/BUILD +++ /dev/null @@ -1,17 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transformer_executor/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transformer_executor/BUILD.docx new file mode 100644 index 000000000..2965bfbd2 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transformer_executor/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transformer_executor/PerCandidateTransformerExecutor.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transformer_executor/PerCandidateTransformerExecutor.docx new file mode 100644 index 000000000..91b1f60cb Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transformer_executor/PerCandidateTransformerExecutor.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transformer_executor/PerCandidateTransformerExecutor.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transformer_executor/PerCandidateTransformerExecutor.scala deleted file mode 100644 index 19a3ed126..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transformer_executor/PerCandidateTransformerExecutor.scala +++ /dev/null @@ -1,35 +0,0 @@ -package com.twitter.product_mixer.core.service.transformer_executor - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.functional_component.transformer.Transformer -import com.twitter.product_mixer.core.service.Executor -import com.twitter.stitch.Arrow -import com.twitter.util.Try -import javax.inject.Inject -import javax.inject.Singleton - -/** - * For wrapping [[Transformer]]s that are applied per-candidate - * - * Records a single span for running all the components, - * but stats per-component. - */ -@Singleton -class PerCandidateTransformerExecutor @Inject() (override val statsReceiver: StatsReceiver) - extends Executor { - - def arrow[In, Out]( - transformer: Transformer[In, Out], - context: Executor.Context, - ): Arrow[Seq[In], Seq[Try[Out]]] = { - val perCandidateArrow = wrapPerCandidateComponentWithExecutorBookkeepingWithoutTracing( - context, - transformer.identifier - )(Arrow.map(transformer.transform)).liftToTry - - wrapComponentsWithTracingOnly( - context, - transformer.identifier - )(Arrow.sequence(perCandidateArrow)) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transformer_executor/TransformerExecutor.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transformer_executor/TransformerExecutor.docx new file mode 100644 index 000000000..2b04700a0 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transformer_executor/TransformerExecutor.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transformer_executor/TransformerExecutor.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transformer_executor/TransformerExecutor.scala deleted file mode 100644 index bdefad6f3..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transformer_executor/TransformerExecutor.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.product_mixer.core.service.transformer_executor - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.functional_component.transformer.Transformer -import com.twitter.product_mixer.core.service.Executor -import com.twitter.stitch.Arrow - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class TransformerExecutor @Inject() (override val statsReceiver: StatsReceiver) extends Executor { - def arrow[In, Out]( - transformer: Transformer[In, Out], - context: Executor.Context - ): Arrow[In, Out] = { - wrapComponentWithExecutorBookkeeping( - context, - transformer.identifier - )(Arrow.map(transformer.transform)) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transport_marshaller_executor/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transport_marshaller_executor/BUILD deleted file mode 100644 index 7cedc62ae..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transport_marshaller_executor/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "stitch/stitch-core", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transport_marshaller_executor/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transport_marshaller_executor/BUILD.docx new file mode 100644 index 000000000..81a2c0a49 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transport_marshaller_executor/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transport_marshaller_executor/TransportMarshallerExecutor.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transport_marshaller_executor/TransportMarshallerExecutor.docx new file mode 100644 index 000000000..852c3bf07 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transport_marshaller_executor/TransportMarshallerExecutor.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transport_marshaller_executor/TransportMarshallerExecutor.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transport_marshaller_executor/TransportMarshallerExecutor.scala deleted file mode 100644 index c93c9000e..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/transport_marshaller_executor/TransportMarshallerExecutor.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.product_mixer.core.service.transport_marshaller_executor - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.service.Executor -import com.twitter.product_mixer.core.service.ExecutorResult -import com.twitter.product_mixer.core.service.transport_marshaller_executor.TransportMarshallerExecutor.Inputs -import com.twitter.product_mixer.core.service.transport_marshaller_executor.TransportMarshallerExecutor.Result -import com.twitter.stitch.Arrow -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Executes a [[TransportMarshaller]]. - * - * @note This is a synchronous transform, so we don't observe it directly. Failures and such - * can be observed at the parent pipeline. - */ -@Singleton -class TransportMarshallerExecutor @Inject() (override val statsReceiver: StatsReceiver) - extends Executor { - - def arrow[DomainResponseType <: HasMarshalling, TransportResponseType]( - marshaller: TransportMarshaller[DomainResponseType, TransportResponseType], - context: Executor.Context - ): Arrow[Inputs[DomainResponseType], Result[TransportResponseType]] = { - val arrow = - Arrow.map[Inputs[DomainResponseType], Result[TransportResponseType]] { - case Inputs(domainResponse) => Result(marshaller(domainResponse)) - } - - wrapComponentWithExecutorBookkeeping(context, marshaller.identifier)(arrow) - } -} - -object TransportMarshallerExecutor { - case class Inputs[DomainResponseType <: HasMarshalling](domainResponse: DomainResponseType) - case class Result[TransportResponseType](result: TransportResponseType) extends ExecutorResult -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urp/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urp/BUILD deleted file mode 100644 index 1f4780222..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urp/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/lihaoyi:pprint", - "3rdparty/jvm/javax/inject:javax.inject", - "configapi/configapi-core", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urp", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urp", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry", - "stitch/stitch-core", - "util/util-core:scala", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urp/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urp/BUILD.docx new file mode 100644 index 000000000..42c6a3784 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urp/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urp/UrpService.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urp/UrpService.docx new file mode 100644 index 000000000..6fef69384 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urp/UrpService.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urp/UrpService.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urp/UrpService.scala deleted file mode 100644 index 0e08f670e..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urp/UrpService.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.product_mixer.core.service.urp - -import com.twitter.pages.render.{thriftscala => urp} -import com.twitter.product_mixer.core.model.marshalling.request.Request -import com.twitter.product_mixer.core.pipeline.product.ProductPipelineRequest -import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.Params - -import javax.inject.Inject -import javax.inject.Singleton -import scala.reflect.runtime.universe.TypeTag - -@Singleton -class UrpService @Inject() (productPipelineRegistry: ProductPipelineRegistry) { - - def getUrpResponse[RequestType <: Request]( - request: RequestType, - params: Params - )( - implicit requestTypeTag: TypeTag[RequestType] - ): Stitch[urp.Page] = - productPipelineRegistry - .getProductPipeline[RequestType, urp.Page](request.product) - .process(ProductPipelineRequest(request, params)) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt/BUILD deleted file mode 100644 index b4c6c49a1..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt/BUILD +++ /dev/null @@ -1,20 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "configapi/configapi-core", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry", - "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", - "stitch/stitch-core", - "util/util-core:scala", - "util/util-jackson/src/main/scala/com/twitter/util/jackson", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt/BUILD.docx new file mode 100644 index 000000000..18cec9f96 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt/UrtService.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt/UrtService.docx new file mode 100644 index 000000000..6c81be70f Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt/UrtService.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt/UrtService.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt/UrtService.scala deleted file mode 100644 index 2f8c79d60..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt/UrtService.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.product_mixer.core.service.urt - -import com.fasterxml.jackson.databind.SerializationFeature -import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.UrtTransportMarshaller -import com.twitter.product_mixer.core.model.marshalling.request.Request -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.pipeline_failure.ProductDisabled -import com.twitter.product_mixer.core.pipeline.product.ProductPipelineRequest -import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry -import com.twitter.product_mixer.core.{thriftscala => t} -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.Params -import com.twitter.timelines.render.{thriftscala => urt} -import com.twitter.util.jackson.ScalaObjectMapper - -import javax.inject.Inject -import javax.inject.Singleton -import scala.reflect.runtime.universe.TypeTag - -/** - * Look up and execute products in the [[ProductPipelineRegistry]] - */ -@Singleton -class UrtService @Inject() (productPipelineRegistry: ProductPipelineRegistry) { - - def getUrtResponse[RequestType <: Request]( - request: RequestType, - params: Params - )( - implicit requestTypeTag: TypeTag[RequestType] - ): Stitch[urt.TimelineResponse] = - productPipelineRegistry - .getProductPipeline[RequestType, urt.TimelineResponse](request.product) - .process(ProductPipelineRequest(request, params)) - .handle { - // Detect ProductDisabled and convert it to TimelineUnavailable - case pipelineFailure: PipelineFailure if pipelineFailure.category == ProductDisabled => - UrtTransportMarshaller.unavailable("") - } - - /** - * Get detailed pipeline execution as a serialized JSON String - */ - def getPipelineExecutionResult[RequestType <: Request]( - request: RequestType, - params: Params - )( - implicit requestTypeTag: TypeTag[RequestType] - ): Stitch[t.PipelineExecutionResult] = - productPipelineRegistry - .getProductPipeline[RequestType, urt.TimelineResponse](request.product) - .arrow(ProductPipelineRequest(request, params)).map { detailedResult => - val mapper = ScalaObjectMapper() - // configure so that exception is not thrown whenever case class is not serializable - mapper.underlying.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false) - val serializedJSON = mapper.writePrettyString(detailedResult) - t.PipelineExecutionResult(serializedJSON) - } - -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util/BUILD deleted file mode 100644 index 14035e637..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util/BUILD +++ /dev/null @@ -1,24 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "finagle/finagle-core/src/main", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "servo/util", - "snowflake:id", - "stitch/stitch-core", - "util/util-core:util-core-util", - "util/util-stats", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "servo/util", - "snowflake:id", - "stitch/stitch-core", - "util/util-core:util-core-util", - "util/util-stats", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util/BUILD.docx new file mode 100644 index 000000000..a4434c335 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util/FuturePools.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util/FuturePools.docx new file mode 100644 index 000000000..db920d7e7 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util/FuturePools.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util/FuturePools.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util/FuturePools.scala deleted file mode 100644 index 52885db34..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util/FuturePools.scala +++ /dev/null @@ -1,101 +0,0 @@ -package com.twitter.product_mixer.core.util - -import com.twitter.concurrent.NamedPoolThreadFactory -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.util.Duration -import com.twitter.util.FuturePool - -import java.util.concurrent.ArrayBlockingQueue -import java.util.concurrent.BlockingQueue -import java.util.concurrent.LinkedBlockingQueue -import java.util.concurrent.ThreadPoolExecutor -import java.util.concurrent.TimeUnit - -/** - * Utility for making [[FuturePool]] with finite thread counts and different work queue options - * while also monitoring the size of the work queue that's used. - * - * This only handles the cases where the number of threads are bounded. - * For unbounded numbers of threads in a [[FuturePool]] use [[FuturePool.interruptibleUnboundedPool]] instead. - */ -object FuturePools { - - /** - * Makes a [[FuturePool]] with a fixed number of threads and a work queue - * with a maximum size of `maxWorkQueueSize`. - * - * @note the [[FuturePool]] returns a failed [[com.twitter.util.Future]]s containing - * [[java.util.concurrent.RejectedExecutionException]] when trying to enqueue - * work when the work queue is full. - */ - def boundedFixedThreadPool( - name: String, - fixedThreadCount: Int, - workQueueSize: Int, - statsReceiver: StatsReceiver - ): FuturePool = - futurePool( - name = name, - minThreads = fixedThreadCount, - maxThreads = fixedThreadCount, - keepIdleThreadsAlive = Duration.Zero, - workQueue = new ArrayBlockingQueue[Runnable](workQueueSize), - statsReceiver = statsReceiver - ) - - /** - * Makes a [[FuturePool]] with a fix number of threads and an unbounded work queue - * - * @note Since the work queue is unbounded, this will fill up memory if the available worker threads can't keep up - */ - def unboundedFixedThreadPool( - name: String, - fixedThreadCount: Int, - statsReceiver: StatsReceiver - ): FuturePool = - futurePool( - name = name, - minThreads = fixedThreadCount, - maxThreads = fixedThreadCount, - keepIdleThreadsAlive = Duration.Zero, - workQueue = new LinkedBlockingQueue[Runnable], - statsReceiver = statsReceiver - ) - - /** - * Makes a [[FuturePool]] with the provided thread configuration and - * who's `workQueue` is monitored by a [[com.twitter.finagle.stats.Gauge]] - * - * @note if `minThreads` == `maxThreads` then the threadpool has a fixed size - * - * @param name name of the threadpool - * @param minThreads minimum number of threads in the pool - * @param maxThreads maximum number of threads in the pool - * @param keepIdleThreadsAlive threads that are idle for this long will be removed from - * the pool if there are more than `minThreads` in the pool. - * If the pool size is fixed this is ignored. - */ - private def futurePool( - name: String, - minThreads: Int, - maxThreads: Int, - keepIdleThreadsAlive: Duration, - workQueue: BlockingQueue[Runnable], - statsReceiver: StatsReceiver - ): FuturePool = { - val gaugeReference = statsReceiver.addGauge("workQueueSize")(workQueue.size()) - - val threadFactory = new NamedPoolThreadFactory(name, makeDaemons = true) - - val executorService = - new ThreadPoolExecutor( - minThreads, - maxThreads, // ignored by ThreadPoolExecutor when an unbounded queue is provided - keepIdleThreadsAlive.inMillis, - TimeUnit.MILLISECONDS, - workQueue, - threadFactory) - - FuturePool.interruptible(executorService) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util/OffloadFuturePools.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util/OffloadFuturePools.docx new file mode 100644 index 000000000..6e6422d72 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util/OffloadFuturePools.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util/OffloadFuturePools.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util/OffloadFuturePools.scala deleted file mode 100644 index cb0a92657..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util/OffloadFuturePools.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.twitter.product_mixer.core.util - -import com.twitter.finagle.offload.OffloadFuturePool -import com.twitter.util.Future - -object OffloadFuturePools { - - def parallelize[In, Out]( - inputSeq: Seq[In], - transformer: In => Out, - parallelism: Int - ): Future[Seq[Out]] = { - parallelize(inputSeq, transformer.andThen(Some(_)), parallelism, None).map(_.flatten) - } - - def parallelize[In, Out]( - inputSeq: Seq[In], - transformer: In => Out, - parallelism: Int, - default: Out - ): Future[Seq[Out]] = { - val threadProcessFutures = (0 until parallelism).map { i => - OffloadFuturePool.getPool(partitionAndProcessInput(inputSeq, transformer, i, parallelism)) - } - - val resultMap = Future.collect(threadProcessFutures).map(_.flatten.toMap) - - Future.collect { - inputSeq.indices.map { idx => - resultMap.map(_.getOrElse(idx, default)) - } - } - } - - private def partitionAndProcessInput[In, Out]( - inputSeq: Seq[In], - transformer: In => Out, - threadId: Int, - parallelism: Int - ): Seq[(Int, Out)] = { - partitionInputForThread(inputSeq, threadId, parallelism) - .map { - case (inputRecord, idx) => - (idx, transformer(inputRecord)) - } - } - - private def partitionInputForThread[In]( - inputSeq: Seq[In], - threadId: Int, - parallelism: Int - ): Seq[(In, Int)] = { - inputSeq.zipWithIndex - .filter { - case (_, idx) => idx % parallelism == threadId - case _ => false - } - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util/SortIndexBuilder.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util/SortIndexBuilder.docx new file mode 100644 index 000000000..af6beff9f Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util/SortIndexBuilder.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util/SortIndexBuilder.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util/SortIndexBuilder.scala deleted file mode 100644 index eb55b231b..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util/SortIndexBuilder.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.twitter.product_mixer.core.util - -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.util.Time - -object SortIndexBuilder { - - /** the [[Time]] from a [[SnowflakeId]] */ - def idToTime(id: Long): Time = - Time.fromMilliseconds(SnowflakeId.unixTimeMillisOrFloorFromId(id)) - - /** the first [[SnowflakeId]] possible for a given [[Time]] */ - def timeToId(time: Time): Long = SnowflakeId.firstIdFor(time) - - /** the first [[SnowflakeId]] possible for a given unix epoch millis */ - def timeToId(timeMillis: Long): Long = SnowflakeId.firstIdFor(timeMillis) -} diff --git a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/http_client/BUILD b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/http_client/BUILD deleted file mode 100644 index 2be84e6a8..000000000 --- a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/http_client/BUILD +++ /dev/null @@ -1,22 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "finagle/finagle-core/src/main", - "finagle/finagle-http/src/main/scala", - "finagle/finagle-thriftmux/src/main/scala", - "finatra-internal/mtls-http/src/main/scala", - "finatra-internal/mtls-thriftmux/src/main/scala", - "util/util-core", - ], - exports = [ - "finagle/finagle-core/src/main", - "finagle/finagle-http/src/main/scala", - "finagle/finagle-thriftmux/src/main/scala", - "finatra-internal/mtls-http/src/main/scala", - "finatra-internal/mtls-thriftmux/src/main/scala", - "util/util-core", - ], -) diff --git a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/http_client/BUILD.docx b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/http_client/BUILD.docx new file mode 100644 index 000000000..2e4c4bf2e Binary files /dev/null and b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/http_client/BUILD.docx differ diff --git a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/http_client/FinagleHttpClientBuilder.docx b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/http_client/FinagleHttpClientBuilder.docx new file mode 100644 index 000000000..61a0fe771 Binary files /dev/null and b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/http_client/FinagleHttpClientBuilder.docx differ diff --git a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/http_client/FinagleHttpClientBuilder.scala b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/http_client/FinagleHttpClientBuilder.scala deleted file mode 100644 index fd203cacf..000000000 --- a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/http_client/FinagleHttpClientBuilder.scala +++ /dev/null @@ -1,57 +0,0 @@ -package com.twitter.product_mixer.shared_library.http_client - -import com.twitter.finagle.Http -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.mtls.client.MtlsStackClient._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.util.Duration - -object FinagleHttpClientBuilder { - - /** - * Build a Finagle HTTP client with S2S Auth / Mutual TLS - * - * @param requestTimeout HTTP client request timeout - * @param connectTimeout HTTP client transport connect timeout - * @param acquisitionTimeout HTTP client session acquisition timeout - * @param serviceIdentifier Service ID used to S2S Auth - * @param statsReceiver Stats - * - * @return Finagle HTTP Client with S2S Auth / Mutual TLS - */ - def buildFinagleHttpClientMutualTls( - requestTimeout: Duration, - connectTimeout: Duration, - acquisitionTimeout: Duration, - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): Http.Client = - buildFinagleHttpClient( - requestTimeout = requestTimeout, - connectTimeout = connectTimeout, - acquisitionTimeout = acquisitionTimeout, - statsReceiver = statsReceiver - ).withMutualTls(serviceIdentifier) - - /** - * Build a Finagle HTTP client - * - * @param requestTimeout HTTP client request timeout - * @param connectTimeout HTTP client transport connect timeout - * @param acquisitionTimeout HTTP client session acquisition timeout - * @param statsReceiver stats - * - * @return Finagle HTTP Client - */ - def buildFinagleHttpClient( - requestTimeout: Duration, - connectTimeout: Duration, - acquisitionTimeout: Duration, - statsReceiver: StatsReceiver, - ): Http.Client = - Http.client - .withStatsReceiver(statsReceiver) - .withRequestTimeout(requestTimeout) - .withTransport.connectTimeout(connectTimeout) - .withSession.acquisitionTimeout(acquisitionTimeout) -} diff --git a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/http_client/FinagleHttpClientWithProxyBuilder.docx b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/http_client/FinagleHttpClientWithProxyBuilder.docx new file mode 100644 index 000000000..1308edfc7 Binary files /dev/null and b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/http_client/FinagleHttpClientWithProxyBuilder.docx differ diff --git a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/http_client/FinagleHttpClientWithProxyBuilder.scala b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/http_client/FinagleHttpClientWithProxyBuilder.scala deleted file mode 100644 index a2f4a5764..000000000 --- a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/http_client/FinagleHttpClientWithProxyBuilder.scala +++ /dev/null @@ -1,97 +0,0 @@ -package com.twitter.product_mixer.shared_library.http_client - -import com.twitter.finagle.Http -import com.twitter.finagle.Service -import com.twitter.finagle.client.Transporter -import com.twitter.finagle.http.ProxyCredentials -import com.twitter.finagle.http.Request -import com.twitter.finagle.http.Response -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.shared_library.http_client.FinagleHttpClientBuilder.buildFinagleHttpClient -import com.twitter.util.Duration - -object FinagleHttpClientWithProxyBuilder { - - /** - * Build a Finagle HTTP client with Egress Proxy support using Credentials - * - * @param twitterProxyHostPort Twitter egress proxy host port - * @param remoteProxyHostPort Remote proxy host port - * @param requestTimeout HTTP client request timeout - * @param connectTimeout HTTP client transport connect timeout - * @param acquisitionTimeout HTTP client session acquisition timeout - * @param proxyCredentials Proxy credentials - * @param statsReceiver Stats - * - * @return Finagle HTTP client with Egress Proxy support using Credentials - */ - def buildFinagleHttpClientWithCredentialProxy( - twitterProxyHostPort: HttpHostPort, - remoteProxyHostPort: HttpHostPort, - requestTimeout: Duration, - connectTimeout: Duration, - acquisitionTimeout: Duration, - proxyCredentials: ProxyCredentials, - statsReceiver: StatsReceiver, - ): Http.Client = { - val httpClient = buildFinagleHttpClient( - requestTimeout = requestTimeout, - connectTimeout = connectTimeout, - acquisitionTimeout = acquisitionTimeout, - statsReceiver = statsReceiver - ) - - httpClient.withTransport - .httpProxyTo( - host = remoteProxyHostPort.toString, - credentials = Transporter.Credentials(proxyCredentials.username, proxyCredentials.password)) - .withTls(remoteProxyHostPort.host) - } - - /** - * Build a Finagle HTTP client with Egress Proxy support - * - * @param twitterProxyHostPort Twitter egress proxy host port - * @param remoteProxyHostPort Remote proxy host port - * @param requestTimeout HTTP client request timeout - * @param connectTimeout HTTP client transport connect timeout - * @param acquisitionTimeout HTTP client session acquisition timeout - * @param statsReceiver Stats - * - * @return Finagle HTTP client with Egress Proxy support - */ - def buildFinagleHttpClientWithProxy( - twitterProxyHostPort: HttpHostPort, - remoteProxyHostPort: HttpHostPort, - requestTimeout: Duration, - connectTimeout: Duration, - acquisitionTimeout: Duration, - statsReceiver: StatsReceiver, - ): Http.Client = { - val httpClient = buildFinagleHttpClient( - requestTimeout = requestTimeout, - connectTimeout = connectTimeout, - acquisitionTimeout = acquisitionTimeout, - statsReceiver = statsReceiver - ) - - httpClient.withTransport - .httpProxyTo(remoteProxyHostPort.toString) - .withTls(remoteProxyHostPort.host) - } - - /** - * Build a Finagle HTTP service with Egress Proxy support - * - * @param finagleHttpClientWithProxy Finagle HTTP client from which to build the service - * @param twitterProxyHostPort Twitter egress proxy host port - * - * @return Finagle HTTP service with Egress Proxy support - */ - def buildFinagleHttpServiceWithProxy( - finagleHttpClientWithProxy: Http.Client, - twitterProxyHostPort: HttpHostPort - ): Service[Request, Response] = { - finagleHttpClientWithProxy.newService(twitterProxyHostPort.toString) - } -} diff --git a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/http_client/HttpHostPort.docx b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/http_client/HttpHostPort.docx new file mode 100644 index 000000000..a60b5a39e Binary files /dev/null and b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/http_client/HttpHostPort.docx differ diff --git a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/http_client/HttpHostPort.scala b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/http_client/HttpHostPort.scala deleted file mode 100644 index ceab3d24e..000000000 --- a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/http_client/HttpHostPort.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.product_mixer.shared_library.http_client - -case class HttpHostPort(host: String, port: Int) { - override val toString: String = s"$host:$port" -} diff --git a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/manhattan_client/BUILD b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/manhattan_client/BUILD deleted file mode 100644 index 3de790d1f..000000000 --- a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/manhattan_client/BUILD +++ /dev/null @@ -1,24 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authorization", - "finagle/finagle-core/src/main", - "finagle/finagle-thriftmux/src/main/scala", - "src/scala/com/twitter/storehaus_internal/manhattan/config", - "src/thrift/com/twitter/manhattan:v1-scala", - "storage/clients/manhattan", - "util/util-core", - ], - exports = [ - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authorization", - "finagle/finagle-core/src/main", - "src/scala/com/twitter/storehaus_internal/manhattan/config", - "storage/clients/manhattan", - "util/util-core", - ], -) diff --git a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/manhattan_client/BUILD.docx b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/manhattan_client/BUILD.docx new file mode 100644 index 000000000..edf9c906a Binary files /dev/null and b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/manhattan_client/BUILD.docx differ diff --git a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/manhattan_client/ManhattanClientBuilder.docx b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/manhattan_client/ManhattanClientBuilder.docx new file mode 100644 index 000000000..16b7acd77 Binary files /dev/null and b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/manhattan_client/ManhattanClientBuilder.docx differ diff --git a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/manhattan_client/ManhattanClientBuilder.scala b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/manhattan_client/ManhattanClientBuilder.scala deleted file mode 100644 index a9903a4e0..000000000 --- a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/manhattan_client/ManhattanClientBuilder.scala +++ /dev/null @@ -1,116 +0,0 @@ -package com.twitter.product_mixer.shared_library.manhattan_client - -import com.twitter.finagle.mtls.authentication.EmptyServiceIdentifier -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.ssl.OpportunisticTls -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.manhattan.v1.{thriftscala => mh} -import com.twitter.storage.client.manhattan.kv.Experiments -import com.twitter.storage.client.manhattan.kv.Experiments.Experiment -import com.twitter.storage.client.manhattan.kv.Guarantee -import com.twitter.storage.client.manhattan.kv.ManhattanKVClient -import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams -import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpoint -import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpointBuilder -import com.twitter.storage.client.manhattan.kv.NoMtlsParams -import com.twitter.storehaus_internal.manhattan.ManhattanCluster -import com.twitter.util.Duration - -object ManhattanClientBuilder { - - /** - * Build a ManhattanKVClient/Endpoint [[ManhattanKVEndpoint]] / [[ManhattanKVClient]] - * - * @param cluster Manhattan cluster - * @param appId Manhattan appid - * @param numTries Max number of times to try - * @param maxTimeout Max request timeout - * @param maxItemsPerRequest Max items per request - * @param guarantee Consistency guarantee - * @param serviceIdentifier Service ID used to S2S Auth - * @param statsReceiver Stats - * @param experiments MH client experiments to include - * @return ManhattanKVEndpoint - */ - def buildManhattanEndpoint( - cluster: ManhattanCluster, - appId: String, - numTries: Int, - maxTimeout: Duration, - guarantee: Guarantee, - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver, - maxItemsPerRequest: Int = 100, - experiments: Seq[Experiment] = Seq(Experiments.ApertureLoadBalancer) - ): ManhattanKVEndpoint = { - val client = buildManhattanClient( - cluster, - appId, - serviceIdentifier, - experiments - ) - - ManhattanKVEndpointBuilder(client) - .defaultGuarantee(guarantee) - .defaultMaxTimeout(maxTimeout) - .maxRetryCount(numTries) - .maxItemsPerRequest(maxItemsPerRequest) - .statsReceiver(statsReceiver) - .build() - } - - /** - * Build a ManhattanKVClient - * - * @param cluster Manhattan cluster - * @param appId Manhattan appid - * @param serviceIdentifier Service ID used to S2S Auth - * @param experiments MH client experiments to include - * - * @return ManhattanKVClient - */ - def buildManhattanClient( - cluster: ManhattanCluster, - appId: String, - serviceIdentifier: ServiceIdentifier, - experiments: Seq[Experiment] = Seq(Experiments.ApertureLoadBalancer) - ): ManhattanKVClient = { - val mtlsParams = serviceIdentifier match { - case EmptyServiceIdentifier => NoMtlsParams - case serviceIdentifier => - ManhattanKVClientMtlsParams( - serviceIdentifier = serviceIdentifier, - opportunisticTls = OpportunisticTls.Required) - } - - val label = s"manhattan/${cluster.prefix}" - - new ManhattanKVClient( - appId = appId, - dest = cluster.wilyName, - mtlsParams = mtlsParams, - label = label, - experiments = experiments - ) - } - - def buildManhattanV1FinagleClient( - cluster: ManhattanCluster, - serviceIdentifier: ServiceIdentifier, - experiments: Seq[Experiment] = Seq(Experiments.ApertureLoadBalancer) - ): mh.ManhattanCoordinator.MethodPerEndpoint = { - val mtlsParams = serviceIdentifier match { - case EmptyServiceIdentifier => NoMtlsParams - case serviceIdentifier => - ManhattanKVClientMtlsParams( - serviceIdentifier = serviceIdentifier, - opportunisticTls = OpportunisticTls.Required) - } - - val label = s"manhattan/${cluster.prefix}" - - Experiments - .clientWithExperiments(experiments, mtlsParams) - .build[mh.ManhattanCoordinator.MethodPerEndpoint](cluster.wilyName, label) - } -} diff --git a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/memcached_client/BUILD b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/memcached_client/BUILD deleted file mode 100644 index dfd7ba2b1..000000000 --- a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/memcached_client/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "finagle/finagle-memcached/src/main/scala", - "finatra-internal/mtls-http/src/main/scala", - "finatra-internal/mtls-thriftmux/src/main/scala", - "servo/repo/src/main/scala", - ], - exports = [ - "finagle/finagle-memcached/src/main/scala", - "finatra-internal/mtls-http/src/main/scala", - "finatra-internal/mtls-thriftmux/src/main/scala", - "servo/repo/src/main/scala", - ], -) diff --git a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/memcached_client/BUILD.docx b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/memcached_client/BUILD.docx new file mode 100644 index 000000000..aace23a9e Binary files /dev/null and b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/memcached_client/BUILD.docx differ diff --git a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/memcached_client/MemcachedClientBuilder.docx b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/memcached_client/MemcachedClientBuilder.docx new file mode 100644 index 000000000..e9de0af45 Binary files /dev/null and b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/memcached_client/MemcachedClientBuilder.docx differ diff --git a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/memcached_client/MemcachedClientBuilder.scala b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/memcached_client/MemcachedClientBuilder.scala deleted file mode 100644 index 62453afbd..000000000 --- a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/memcached_client/MemcachedClientBuilder.scala +++ /dev/null @@ -1,117 +0,0 @@ -package com.twitter.product_mixer.shared_library.memcached_client - -import com.twitter.finagle.memcached.Client -import com.twitter.finagle.memcached.protocol.Command -import com.twitter.finagle.memcached.protocol.Response -import com.twitter.finagle.mtls.client.MtlsStackClient._ -import com.twitter.finagle.service.RetryExceptionsFilter -import com.twitter.finagle.service.RetryPolicy -import com.twitter.finagle.service.TimeoutFilter -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.util.DefaultTimer -import com.twitter.finagle.GlobalRequestTimeoutException -import com.twitter.finagle.Memcached -import com.twitter.finagle.liveness.FailureAccrualFactory -import com.twitter.finagle.liveness.FailureAccrualPolicy -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.hashing.KeyHasher -import com.twitter.util.Duration - -object MemcachedClientBuilder { - - /** - * Build a Finagle Memcached [[Client]]. - * - * @param destName Destination as a Wily path e.g. "/s/sample/sample". - * @param numTries Maximum number of times to try. - * @param requestTimeout Thrift client timeout per request. The Finagle default - * is unbounded which is almost never optimal. - * @param globalTimeout Thrift client total timeout. The Finagle default is - * unbounded which is almost never optimal. - * @param connectTimeout Thrift client transport connect timeout. The Finagle - * default of one second is reasonable but we lower this - * to match acquisitionTimeout for consistency. - * @param acquisitionTimeout Thrift client session acquisition timeout. The Finagle - * default is unbounded which is almost never optimal. - * @param serviceIdentifier Service ID used to S2S Auth. - * @param statsReceiver Stats. - * @param failureAccrualPolicy Policy to determine when to mark a cache server as dead. - * Memcached client will use default failure accrual policy - * if it is not set. - * @param keyHasher Hash algorithm that hashes a key into a 32-bit or 64-bit - * number. Memcached client will use default hash algorithm - * if it is not set. - * - * @see [[https://confluence.twitter.biz/display/CACHE/Finagle-memcached+User+Guide user guide]] - * @return Finagle Memcached [[Client]] - */ - def buildMemcachedClient( - destName: String, - numTries: Int, - requestTimeout: Duration, - globalTimeout: Duration, - connectTimeout: Duration, - acquisitionTimeout: Duration, - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver, - failureAccrualPolicy: Option[FailureAccrualPolicy] = None, - keyHasher: Option[KeyHasher] = None - ): Client = { - buildRawMemcachedClient( - numTries, - requestTimeout, - globalTimeout, - connectTimeout, - acquisitionTimeout, - serviceIdentifier, - statsReceiver, - failureAccrualPolicy, - keyHasher - ).newRichClient(destName) - } - - def buildRawMemcachedClient( - numTries: Int, - requestTimeout: Duration, - globalTimeout: Duration, - connectTimeout: Duration, - acquisitionTimeout: Duration, - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver, - failureAccrualPolicy: Option[FailureAccrualPolicy] = None, - keyHasher: Option[KeyHasher] = None - ): Memcached.Client = { - val globalTimeoutFilter = new TimeoutFilter[Command, Response]( - timeout = globalTimeout, - exception = new GlobalRequestTimeoutException(globalTimeout), - timer = DefaultTimer) - val retryFilter = new RetryExceptionsFilter[Command, Response]( - RetryPolicy.tries(numTries), - DefaultTimer, - statsReceiver) - - val client = Memcached.client.withTransport - .connectTimeout(connectTimeout) - .withMutualTls(serviceIdentifier) - .withSession - .acquisitionTimeout(acquisitionTimeout) - .withRequestTimeout(requestTimeout) - .withStatsReceiver(statsReceiver) - .filtered(globalTimeoutFilter.andThen(retryFilter)) - - (keyHasher, failureAccrualPolicy) match { - case (Some(hasher), Some(policy)) => - client - .withKeyHasher(hasher) - .configured(FailureAccrualFactory.Param(() => policy)) - case (Some(hasher), None) => - client - .withKeyHasher(hasher) - case (None, Some(policy)) => - client - .configured(FailureAccrualFactory.Param(() => policy)) - case _ => - client - } - } -} diff --git a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/observer/BUILD b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/observer/BUILD deleted file mode 100644 index 325ddbae3..000000000 --- a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/observer/BUILD +++ /dev/null @@ -1,19 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "servo/util", - "stitch/stitch-core", - "util/util-core:util-core-util", - "util/util-stats", - ], - exports = [ - "servo/util", - "stitch/stitch-core", - "util/util-core:util-core-util", - "util/util-stats", - ], -) diff --git a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/observer/BUILD.docx b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/observer/BUILD.docx new file mode 100644 index 000000000..da9e0a41c Binary files /dev/null and b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/observer/BUILD.docx differ diff --git a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/observer/Observer.docx b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/observer/Observer.docx new file mode 100644 index 000000000..befc8cd7a Binary files /dev/null and b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/observer/Observer.docx differ diff --git a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/observer/Observer.scala b/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/observer/Observer.scala deleted file mode 100644 index df9e8389d..000000000 --- a/product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/observer/Observer.scala +++ /dev/null @@ -1,203 +0,0 @@ -package com.twitter.product_mixer.shared_library.observer - -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.RollupStatsReceiver -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.servo.util.CancelledExceptionExtractor -import com.twitter.stitch.Arrow -import com.twitter.stitch.Stitch -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.util.Throwables -import com.twitter.util.Try - -/** - * Helper functions to observe requests, success, failures, cancellations, exceptions, and latency. - * Supports native functions and asynchronous operations. - */ -object Observer { - val Requests = "requests" - val Success = "success" - val Failures = "failures" - val Cancelled = "cancelled" - val Latency = "latency_ms" - - /** - * Helper function to observe a stitch - * - * @see [[StitchObserver]] - */ - def stitch[T](statsReceiver: StatsReceiver, scopes: String*): StitchObserver[T] = - new StitchObserver[T](statsReceiver, scopes) - - /** - * Helper function to observe an arrow - * - * @see [[ArrowObserver]] - */ - def arrow[In, Out](statsReceiver: StatsReceiver, scopes: String*): ArrowObserver[In, Out] = - new ArrowObserver[In, Out](statsReceiver, scopes) - - /** - * Helper function to observe a future - * - * @see [[FutureObserver]] - */ - def future[T](statsReceiver: StatsReceiver, scopes: String*): FutureObserver[T] = - new FutureObserver[T](statsReceiver, scopes) - - /** - * Helper function to observe a function - * - * @see [[FunctionObserver]] - */ - def function[T](statsReceiver: StatsReceiver, scopes: String*): FunctionObserver[T] = - new FunctionObserver[T](statsReceiver, scopes) - - /** - * [[StitchObserver]] can record latency stats, success counters, and - * detailed failure stats for the results of a Stitch computation. - */ - class StitchObserver[T]( - override val statsReceiver: StatsReceiver, - override val scopes: Seq[String]) - extends Observer[T] { - - /** - * Record stats for the provided Stitch. - * The result of the computation is passed through. - * - * @note the provided Stitch must contain the parts that need to be timed. - * Using this on just the result of the computation the latency stat - * will be incorrect. - */ - def apply(stitch: => Stitch[T]): Stitch[T] = - Stitch.time(stitch).map(observe.tupled).lowerFromTry - } - - /** - * [[ArrowObserver]] can record the latency stats, success counters, and - * detailed failure stats for the result of an Arrow computation. - * The result of the computation is passed through. - */ - class ArrowObserver[In, Out]( - override val statsReceiver: StatsReceiver, - override val scopes: Seq[String]) - extends Observer[Out] { - - /** - * Returns a new Arrow that records stats when it's run. - * The result of the Arrow is passed through. - * - * @note the provided Arrow must contain the parts that need to be timed. - * Using this on just the result of the computation the latency stat - * will be incorrect. - */ - def apply(arrow: Arrow[In, Out]): Arrow[In, Out] = - Arrow.time(arrow).map(observe.tupled).lowerFromTry - } - - /** - * [[FutureObserver]] can record latency stats, success counters, and - * detailed failure stats for the results of a Future computation. - */ - class FutureObserver[T]( - override val statsReceiver: StatsReceiver, - override val scopes: Seq[String]) - extends Observer[T] { - - /** - * Record stats for the provided Future. - * The result of the computation is passed through. - * - * @note the provided Future must contain the parts that need to be timed. - * Using this on just the result of the computation the latency stat - * will be incorrect. - */ - def apply(future: => Future[T]): Future[T] = - Stat - .timeFuture(latencyStat)(future) - .onSuccess(observeSuccess) - .onFailure(observeFailure) - } - - /** - * [[FunctionObserver]] can record latency stats, success counters, and - * detailed failure stats for the results of a computation computation. - */ - class FunctionObserver[T]( - override val statsReceiver: StatsReceiver, - override val scopes: Seq[String]) - extends Observer[T] { - - /** - * Record stats for the provided computation. - * The result of the computation is passed through. - * - * @note the provided computation must contain the parts that need to be timed. - * Using this on just the result of the computation the latency stat - * will be incorrect. - */ - def apply(f: => T): T = { - Try(Stat.time(latencyStat)(f)) - .onSuccess(observeSuccess) - .onFailure(observeFailure) - .apply() - } - } - - /** [[Observer]] provides methods for recording latency, success, and failure stats */ - trait Observer[T] { - protected val statsReceiver: StatsReceiver - - /** Scopes that prefix all stats */ - protected val scopes: Seq[String] - - private val rollupStatsReceiver = new RollupStatsReceiver(statsReceiver.scope(scopes: _*)) - private val requestsCounter: Counter = statsReceiver.counter(scopes :+ Requests: _*) - private val successCounter: Counter = statsReceiver.counter(scopes :+ Success: _*) - - // create the stats so their metrics paths are always present but - // defer to the [[RollupStatsReceiver]] to increment these stats - rollupStatsReceiver.counter(Failures) - rollupStatsReceiver.counter(Cancelled) - - /** Serialize a throwable and it's causes into a seq of Strings for scoping metrics */ - protected def serializeThrowable(throwable: Throwable): Seq[String] = - Throwables.mkString(throwable) - - /** Used to record latency in milliseconds */ - protected val latencyStat: Stat = statsReceiver.stat(scopes :+ Latency: _*) - - /** Records the latency from a [[Duration]] */ - protected val observeLatency: Duration => Unit = { latency => - latencyStat.add(latency.inMilliseconds) - } - - /** Records successes */ - protected val observeSuccess: T => Unit = { _ => - requestsCounter.incr() - successCounter.incr() - } - - /** Records failures and failure details */ - protected val observeFailure: Throwable => Unit = { - case CancelledExceptionExtractor(throwable) => - requestsCounter.incr() - rollupStatsReceiver.counter(Cancelled +: serializeThrowable(throwable): _*).incr() - case throwable => - requestsCounter.incr() - rollupStatsReceiver.counter(Failures +: serializeThrowable(throwable): _*).incr() - } - - /** Records the latency, successes, and failures */ - protected val observe: (Try[T], Duration) => Try[T] = - (response: Try[T], runDuration: Duration) => { - observeLatency(runDuration) - response - .onSuccess(observeSuccess) - .onFailure(observeFailure) - } - } -}