diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ClearCacheInstructionBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ClearCacheInstructionBuilder.docx new file mode 100644 index 000000000..c3d1831a4 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ClearCacheInstructionBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ClearCacheInstructionBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ClearCacheInstructionBuilder.scala deleted file mode 100644 index 458c25ff5..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ClearCacheInstructionBuilder.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.product_mixer.component_library.premarshaller.urt.builder - -import com.twitter.product_mixer.core.model.marshalling.response.urt.ClearCacheTimelineInstruction -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class ClearCacheInstructionBuilder[Query <: PipelineQuery]( - override val includeInstruction: IncludeInstruction[Query] = AlwaysInclude) - extends UrtInstructionBuilder[Query, ClearCacheTimelineInstruction] { - - override def build( - query: Query, - entries: Seq[TimelineEntry] - ): Seq[ClearCacheTimelineInstruction] = - if (includeInstruction(query, entries)) Seq(ClearCacheTimelineInstruction()) else Seq.empty -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/FeaturePassThroughCursorBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/FeaturePassThroughCursorBuilder.docx new file mode 100644 index 000000000..82298d745 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/FeaturePassThroughCursorBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/FeaturePassThroughCursorBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/FeaturePassThroughCursorBuilder.scala deleted file mode 100644 index 368336b2e..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/FeaturePassThroughCursorBuilder.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.product_mixer.component_library.premarshaller.urt.builder - -import com.twitter.product_mixer.component_library.model.cursor.UrtPassThroughCursor -import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorType -import com.twitter.product_mixer.core.pipeline.HasPipelineCursor -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class PassThroughCursorBuilder[ - -Query <: PipelineQuery with HasPipelineCursor[UrtPassThroughCursor] -]( - cursorFeature: Feature[Query, String], - override val cursorType: CursorType) - extends UrtCursorBuilder[Query] { - - override val includeOperation: IncludeInstruction[Query] = { (query, _) => - query.features.exists(_.getOrElse(cursorFeature, "").nonEmpty) - } - - override def cursorValue( - query: Query, - entries: Seq[TimelineEntry] - ): String = - UrtCursorSerializer.serializeCursor( - UrtPassThroughCursor( - cursorSortIndex(query, entries), - query.features.map(_.get(cursorFeature)).getOrElse(""), - cursorType = Some(cursorType) - ) - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/IncludeInstruction.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/IncludeInstruction.docx new file mode 100644 index 000000000..07756cb4d Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/IncludeInstruction.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/IncludeInstruction.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/IncludeInstruction.scala deleted file mode 100644 index e34e7b85d..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/IncludeInstruction.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.product_mixer.component_library.premarshaller.urt.builder - -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.pipeline.HasPipelineCursor -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -trait IncludeInstruction[-Query <: PipelineQuery] { self => - def apply(query: Query, entries: Seq[TimelineEntry]): Boolean - - def inverse(): IncludeInstruction[Query] = new IncludeInstruction[Query] { - def apply(query: Query, entries: Seq[TimelineEntry]): Boolean = !self.apply(query, entries) - } -} - -object AlwaysInclude extends IncludeInstruction[PipelineQuery] { - override def apply(query: PipelineQuery, entries: Seq[TimelineEntry]): Boolean = true -} - -object IncludeOnFirstPage extends IncludeInstruction[PipelineQuery with HasPipelineCursor[_]] { - override def apply( - query: PipelineQuery with HasPipelineCursor[_], - entries: Seq[TimelineEntry] - ): Boolean = query.isFirstPage -} - -object IncludeAfterFirstPage extends IncludeInstruction[PipelineQuery with HasPipelineCursor[_]] { - override def apply( - query: PipelineQuery with HasPipelineCursor[_], - entries: Seq[TimelineEntry] - ): Boolean = !query.isFirstPage -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/MarkUnreadInstructionBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/MarkUnreadInstructionBuilder.docx new file mode 100644 index 000000000..263d0377e Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/MarkUnreadInstructionBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/MarkUnreadInstructionBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/MarkUnreadInstructionBuilder.scala deleted file mode 100644 index a4b8d0f5b..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/MarkUnreadInstructionBuilder.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.product_mixer.component_library.premarshaller.urt.builder - -import com.twitter.product_mixer.core.model.marshalling.response.urt.MarkEntriesUnreadInstruction -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.MarkUnreadableEntry - -/** - * Build a MarkUnreadEntries instruction - * - * Note that this implementation currently supports top-level entries, but not module item entries. - */ -case class MarkUnreadInstructionBuilder[Query <: PipelineQuery]( - override val includeInstruction: IncludeInstruction[Query] = AlwaysInclude) - extends UrtInstructionBuilder[Query, MarkEntriesUnreadInstruction] { - - override def build( - query: Query, - entries: Seq[TimelineEntry] - ): Seq[MarkEntriesUnreadInstruction] = { - if (includeInstruction(query, entries)) { - val filteredEntries = entries.collect { - case entry: MarkUnreadableEntry if entry.isMarkUnread.contains(true) => - entry.entryIdentifier - } - if (filteredEntries.nonEmpty) Seq(MarkEntriesUnreadInstruction(filteredEntries)) - else Seq.empty - } else { - Seq.empty - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/OrderedBottomCursorBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/OrderedBottomCursorBuilder.docx new file mode 100644 index 000000000..47b01e957 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/OrderedBottomCursorBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/OrderedBottomCursorBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/OrderedBottomCursorBuilder.scala deleted file mode 100644 index 85b7b5e47..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/OrderedBottomCursorBuilder.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.product_mixer.component_library.premarshaller.urt.builder - -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.BottomCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorType -import com.twitter.product_mixer.core.pipeline.HasPipelineCursor -import com.twitter.product_mixer.core.pipeline.PipelineCursorSerializer -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * Builds [[UrtOrderedCursor]] in the Bottom position - * - * @param idSelector Specifies the entry from which to derive the `id` field - * @param includeOperation Logic to determine whether or not to build the bottom cursor, which only - * applies if gap cursors are required (e.g. Home Latest). When applicable, - * this logic should always be the inverse of the logic used to decide - * whether or not to build the gap cursor via [[OrderedGapCursorBuilder]], - * since either the gap or the bottom cursor must always be returned. - * @param serializer Converts the cursor to an encoded string - */ -case class OrderedBottomCursorBuilder[ - -Query <: PipelineQuery with HasPipelineCursor[UrtOrderedCursor] -]( - idSelector: PartialFunction[TimelineEntry, Long], - override val includeOperation: IncludeInstruction[Query] = AlwaysInclude, - serializer: PipelineCursorSerializer[UrtOrderedCursor] = UrtCursorSerializer) - extends UrtCursorBuilder[Query] { - override val cursorType: CursorType = BottomCursor - - override def cursorValue(query: Query, timelineEntries: Seq[TimelineEntry]): String = { - val bottomId = timelineEntries.reverseIterator.collectFirst(idSelector) - - val id = bottomId.orElse(query.pipelineCursor.flatMap(_.id)) - - val cursor = UrtOrderedCursor( - initialSortIndex = nextBottomInitialSortIndex(query, timelineEntries), - id = id, - cursorType = Some(cursorType) - ) - - serializer.serializeCursor(cursor) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/OrderedGapCursorBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/OrderedGapCursorBuilder.docx new file mode 100644 index 000000000..53275f696 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/OrderedGapCursorBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/OrderedGapCursorBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/OrderedGapCursorBuilder.scala deleted file mode 100644 index c5136c1b7..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/OrderedGapCursorBuilder.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.product_mixer.component_library.premarshaller.urt.builder - -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorType -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor -import com.twitter.product_mixer.core.pipeline.HasPipelineCursor -import com.twitter.product_mixer.core.pipeline.PipelineCursorSerializer -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * Builds [[UrtOrderedCursor]] in the Bottom position as a Gap cursor. - * - * @param idSelector Specifies the entry from which to derive the `id` field - * @param includeOperation Logic to determine whether or not to build the gap cursor, which should - * always be the inverse of the logic used to decide whether or not to build - * the bottom cursor via [[OrderedBottomCursorBuilder]], since either the - * gap or the bottom cursor must always be returned. - * @param serializer Converts the cursor to an encoded string - */ -case class OrderedGapCursorBuilder[ - -Query <: PipelineQuery with HasPipelineCursor[UrtOrderedCursor] -]( - idSelector: PartialFunction[TimelineEntry, Long], - override val includeOperation: IncludeInstruction[Query], - serializer: PipelineCursorSerializer[UrtOrderedCursor] = UrtCursorSerializer) - extends UrtCursorBuilder[Query] { - override val cursorType: CursorType = GapCursor - - override def cursorValue( - query: Query, - timelineEntries: Seq[TimelineEntry] - ): String = { - // To determine the gap boundary, use any existing cursor gap boundary id (i.e. if submitted - // from a previous gap cursor, else use the existing cursor id (i.e. from a previous top cursor) - val gapBoundaryId = query.pipelineCursor.flatMap(_.gapBoundaryId).orElse { - query.pipelineCursor.flatMap(_.id) - } - - val bottomId = timelineEntries.reverseIterator.collectFirst(idSelector) - - val id = bottomId.orElse(gapBoundaryId) - - val cursor = UrtOrderedCursor( - initialSortIndex = nextBottomInitialSortIndex(query, timelineEntries), - id = id, - cursorType = Some(cursorType), - gapBoundaryId = gapBoundaryId - ) - - serializer.serializeCursor(cursor) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/OrderedTopCursorBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/OrderedTopCursorBuilder.docx new file mode 100644 index 000000000..0a05a5310 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/OrderedTopCursorBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/OrderedTopCursorBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/OrderedTopCursorBuilder.scala deleted file mode 100644 index 2939f8d5e..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/OrderedTopCursorBuilder.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.product_mixer.component_library.premarshaller.urt.builder - -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedTopCursorBuilder.TopCursorOffset -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorType -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor -import com.twitter.product_mixer.core.pipeline.HasPipelineCursor -import com.twitter.product_mixer.core.pipeline.PipelineCursorSerializer -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case object OrderedTopCursorBuilder { - // Ensure that the next initial sort index is at least 10000 entries away from top cursor's - // current sort index. This is to ensure that the contents of the next page can be populated - // without being assigned sort indices which conflict with that of the current page. This assumes - // that each page will have fewer than 10000 entries. - val TopCursorOffset = 10000L -} - -/** - * Builds [[UrtOrderedCursor]] in the Top position - * - * @param idSelector Specifies the entry from which to derive the `id` field - * @param serializer Converts the cursor to an encoded string - */ -case class OrderedTopCursorBuilder( - idSelector: PartialFunction[UniversalNoun[_], Long], - serializer: PipelineCursorSerializer[UrtOrderedCursor] = UrtCursorSerializer) - extends UrtCursorBuilder[ - PipelineQuery with HasPipelineCursor[UrtOrderedCursor] - ] { - override val cursorType: CursorType = TopCursor - - override def cursorValue( - query: PipelineQuery with HasPipelineCursor[UrtOrderedCursor], - timelineEntries: Seq[TimelineEntry] - ): String = { - val topId = timelineEntries.collectFirst(idSelector) - - val id = topId.orElse(query.pipelineCursor.flatMap(_.id)) - - val cursor = UrtOrderedCursor( - initialSortIndex = cursorSortIndex(query, timelineEntries) + TopCursorOffset, - id = id, - cursorType = Some(cursorType) - ) - - serializer.serializeCursor(cursor) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/PinEntryInstructionBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/PinEntryInstructionBuilder.docx new file mode 100644 index 000000000..6a1b7c208 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/PinEntryInstructionBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/PinEntryInstructionBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/PinEntryInstructionBuilder.scala deleted file mode 100644 index 9fc4cea24..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/PinEntryInstructionBuilder.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.product_mixer.component_library.premarshaller.urt.builder - -import com.twitter.product_mixer.core.model.marshalling.response.urt.PinEntryTimelineInstruction -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.PinnableEntry - -case class PinEntryInstructionBuilder() - extends UrtInstructionBuilder[PipelineQuery, PinEntryTimelineInstruction] { - - override def build( - query: PipelineQuery, - entries: Seq[TimelineEntry] - ): Seq[PinEntryTimelineInstruction] = { - // Only one entry can be pinned and the desirable behavior is to pick the entry with the highest - // sort index in the event that multiple pinned items exist. Since the entries are already - // sorted we can accomplish this by picking the first one. - entries.collectFirst { - case entry: PinnableEntry if entry.isPinned.getOrElse(false) => - PinEntryTimelineInstruction(entry) - }.toSeq - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/PlaceholderTopCursorBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/PlaceholderTopCursorBuilder.docx new file mode 100644 index 000000000..4acfd6ab1 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/PlaceholderTopCursorBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/PlaceholderTopCursorBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/PlaceholderTopCursorBuilder.scala deleted file mode 100644 index adfe085aa..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/PlaceholderTopCursorBuilder.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.product_mixer.component_library.premarshaller.urt.builder - -import com.twitter.product_mixer.component_library.model.cursor.UrtPlaceholderCursor -import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.PlaceholderTopCursorBuilder.DefaultPlaceholderCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorType -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor -import com.twitter.product_mixer.core.pipeline.HasPipelineCursor -import com.twitter.product_mixer.core.pipeline.PipelineCursorSerializer -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.UrtPipelineCursor - -object PlaceholderTopCursorBuilder { - val DefaultPlaceholderCursor = UrtPlaceholderCursor() -} - -/** - * Top cursor builder that can be used when the Product does not support paging up. The URT spec - * requires that both bottom and top cursors always be present on each page. Therefore, if the - * product does not support paging up, then we can use a cursor value that is not deserializable. - * This way if the client submits a TopCursor, the backend will treat the the request as if no - * cursor was submitted. - */ -case class PlaceholderTopCursorBuilder( - serializer: PipelineCursorSerializer[UrtPipelineCursor] = UrtCursorSerializer) - extends UrtCursorBuilder[PipelineQuery with HasPipelineCursor[UrtPipelineCursor]] { - override val cursorType: CursorType = TopCursor - - override def cursorValue( - query: PipelineQuery with HasPipelineCursor[UrtPipelineCursor], - timelineEntries: Seq[TimelineEntry] - ): String = serializer.serializeCursor(DefaultPlaceholderCursor) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ReplaceEntryInstructionBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ReplaceEntryInstructionBuilder.docx new file mode 100644 index 000000000..46d6b68e5 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ReplaceEntryInstructionBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ReplaceEntryInstructionBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ReplaceEntryInstructionBuilder.scala deleted file mode 100644 index e93553780..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ReplaceEntryInstructionBuilder.scala +++ /dev/null @@ -1,63 +0,0 @@ -package com.twitter.product_mixer.component_library.premarshaller.urt.builder - -import com.twitter.product_mixer.core.model.marshalling.response.urt.ReplaceEntryTimelineInstruction -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorOperation -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorType -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * Selects one or more [[TimelineEntry]] instance from the input timeline entries. - * - * @tparam Query The domain model for the [[PipelineQuery]] used as input. - */ -trait EntriesToReplace[-Query <: PipelineQuery] { - def apply(query: Query, entries: Seq[TimelineEntry]): Seq[TimelineEntry] -} - -/** - * Selects all entries with a non-empty valid entryIdToReplace. - * - * @note this will result in multiple [[ReplaceEntryTimelineInstruction]]s - */ -case object ReplaceAllEntries extends EntriesToReplace[PipelineQuery] { - def apply(query: PipelineQuery, entries: Seq[TimelineEntry]): Seq[TimelineEntry] = - entries.filter(_.entryIdToReplace.isDefined) -} - -/** - * Selects a replaceable URT [[CursorOperation]] from the timeline entries, that matches the - * input cursorType. - */ -case class ReplaceUrtCursor(cursorType: CursorType) extends EntriesToReplace[PipelineQuery] { - override def apply(query: PipelineQuery, entries: Seq[TimelineEntry]): Seq[TimelineEntry] = - entries.collectFirst { - case cursorOperation: CursorOperation - if cursorOperation.cursorType == cursorType && cursorOperation.entryIdToReplace.isDefined => - cursorOperation - }.toSeq -} - -/** - * Create a ReplaceEntry instruction - * - * @param entriesToReplace each replace instruction can contain only one entry. Users specify which - * entry to replace using [[EntriesToReplace]]. If multiple entries are - * specified, multiple [[ReplaceEntryTimelineInstruction]]s will be created. - * @param includeInstruction whether the instruction should be included in the response - */ -case class ReplaceEntryInstructionBuilder[Query <: PipelineQuery]( - entriesToReplace: EntriesToReplace[Query], - override val includeInstruction: IncludeInstruction[Query] = AlwaysInclude) - extends UrtInstructionBuilder[Query, ReplaceEntryTimelineInstruction] { - - override def build( - query: Query, - entries: Seq[TimelineEntry] - ): Seq[ReplaceEntryTimelineInstruction] = { - if (includeInstruction(query, entries)) - entriesToReplace(query, entries).map(ReplaceEntryTimelineInstruction) - else - Seq.empty - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ShowAlertInstructionBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ShowAlertInstructionBuilder.docx new file mode 100644 index 000000000..bd6d9c0b4 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ShowAlertInstructionBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ShowAlertInstructionBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ShowAlertInstructionBuilder.scala deleted file mode 100644 index 678904755..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ShowAlertInstructionBuilder.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.product_mixer.component_library.premarshaller.urt.builder - -import com.twitter.product_mixer.core.model.marshalling.response.urt.ShowAlert -import com.twitter.product_mixer.core.model.marshalling.response.urt.ShowAlertInstruction -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class ShowAlertInstructionBuilder[Query <: PipelineQuery]( - override val includeInstruction: IncludeInstruction[Query] = AlwaysInclude) - extends UrtInstructionBuilder[Query, ShowAlertInstruction] { - - override def build( - query: Query, - entries: Seq[TimelineEntry] - ): Seq[ShowAlertInstruction] = { - if (includeInstruction(query, entries)) { - // Currently only one Alert is supported per response - entries.collectFirst { - case alertEntry: ShowAlert => ShowAlertInstruction(alertEntry) - }.toSeq - } else Seq.empty - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ShowCoverInstructionBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ShowCoverInstructionBuilder.docx new file mode 100644 index 000000000..2ef7130c5 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ShowCoverInstructionBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ShowCoverInstructionBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ShowCoverInstructionBuilder.scala deleted file mode 100644 index fe7e4364e..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ShowCoverInstructionBuilder.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.product_mixer.component_library.premarshaller.urt.builder - -import com.twitter.product_mixer.core.model.marshalling.response.urt.ShowCoverInstruction -import com.twitter.product_mixer.core.model.marshalling.response.urt.Cover -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class ShowCoverInstructionBuilder[Query <: PipelineQuery]( - override val includeInstruction: IncludeInstruction[Query] = AlwaysInclude) - extends UrtInstructionBuilder[Query, ShowCoverInstruction] { - override def build( - query: Query, - entries: Seq[TimelineEntry] - ): Seq[ShowCoverInstruction] = { - if (includeInstruction(query, entries)) { - // Currently only one cover is supported per response - entries.collectFirst { - case coverEntry: Cover => ShowCoverInstruction(coverEntry) - }.toSeq - } else { - Seq.empty - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/StaticTimelineScribeConfigBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/StaticTimelineScribeConfigBuilder.docx new file mode 100644 index 000000000..97d37b8a9 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/StaticTimelineScribeConfigBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/StaticTimelineScribeConfigBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/StaticTimelineScribeConfigBuilder.scala deleted file mode 100644 index 2b374f2d0..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/StaticTimelineScribeConfigBuilder.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.product_mixer.component_library.premarshaller.urt.builder - -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class StaticTimelineScribeConfigBuilder( - timelineScribeConfig: TimelineScribeConfig) - extends TimelineScribeConfigBuilder[PipelineQuery] { - - def build( - query: PipelineQuery, - entries: Seq[TimelineEntry] - ): Option[TimelineScribeConfig] = Some(timelineScribeConfig) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/TerminateInstructionBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/TerminateInstructionBuilder.docx new file mode 100644 index 000000000..b4da362ef Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/TerminateInstructionBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/TerminateInstructionBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/TerminateInstructionBuilder.scala deleted file mode 100644 index 0e6831ca2..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/TerminateInstructionBuilder.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.twitter.product_mixer.component_library.premarshaller.urt.builder - -import com.twitter.product_mixer.core.model.marshalling.response.urt.BottomTermination -import com.twitter.product_mixer.core.model.marshalling.response.urt.TerminateTimelineInstruction -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineTerminationDirection -import com.twitter.product_mixer.core.model.marshalling.response.urt.TopAndBottomTermination -import com.twitter.product_mixer.core.model.marshalling.response.urt.TopTermination -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -sealed trait TerminateInstructionBuilder[Query <: PipelineQuery] - extends UrtInstructionBuilder[Query, TerminateTimelineInstruction] { - - def direction: TimelineTerminationDirection - - override def build( - query: Query, - entries: Seq[TimelineEntry] - ): Seq[TerminateTimelineInstruction] = - if (includeInstruction(query, entries)) - Seq(TerminateTimelineInstruction(terminateTimelineDirection = direction)) - else Seq.empty -} - -case class TerminateTopInstructionBuilder[Query <: PipelineQuery]( - override val includeInstruction: IncludeInstruction[Query] = AlwaysInclude) - extends TerminateInstructionBuilder[Query] { - - override val direction = TopTermination -} - -case class TerminateBottomInstructionBuilder[Query <: PipelineQuery]( - override val includeInstruction: IncludeInstruction[Query] = AlwaysInclude) - extends TerminateInstructionBuilder[Query] { - - override val direction = BottomTermination -} - -case class TerminateTopAndBottomInstructionBuilder[Query <: PipelineQuery]( - override val includeInstruction: IncludeInstruction[Query] = AlwaysInclude) - extends TerminateInstructionBuilder[Query] { - - override val direction = TopAndBottomTermination -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/TimelineScribeConfigBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/TimelineScribeConfigBuilder.docx new file mode 100644 index 000000000..106d125d2 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/TimelineScribeConfigBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/TimelineScribeConfigBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/TimelineScribeConfigBuilder.scala deleted file mode 100644 index 98c05ac4a..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/TimelineScribeConfigBuilder.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.product_mixer.component_library.premarshaller.urt.builder - -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * Trait for our builder which given a query and entries will return an `Option[TimelineScribeConfig]` - * - * @tparam Query - */ -trait TimelineScribeConfigBuilder[-Query <: PipelineQuery] { - - def build( - query: Query, - entries: Seq[TimelineEntry] - ): Option[TimelineScribeConfig] -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UnorderedBloomFilterBottomCursorBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UnorderedBloomFilterBottomCursorBuilder.docx new file mode 100644 index 000000000..fb01e82ae Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UnorderedBloomFilterBottomCursorBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UnorderedBloomFilterBottomCursorBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UnorderedBloomFilterBottomCursorBuilder.scala deleted file mode 100644 index aeb333373..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UnorderedBloomFilterBottomCursorBuilder.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.twitter.product_mixer.component_library.premarshaller.urt.builder - -import com.twitter.product_mixer.component_library.model.cursor.UrtUnorderedBloomFilterCursor -import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.BottomCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorType -import com.twitter.product_mixer.core.pipeline.HasPipelineCursor -import com.twitter.product_mixer.core.pipeline.PipelineCursorSerializer -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.search.common.util.bloomfilter.AdaptiveLongIntBloomFilterBuilder - -/** - * Builds [[UrtUnorderedBloomFilterCursor]] in the Bottom position - * - * @param idSelector Specifies the entry from which to derive the `id` field - * @param serializer Converts the cursor to an encoded string - */ -case class UnorderedBloomFilterBottomCursorBuilder( - idSelector: PartialFunction[UniversalNoun[_], Long], - serializer: PipelineCursorSerializer[UrtUnorderedBloomFilterCursor] = UrtCursorSerializer) - extends UrtCursorBuilder[ - PipelineQuery with HasPipelineCursor[UrtUnorderedBloomFilterCursor] - ] { - - override val cursorType: CursorType = BottomCursor - - override def cursorValue( - query: PipelineQuery with HasPipelineCursor[UrtUnorderedBloomFilterCursor], - entries: Seq[TimelineEntry] - ): String = { - val bloomFilter = query.pipelineCursor.map(_.longIntBloomFilter) - val ids = entries.collect(idSelector) - - val cursor = UrtUnorderedBloomFilterCursor( - initialSortIndex = nextBottomInitialSortIndex(query, entries), - longIntBloomFilter = AdaptiveLongIntBloomFilterBuilder.build(ids, bloomFilter) - ) - - serializer.serializeCursor(cursor) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UnorderedExcludeIdsBottomCursorBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UnorderedExcludeIdsBottomCursorBuilder.docx new file mode 100644 index 000000000..3462d52aa Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UnorderedExcludeIdsBottomCursorBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UnorderedExcludeIdsBottomCursorBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UnorderedExcludeIdsBottomCursorBuilder.scala deleted file mode 100644 index 5a25da032..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UnorderedExcludeIdsBottomCursorBuilder.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.product_mixer.component_library.premarshaller.urt.builder - -import com.twitter.product_mixer.component_library.model.cursor.UrtUnorderedExcludeIdsCursor -import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.pipeline.PipelineCursorSerializer -import com.twitter.timelines.configapi.Param - -/** - * Builds [[UrtUnorderedExcludeIdsCursor]] in the Bottom position - * - * @param excludedIdsMaxLengthParam The maximum length of the cursor - * @param excludeIdsSelector Specifies the entry Ids to populate on the `excludedIds` field - * @param serializer Converts the cursor to an encoded string - */ -case class UnorderedExcludeIdsBottomCursorBuilder( - override val excludedIdsMaxLengthParam: Param[Int], - excludeIdsSelector: PartialFunction[UniversalNoun[_], Long], - override val serializer: PipelineCursorSerializer[UrtUnorderedExcludeIdsCursor] = - UrtCursorSerializer) - extends BaseUnorderedExcludeIdsBottomCursorBuilder { - - override def excludeEntriesCollector(entries: Seq[TimelineEntry]): Seq[Long] = - entries.collect(excludeIdsSelector) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UnorderedExcludeIdsSeqBottomCursorBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UnorderedExcludeIdsSeqBottomCursorBuilder.docx new file mode 100644 index 000000000..1e344c159 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UnorderedExcludeIdsSeqBottomCursorBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UnorderedExcludeIdsSeqBottomCursorBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UnorderedExcludeIdsSeqBottomCursorBuilder.scala deleted file mode 100644 index 468b5076b..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UnorderedExcludeIdsSeqBottomCursorBuilder.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.product_mixer.component_library.premarshaller.urt.builder - -import com.twitter.product_mixer.component_library.model.cursor.UrtUnorderedExcludeIdsCursor -import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.pipeline.PipelineCursorSerializer -import com.twitter.timelines.configapi.Param - -/** - * Builds [[UrtUnorderedExcludeIdsCursor]] in the Bottom position when we want to also exclude ids - * of items inside a module. The reason we cannot use [[UnorderedExcludeIdsBottomCursorBuilder]] in - * such case is that the excludeIdsSelector of [[UnorderedExcludeIdsBottomCursorBuilder]] is doing a - * one to one mapping between entries and excluded ids, but in case of having a module, a module - * entry can result in excluding a sequence of entries. - * - * @param excludedIdsMaxLengthParam The maximum length of the cursor - * @param excludeIdsSelector Specifies the entry Ids to populate on the `excludedIds` field - * @param serializer Converts the cursor to an encoded string - */ -case class UnorderedExcludeIdsSeqBottomCursorBuilder( - override val excludedIdsMaxLengthParam: Param[Int], - excludeIdsSelector: PartialFunction[UniversalNoun[_], Seq[Long]], - override val serializer: PipelineCursorSerializer[UrtUnorderedExcludeIdsCursor] = - UrtCursorSerializer) - extends BaseUnorderedExcludeIdsBottomCursorBuilder { - - override def excludeEntriesCollector(entries: Seq[TimelineEntry]): Seq[Long] = - entries.collect(excludeIdsSelector).flatten -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtBuilder.docx new file mode 100644 index 000000000..8278f9e34 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtBuilder.scala deleted file mode 100644 index e47bf476b..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtBuilder.scala +++ /dev/null @@ -1,94 +0,0 @@ -package com.twitter.product_mixer.component_library.premarshaller.urt.builder - -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorOperation -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineInstruction -import com.twitter.product_mixer.core.pipeline.HasPipelineCursor -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.UrtPipelineCursor -import com.twitter.product_mixer.core.util.SortIndexBuilder - -trait UrtBuilder[-Query <: PipelineQuery, +Instruction <: TimelineInstruction] { - private val TimelineIdSuffix = "-Timeline" - - def instructionBuilders: Seq[UrtInstructionBuilder[Query, Instruction]] - - def cursorBuilders: Seq[UrtCursorBuilder[Query]] - def cursorUpdaters: Seq[UrtCursorUpdater[Query]] - - def metadataBuilder: Option[BaseUrtMetadataBuilder[Query]] - - // Timeline entry sort indexes will count down by this value. Values higher than 1 are useful to - // leave room in the sequence for dynamically injecting content in between existing entries. - def sortIndexStep: Int = 1 - - final def buildTimeline( - query: Query, - entries: Seq[TimelineEntry] - ): Timeline = { - val initialSortIndex = getInitialSortIndex(query) - - // Set the sort indexes of the entries before we pass them to the cursor builders, since many - // cursor implementations use the sort index of the first/last entry as part of the cursor value - val sortIndexedEntries = updateSortIndexes(initialSortIndex, entries) - - // Iterate over the cursorUpdaters in the order they were defined. Note that each updater will - // be passed the timelineEntries updated by the previous cursorUpdater. - val updatedCursorEntries: Seq[TimelineEntry] = - cursorUpdaters.foldLeft(sortIndexedEntries) { (timelineEntries, cursorUpdater) => - cursorUpdater.update(query, timelineEntries) - } - - val allCursoredEntries = - updatedCursorEntries ++ cursorBuilders.flatMap(_.build(query, updatedCursorEntries)) - - val instructions: Seq[Instruction] = - instructionBuilders.flatMap(_.build(query, allCursoredEntries)) - - val metadata = metadataBuilder.map(_.build(query, allCursoredEntries)) - - Timeline( - id = query.product.identifier.toString + TimelineIdSuffix, - instructions = instructions, - metadata = metadata - ) - } - - final def getInitialSortIndex(query: Query): Long = - query match { - case cursorQuery: HasPipelineCursor[_] => - UrtPipelineCursor - .getCursorInitialSortIndex(cursorQuery) - .getOrElse(SortIndexBuilder.timeToId(query.queryTime)) - case _ => SortIndexBuilder.timeToId(query.queryTime) - } - - /** - * Updates the sort indexes in the timeline entries starting from the given initial sort index - * value and decreasing by the value defined in the sort index step field - * - * @param initialSortIndex The initial value of the sort index - * @param timelineEntries Timeline entries to update - */ - final def updateSortIndexes( - initialSortIndex: Long, - timelineEntries: Seq[TimelineEntry] - ): Seq[TimelineEntry] = { - val indexRange = - initialSortIndex to (initialSortIndex - (timelineEntries.size * sortIndexStep)) by -sortIndexStep - - // Skip any existing cursors because their sort indexes will be managed by their cursor updater. - // If the cursors are not removed first, then the remaining entries would have a gap everywhere - // an existing cursor was present. - val (cursorEntries, nonCursorEntries) = timelineEntries.partition { - case _: CursorOperation => true - case _ => false - } - - nonCursorEntries.zip(indexRange).map { - case (entry, index) => - entry.withSortIndex(index) - } ++ cursorEntries - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtCursorBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtCursorBuilder.docx new file mode 100644 index 000000000..39027fa0a Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtCursorBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtCursorBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtCursorBuilder.scala deleted file mode 100644 index 9142bbd05..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtCursorBuilder.scala +++ /dev/null @@ -1,134 +0,0 @@ -package com.twitter.product_mixer.component_library.premarshaller.urt.builder - -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtCursorBuilder.DefaultSortIndex -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtCursorBuilder.NextPageTopCursorEntryOffset -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtCursorBuilder.UrtEntryOffset -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.BottomCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorItem -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorOperation -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorType -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor -import com.twitter.product_mixer.core.pipeline.HasPipelineCursor -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.UrtPipelineCursor -import com.twitter.product_mixer.core.util.SortIndexBuilder - -object UrtCursorBuilder { - val NextPageTopCursorEntryOffset = 1L - val UrtEntryOffset = 1L - val DefaultSortIndex = (query: PipelineQuery) => SortIndexBuilder.timeToId(query.queryTime) -} - -trait UrtCursorBuilder[-Query <: PipelineQuery] { - - val includeOperation: IncludeInstruction[Query] = AlwaysInclude - - def cursorType: CursorType - def cursorValue(query: Query, entries: Seq[TimelineEntry]): String - - /** - * Identifier of an *existing* timeline cursor that this new cursor would replace, if this cursor - * is returned in a `ReplaceEntry` timeline instruction. - * - * Note: - * - This id is used to populate the `entryIdToReplace` field on the URT TimelineEntry - * generated. More details at [[CursorOperation.entryIdToReplace]]. - * - As a convention, we use the sortIndex of the cursor for its id/entryId fields. So the - * `idToReplace` should represent the sortIndex of the existing cursor to be replaced. - */ - def idToReplace(query: Query): Option[Long] = None - - def cursorSortIndex(query: Query, entries: Seq[TimelineEntry]): Long = - (query, cursorType) match { - case (query: PipelineQuery with HasPipelineCursor[_], TopCursor) => - topCursorSortIndex(query, entries) - case (query: PipelineQuery with HasPipelineCursor[_], BottomCursor | GapCursor) => - bottomCursorSortIndex(query, entries) - case _ => - throw new UnsupportedOperationException( - "Automatic sort index support limited to top and bottom cursors") - } - - def build(query: Query, entries: Seq[TimelineEntry]): Option[CursorOperation] = { - if (includeOperation(query, entries)) { - val sortIndex = cursorSortIndex(query, entries) - - val cursorOperation = CursorOperation( - id = sortIndex, - sortIndex = Some(sortIndex), - value = cursorValue(query, entries), - cursorType = cursorType, - displayTreatment = None, - idToReplace = idToReplace(query), - ) - - Some(cursorOperation) - } else None - } - - /** - * Build the top cursor sort index which handles the following cases: - * 1. When there is at least one non-cursor entry, use the first entry's sort index + UrtEntryOffset - * 2. When there are no non-cursor entries, and initialSortIndex is not set which indicates that - * it is the first page, use DefaultSortIndex + UrtEntryOffset - * 3. When there are no non-cursor entries, and initialSortIndex is set which indicates that it is - * not the first page, use the query.initialSortIndex from the passed-in cursor + UrtEntryOffset - */ - protected def topCursorSortIndex( - query: PipelineQuery with HasPipelineCursor[_], - entries: Seq[TimelineEntry] - ): Long = { - val nonCursorEntries = entries.filter { - case _: CursorOperation => false - case _: CursorItem => false - case _ => true - } - - lazy val initialSortIndex = - UrtPipelineCursor.getCursorInitialSortIndex(query).getOrElse(DefaultSortIndex(query)) - - nonCursorEntries.headOption.flatMap(_.sortIndex).getOrElse(initialSortIndex) + UrtEntryOffset - } - - /** - * Specifies the point at which the next page's entries' sort indices will start counting. - * - * Note that in the case of URT, the next page's entries' does not include the top cursor. As - * such, the value of initialSortIndex passed back in the cursor is typically the bottom cursor's - * sort index - 2. Subtracting 2 leaves room for the next page's top cursor, which will have a - * sort index of top entry + 1. - */ - protected def nextBottomInitialSortIndex( - query: PipelineQuery with HasPipelineCursor[_], - entries: Seq[TimelineEntry] - ): Long = { - bottomCursorSortIndex(query, entries) - NextPageTopCursorEntryOffset - UrtEntryOffset - } - - /** - * Build the bottom cursor sort index which handles the following cases: - * 1. When there is at least one non-cursor entry, use the last entry's sort index - UrtEntryOffset - * 2. When there are no non-cursor entries, and initialSortIndex is not set which indicates that - * it is the first page, use DefaultSortIndex - * 3. When there are no non-cursor entries, and initialSortIndex is set which indicates that it is - * not the first page, use the query.initialSortIndex from the passed-in cursor - */ - protected def bottomCursorSortIndex( - query: PipelineQuery with HasPipelineCursor[_], - entries: Seq[TimelineEntry] - ): Long = { - val nonCursorEntries = entries.filter { - case _: CursorOperation => false - case _: CursorItem => false - case _ => true - } - - lazy val initialSortIndex = - UrtPipelineCursor.getCursorInitialSortIndex(query).getOrElse(DefaultSortIndex(query)) - - nonCursorEntries.lastOption - .flatMap(_.sortIndex).map(_ - UrtEntryOffset).getOrElse(initialSortIndex) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtCursorUpdater.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtCursorUpdater.docx new file mode 100644 index 000000000..b9941e6f5 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtCursorUpdater.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtCursorUpdater.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtCursorUpdater.scala deleted file mode 100644 index 0885ff852..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtCursorUpdater.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.twitter.product_mixer.component_library.premarshaller.urt.builder - -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtCursorUpdater.getCursorByType -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorOperation -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorType -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object UrtCursorUpdater { - - def getCursorByType( - entries: Seq[TimelineEntry], - cursorType: CursorType - ): Option[CursorOperation] = { - entries.collectFirst { - case cursor: CursorOperation if cursor.cursorType == cursorType => cursor - } - } -} - -// If a CursorCandidate is returned by a Candidate Source, use this trait to update that Cursor as -// necessary (as opposed to building a new cursor which is done with the UrtCursorBuilder) -trait UrtCursorUpdater[-Query <: PipelineQuery] extends UrtCursorBuilder[Query] { self => - - def getExistingCursor(entries: Seq[TimelineEntry]): Option[CursorOperation] = { - getCursorByType(entries, self.cursorType) - } - - def update(query: Query, entries: Seq[TimelineEntry]): Seq[TimelineEntry] = { - if (includeOperation(query, entries)) { - getExistingCursor(entries) - .map { existingCursor => - // Safe .get because includeOperation() is shared in this context - // build() method creates a new CursorOperation. We copy over the `idToReplace` - // from the existing cursor. - val newCursor = - build(query, entries).get - .copy(idToReplace = existingCursor.idToReplace) - - entries.filterNot(_ == existingCursor) :+ newCursor - }.getOrElse(entries) - } else entries - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtInstructionBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtInstructionBuilder.docx new file mode 100644 index 000000000..933987847 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtInstructionBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtInstructionBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtInstructionBuilder.scala deleted file mode 100644 index ac1b9da31..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtInstructionBuilder.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.product_mixer.component_library.premarshaller.urt.builder - -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineInstruction -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -trait UrtInstructionBuilder[-Query <: PipelineQuery, +Instruction <: TimelineInstruction] { - - def includeInstruction: IncludeInstruction[Query] = AlwaysInclude - - def build( - query: Query, - entries: Seq[TimelineEntry] - ): Seq[Instruction] -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtMetadataBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtMetadataBuilder.docx new file mode 100644 index 000000000..ae39c269f Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtMetadataBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtMetadataBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtMetadataBuilder.scala deleted file mode 100644 index a59efea0c..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtMetadataBuilder.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.twitter.product_mixer.component_library.premarshaller.urt.builder - -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineMetadata -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stringcenter.client.StringCenter -import com.twitter.stringcenter.client.core.ExternalString - -trait BaseUrtMetadataBuilder[-Query <: PipelineQuery] { - def build( - query: Query, - entries: Seq[TimelineEntry] - ): TimelineMetadata -} - -case class UrtMetadataBuilder( - title: Option[String] = None, - scribeConfigBuilder: Option[TimelineScribeConfigBuilder[PipelineQuery]]) - extends BaseUrtMetadataBuilder[PipelineQuery] { - - override def build( - query: PipelineQuery, - entries: Seq[TimelineEntry] - ): TimelineMetadata = TimelineMetadata( - title = title, - scribeConfig = scribeConfigBuilder.flatMap(_.build(query, entries)) - ) -} - -case class UrtMetadataStringCenterBuilder( - titleKey: ExternalString, - scribeConfigBuilder: Option[TimelineScribeConfigBuilder[PipelineQuery]], - stringCenter: StringCenter) - extends BaseUrtMetadataBuilder[PipelineQuery] { - - override def build( - query: PipelineQuery, - entries: Seq[TimelineEntry] - ): TimelineMetadata = TimelineMetadata( - title = Some(stringCenter.prepare(titleKey)), - scribeConfig = scribeConfigBuilder.flatMap(_.build(query, entries)) - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/BUILD deleted file mode 100644 index ed7e90b3f..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/BUILD +++ /dev/null @@ -1,30 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/io/grpc:grpc-netty", - "3rdparty/jvm/io/netty:netty4-tcnative-boringssl-static", - "3rdparty/jvm/io/opil:tensorflow-serving-client", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/jvm/triton/inference:triton-grpc", - "finagle-internal/finagle-grpc/src/main/scala", - "finagle-internal/finagle-grpc/src/test/java", - "finagle-internal/finagle-grpc/src/test/proto", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/client", - "finagle/finagle-http/src/main/scala", - "finatra-internal/mtls/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "ml-serving/scala:kfserving-tfserving-converter", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "stitch/stitch-core", - ], - exports = [ - "3rdparty/jvm/triton/inference:triton-grpc", - "finagle/finagle-http/src/main/scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/BUILD.docx new file mode 100644 index 000000000..6ab5793b2 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/MLModelInferenceClient.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/MLModelInferenceClient.docx new file mode 100644 index 000000000..10b6fd2b2 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/MLModelInferenceClient.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/MLModelInferenceClient.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/MLModelInferenceClient.scala deleted file mode 100644 index ee763759c..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/MLModelInferenceClient.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.common - -import com.twitter.stitch.Stitch -import inference.GrpcService.ModelInferRequest -import inference.GrpcService.ModelInferResponse - -/** - * MLModelInferenceClient for calling different Inference Service such as ManagedModelClient or NaviModelClient. - */ -trait MLModelInferenceClient { - def score(request: ModelInferRequest): Stitch[ModelInferResponse] -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/ManagedModelClient.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/ManagedModelClient.docx new file mode 100644 index 000000000..fc51ca43a Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/ManagedModelClient.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/ManagedModelClient.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/ManagedModelClient.scala deleted file mode 100644 index cd71a072a..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/ManagedModelClient.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.common - -import com.twitter.finagle.Http -import com.twitter.finagle.grpc.FinagleChannelBuilder -import com.twitter.finagle.grpc.FutureConverters -import com.twitter.stitch.Stitch -import inference.GRPCInferenceServiceGrpc -import inference.GrpcService.ModelInferRequest -import inference.GrpcService.ModelInferResponse -import io.grpc.ManagedChannel - -/** - * Client wrapper for calling a Cortex Managed Inference Service (go/cmis) ML Model using GRPC. - * @param httpClient Finagle HTTP Client to use for connection. - * @param modelPath Wily path to the ML Model service (e.g. /cluster/local/role/service/instance). - */ -case class ManagedModelClient( - httpClient: Http.Client, - modelPath: String) - extends MLModelInferenceClient { - - private val channel: ManagedChannel = - FinagleChannelBuilder.forTarget(modelPath).httpClient(httpClient).build() - - private val inferenceServiceStub = GRPCInferenceServiceGrpc.newFutureStub(channel) - - def score(request: ModelInferRequest): Stitch[ModelInferResponse] = { - Stitch - .callFuture( - FutureConverters - .RichListenableFuture(inferenceServiceStub.modelInfer(request)).toTwitter) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/ModelSelector.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/ModelSelector.docx new file mode 100644 index 000000000..809c99203 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/ModelSelector.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/ModelSelector.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/ModelSelector.scala deleted file mode 100644 index 9dfdf7415..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/ModelSelector.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.common - -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.Param - -/** - * Selector for choosing which Model ID/Name to use when calling an underlying ML Model Service. - */ -trait ModelSelector[-Query <: PipelineQuery] { - def apply(query: Query): Option[String] -} - -/** - * Simple Model ID Selector that chooses model based off of a Param object. - * @param param ConfigAPI Param that decides the model id. - */ -case class ParamModelSelector[Query <: PipelineQuery](param: Param[String]) - extends ModelSelector[Query] { - override def apply(query: Query): Option[String] = Some(query.params(param)) -} - -/** - * Static Selector that chooses the same model name always - * @param modelName The model name to use. - */ -case class StaticModelSelector(modelName: String) extends ModelSelector[PipelineQuery] { - override def apply(query: PipelineQuery): Option[String] = Some(modelName) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/NaviModelClient.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/NaviModelClient.docx new file mode 100644 index 000000000..44793e7ac Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/NaviModelClient.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/NaviModelClient.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/NaviModelClient.scala deleted file mode 100644 index d5a5ccb50..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/NaviModelClient.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.common - -import com.twitter.finagle.Http -import com.twitter.finagle.grpc.FinagleChannelBuilder -import com.twitter.finagle.grpc.FutureConverters -import com.twitter.mlserving.frontend.TFServingInferenceServiceImpl -import com.twitter.stitch.Stitch -import tensorflow.serving.PredictionServiceGrpc -import inference.GrpcService.ModelInferRequest -import inference.GrpcService.ModelInferResponse -import io.grpc.ManagedChannel -import io.grpc.Status - -/** - * Client wrapper for calling a Navi Inference Service (go/navi). - * @param httpClient Finagle HTTP Client to use for connection. - * @param modelPath Wily path to the ML Model service (e.g. /s/role/service). - */ -case class NaviModelClient( - httpClient: Http.Client, - modelPath: String) - extends MLModelInferenceClient { - - private val channel: ManagedChannel = - FinagleChannelBuilder - .forTarget(modelPath) - .httpClient(httpClient) - // Navi enforces an authority name. - .overrideAuthority("rustserving") - // certain GRPC errors need to be retried. - .enableRetryForStatus(Status.UNKNOWN) - .enableRetryForStatus(Status.RESOURCE_EXHAUSTED) - // this is required at channel level as mTLS is enabled at httpClient level - .usePlaintext() - .build() - - private val inferenceServiceStub = PredictionServiceGrpc.newFutureStub(channel) - - def score(request: ModelInferRequest): Stitch[ModelInferResponse] = { - val tfServingRequest = TFServingInferenceServiceImpl.adaptModelInferRequest(request) - Stitch - .callFuture( - FutureConverters - .RichListenableFuture(inferenceServiceStub.predict(tfServingRequest)).toTwitter - .map { response => - TFServingInferenceServiceImpl.adaptModelInferResponse(response) - } - ) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/BUILD deleted file mode 100644 index 80e97e4a4..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/BUILD +++ /dev/null @@ -1,35 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "finagle/finagle-http/src/main/scala", - "finatra-internal/mtls/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/module/http", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/scorer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "src/scala/com/twitter/ml/featurestore/lib", - "src/thrift/com/twitter/ml/prediction_service:prediction_service-java", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/module/http", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/scorer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/BUILD.docx new file mode 100644 index 000000000..887967850 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceDataRecordScorer.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceDataRecordScorer.docx new file mode 100644 index 000000000..f21c53285 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceDataRecordScorer.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceDataRecordScorer.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceDataRecordScorer.scala deleted file mode 100644 index 67c85407d..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceDataRecordScorer.scala +++ /dev/null @@ -1,137 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.cortex - -import com.google.protobuf.ByteString -import com.twitter.ml.prediction_service.BatchPredictionRequest -import com.twitter.ml.prediction_service.BatchPredictionResponse -import com.twitter.product_mixer.component_library.scorer.common.ManagedModelClient -import com.twitter.product_mixer.component_library.scorer.common.ModelSelector -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.datarecord.BaseDataRecordFeature -import com.twitter.product_mixer.core.feature.datarecord.TensorDataRecordCompatible -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordConverter -import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordExtractor -import com.twitter.product_mixer.core.feature.featuremap.datarecord.FeaturesScope -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.pipeline_failure.IllegalStateFailure -import inference.GrpcService -import inference.GrpcService.ModelInferRequest -import inference.GrpcService.ModelInferResponse -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.stitch.Stitch -import org.apache.thrift.TDeserializer -import org.apache.thrift.TSerializer -import scala.collection.JavaConverters._ - -private[cortex] class CortexManagedDataRecordScorer[ - Query <: PipelineQuery, - Candidate <: UniversalNoun[Any], - QueryFeatures <: BaseDataRecordFeature[Query, _], - CandidateFeatures <: BaseDataRecordFeature[Candidate, _], - ResultFeatures <: BaseDataRecordFeature[Candidate, _] with TensorDataRecordCompatible[_] -]( - override val identifier: ScorerIdentifier, - modelSignature: String, - modelSelector: ModelSelector[Query], - modelClient: ManagedModelClient, - queryFeatures: FeaturesScope[QueryFeatures], - candidateFeatures: FeaturesScope[CandidateFeatures], - resultFeatures: Set[ResultFeatures]) - extends Scorer[Query, Candidate] { - - require(resultFeatures.nonEmpty, "Result features cannot be empty") - override val features: Set[Feature[_, _]] = resultFeatures.asInstanceOf[Set[Feature[_, _]]] - - private val queryDataRecordAdapter = new DataRecordConverter(queryFeatures) - private val candidatesDataRecordAdapter = new DataRecordConverter(candidateFeatures) - private val resultDataRecordExtractor = new DataRecordExtractor(resultFeatures) - - private val localTSerializer = new ThreadLocal[TSerializer] { - override protected def initialValue: TSerializer = new TSerializer() - } - - private val localTDeserializer = new ThreadLocal[TDeserializer] { - override protected def initialValue: TDeserializer = new TDeserializer() - } - - override def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[Candidate]] - ): Stitch[Seq[FeatureMap]] = { - modelClient.score(buildRequest(query, candidates)).map(buildResponse(candidates, _)) - } - - /** - * Takes candidates to be scored and converts it to a ModelInferRequest that can be passed to the - * managed ML service - */ - private def buildRequest( - query: Query, - scorerCandidates: Seq[CandidateWithFeatures[Candidate]] - ): ModelInferRequest = { - // Convert the feature maps to thrift data records and construct thrift request. - val thriftDataRecords = scorerCandidates.map { candidate => - candidatesDataRecordAdapter.toDataRecord(candidate.features) - } - val batchRequest = new BatchPredictionRequest(thriftDataRecords.asJava) - query.features.foreach { featureMap => - batchRequest.setCommonFeatures(queryDataRecordAdapter.toDataRecord(featureMap)) - } - val serializedBatchRequest = localTSerializer.get().serialize(batchRequest) - - // Build Tensor Request - val requestBuilder = ModelInferRequest - .newBuilder() - - modelSelector.apply(query).foreach { modelName => - requestBuilder.setModelName(modelName) // model name in the model config - } - - val inputTensorBuilder = ModelInferRequest.InferInputTensor - .newBuilder() - .setName("request") - .setDatatype("UINT8") - .addShape(serializedBatchRequest.length) - - val inferParameter = GrpcService.InferParameter - .newBuilder() - .setStringParam(modelSignature) // signature of exported tf function - .build() - - requestBuilder - .addInputs(inputTensorBuilder) - .addRawInputContents(ByteString.copyFrom(serializedBatchRequest)) - .putParameters("signature_name", inferParameter) - .build() - } - - private def buildResponse( - scorerCandidates: Seq[CandidateWithFeatures[Candidate]], - response: ModelInferResponse - ): Seq[FeatureMap] = { - - val responseByteString = if (response.getRawOutputContentsList.isEmpty()) { - throw PipelineFailure( - IllegalStateFailure, - "Model inference response has empty raw outputContents") - } else { - response.getRawOutputContents(0) - } - val batchPredictionResponse: BatchPredictionResponse = new BatchPredictionResponse() - localTDeserializer.get().deserialize(batchPredictionResponse, responseByteString.toByteArray) - - // get the prediction values from the batch prediction response - val resultScoreMaps = - batchPredictionResponse.predictions.asScala.map(resultDataRecordExtractor.fromDataRecord) - - if (resultScoreMaps.size != scorerCandidates.size) { - throw PipelineFailure(IllegalStateFailure, "Result Size mismatched candidates size") - } - - resultScoreMaps - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceDataRecordScorerBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceDataRecordScorerBuilder.docx new file mode 100644 index 000000000..974b7c061 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceDataRecordScorerBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceDataRecordScorerBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceDataRecordScorerBuilder.scala deleted file mode 100644 index 026cfa696..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceDataRecordScorerBuilder.scala +++ /dev/null @@ -1,67 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.cortex - -import com.twitter.finagle.Http -import com.twitter.product_mixer.component_library.module.http.FinagleHttpClientModule.FinagleHttpClientModule -import com.twitter.product_mixer.component_library.scorer.common.ManagedModelClient -import com.twitter.product_mixer.component_library.scorer.common.ModelSelector -import com.twitter.product_mixer.core.feature.datarecord.BaseDataRecordFeature -import com.twitter.product_mixer.core.feature.datarecord.TensorDataRecordCompatible -import com.twitter.product_mixer.core.feature.featuremap.datarecord.FeaturesScope -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -@Singleton -class CortexManagedInferenceServiceDataRecordScorerBuilder @Inject() ( - @Named(FinagleHttpClientModule) httpClient: Http.Client) { - - /** - * Builds a configurable Scorer to call into your desired DataRecord-backed Cortex Managed ML Model Service. - * - * If your service does not bind an Http.Client implementation, add - * [[com.twitter.product_mixer.component_library.module.http.FinagleHttpClientModule]] - * to your server module list - * - * @param scorerIdentifier Unique identifier for the scorer - * @param modelPath MLS path to model - * @param modelSignature Model Signature Key - * @param modelSelector [[ModelSelector]] for choosing the model name, can be an anon function. - * @param candidateFeatures Desired candidate level feature store features to pass to the model. - * @param resultFeatures Desired candidate level feature store features to extract from the model. - * Since the Cortex Managed Platform always returns tensor values, the - * feature must use a [[TensorDataRecordCompatible]]. - * @tparam Query Type of pipeline query. - * @tparam Candidate Type of candidates to score. - * @tparam QueryFeatures type of the query level features consumed by the scorer. - * @tparam CandidateFeatures type of the candidate level features consumed by the scorer. - * @tparam ResultFeatures type of the candidate level features returned by the scorer. - */ - def build[ - Query <: PipelineQuery, - Candidate <: UniversalNoun[Any], - QueryFeatures <: BaseDataRecordFeature[Query, _], - CandidateFeatures <: BaseDataRecordFeature[Candidate, _], - ResultFeatures <: BaseDataRecordFeature[Candidate, _] with TensorDataRecordCompatible[_] - ]( - scorerIdentifier: ScorerIdentifier, - modelPath: String, - modelSignature: String, - modelSelector: ModelSelector[Query], - queryFeatures: FeaturesScope[QueryFeatures], - candidateFeatures: FeaturesScope[CandidateFeatures], - resultFeatures: Set[ResultFeatures] - ): Scorer[Query, Candidate] = - new CortexManagedDataRecordScorer( - identifier = scorerIdentifier, - modelSignature = modelSignature, - modelSelector = modelSelector, - modelClient = ManagedModelClient(httpClient, modelPath), - queryFeatures = queryFeatures, - candidateFeatures = candidateFeatures, - resultFeatures = resultFeatures - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceTensorScorer.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceTensorScorer.docx new file mode 100644 index 000000000..a667dab09 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceTensorScorer.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceTensorScorer.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceTensorScorer.scala deleted file mode 100644 index be96e78be..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceTensorScorer.scala +++ /dev/null @@ -1,97 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.cortex - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.component_library.scorer.common.MLModelInferenceClient -import com.twitter.product_mixer.component_library.scorer.tensorbuilder.ModelInferRequestBuilder -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.pipeline_failure.IllegalStateFailure -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.stitch.Stitch -import com.twitter.util.logging.Logging -import inference.GrpcService.ModelInferRequest -import inference.GrpcService.ModelInferResponse.InferOutputTensor -import scala.collection.convert.ImplicitConversions.`collection AsScalaIterable` - -private[scorer] class CortexManagedInferenceServiceTensorScorer[ - Query <: PipelineQuery, - Candidate <: UniversalNoun[Any] -]( - override val identifier: ScorerIdentifier, - modelInferRequestBuilder: ModelInferRequestBuilder[ - Query, - Candidate - ], - resultFeatureExtractors: Seq[FeatureWithExtractor[Query, Candidate, _]], - client: MLModelInferenceClient, - statsReceiver: StatsReceiver) - extends Scorer[Query, Candidate] - with Logging { - - require(resultFeatureExtractors.nonEmpty, "Result Extractors cannot be empty") - - private val managedServiceRequestFailures = statsReceiver.counter("managedServiceRequestFailures") - override val features: Set[Feature[_, _]] = - resultFeatureExtractors.map(_.feature).toSet.asInstanceOf[Set[Feature[_, _]]] - - override def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[Candidate]] - ): Stitch[Seq[FeatureMap]] = { - val batchInferRequest: ModelInferRequest = modelInferRequestBuilder(query, candidates) - - val managedServiceResponse: Stitch[Seq[InferOutputTensor]] = - client.score(batchInferRequest).map(_.getOutputsList.toSeq).onFailure { e => - error(s"request to ML Managed Service Failed: $e") - managedServiceRequestFailures.incr() - } - - managedServiceResponse.map { responses => - extractResponse(query, candidates.map(_.candidate), responses) - } - } - - def extractResponse( - query: Query, - candidates: Seq[Candidate], - tensorOutput: Seq[InferOutputTensor] - ): Seq[FeatureMap] = { - val featureMapBuilders = candidates.map { _ => FeatureMapBuilder.apply() } - // Extract the feature for each candidate from the tensor outputs - resultFeatureExtractors.foreach { - case FeatureWithExtractor(feature, extractor) => - val extractedValues = extractor.apply(query, tensorOutput) - if (candidates.size != extractedValues.size) { - throw PipelineFailure( - IllegalStateFailure, - s"Managed Service returned a different number of $feature than the number of candidates." + - s"Returned ${extractedValues.size} scores but there were ${candidates.size} candidates." - ) - } - // Go through the extracted features list one by one and update the feature map result for each candidate. - featureMapBuilders.zip(extractedValues).foreach { - case (builder, value) => - builder.add(feature, Some(value)) - } - } - - featureMapBuilders.map(_.build()) - } -} - -case class FeatureWithExtractor[ - -Query <: PipelineQuery, - -Candidate <: UniversalNoun[Any], - ResultType -]( - feature: Feature[Candidate, Option[ResultType]], - featureExtractor: ModelFeatureExtractor[Query, ResultType]) - -class UnexpectedFeatureTypeException(feature: Feature[_, _]) - extends UnsupportedOperationException(s"Unsupported Feature type passed in $feature") diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceTensorScorerBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceTensorScorerBuilder.docx new file mode 100644 index 000000000..b7c7f7275 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceTensorScorerBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceTensorScorerBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceTensorScorerBuilder.scala deleted file mode 100644 index 84ef39fd7..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceTensorScorerBuilder.scala +++ /dev/null @@ -1,47 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.cortex - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.component_library.scorer.common.MLModelInferenceClient -import com.twitter.product_mixer.component_library.scorer.tensorbuilder.ModelInferRequestBuilder -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class CortexManagedInferenceServiceTensorScorerBuilder @Inject() ( - statsReceiver: StatsReceiver) { - - /** - * Builds a configurable Scorer to call into your desired Cortex Managed ML Model Service. - * - * If your service does not bind an Http.Client implementation, add - * [[com.twitter.product_mixer.component_library.module.http.FinagleHttpClientModule]] - * to your server module list - * - * @param scorerIdentifier Unique identifier for the scorer - * @param resultFeatureExtractors The result features an their tensor extractors for each candidate. - * @tparam Query Type of pipeline query. - * @tparam Candidate Type of candidates to score. - * @tparam QueryFeatures type of the query level features consumed by the scorer. - * @tparam CandidateFeatures type of the candidate level features consumed by the scorer. - */ - def build[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]( - scorerIdentifier: ScorerIdentifier, - modelInferRequestBuilder: ModelInferRequestBuilder[ - Query, - Candidate - ], - resultFeatureExtractors: Seq[FeatureWithExtractor[Query, Candidate, _]], - client: MLModelInferenceClient - ): Scorer[Query, Candidate] = - new CortexManagedInferenceServiceTensorScorer( - scorerIdentifier, - modelInferRequestBuilder, - resultFeatureExtractors, - client, - statsReceiver.scope(scorerIdentifier.name) - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/ModelFeatureExtractor.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/ModelFeatureExtractor.docx new file mode 100644 index 000000000..1af73980d Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/ModelFeatureExtractor.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/ModelFeatureExtractor.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/ModelFeatureExtractor.scala deleted file mode 100644 index 3a44d5ceb..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/ModelFeatureExtractor.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.cortex - -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import inference.GrpcService.ModelInferResponse.InferOutputTensor - -/** - * Extractor defining how a Scorer should go from outputted tensors to the individual results - * for each candidate being scored. - * - * @tparam Result the type of the Value being returned. - * Users can pass in an anonymous function - */ -trait ModelFeatureExtractor[-Query <: PipelineQuery, Result] { - def apply(query: Query, tensorOutput: Seq[InferOutputTensor]): Seq[Result] -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cr_ml_ranker/BUILD.bazel b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cr_ml_ranker/BUILD.bazel deleted file mode 100644 index ce01a28e6..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cr_ml_ranker/BUILD.bazel +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "cr-ml-ranker/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/cr_ml_ranker", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/scorer", - ], - exports = [ - "cr-ml-ranker/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/cr_ml_ranker", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/scorer", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cr_ml_ranker/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cr_ml_ranker/BUILD.docx new file mode 100644 index 000000000..2b19675ff Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cr_ml_ranker/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cr_ml_ranker/CrMlRankerScorer.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cr_ml_ranker/CrMlRankerScorer.docx new file mode 100644 index 000000000..838b4dd7b Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cr_ml_ranker/CrMlRankerScorer.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cr_ml_ranker/CrMlRankerScorer.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cr_ml_ranker/CrMlRankerScorer.scala deleted file mode 100644 index 9bc1b6aae..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cr_ml_ranker/CrMlRankerScorer.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.cr_ml_ranker - -import com.twitter.product_mixer.component_library.feature_hydrator.query.cr_ml_ranker.CrMlRankerCommonFeatures -import com.twitter.product_mixer.component_library.feature_hydrator.query.cr_ml_ranker.CrMlRankerRankingConfig -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -object CrMlRankerScore extends Feature[TweetCandidate, Double] - -/** - * Scorer that scores tweets using the Content Recommender ML Light Ranker: http://go/cr-ml-ranker - */ -@Singleton -class CrMlRankerScorer @Inject() (crMlRanker: CrMlRankerScoreStitchClient) - extends Scorer[PipelineQuery, TweetCandidate] { - - override val identifier: ScorerIdentifier = ScorerIdentifier("CrMlRanker") - - override val features: Set[Feature[_, _]] = Set(CrMlRankerScore) - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - val queryFeatureMap = query.features.getOrElse(FeatureMap.empty) - val rankingConfig = queryFeatureMap.get(CrMlRankerRankingConfig) - val commonFeatures = queryFeatureMap.get(CrMlRankerCommonFeatures) - val userId = query.getRequiredUserId - - val scoresStitch = Stitch.collect(candidates.map { candidateWithFeatures => - crMlRanker - .getScore(userId, candidateWithFeatures.candidate, rankingConfig, commonFeatures).map( - _.score) - }) - scoresStitch.map { scores => - scores.map { score => - FeatureMapBuilder() - .add(CrMlRankerScore, score) - .build() - } - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cr_ml_ranker/CrMlRankerStitchClient.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cr_ml_ranker/CrMlRankerStitchClient.docx new file mode 100644 index 000000000..a241e8c1c Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cr_ml_ranker/CrMlRankerStitchClient.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cr_ml_ranker/CrMlRankerStitchClient.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cr_ml_ranker/CrMlRankerStitchClient.scala deleted file mode 100644 index 15e9103f1..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cr_ml_ranker/CrMlRankerStitchClient.scala +++ /dev/null @@ -1,79 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.cr_ml_ranker - -import com.twitter.cr_ml_ranker.{thriftscala => t} -import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate -import com.twitter.stitch.SeqGroup -import com.twitter.stitch.Stitch -import com.twitter.util.Future -import com.twitter.util.Return -import com.twitter.util.Try - -case class CrMlRankerResult( - tweetId: Long, - score: Double) - -class CrMlRankerScoreStitchClient( - crMLRanker: t.CrMLRanker.MethodPerEndpoint, - maxBatchSize: Int) { - - def getScore( - userId: Long, - tweetCandidate: BaseTweetCandidate, - rankingConfig: t.RankingConfig, - commonFeatures: t.CommonFeatures - ): Stitch[CrMlRankerResult] = { - Stitch.call( - tweetCandidate, - CrMlRankerGroup( - userId = userId, - rankingConfig = rankingConfig, - commonFeatures = commonFeatures - ) - ) - } - - private case class CrMlRankerGroup( - userId: Long, - rankingConfig: t.RankingConfig, - commonFeatures: t.CommonFeatures) - extends SeqGroup[BaseTweetCandidate, CrMlRankerResult] { - - override val maxSize: Int = maxBatchSize - - override protected def run( - tweetCandidates: Seq[BaseTweetCandidate] - ): Future[Seq[Try[CrMlRankerResult]]] = { - val crMlRankerCandidates = - tweetCandidates.map { tweetCandidate => - t.RankingCandidate( - tweetId = tweetCandidate.id, - hydrationContext = Some( - t.FeatureHydrationContext.HomeHydrationContext(t - .HomeFeatureHydrationContext(tweetAuthor = None))) - ) - } - - val thriftResults = crMLRanker.getRankedResults( - t.RankingRequest( - requestContext = t.RankingRequestContext( - userId = userId, - config = rankingConfig - ), - candidates = crMlRankerCandidates, - commonFeatures = commonFeatures.commonFeatures - ) - ) - - thriftResults.map { response => - response.scoredTweets.map { scoredTweet => - Return( - CrMlRankerResult( - tweetId = scoredTweet.tweetId, - score = scoredTweet.score - ) - ) - } - } - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/BUILD deleted file mode 100644 index 355427143..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/BUILD +++ /dev/null @@ -1,42 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "cortex-deepbird/thrift/src/main/thrift:thrift-java", - "finagle/finagle-http/src/main/scala", - "finatra-internal/mtls/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/featurestorev1", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/scorer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "scrooge/scrooge-serializer", - "src/java/com/twitter/ml/api:api-base", - "src/java/com/twitter/ml/common/base", - "src/java/com/twitter/ml/prediction/core", - "src/thrift/com/twitter/ml/prediction_service:prediction_service-java", - "src/thrift/com/twitter/ml/prediction_service:prediction_service-scala", - "twml/runtime/src/main/scala/com/twitter/deepbird/runtime/prediction_engine", - ], - exports = [ - "cortex-deepbird/thrift/src/main/thrift:thrift-java", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/scorer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "src/java/com/twitter/ml/prediction/core", - "src/thrift/com/twitter/ml/prediction_service:prediction_service-java", - "twml/runtime/src/main/scala/com/twitter/deepbird/runtime/prediction_engine", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/BUILD.docx new file mode 100644 index 000000000..d98885673 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/BaseDeepbirdV2Scorer.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/BaseDeepbirdV2Scorer.docx new file mode 100644 index 000000000..c578da41c Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/BaseDeepbirdV2Scorer.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/BaseDeepbirdV2Scorer.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/BaseDeepbirdV2Scorer.scala deleted file mode 100644 index 59b1f29bf..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/BaseDeepbirdV2Scorer.scala +++ /dev/null @@ -1,91 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.deepbird - -import com.twitter.product_mixer.core.feature.datarecord.BaseDataRecordFeature -import com.twitter.ml.prediction_service.BatchPredictionRequest -import com.twitter.ml.prediction_service.BatchPredictionResponse -import com.twitter.cortex.deepbird.thriftjava.{ModelSelector => TModelSelector} -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.scorer.common.ModelSelector -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordConverter -import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordExtractor -import com.twitter.product_mixer.core.feature.featuremap.datarecord.FeaturesScope -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier -import scala.collection.JavaConverters._ -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.pipeline_failure.IllegalStateFailure -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.stitch.Stitch -import com.twitter.util.Future - -abstract class BaseDeepbirdV2Scorer[ - Query <: PipelineQuery, - Candidate <: UniversalNoun[Any], - QueryFeatures <: BaseDataRecordFeature[Query, _], - CandidateFeatures <: BaseDataRecordFeature[Candidate, _], - ResultFeatures <: BaseDataRecordFeature[Candidate, _] -]( - override val identifier: ScorerIdentifier, - modelIdSelector: ModelSelector[Query], - queryFeatures: FeaturesScope[QueryFeatures], - candidateFeatures: FeaturesScope[CandidateFeatures], - resultFeatures: Set[ResultFeatures]) - extends Scorer[Query, Candidate] { - - private val queryDataRecordConverter = new DataRecordConverter(queryFeatures) - private val candidateDataRecordConverter = new DataRecordConverter(candidateFeatures) - private val resultDataRecordExtractor = new DataRecordExtractor(resultFeatures) - - require(resultFeatures.nonEmpty, "Result features cannot be empty") - override val features: Set[Feature[_, _]] = resultFeatures.asInstanceOf[Set[Feature[_, _]]] - def getBatchPredictions( - request: BatchPredictionRequest, - modelSelector: TModelSelector - ): Future[BatchPredictionResponse] - - override def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[Candidate]] - ): Stitch[Seq[FeatureMap]] = { - // Convert all candidate feature maps to java datarecords then to scala datarecords. - val thriftCandidateDataRecords = candidates.map { candidate => - candidateDataRecordConverter.toDataRecord(candidate.features) - } - - val request = new BatchPredictionRequest(thriftCandidateDataRecords.asJava) - - // Convert the query feature map to data record if available. - query.features.foreach { featureMap => - request.setCommonFeatures(queryDataRecordConverter.toDataRecord(featureMap)) - } - - val modelSelector = modelIdSelector - .apply(query).map { id => - val selector = new TModelSelector() - selector.setId(id) - selector - }.orNull - - Stitch.callFuture(getBatchPredictions(request, modelSelector)).map { response => - val dataRecords = Option(response.predictions).map(_.asScala).getOrElse(Seq.empty) - buildResults(candidates, dataRecords) - } - } - - private def buildResults( - candidates: Seq[CandidateWithFeatures[Candidate]], - dataRecords: Seq[DataRecord] - ): Seq[FeatureMap] = { - if (dataRecords.size != candidates.size) { - throw PipelineFailure(IllegalStateFailure, "Result Size mismatched candidates size") - } - - dataRecords.map { resultDataRecord => - resultDataRecordExtractor.fromDataRecord(resultDataRecord) - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/DeepbirdV2PredictionServerScorer.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/DeepbirdV2PredictionServerScorer.docx new file mode 100644 index 000000000..b4ae0720d Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/DeepbirdV2PredictionServerScorer.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/DeepbirdV2PredictionServerScorer.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/DeepbirdV2PredictionServerScorer.scala deleted file mode 100644 index 330706490..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/DeepbirdV2PredictionServerScorer.scala +++ /dev/null @@ -1,55 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.deepbird - -import com.twitter.cortex.deepbird.{thriftjava => t} -import com.twitter.ml.prediction_service.BatchPredictionRequest -import com.twitter.ml.prediction_service.BatchPredictionResponse -import com.twitter.product_mixer.component_library.scorer.common.ModelSelector -import com.twitter.product_mixer.core.feature.datarecord.BaseDataRecordFeature -import com.twitter.product_mixer.core.feature.featuremap.datarecord.FeaturesScope -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.util.Future - -/** - * Configurable Scorer that calls any Deepbird Prediction Service thrift. - * @param identifier Unique identifier for the scorer - * @param predictionService The Prediction Thrift Service - * @param modelSelector Model ID Selector to decide which model to select, can also be represented - * as an anonymous function: { query: Query => Some("Ex") } - * @param queryFeatures The Query Features to convert and pass to the deepbird model. - * @param candidateFeatures The Candidate Features to convert and pass to the deepbird model. - * @param resultFeatures The Candidate features returned by the model. - * @tparam Query Type of pipeline query. - * @tparam Candidate Type of candidates to score. - * @tparam QueryFeatures type of the query level features consumed by the scorer. - * @tparam CandidateFeatures type of the candidate level features consumed by the scorer. - * @tparam ResultFeatures type of the candidate level features returned by the scorer. - */ -case class DeepbirdV2PredictionServerScorer[ - Query <: PipelineQuery, - Candidate <: UniversalNoun[Any], - QueryFeatures <: BaseDataRecordFeature[Query, _], - CandidateFeatures <: BaseDataRecordFeature[Candidate, _], - ResultFeatures <: BaseDataRecordFeature[Candidate, _] -]( - override val identifier: ScorerIdentifier, - predictionService: t.DeepbirdPredictionService.ServiceToClient, - modelSelector: ModelSelector[Query], - queryFeatures: FeaturesScope[QueryFeatures], - candidateFeatures: FeaturesScope[CandidateFeatures], - resultFeatures: Set[ResultFeatures]) - extends BaseDeepbirdV2Scorer[ - Query, - Candidate, - QueryFeatures, - CandidateFeatures, - ResultFeatures - ](identifier, modelSelector, queryFeatures, candidateFeatures, resultFeatures) { - - override def getBatchPredictions( - request: BatchPredictionRequest, - modelSelector: t.ModelSelector - ): Future[BatchPredictionResponse] = - predictionService.batchPredictFromModel(request, modelSelector) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/LollyPredictionEngineScorer.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/LollyPredictionEngineScorer.docx new file mode 100644 index 000000000..8739307df Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/LollyPredictionEngineScorer.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/LollyPredictionEngineScorer.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/LollyPredictionEngineScorer.scala deleted file mode 100644 index c60651be9..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/LollyPredictionEngineScorer.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.deepbird - -import com.twitter.ml.prediction.core.PredictionEngine -import com.twitter.ml.prediction_service.PredictionRequest -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.datarecord.BaseDataRecordFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordConverter -import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordExtractor -import com.twitter.product_mixer.core.feature.featuremap.datarecord.FeaturesScope -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -/** - * Scorer that locally loads a Deepbird model. - * @param identifier Unique identifier for the scorer - * @param predictionEngine Prediction Engine hosting the Deepbird model. - * @param candidateFeatures The Candidate Features to convert and pass to the deepbird model. - * @param resultFeatures The Candidate features returned by the model. - * @tparam Query Type of pipeline query. - * @tparam Candidate Type of candidates to score. - * @tparam QueryFeatures type of the query level features consumed by the scorer. - * @tparam CandidateFeatures type of the candidate level features consumed by the scorer. - * @tparam ResultFeatures type of the candidate level features returned by the scorer. - */ -class LollyPredictionEngineScorer[ - Query <: PipelineQuery, - Candidate <: UniversalNoun[Any], - QueryFeatures <: BaseDataRecordFeature[Query, _], - CandidateFeatures <: BaseDataRecordFeature[Candidate, _], - ResultFeatures <: BaseDataRecordFeature[Candidate, _] -]( - override val identifier: ScorerIdentifier, - predictionEngine: PredictionEngine, - candidateFeatures: FeaturesScope[CandidateFeatures], - resultFeatures: Set[ResultFeatures]) - extends Scorer[Query, Candidate] { - - private val dataRecordAdapter = new DataRecordConverter(candidateFeatures) - - require(resultFeatures.nonEmpty, "Result features cannot be empty") - override val features: Set[Feature[_, _]] = resultFeatures.asInstanceOf[Set[Feature[_, _]]] - - private val resultsDataRecordExtractor = new DataRecordExtractor(resultFeatures) - - override def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[Candidate]] - ): Stitch[Seq[FeatureMap]] = { - val featureMaps = candidates.map { candidateWithFeatures => - val dataRecord = dataRecordAdapter.toDataRecord(candidateWithFeatures.features) - val predictionResponse = predictionEngine.apply(new PredictionRequest(dataRecord), true) - resultsDataRecordExtractor.fromDataRecord(predictionResponse.getPrediction) - } - Stitch.value(featureMaps) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/TensorflowPredictionEngineScorer.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/TensorflowPredictionEngineScorer.docx new file mode 100644 index 000000000..1c379a15c Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/TensorflowPredictionEngineScorer.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/TensorflowPredictionEngineScorer.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/TensorflowPredictionEngineScorer.scala deleted file mode 100644 index 847d62c89..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/TensorflowPredictionEngineScorer.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.deepbird - -import com.twitter.cortex.deepbird.runtime.prediction_engine.TensorflowPredictionEngine -import com.twitter.cortex.deepbird.thriftjava.ModelSelector -import com.twitter.ml.prediction_service.BatchPredictionRequest -import com.twitter.ml.prediction_service.BatchPredictionResponse -import com.twitter.product_mixer.core.feature.datarecord.BaseDataRecordFeature -import com.twitter.product_mixer.core.feature.featuremap.datarecord.FeaturesScope -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.util.Future - -/** - * Configurable Scorer that calls a TensorflowPredictionEngine. - * @param identifier Unique identifier for the scorer - * @param tensorflowPredictionEngine The TensorFlow Prediction Engine - * @param queryFeatures The Query Features to convert and pass to the deepbird model. - * @param candidateFeatures The Candidate Features to convert and pass to the deepbird model. - * @param resultFeatures The Candidate features returned by the model. - * @tparam Query Type of pipeline query. - * @tparam Candidate Type of candidates to score. - * @tparam QueryFeatures type of the query level features consumed by the scorer. - * @tparam CandidateFeatures type of the candidate level features consumed by the scorer. - * @tparam ResultFeatures type of the candidate level features returned by the scorer. - */ -class TensorflowPredictionEngineScorer[ - Query <: PipelineQuery, - Candidate <: UniversalNoun[Any], - QueryFeatures <: BaseDataRecordFeature[Query, _], - CandidateFeatures <: BaseDataRecordFeature[Candidate, _], - ResultFeatures <: BaseDataRecordFeature[Candidate, _] -]( - override val identifier: ScorerIdentifier, - tensorflowPredictionEngine: TensorflowPredictionEngine, - queryFeatures: FeaturesScope[QueryFeatures], - candidateFeatures: FeaturesScope[CandidateFeatures], - resultFeatures: Set[ResultFeatures]) - extends BaseDeepbirdV2Scorer[ - Query, - Candidate, - QueryFeatures, - CandidateFeatures, - ResultFeatures - ]( - identifier, - { _: Query => - None - }, - queryFeatures, - candidateFeatures, - resultFeatures) { - - override def getBatchPredictions( - request: BatchPredictionRequest, - modelSelector: ModelSelector - ): Future[BatchPredictionResponse] = tensorflowPredictionEngine.getBatchPrediction(request) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/param_gated/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/param_gated/BUILD deleted file mode 100644 index af69ac038..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/param_gated/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/scorer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/scorer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/param_gated/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/param_gated/BUILD.docx new file mode 100644 index 000000000..2108a3377 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/param_gated/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/param_gated/ParamGatedScorer.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/param_gated/ParamGatedScorer.docx new file mode 100644 index 000000000..023547857 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/param_gated/ParamGatedScorer.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/param_gated/ParamGatedScorer.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/param_gated/ParamGatedScorer.scala deleted file mode 100644 index 3185ce019..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/param_gated/ParamGatedScorer.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.param_gated - -import com.twitter.product_mixer.component_library.scorer.param_gated.ParamGatedScorer.IdentifierPrefix -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.Conditionally -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.Param - -/** - * A [[scorer]] with [[Conditionally]] based on a [[Param]] - * - * @param enabledParam the param to turn this [[scorer]] on and off - * @param scorer the underlying [[scorer]] to run when `enabledParam` is true - * @tparam Query The domain model for the query or request - * @tparam Result The type of the candidates - */ -case class ParamGatedScorer[-Query <: PipelineQuery, Result <: UniversalNoun[Any]]( - enabledParam: Param[Boolean], - scorer: Scorer[Query, Result]) - extends Scorer[Query, Result] - with Conditionally[Query] { - override val identifier: ScorerIdentifier = ScorerIdentifier( - IdentifierPrefix + scorer.identifier.name) - override val alerts: Seq[Alert] = scorer.alerts - override val features: Set[Feature[_, _]] = scorer.features - override def onlyIf(query: Query): Boolean = - Conditionally.and(query, scorer, query.params(enabledParam)) - override def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[Result]] - ): Stitch[Seq[FeatureMap]] = scorer(query, candidates) -} - -object ParamGatedScorer { - val IdentifierPrefix = "ParamGated" -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/qualityfactor_gated/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/qualityfactor_gated/BUILD deleted file mode 100644 index af69ac038..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/qualityfactor_gated/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/scorer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/scorer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/qualityfactor_gated/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/qualityfactor_gated/BUILD.docx new file mode 100644 index 000000000..2108a3377 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/qualityfactor_gated/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/qualityfactor_gated/QualityFactorGatedScorer.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/qualityfactor_gated/QualityFactorGatedScorer.docx new file mode 100644 index 000000000..9936eb2f9 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/qualityfactor_gated/QualityFactorGatedScorer.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/qualityfactor_gated/QualityFactorGatedScorer.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/qualityfactor_gated/QualityFactorGatedScorer.scala deleted file mode 100644 index 3309fbdca..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/qualityfactor_gated/QualityFactorGatedScorer.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.qualityfactor_gated - -import com.twitter.product_mixer.component_library.scorer.qualityfactor_gated.QualityFactorGatedScorer.IdentifierPrefix -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.Conditionally -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.Param - -/** - * A [[scorer]] with [[Conditionally]] based on quality factor value and threshold - * - * @param qualityFactorThreshold quliaty factor threshold that turn off the scorer - * @param pipelineIdentifier identifier of the pipeline that quality factor is based on - * @param scorer the underlying [[scorer]] to run when `enabledParam` is true - * @tparam Query The domain model for the query or request - * @tparam Result The type of the candidates - */ -case class QualityFactorGatedScorer[ - -Query <: PipelineQuery with HasQualityFactorStatus, - Result <: UniversalNoun[Any] -]( - pipelineIdentifier: ComponentIdentifier, - qualityFactorThresholdParam: Param[Double], - scorer: Scorer[Query, Result]) - extends Scorer[Query, Result] - with Conditionally[Query] { - - override val identifier: ScorerIdentifier = ScorerIdentifier( - IdentifierPrefix + scorer.identifier.name) - - override val alerts: Seq[Alert] = scorer.alerts - - override val features: Set[Feature[_, _]] = scorer.features - - override def onlyIf(query: Query): Boolean = - Conditionally.and( - query, - scorer, - query.getQualityFactorCurrentValue(pipelineIdentifier) >= query.params( - qualityFactorThresholdParam)) - - override def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[Result]] - ): Stitch[Seq[FeatureMap]] = scorer(query, candidates) -} - -object QualityFactorGatedScorer { - val IdentifierPrefix = "QualityFactorGated" -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/BUILD deleted file mode 100644 index e18c39662..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/featurestorev1", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/thrift/com/twitter/ml/api:embedding-scala", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/BUILD.docx new file mode 100644 index 000000000..a1074dbe5 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/BooleanInferInputTensorBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/BooleanInferInputTensorBuilder.docx new file mode 100644 index 000000000..795ec87de Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/BooleanInferInputTensorBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/BooleanInferInputTensorBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/BooleanInferInputTensorBuilder.scala deleted file mode 100644 index bbdf2b5c6..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/BooleanInferInputTensorBuilder.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.tensorbuilder - -import inference.GrpcService.ModelInferRequest.InferInputTensor - -case object BooleanInferInputTensorBuilder extends InferInputTensorBuilder[Boolean] { - def apply( - featureName: String, - featureValues: Seq[Boolean] - ): Seq[InferInputTensor] = { - val tensorShape = Seq(featureValues.size, 1) - InferInputTensorBuilder.buildBoolInferInputTensor(featureName, featureValues, tensorShape) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/BytesInferInputTensorBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/BytesInferInputTensorBuilder.docx new file mode 100644 index 000000000..06d68d158 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/BytesInferInputTensorBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/BytesInferInputTensorBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/BytesInferInputTensorBuilder.scala deleted file mode 100644 index 8b36c514f..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/BytesInferInputTensorBuilder.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.tensorbuilder - -import inference.GrpcService.ModelInferRequest.InferInputTensor - -case object BytesInferInputTensorBuilder extends InferInputTensorBuilder[String] { - def apply( - featureName: String, - featureValues: Seq[String] - ): Seq[InferInputTensor] = { - val tensorShape = Seq(featureValues.size, 1) - InferInputTensorBuilder.buildBytesInferInputTensor(featureName, featureValues, tensorShape) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/CandidateInferInputTensorBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/CandidateInferInputTensorBuilder.docx new file mode 100644 index 000000000..cd987bf6d Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/CandidateInferInputTensorBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/CandidateInferInputTensorBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/CandidateInferInputTensorBuilder.scala deleted file mode 100644 index ed7c28cdb..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/CandidateInferInputTensorBuilder.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.tensorbuilder - -import com.twitter.ml.api.thriftscala.FloatTensor -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.ModelFeatureName -import com.twitter.product_mixer.core.feature.featuremap.featurestorev1.FeatureStoreV1FeatureMap._ -import com.twitter.product_mixer.core.feature.featurestorev1.FeatureStoreV1CandidateFeature -import com.twitter.product_mixer.core.feature.featurestorev1.FeatureStoreV1QueryFeature -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import inference.GrpcService.ModelInferRequest.InferInputTensor - -class CandidateInferInputTensorBuilder[-Candidate <: UniversalNoun[Any], +Value]( - builder: InferInputTensorBuilder[Value], - features: Set[_ <: Feature[Candidate, _] with ModelFeatureName]) { - def apply( - candidates: Seq[CandidateWithFeatures[Candidate]], - ): Seq[InferInputTensor] = { - features.flatMap { feature => - val featureValues: Seq[Value] = feature match { - case feature: FeatureStoreV1CandidateFeature[_, Candidate, _, Value] => - candidates.map(_.features.getFeatureStoreV1CandidateFeature(feature)) - case feature: FeatureStoreV1QueryFeature[_, _, _] => - throw new UnexpectedFeatureTypeException(feature) - case feature: FeatureWithDefaultOnFailure[Candidate, Value] => - candidates.map(_.features.getTry(feature).toOption.getOrElse(feature.defaultValue)) - case feature: Feature[Candidate, Value] => - candidates.map(_.features.get(feature)) - } - builder.apply(feature.featureName, featureValues) - }.toSeq - } -} - -case class CandidateBooleanInferInputTensorBuilder[-Candidate <: UniversalNoun[Any]]( - features: Set[_ <: Feature[Candidate, Boolean] with ModelFeatureName]) - extends CandidateInferInputTensorBuilder[Candidate, Boolean]( - BooleanInferInputTensorBuilder, - features) - -case class CandidateBytesInferInputTensorBuilder[-Candidate <: UniversalNoun[Any]]( - features: Set[_ <: Feature[Candidate, String] with ModelFeatureName]) - extends CandidateInferInputTensorBuilder[Candidate, String]( - BytesInferInputTensorBuilder, - features) - -case class CandidateFloat32InferInputTensorBuilder[-Candidate <: UniversalNoun[Any]]( - features: Set[_ <: Feature[Candidate, _ <: AnyVal] with ModelFeatureName]) - extends CandidateInferInputTensorBuilder[Candidate, AnyVal]( - Float32InferInputTensorBuilder, - features) - -case class CandidateFloatTensorInferInputTensorBuilder[-Candidate <: UniversalNoun[Any]]( - features: Set[_ <: Feature[Candidate, FloatTensor] with ModelFeatureName]) - extends CandidateInferInputTensorBuilder[Candidate, FloatTensor]( - FloatTensorInferInputTensorBuilder, - features) - -case class CandidateInt64InferInputTensorBuilder[-Candidate <: UniversalNoun[Any]]( - features: Set[_ <: Feature[Candidate, _ <: AnyVal] with ModelFeatureName]) - extends CandidateInferInputTensorBuilder[Candidate, AnyVal]( - Int64InferInputTensorBuilder, - features) - -case class CandidateSparseMapInferInputTensorBuilder[-Candidate <: UniversalNoun[Any]]( - features: Set[_ <: Feature[Candidate, Option[Map[Int, Double]]] with ModelFeatureName]) - extends CandidateInferInputTensorBuilder[Candidate, Option[Map[Int, Double]]]( - SparseMapInferInputTensorBuilder, - features) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/Float32InferInputTensorBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/Float32InferInputTensorBuilder.docx new file mode 100644 index 000000000..8ca507deb Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/Float32InferInputTensorBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/Float32InferInputTensorBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/Float32InferInputTensorBuilder.scala deleted file mode 100644 index a6fb674bc..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/Float32InferInputTensorBuilder.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.tensorbuilder - -import inference.GrpcService.ModelInferRequest.InferInputTensor - -case object Float32InferInputTensorBuilder extends InferInputTensorBuilder[AnyVal] { - - private def toFloat(x: AnyVal): Float = { - x match { - case y: Float => y - case y: Int => y.toFloat - case y: Long => y.toFloat - case y: Double => y.toFloat - case y => throw new UnexpectedDataTypeException(y, this) - } - } - - def apply( - featureName: String, - featureValues: Seq[AnyVal] - ): Seq[InferInputTensor] = { - val tensorShape = Seq(featureValues.size, 1) - InferInputTensorBuilder.buildFloat32InferInputTensor( - featureName, - featureValues.map(toFloat), - tensorShape) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/FloatTensorInferInputTensorBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/FloatTensorInferInputTensorBuilder.docx new file mode 100644 index 000000000..24114a29f Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/FloatTensorInferInputTensorBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/FloatTensorInferInputTensorBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/FloatTensorInferInputTensorBuilder.scala deleted file mode 100644 index d1f4f021f..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/FloatTensorInferInputTensorBuilder.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.tensorbuilder - -import com.twitter.ml.api.thriftscala.FloatTensor -import inference.GrpcService.ModelInferRequest.InferInputTensor - -case object FloatTensorInferInputTensorBuilder extends InferInputTensorBuilder[FloatTensor] { - - private[tensorbuilder] def extractTensorShape(featureValues: Seq[FloatTensor]): Seq[Int] = { - val headFloatTensor = featureValues.head - if (headFloatTensor.shape.isEmpty) { - Seq( - featureValues.size, - featureValues.head.floats.size - ) - } else { - Seq(featureValues.size) ++ headFloatTensor.shape.get.map(_.toInt) - } - } - - def apply( - featureName: String, - featureValues: Seq[FloatTensor] - ): Seq[InferInputTensor] = { - if (featureValues.isEmpty) throw new EmptyFloatTensorException(featureName) - val tensorShape = extractTensorShape(featureValues) - val floatValues = featureValues.flatMap { featureValue => - featureValue.floats.map(_.toFloat) - } - InferInputTensorBuilder.buildFloat32InferInputTensor(featureName, floatValues, tensorShape) - } -} -class EmptyFloatTensorException(featureName: String) - extends RuntimeException(s"FloatTensor in feature $featureName is empty!") diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/InferInputTensorBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/InferInputTensorBuilder.docx new file mode 100644 index 000000000..447aa4d3b Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/InferInputTensorBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/InferInputTensorBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/InferInputTensorBuilder.scala deleted file mode 100644 index e83fea4e2..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/InferInputTensorBuilder.scala +++ /dev/null @@ -1,151 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.tensorbuilder - -import com.google.protobuf.ByteString -import com.twitter.product_mixer.core.feature.Feature -import inference.GrpcService.InferTensorContents -import inference.GrpcService.ModelInferRequest.InferInputTensor - -// This class contains most of common versions at Twitter, but in the future we can add more: -// https://github.com/kserve/kserve/blob/master/docs/predict-api/v2/required_api.md#tensor-data-1 - -trait InferInputTensorBuilder[Value] { - - def apply( - featureName: String, - featureValues: Seq[Value] - ): Seq[InferInputTensor] - -} - -object InferInputTensorBuilder { - - def checkTensorShapeMatchesValueLength( - featureName: String, - featureValues: Seq[Any], - tensorShape: Seq[Int] - ): Unit = { - val featureValuesSize = featureValues.size - val tensorShapeSize = tensorShape.product - if (featureValuesSize != tensorShapeSize) { - throw new FeatureValuesAndShapeMismatchException( - featureName, - featureValuesSize, - tensorShapeSize) - } - } - - def buildBoolInferInputTensor( - featureName: String, - featureValues: Seq[Boolean], - tensorShape: Seq[Int] - ): Seq[InferInputTensor] = { - - checkTensorShapeMatchesValueLength(featureName, featureValues, tensorShape) - - val inputTensorBuilder = InferInputTensor.newBuilder().setName(featureName) - tensorShape.foreach { shape => - inputTensorBuilder.addShape(shape) - } - val inputTensor = inputTensorBuilder - .setDatatype("BOOL") - .setContents { - val contents = InferTensorContents.newBuilder() - featureValues.foreach { featureValue => - contents.addBoolContents(featureValue) - } - contents - } - .build() - Seq(inputTensor) - } - - def buildBytesInferInputTensor( - featureName: String, - featureValues: Seq[String], - tensorShape: Seq[Int] - ): Seq[InferInputTensor] = { - - checkTensorShapeMatchesValueLength(featureName, featureValues, tensorShape) - - val inputTensorBuilder = InferInputTensor.newBuilder().setName(featureName) - tensorShape.foreach { shape => - inputTensorBuilder.addShape(shape) - } - val inputTensor = inputTensorBuilder - .setDatatype("BYTES") - .setContents { - val contents = InferTensorContents.newBuilder() - featureValues.foreach { featureValue => - val featureValueBytes = ByteString.copyFromUtf8(featureValue) - contents.addByteContents(featureValueBytes) - } - contents - } - .build() - Seq(inputTensor) - } - - def buildFloat32InferInputTensor( - featureName: String, - featureValues: Seq[Float], - tensorShape: Seq[Int] - ): Seq[InferInputTensor] = { - - checkTensorShapeMatchesValueLength(featureName, featureValues, tensorShape) - - val inputTensorBuilder = InferInputTensor.newBuilder().setName(featureName) - tensorShape.foreach { shape => - inputTensorBuilder.addShape(shape) - } - val inputTensor = inputTensorBuilder - .setDatatype("FP32") - .setContents { - val contents = InferTensorContents.newBuilder() - featureValues.foreach { featureValue => - contents.addFp32Contents(featureValue.floatValue) - } - contents - } - .build() - Seq(inputTensor) - } - - def buildInt64InferInputTensor( - featureName: String, - featureValues: Seq[Long], - tensorShape: Seq[Int] - ): Seq[InferInputTensor] = { - - checkTensorShapeMatchesValueLength(featureName, featureValues, tensorShape) - - val inputTensorBuilder = InferInputTensor.newBuilder().setName(featureName) - tensorShape.foreach { shape => - inputTensorBuilder.addShape(shape) - } - val inputTensor = inputTensorBuilder - .setDatatype("INT64") - .setContents { - val contents = InferTensorContents.newBuilder() - featureValues.foreach { featureValue => - contents.addInt64Contents(featureValue) - } - contents - } - .build() - Seq(inputTensor) - } -} - -class UnexpectedFeatureTypeException(feature: Feature[_, _]) - extends UnsupportedOperationException(s"Unsupported Feature type passed in $feature") - -class FeatureValuesAndShapeMismatchException( - featureName: String, - featureValuesSize: Int, - tensorShapeSize: Int) - extends UnsupportedOperationException( - s"Feature $featureName has mismatching FeatureValues (size: $featureValuesSize) and TensorShape (size: $tensorShapeSize)!") - -class UnexpectedDataTypeException[T](value: T, builder: InferInputTensorBuilder[_]) - extends UnsupportedOperationException( - s"Unsupported data type ${value} passed in at ${builder.getClass.toString}") diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/Int64InferInputTensorBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/Int64InferInputTensorBuilder.docx new file mode 100644 index 000000000..846217dbb Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/Int64InferInputTensorBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/Int64InferInputTensorBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/Int64InferInputTensorBuilder.scala deleted file mode 100644 index 9743317d8..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/Int64InferInputTensorBuilder.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.tensorbuilder - -import com.twitter.ml.featurestore.lib.Discrete -import inference.GrpcService.ModelInferRequest.InferInputTensor - -case object Int64InferInputTensorBuilder extends InferInputTensorBuilder[AnyVal] { - - private def toLong(x: AnyVal): Long = { - x match { - case y: Int => y.toLong - case y: Long => y - case y: Discrete => y.value - case y => throw new UnexpectedDataTypeException(y, this) - } - } - def apply( - featureName: String, - featureValues: Seq[AnyVal] - ): Seq[InferInputTensor] = { - val tensorShape = Seq(featureValues.size, 1) - InferInputTensorBuilder.buildInt64InferInputTensor( - featureName, - featureValues.map(toLong), - tensorShape) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/ModelInferRequestBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/ModelInferRequestBuilder.docx new file mode 100644 index 000000000..83bcf131d Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/ModelInferRequestBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/ModelInferRequestBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/ModelInferRequestBuilder.scala deleted file mode 100644 index e37c145fb..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/ModelInferRequestBuilder.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.tensorbuilder - -import com.twitter.product_mixer.component_library.scorer.common.ModelSelector -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import inference.GrpcService.InferParameter -import inference.GrpcService.ModelInferRequest -import scala.collection.JavaConverters._ - -class ModelInferRequestBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( - queryInferInputTensorBuilders: Seq[QueryInferInputTensorBuilder[Query, Any]], - candidateInferInputTensorBuilders: Seq[ - CandidateInferInputTensorBuilder[Candidate, Any] - ], - modelSignatureName: String, - modelSelector: ModelSelector[Query]) { - - private val modelSignature: InferParameter = - InferParameter.newBuilder().setStringParam(modelSignatureName).build() - - def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[Candidate]], - ): ModelInferRequest = { - val inferRequest = ModelInferRequest - .newBuilder() - .putParameters("signature_name", modelSignature) - modelSelector.apply(query).foreach { modelName => - inferRequest.setModelName(modelName) - } - queryInferInputTensorBuilders.foreach { builder => - inferRequest.addAllInputs(builder(query).asJava) - } - candidateInferInputTensorBuilders.foreach { builder => - inferRequest.addAllInputs(builder(candidates).asJava) - } - inferRequest.build() - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/QueryInferInputTensorBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/QueryInferInputTensorBuilder.docx new file mode 100644 index 000000000..99db5a4f6 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/QueryInferInputTensorBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/QueryInferInputTensorBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/QueryInferInputTensorBuilder.scala deleted file mode 100644 index 4415d61a1..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/QueryInferInputTensorBuilder.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.tensorbuilder - -import com.twitter.ml.api.thriftscala.FloatTensor -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.ModelFeatureName -import com.twitter.product_mixer.core.feature.featuremap.featurestorev1.FeatureStoreV1FeatureMap._ -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featurestorev1.FeatureStoreV1CandidateFeature -import com.twitter.product_mixer.core.feature.featurestorev1.FeatureStoreV1QueryFeature -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import inference.GrpcService.ModelInferRequest.InferInputTensor - -class QueryInferInputTensorBuilder[-Query <: PipelineQuery, +Value]( - builder: InferInputTensorBuilder[Value], - features: Set[_ <: Feature[Query, _] with ModelFeatureName]) { - def apply(query: Query): Seq[InferInputTensor] = { - val featureMap = query.features.getOrElse(FeatureMap.empty) - features.flatMap { feature => - val queryFeatureValue: Value = feature match { - case feature: FeatureStoreV1QueryFeature[Query, _, Value] => - featureMap.getFeatureStoreV1QueryFeature(feature) - case feature: FeatureStoreV1CandidateFeature[Query, _, _, Value] => - throw new UnexpectedFeatureTypeException(feature) - case feature: FeatureWithDefaultOnFailure[Query, Value] => - featureMap.getTry(feature).toOption.getOrElse(feature.defaultValue) - case feature: Feature[Query, Value] => - featureMap.get(feature) - } - builder.apply(feature.featureName, Seq(queryFeatureValue)) - }.toSeq - } -} - -case class QueryBooleanInferInputTensorBuilder[-Query <: PipelineQuery]( - features: Set[_ <: Feature[Query, Boolean] with ModelFeatureName]) - extends QueryInferInputTensorBuilder[Query, Boolean](BooleanInferInputTensorBuilder, features) - -case class QueryBytesInferInputTensorBuilder[-Query <: PipelineQuery]( - features: Set[_ <: Feature[Query, String] with ModelFeatureName]) - extends QueryInferInputTensorBuilder[Query, String](BytesInferInputTensorBuilder, features) - -case class QueryFloat32InferInputTensorBuilder[-Query <: PipelineQuery]( - features: Set[_ <: Feature[Query, _ <: AnyVal] with ModelFeatureName]) - extends QueryInferInputTensorBuilder[Query, AnyVal](Float32InferInputTensorBuilder, features) - -case class QueryFloatTensorInferInputTensorBuilder[-Query <: PipelineQuery]( - features: Set[_ <: Feature[Query, FloatTensor] with ModelFeatureName]) - extends QueryInferInputTensorBuilder[Query, FloatTensor]( - FloatTensorInferInputTensorBuilder, - features) - -case class QueryInt64InferInputTensorBuilder[-Query <: PipelineQuery]( - features: Set[_ <: Feature[Query, _ <: AnyVal] with ModelFeatureName]) - extends QueryInferInputTensorBuilder[Query, AnyVal](Int64InferInputTensorBuilder, features) - -case class QuerySparseMapInferInputTensorBuilder[-Query <: PipelineQuery]( - features: Set[_ <: Feature[Query, Option[Map[Int, Double]]] with ModelFeatureName]) - extends QueryInferInputTensorBuilder[Query, Option[Map[Int, Double]]]( - SparseMapInferInputTensorBuilder, - features) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/SparseMapInferInputTensorBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/SparseMapInferInputTensorBuilder.docx new file mode 100644 index 000000000..714219d86 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/SparseMapInferInputTensorBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/SparseMapInferInputTensorBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/SparseMapInferInputTensorBuilder.scala deleted file mode 100644 index 038ff90db..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/SparseMapInferInputTensorBuilder.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.tensorbuilder - -import inference.GrpcService.InferTensorContents -import inference.GrpcService.ModelInferRequest.InferInputTensor - -case object SparseMapInferInputTensorBuilder - extends InferInputTensorBuilder[Option[Map[Int, Double]]] { - - private final val batchFeatureNameSuffix: String = "batch" - private final val keyFeatureNameSuffix: String = "key" - private final val valueFeatureNameSuffix: String = "value" - - def apply( - featureName: String, - featureValues: Seq[Option[Map[Int, Double]]] - ): Seq[InferInputTensor] = { - val batchIdsTensorContents = InferTensorContents.newBuilder() - val sparseKeysTensorContents = InferTensorContents.newBuilder() - val sparseValuesTensorContents = InferTensorContents.newBuilder() - featureValues.zipWithIndex.foreach { - case (featureValueOption, batchIndex) => - featureValueOption.foreach { featureValue => - featureValue.foreach { - case (sparseKey, sparseValue) => - batchIdsTensorContents.addInt64Contents(batchIndex.toLong) - sparseKeysTensorContents.addInt64Contents(sparseKey.toLong) - sparseValuesTensorContents.addFp32Contents(sparseValue.floatValue) - } - } - } - - val batchIdsInputTensor = InferInputTensor - .newBuilder() - .setName(Seq(featureName, batchFeatureNameSuffix).mkString("_")) - .addShape(batchIdsTensorContents.getInt64ContentsCount) - .addShape(1) - .setDatatype("INT64") - .setContents(batchIdsTensorContents) - .build() - - val sparseKeysInputTensor = InferInputTensor - .newBuilder() - .setName(Seq(featureName, keyFeatureNameSuffix).mkString("_")) - .addShape(sparseKeysTensorContents.getInt64ContentsCount) - .addShape(1) - .setDatatype("INT64") - .setContents(sparseKeysTensorContents) - .build() - - val sparseValuesInputTensor = InferInputTensor - .newBuilder() - .setName(Seq(featureName, valueFeatureNameSuffix).mkString("_")) - .addShape(sparseValuesTensorContents.getFp32ContentsCount) - .addShape(1) - .setDatatype("FP32") - .setContents(sparseValuesTensorContents) - .build() - - Seq(batchIdsInputTensor, sparseKeysInputTensor, sparseValuesInputTensor) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tweet_tlx/BUILD.bazel b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tweet_tlx/BUILD.bazel deleted file mode 100644 index e306724d0..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tweet_tlx/BUILD.bazel +++ /dev/null @@ -1,26 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/scorer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/thrift/com/twitter/ml/featurestore/timelines:ml-features-timelines-scala", - "src/thrift/com/twitter/timelinescorer:thrift-scala", - "src/thrift/com/twitter/timelinescorer/server/internal:thrift-scala", - "strato/config/columns/ml/featureStore:featureStore-strato-client", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/scorer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/thrift/com/twitter/ml/featurestore/timelines:ml-features-timelines-scala", - "src/thrift/com/twitter/timelinescorer:thrift-scala", - "src/thrift/com/twitter/timelinescorer/server/internal:thrift-scala", - "strato/config/columns/ml/featureStore:featureStore-strato-client", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tweet_tlx/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tweet_tlx/BUILD.docx new file mode 100644 index 000000000..d30cb1956 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tweet_tlx/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tweet_tlx/TweetTLXStratoScorer.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tweet_tlx/TweetTLXStratoScorer.docx new file mode 100644 index 000000000..0aecda376 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tweet_tlx/TweetTLXStratoScorer.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tweet_tlx/TweetTLXStratoScorer.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tweet_tlx/TweetTLXStratoScorer.scala deleted file mode 100644 index 69a803c90..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tweet_tlx/TweetTLXStratoScorer.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.tweet_tlx - -import com.twitter.ml.featurestore.timelines.thriftscala.TimelineScorerScoreView -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.strato.catalog.Fetch.Result -import com.twitter.strato.generated.client.ml.featureStore.TimelineScorerTweetScoresV1ClientColumn -import com.twitter.timelinescorer.thriftscala.v1 -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Score Tweets via Timeline Scorer (TLX) Strato API - * - * @note This results in an additional hop through Strato Server - * @note This is the [[Scorer]] version of - * [[com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_tlx.TweetTLXScoreCandidateFeatureHydrator]] - */ -@Singleton -class TweetTLXStratoScorer @Inject() (column: TimelineScorerTweetScoresV1ClientColumn) - extends Scorer[PipelineQuery, TweetCandidate] { - - override val identifier: ScorerIdentifier = ScorerIdentifier("TweetTLX") - - override val features: Set[Feature[_, _]] = Set(TLXScore) - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = query.getOptionalUserId match { - case Some(userId) => getScoredTweetsFromTLX(userId, candidates.map(_.candidate)) - case _ => - val defaultFeatureMap = FeatureMapBuilder().add(TLXScore, None).build() - Stitch.value(candidates.map(_ => defaultFeatureMap)) - } - - def getScoredTweetsFromTLX( - userId: Long, - tweetCandidates: Seq[TweetCandidate] - ): Stitch[Seq[FeatureMap]] = Stitch.collect(tweetCandidates.map { candidate => - column.fetcher - .fetch(candidate.id, TimelineScorerScoreView(Some(userId))) - .map { - case Result(Some(v1.ScoredTweet(_, score, _, _)), _) => - FeatureMapBuilder() - .add(TLXScore, score) - .build() - case fetchResult => throw new Exception(s"Invalid response from TLX: ${fetchResult.v}") - } - }) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tweet_tlx/TweetTLXThriftScorer.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tweet_tlx/TweetTLXThriftScorer.docx new file mode 100644 index 000000000..656f58f8b Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tweet_tlx/TweetTLXThriftScorer.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tweet_tlx/TweetTLXThriftScorer.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tweet_tlx/TweetTLXThriftScorer.scala deleted file mode 100644 index e53f3dcfc..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tweet_tlx/TweetTLXThriftScorer.scala +++ /dev/null @@ -1,79 +0,0 @@ -package com.twitter.product_mixer.component_library.scorer.tweet_tlx - -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelinescorer.thriftscala.v1 -import com.twitter.timelinescorer.{thriftscala => t} -import javax.inject.Inject -import javax.inject.Singleton - -/** - * @note This Feature is shared with - * [[com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_tlx.TweetTLXScoreCandidateFeatureHydrator]] - * and - * [[com.twitter.product_mixer.component_library.scorer.tweet_tlx.TweetTLXStratoScorer]] - * as the these components should not be used at the same time by the same Product - */ -object TLXScore extends FeatureWithDefaultOnFailure[TweetCandidate, Option[Double]] { - override val defaultValue = None -} - -/** - * Score Tweets via Timeline Scorer (TLX) Thrift API - * - * @note This is the [[Scorer]] version of - * [[com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_tlx.TweetTLXScoreCandidateFeatureHydrator]] - */ -@Singleton -class TweetTLXThriftScorer @Inject() (timelineScorerClient: t.TimelineScorer.MethodPerEndpoint) - extends Scorer[PipelineQuery, TweetCandidate] { - - override val identifier: ScorerIdentifier = ScorerIdentifier("TLX") - - override val features: Set[Feature[_, _]] = Set(TLXScore) - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - val userId = query.getOptionalUserId - val tweetScoringQuery = v1.TweetScoringQuery( - predictionPipeline = v1.PredictionPipeline.Recap, - tweetIds = candidates.map(_.candidate.id)) - - val tweetScoringRequest = t.TweetScoringRequest.V1( - v1.TweetScoringRequest( - tweetScoringRequestContext = Some(v1.TweetScoringRequestContext(userId = userId)), - tweetScoringQueries = Some(Seq(tweetScoringQuery)), - retrieveFeatures = Some(false) - )) - - Stitch.callFuture(timelineScorerClient.getTweetScores(tweetScoringRequest)).map { - case t.TweetScoringResponse.V1(response) => - val tweetIdScoreMap = response.tweetScoringResults - .flatMap { - _.headOption.map { - _.scoredTweets.flatMap(tweet => tweet.tweetId.map(_ -> tweet.score)) - } - }.getOrElse(Seq.empty).toMap - - candidates.map { candidateWithFeatures => - val score = tweetIdScoreMap.getOrElse(candidateWithFeatures.candidate.id, None) - FeatureMapBuilder() - .add(TLXScore, score) - .build() - - } - case t.TweetScoringResponse.UnknownUnionField(field) => - throw new UnsupportedOperationException(s"Unknown response type: ${field.field.name}") - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/BUILD deleted file mode 100644 index c07a21e65..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/BUILD +++ /dev/null @@ -1,47 +0,0 @@ -scala_library( - name = "insert_append_results", - sources = ["InsertAppendResults.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector", - ], -) - -scala_library( - sources = [ - "!InsertAppendResults.scala", - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - scalac_plugins = ["no-roomba"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - ":insert_append_results", - "3rdparty/jvm/com/google/inject:guice", - "finatra/inject/inject-slf4j/src/main/scala/com/twitter/inject", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector", - ], - exports = [ - ":insert_append_results", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/BUILD.docx new file mode 100644 index 000000000..57932e348 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/Bucketer.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/Bucketer.docx new file mode 100644 index 000000000..220e910be Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/Bucketer.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/Bucketer.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/Bucketer.scala deleted file mode 100644 index d50394a2b..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/Bucketer.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails - -/** - * Given a [[CandidateWithDetails]] return the corresponding [[Bucket]] - * it should be associated with when used in a `pattern` or `ratio` - * in [[InsertAppendPatternResults]] or [[InsertAppendRatioResults]] - */ -trait Bucketer[Bucket] { - def apply(candidateWithDetails: CandidateWithDetails): Bucket -} - -object Bucketer { - - /** A [[Bucketer]] that buckets by [[CandidateWithDetails.source]] */ - val ByCandidateSource: Bucketer[CandidatePipelineIdentifier] = - (candidateWithDetails: CandidateWithDetails) => candidateWithDetails.source -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/CandidateMergeStrategy.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/CandidateMergeStrategy.docx new file mode 100644 index 000000000..546511fcd Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/CandidateMergeStrategy.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/CandidateMergeStrategy.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/CandidateMergeStrategy.scala deleted file mode 100644 index 2ef321cd6..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/CandidateMergeStrategy.scala +++ /dev/null @@ -1,82 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate -import com.twitter.product_mixer.component_library.model.candidate.IsPinnedFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.CandidatePipelines -import com.twitter.product_mixer.core.model.common.presentation.CandidateSources -import com.twitter.product_mixer.core.model.common.presentation.CandidateSourcePosition - -/** - * Once a pair of duplicate candidates has been found we need to someone 'resolve' the duplication. - * This may be as simple as picking whichever candidate came first (see [[PickFirstCandidateMerger]] - * but this strategy could mean losing important candidate information. Candidates might, for - * example, have different features. [[CandidateMergeStrategy]] lets you define a custom behavior - * for resolving duplication to help support these more nuanced situations. - */ -trait CandidateMergeStrategy { - def apply( - existingCandidate: ItemCandidateWithDetails, - newCandidate: ItemCandidateWithDetails - ): ItemCandidateWithDetails -} - -/** - * Keep whichever candidate was encountered first. - */ -object PickFirstCandidateMerger extends CandidateMergeStrategy { - override def apply( - existingCandidate: ItemCandidateWithDetails, - newCandidate: ItemCandidateWithDetails - ): ItemCandidateWithDetails = existingCandidate -} - -/** - * Keep the candidate encountered first but combine all candidate feature maps. - */ -object CombineFeatureMapsCandidateMerger extends CandidateMergeStrategy { - override def apply( - existingCandidate: ItemCandidateWithDetails, - newCandidate: ItemCandidateWithDetails - ): ItemCandidateWithDetails = { - // Prepend new because list set keeps insertion order, and last operations in ListSet are O(1) - val mergedCandidateSourceIdentifiers = - newCandidate.features.get(CandidateSources) ++ existingCandidate.features - .get(CandidateSources) - val mergedCandidatePipelineIdentifiers = - newCandidate.features.get(CandidatePipelines) ++ existingCandidate.features - .get(CandidatePipelines) - - // the unitary features are pulled from the existing candidate as explained above, while - // Set Features are merged/accumulated. - val mergedCommonFeatureMap = FeatureMapBuilder() - .add(CandidatePipelines, mergedCandidatePipelineIdentifiers) - .add(CandidateSources, mergedCandidateSourceIdentifiers) - .add(CandidateSourcePosition, existingCandidate.sourcePosition) - .build() - - existingCandidate.copy(features = - existingCandidate.features ++ newCandidate.features ++ mergedCommonFeatureMap) - } -} - -/** - * Keep the pinnable candidate. For cases where we are dealing with duplicate entries across - * different candidate types, such as different sub-classes of - * [[com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate]], we will - * prioritize the candidate with [[IsPinnedFeature]] because it contains additional information - * needed for the positioning of a pinned entry on a timeline. - */ -object PickPinnedCandidateMerger extends CandidateMergeStrategy { - override def apply( - existingCandidate: ItemCandidateWithDetails, - newCandidate: ItemCandidateWithDetails - ): ItemCandidateWithDetails = - Seq(existingCandidate, newCandidate) - .collectFirst { - case candidate @ ItemCandidateWithDetails(_: BaseTweetCandidate, _, features) - if features.getTry(IsPinnedFeature).toOption.contains(true) => - candidate - }.getOrElse(existingCandidate) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/CandidatePositionInResults.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/CandidatePositionInResults.docx new file mode 100644 index 000000000..8b969e2bf Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/CandidatePositionInResults.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/CandidatePositionInResults.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/CandidatePositionInResults.scala deleted file mode 100644 index f73b9bd37..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/CandidatePositionInResults.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.product_mixer.component_library.selector -import com.twitter.product_mixer.component_library.model.candidate.RelevancePromptCandidate -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * Compute a position for inserting a specific candidate into the result sequence originally provided to the Selector. - * If a `None` is returned, the Selector using this would not insert that candidate into the result. - */ -trait CandidatePositionInResults[-Query <: PipelineQuery] { - def apply( - query: Query, - candidate: CandidateWithDetails, - result: Seq[CandidateWithDetails] - ): Option[Int] -} - -object PromptCandidatePositionInResults extends CandidatePositionInResults[PipelineQuery] { - override def apply( - query: PipelineQuery, - candidate: CandidateWithDetails, - result: Seq[CandidateWithDetails] - ): Option[Int] = candidate match { - case ItemCandidateWithDetails(candidate, _, _) => - candidate match { - case relevancePromptCandidate: RelevancePromptCandidate => relevancePromptCandidate.position - case _ => None - } - // not supporting ModuleCandidateWithDetails right now as RelevancePromptCandidate shouldn't be in a module - case _ => None - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DeduplicationKey.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DeduplicationKey.docx new file mode 100644 index 000000000..38be1c1fc Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DeduplicationKey.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DeduplicationKey.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DeduplicationKey.scala deleted file mode 100644 index 44c92826f..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DeduplicationKey.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails - -/** - * [[DropSelector]] detects duplicates by looking for candidates with the same key. A key can be - * anything but is typically derived from a candidate's id and class. This approach is not always - * appropriate. For example, two candidate sources might both return different sub-classes of - * [[com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate]] resulting in - * them not being treated as duplicates. - */ -trait DeduplicationKey[Key] { - def apply(candidate: ItemCandidateWithDetails): Key -} - -/** - * Use candidate id and class to determine duplicates. - */ -object IdAndClassDuplicationKey extends DeduplicationKey[(String, Class[_ <: UniversalNoun[Any]])] { - def apply(item: ItemCandidateWithDetails): (String, Class[_ <: UniversalNoun[Any]]) = - (item.candidate.id.toString, item.candidate.getClass) -} - -/** - * Use candidate id to determine duplicates. - * This should be used instead of [[IdAndClassDuplicationKey]] in order to deduplicate across - * different candidate types, such as different implementations of - * [[com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate]]. - */ -object IdDuplicationKey extends DeduplicationKey[String] { - def apply(item: ItemCandidateWithDetails): String = item.candidate.id.toString -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropAllCandidates.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropAllCandidates.docx new file mode 100644 index 000000000..5aa7a08d1 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropAllCandidates.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropAllCandidates.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropAllCandidates.scala deleted file mode 100644 index a6c239151..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropAllCandidates.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.core.functional_component.common.AllPipelines -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.common.CandidateScope.PartitionedCandidates -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * Drops all Candidates on the `remainingCandidates` side which are in the [[pipelineScope]] - * - * This is typically used as a placeholder when templating out a new pipeline or - * as a simple filter to drop candidates based only on the [[CandidateScope]] - */ -case class DropAllCandidates(override val pipelineScope: CandidateScope = AllPipelines) - extends Selector[PipelineQuery] { - - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val PartitionedCandidates(inScope, outOfScope) = pipelineScope.partition(remainingCandidates) - - SelectorResult(remainingCandidates = outOfScope, result = result) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropDuplicateCandidates.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropDuplicateCandidates.docx new file mode 100644 index 000000000..fd3e239bc Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropDuplicateCandidates.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropDuplicateCandidates.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropDuplicateCandidates.scala deleted file mode 100644 index e52956ad3..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropDuplicateCandidates.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.component_library.selector.DropSelector.dropDuplicates -import com.twitter.product_mixer.core.functional_component.common.AllPipelines -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * Keep only the first instance of a candidate in the `remainingCandidates` as determined by comparing - * the contained candidate ID and class type. Subsequent matching instances will be dropped. For - * more details, see DropSelector#dropDuplicates - * - * @param duplicationKey how to generate the key used to identify duplicate candidates (by default use id and class name) - * @param mergeStrategy how to merge two candidates with the same key (by default pick the first one) - * - * @note [[com.twitter.product_mixer.component_library.model.candidate.CursorCandidate]] are ignored. - * @note [[com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails]] are ignored. - * - * @example if `remainingCandidates` - * `Seq(sourceA_Id1, sourceA_Id1, sourceA_Id2, sourceB_id1, sourceB_id2, sourceB_id3, sourceC_id4)` - * then the output candidates will be `Seq(sourceA_Id1, sourceA_Id2, sourceB_id3, sourceC_id4)` - */ -case class DropDuplicateCandidates( - override val pipelineScope: CandidateScope = AllPipelines, - duplicationKey: DeduplicationKey[_] = IdAndClassDuplicationKey, - mergeStrategy: CandidateMergeStrategy = PickFirstCandidateMerger) - extends Selector[PipelineQuery] { - - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val dedupedCandidates = dropDuplicates( - pipelineScope = pipelineScope, - candidates = remainingCandidates, - duplicationKey = duplicationKey, - mergeStrategy = mergeStrategy) - - SelectorResult(remainingCandidates = dedupedCandidates, result = result) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropDuplicateModuleItemCandidates.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropDuplicateModuleItemCandidates.docx new file mode 100644 index 000000000..818d7dc77 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropDuplicateModuleItemCandidates.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropDuplicateModuleItemCandidates.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropDuplicateModuleItemCandidates.scala deleted file mode 100644 index 8c2c0002b..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropDuplicateModuleItemCandidates.scala +++ /dev/null @@ -1,88 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.component_library.selector.DropSelector.dropDuplicates -import com.twitter.product_mixer.core.functional_component.common.AllPipelines -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.selector._ -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object DropDuplicateModuleItemCandidates { - - /** - * Limit the number of module item candidates (for 1 or more modules) from a certain candidate - * source. See [[DropDuplicateModuleItemCandidates]] for more details. - * - * @param candidatePipeline pipelines on which to run the selector - * - * @note Scala doesn't allow overloaded methods with default arguments. Users wanting to customize - * the de-dupe logic should use the default constructor. We could provide multiple - * constructors but that seemed more confusing (five ways to instantiate the selector) or not - * necessarily less verbose (if we picked specific use-cases rather than trying to support - * everything). - */ - def apply(candidatePipeline: CandidatePipelineIdentifier) = new DropDuplicateModuleItemCandidates( - SpecificPipeline(candidatePipeline), - IdAndClassDuplicationKey, - PickFirstCandidateMerger) - - def apply(candidatePipelines: Set[CandidatePipelineIdentifier]) = - new DropDuplicateModuleItemCandidates( - SpecificPipelines(candidatePipelines), - IdAndClassDuplicationKey, - PickFirstCandidateMerger) -} - -/** - * Limit the number of module item candidates (for 1 or more modules) from certain candidate - * pipelines. - * - * This acts like a [[DropDuplicateCandidates]] but for modules in `remainingCandidates` - * from any of the provided [[candidatePipelines]]. Similar to [[DropDuplicateCandidates]], it - * keeps only the first instance of a candidate within a module as determined by comparing - * the contained candidate ID and class type. - * - * @param pipelineScope pipeline scope on which to run the selector - * @param duplicationKey how to generate the key used to identify duplicate candidates (by default use id and class name) - * @param mergeStrategy how to merge two candidates with the same key (by default pick the first one) - * - * For example, if a candidatePipeline returned 5 modules each - * containing duplicate items in the candidate pool, then the module items in each of the - * 5 modules will be filtered to the unique items within each module. - * - * Another example is if you have 2 modules each with the same items as the other, - * it won't deduplicate across modules. - * - * @note this updates the module in the `remainingCandidates` - */ -case class DropDuplicateModuleItemCandidates( - override val pipelineScope: CandidateScope, - duplicationKey: DeduplicationKey[_] = IdAndClassDuplicationKey, - mergeStrategy: CandidateMergeStrategy = PickFirstCandidateMerger) - extends Selector[PipelineQuery] { - - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - - val remainingCandidatesLimited = remainingCandidates.map { - case module: ModuleCandidateWithDetails if pipelineScope.contains(module) => - // this applies to all candidates in a module, even if they are from a different - // candidate source, which can happen if items are added to a module during selection - module.copy(candidates = dropDuplicates( - pipelineScope = AllPipelines, - candidates = module.candidates, - duplicationKey = duplicationKey, - mergeStrategy = mergeStrategy)) - case candidate => candidate - } - - SelectorResult(remainingCandidates = remainingCandidatesLimited, result = result) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropDuplicateResults.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropDuplicateResults.docx new file mode 100644 index 000000000..54b918be7 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropDuplicateResults.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropDuplicateResults.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropDuplicateResults.scala deleted file mode 100644 index bf15b3b5d..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropDuplicateResults.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.component_library.selector.DropSelector.dropDuplicates -import com.twitter.product_mixer.core.functional_component.common.AllPipelines -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * Keep only the first instance of a candidate in the `result` as determined by comparing - * the contained candidate ID and class type. Subsequent matching instances will be dropped. For - * more details, see DropSelector#dropDuplicates - * - * @param duplicationKey how to generate the key used to identify duplicate candidates (by default use id and class name) - * @param mergeStrategy how to merge two candidates with the same key (by default pick the first one) - * - * @note [[com.twitter.product_mixer.component_library.model.candidate.CursorCandidate]] are ignored. - * @note [[com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails]] are ignored. - * - * @example if `result` - * `Seq(sourceA_Id1, sourceA_Id1, sourceA_Id2, sourceB_id1, sourceB_id2, sourceB_id3, sourceC_id4)` - * then the output result will be `Seq(sourceA_Id1, sourceA_Id2, sourceB_id3, sourceC_id4)` - */ -case class DropDuplicateResults( - duplicationKey: DeduplicationKey[_] = IdAndClassDuplicationKey, - mergeStrategy: CandidateMergeStrategy = PickFirstCandidateMerger) - extends Selector[PipelineQuery] { - - override val pipelineScope: CandidateScope = AllPipelines - - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val dedupedResults = dropDuplicates( - pipelineScope = pipelineScope, - candidates = result, - duplicationKey = duplicationKey, - mergeStrategy = mergeStrategy) - - SelectorResult(remainingCandidates = remainingCandidates, result = dedupedResults) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropFilteredCandidates.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropFilteredCandidates.docx new file mode 100644 index 000000000..05ab18d07 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropFilteredCandidates.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropFilteredCandidates.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropFilteredCandidates.scala deleted file mode 100644 index fe893fbc3..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropFilteredCandidates.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * Predicate which will be applied to each candidate. True indicates that the candidate will be - * kept. - */ -trait ShouldKeepCandidate { - def apply(candidateWithDetails: CandidateWithDetails): Boolean -} - -object DropFilteredCandidates { - def apply(candidatePipeline: CandidatePipelineIdentifier, filter: ShouldKeepCandidate) = - new DropFilteredCandidates(SpecificPipeline(candidatePipeline), filter) - - def apply(candidatePipelines: Set[CandidatePipelineIdentifier], filter: ShouldKeepCandidate) = - new DropFilteredCandidates(SpecificPipelines(candidatePipelines), filter) -} - -/** - * Limit candidates from certain candidates sources to those which satisfy the provided predicate. - */ -case class DropFilteredCandidates( - override val pipelineScope: CandidateScope, - filter: ShouldKeepCandidate) - extends Selector[PipelineQuery] { - - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val candidatesUpdated = remainingCandidates.filter { candidate => - if (pipelineScope.contains(candidate)) filter.apply(candidate) - else true - } - - SelectorResult(remainingCandidates = candidatesUpdated, result = result) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropFilteredModuleItemCandidates.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropFilteredModuleItemCandidates.docx new file mode 100644 index 000000000..58aa4ceec Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropFilteredModuleItemCandidates.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropFilteredModuleItemCandidates.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropFilteredModuleItemCandidates.scala deleted file mode 100644 index 5a9b0f410..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropFilteredModuleItemCandidates.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object DropFilteredModuleItemCandidates { - def apply(candidatePipeline: CandidatePipelineIdentifier, filter: ShouldKeepCandidate) = - new DropFilteredModuleItemCandidates(SpecificPipeline(candidatePipeline), filter) - - def apply(candidatePipelines: Set[CandidatePipelineIdentifier], filter: ShouldKeepCandidate) = - new DropFilteredModuleItemCandidates(SpecificPipelines(candidatePipelines), filter) -} - -/** - * Limit candidates in modules from certain candidates sources to those which satisfy - * the provided predicate. - * - * This acts like a [[DropFilteredCandidates]] but for modules in `remainingCandidates` - * from any of the provided [[candidatePipelines]]. - * - * @note this updates the module in the `remainingCandidates` - */ -case class DropFilteredModuleItemCandidates( - override val pipelineScope: CandidateScope, - filter: ShouldKeepCandidate) - extends Selector[PipelineQuery] { - - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val candidatesUpdated = remainingCandidates.map { - case module: ModuleCandidateWithDetails if pipelineScope.contains(module) => - // this applies to all candidates in a module, even if they are from a different - // candidate source, which can happen if items are added to a module during selection - module.copy(candidates = module.candidates.filter(filter.apply)) - case candidate => candidate - } - - SelectorResult(remainingCandidates = candidatesUpdated, result = result) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropMaxCandidates.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropMaxCandidates.docx new file mode 100644 index 000000000..490a7a013 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropMaxCandidates.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropMaxCandidates.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropMaxCandidates.scala deleted file mode 100644 index 4982ac49a..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropMaxCandidates.scala +++ /dev/null @@ -1,84 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.Param - -trait MaxSelector[-Query <: PipelineQuery] { - def apply( - query: Query, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): Int -} - -object DropMaxCandidates { - - /** - * A [[DropMaxCandidates]] Selector based on a [[Param]] applied to a single candidate pipeline - */ - def apply[Query <: PipelineQuery]( - candidatePipeline: CandidatePipelineIdentifier, - maxSelectionsParam: Param[Int] - ) = new DropMaxCandidates[Query]( - SpecificPipeline(candidatePipeline), - (query, _, _) => query.params(maxSelectionsParam)) - - /** - * A [[DropMaxCandidates]] Selector based on a [[Param]] with multiple candidate pipelines - */ - def apply[Query <: PipelineQuery]( - candidatePipelines: Set[CandidatePipelineIdentifier], - maxSelectionsParam: Param[Int] - ) = new DropMaxCandidates[Query]( - SpecificPipelines(candidatePipelines), - (query, _, _) => query.params(maxSelectionsParam)) - - /** - * A [[DropMaxCandidates]] Selector based on a [[Param]] that applies to a [[CandidateScope]] - */ - def apply[Query <: PipelineQuery]( - pipelineScope: CandidateScope, - maxSelectionsParam: Param[Int] - ) = new DropMaxCandidates[Query](pipelineScope, (query, _, _) => query.params(maxSelectionsParam)) -} - -/** - * Limit the number of item and module (not items inside modules) candidates from the - * specified pipelines based on the value provided by the [[MaxSelector]] - * - * For example, if value from the [[MaxSelector]] is 3, and a candidatePipeline returned 10 items - * in the candidate pool, then these items will be reduced to the first 3 items. Note that to - * update the ordering of the candidates, an UpdateCandidateOrderingSelector may be used prior to - * using this Selector. - * - * Another example, if the [[MaxSelector]] value is 3, and a candidatePipeline returned 10 modules - * in the candidate pool, then these will be reduced to the first 3 modules. The items inside the - * modeles will not be affected by this selector. To control the number of items inside modules see - * [[DropMaxModuleItemCandidates]]. - */ -case class DropMaxCandidates[-Query <: PipelineQuery]( - override val pipelineScope: CandidateScope, - maxSelector: MaxSelector[Query]) - extends Selector[Query] { - - override def apply( - query: Query, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val maxSelections = maxSelector(query, remainingCandidates, result) - assert(maxSelections > 0, "Max selections must be greater than zero") - - val remainingCandidatesLimited = - DropSelector.takeUntil(maxSelections, remainingCandidates, pipelineScope) - - SelectorResult(remainingCandidates = remainingCandidatesLimited, result = result) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropMaxModuleItemCandidates.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropMaxModuleItemCandidates.docx new file mode 100644 index 000000000..311bc746e Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropMaxModuleItemCandidates.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropMaxModuleItemCandidates.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropMaxModuleItemCandidates.scala deleted file mode 100644 index f343d0e97..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropMaxModuleItemCandidates.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.Param - -/** - * Limit the number of module item candidates (for 1 or more modules) from a certain candidate - * source. - * - * For example, if maxModuleItemsParam is 3, and a candidatePipeline returned 1 module containing 10 - * items in the candidate pool, then these module items will be reduced to the first 3 module items. - * Note that to update the ordering of the candidates, an UpdateModuleItemsCandidateOrderingSelector - * may be used prior to using this selector. - * - * Another example, if maxModuleItemsParam is 3, and a candidatePipeline returned 5 modules each - * containing 10 items in the candidate pool, then the module items in each of the 5 modules will be - * reduced to the first 3 module items. - * - * @note this updates the module in the `remainingCandidates` - */ -case class DropMaxModuleItemCandidates( - candidatePipeline: CandidatePipelineIdentifier, - maxModuleItemsParam: Param[Int]) - extends Selector[PipelineQuery] { - - override val pipelineScope: CandidateScope = SpecificPipelines(candidatePipeline) - - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - - val maxModuleItemSelections = query.params(maxModuleItemsParam) - assert(maxModuleItemSelections > 0, "Max module item selections must be greater than zero") - - val remainingCandidatesLimited = remainingCandidates.map { - case module: ModuleCandidateWithDetails if pipelineScope.contains(module) => - // this applies to all candidates in a module, even if they are from a different - // candidate source which can happen if items are added to a module during selection - module.copy(candidates = DropSelector.takeUntil(maxModuleItemSelections, module.candidates)) - case candidate => candidate - } - - SelectorResult(remainingCandidates = remainingCandidatesLimited, result = result) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropMaxResults.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropMaxResults.docx new file mode 100644 index 000000000..8ff45fdd8 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropMaxResults.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropMaxResults.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropMaxResults.scala deleted file mode 100644 index d5c655390..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropMaxResults.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.core.functional_component.common.AllPipelines -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.Param - -/** - * Limit the number of results - * - * For example, if maxResultsParam is 3, and the results contain 10 items, then these items will be - * reduced to the first 3 selected items. Note that the ordering of results is determined by the - * selector configuration. - * - * Another example, if maxResultsParam is 3, and the results contain 10 modules, then these will be - * reduced to the first 3 modules. The items inside the modules will not be affected by this - * selector. - */ -case class DropMaxResults( - maxResultsParam: Param[Int]) - extends Selector[PipelineQuery] { - - override val pipelineScope: CandidateScope = AllPipelines - - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val maxResults = query.params(maxResultsParam) - assert(maxResults > 0, "Max results must be greater than zero") - - val resultUpdated = DropSelector.takeUntil(maxResults, result) - - SelectorResult(remainingCandidates = remainingCandidates, result = resultUpdated) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropModuleTooFewModuleItemResults.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropModuleTooFewModuleItemResults.docx new file mode 100644 index 000000000..123dcda4b Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropModuleTooFewModuleItemResults.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropModuleTooFewModuleItemResults.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropModuleTooFewModuleItemResults.scala deleted file mode 100644 index 18f8989d8..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropModuleTooFewModuleItemResults.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.component_library.model.candidate.CursorCandidate -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.Param - -/** - * Drop the module from the `result` if it doesn't contain enough item candidates. - * - * For example, for a given module, if minResultsParam is 3, and the results contain 2 items, - * then that module will be entirely dropped from the results. - */ -case class DropModuleTooFewModuleItemResults( - candidatePipeline: CandidatePipelineIdentifier, - minModuleItemsParam: Param[Int]) - extends Selector[PipelineQuery] { - - override val pipelineScope: CandidateScope = SpecificPipelines(candidatePipeline) - - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val minModuleItemSelections = query.params(minModuleItemsParam) - assert(minModuleItemSelections > 0, "Min results must be greater than zero") - - val updatedResults = result.filter { - case module: ModuleCandidateWithDetails - if pipelineScope.contains(module) && module.candidates.count { candidateWithDetails => - !candidateWithDetails.candidate.isInstanceOf[CursorCandidate] - } < minModuleItemSelections => - false - case _ => true - } - - SelectorResult(remainingCandidates = remainingCandidates, result = updatedResults) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropNonDuplicateCandidates.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropNonDuplicateCandidates.docx new file mode 100644 index 000000000..d8f2ad887 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropNonDuplicateCandidates.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropNonDuplicateCandidates.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropNonDuplicateCandidates.scala deleted file mode 100644 index 4bbfbe076..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropNonDuplicateCandidates.scala +++ /dev/null @@ -1,75 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.component_library.model.candidate.CursorCandidate -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * Keep only the candidates in `remainingCandidates` that appear multiple times. - * This ignores modules and cursors from being removed. - * - * @param duplicationKey how to generate the key used to identify duplicate candidates - * - * @note [[com.twitter.product_mixer.component_library.model.candidate.CursorCandidate]] are ignored. - * @note [[com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails]] are ignored. - * - * @example if `remainingCandidates` - * `Seq(sourceA_Id1, sourceA_Id1, sourceA_Id2, sourceB_id1, sourceB_id2, sourceB_id3, sourceC_id4)` - * then the output result will be `Seq(sourceA_Id1, sourceA_Id2)` - */ -case class DropNonDuplicateCandidates( - override val pipelineScope: CandidateScope, - duplicationKey: DeduplicationKey[_] = IdAndClassDuplicationKey) - extends Selector[PipelineQuery] { - - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val duplicateCandidates = dropNonDuplicates( - pipelineScope = pipelineScope, - candidates = remainingCandidates, - duplicationKey = duplicationKey) - - SelectorResult(remainingCandidates = duplicateCandidates, result = result) - } - - /** - * Identify and keep candidates using the supplied key extraction and merger functions. By default - * this will keep only candidates that appear multiple times as determined by comparing - * the contained candidate ID and class type. Candidates appearing only once will be dropped. - * - * @note [[com.twitter.product_mixer.component_library.model.candidate.CursorCandidate]] are ignored. - * @note [[com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails]] are ignored. - * - * @param candidates which may have elements to drop - * @param duplicationKey how to generate a key for a candidate for identifying duplicates - */ - private[this] def dropNonDuplicates[Candidate <: CandidateWithDetails, Key]( - pipelineScope: CandidateScope, - candidates: Seq[Candidate], - duplicationKey: DeduplicationKey[Key], - ): Seq[Candidate] = { - // Here we are checking if each candidate has multiple appearances or not - val isCandidateADuplicate: Map[Key, Boolean] = candidates - .collect { - case item: ItemCandidateWithDetails - if pipelineScope.contains(item) && !item.candidate.isInstanceOf[CursorCandidate] => - item - }.groupBy(duplicationKey(_)) - .mapValues(_.length > 1) - - candidates.filter { - case item: ItemCandidateWithDetails => - isCandidateADuplicate.getOrElse(duplicationKey(item), true) - case _: ModuleCandidateWithDetails => true - case _ => false - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropOrthogonalCandidates.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropOrthogonalCandidates.docx new file mode 100644 index 000000000..46a0b20d5 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropOrthogonalCandidates.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropOrthogonalCandidates.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropOrthogonalCandidates.scala deleted file mode 100644 index 036f91bf8..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropOrthogonalCandidates.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * Limit candidates to the first candidate source in the provided orthogonalCandidatePipelines - * seq that has candidates in the candidate pool. For the subsequent candidate sources in the seq, - * remove their candidates from the candidate pool. - * - * @example if [[orthogonalCandidatePipelines]] is `Seq(D, A, C)`, and the remaining candidates - * component identifiers are `Seq(A, A, A, B, B, C, C, D, D, D)`, then `Seq(B, B, D, D, D)` will remain - * in the candidate pool. - * - * @example if [[orthogonalCandidatePipelines]] is `Seq(D, A, C)`, and the remaining candidates - * component identifiers are `Seq(A, A, A, B, B, C, C)`, then `Seq(A, A, A, B, B)` will remain - * in the candidate pool. - */ -case class DropOrthogonalCandidates( - orthogonalCandidatePipelines: Seq[CandidatePipelineIdentifier]) - extends Selector[PipelineQuery] { - - override val pipelineScope: CandidateScope = - SpecificPipelines(orthogonalCandidatePipelines.toSet) - - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val firstMatchingOrthogonalSourceOpt = orthogonalCandidatePipelines - .find { orthogonalCandidatePipeline => - remainingCandidates.exists(_.source == orthogonalCandidatePipeline) - } - - val remainingCandidatesLimited = firstMatchingOrthogonalSourceOpt match { - case Some(firstMatchingOrthogonalSource) => - val subsequentOrthogonalSources = - orthogonalCandidatePipelines.toSet - firstMatchingOrthogonalSource - - remainingCandidates.filterNot { candidate => - subsequentOrthogonalSources.contains(candidate.source) - } - case None => remainingCandidates - } - - SelectorResult(remainingCandidates = remainingCandidatesLimited, result = result) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropRequestedMaxModuleItemCandidates.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropRequestedMaxModuleItemCandidates.docx new file mode 100644 index 000000000..511da43c2 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropRequestedMaxModuleItemCandidates.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropRequestedMaxModuleItemCandidates.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropRequestedMaxModuleItemCandidates.scala deleted file mode 100644 index 47974aff9..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropRequestedMaxModuleItemCandidates.scala +++ /dev/null @@ -1,68 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.Param - -/** - * Limit the number of results (for 1 or more modules) from a certain candidate - * source to PipelineQuery.requestedMaxResults. - * - * PipelineQuery.requestedMaxResults is optionally set in the pipelineQuery. - * If it is not set, then the default value of DefaultRequestedMaxModuleItemsParam is used. - * - * For example, if PipelineQuery.requestedMaxResults is 3, and a candidatePipeline returned 1 module - * containing 10 items in the candidate pool, then these module items will be reduced to the first 3 - * module items. Note that to update the ordering of the candidates, an - * UpdateModuleItemsCandidateOrderingSelector may be used prior to using this selector. - * - * Another example, if PipelineQuery.requestedMaxResults is 3, and a candidatePipeline returned 5 - * modules each containing 10 items in the candidate pool, then the module items in each of the 5 - * modules will be reduced to the first 3 module items. - * - * @note this updates the module in the `remainingCandidates` - */ -case class DropRequestedMaxModuleItemCandidates( - override val pipelineScope: CandidateScope, - defaultRequestedMaxModuleItemResultsParam: Param[Int]) - extends Selector[PipelineQuery] { - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - - val requestedMaxModuleItemSelections = - query.maxResults(defaultRequestedMaxModuleItemResultsParam) - assert( - requestedMaxModuleItemSelections > 0, - "Requested Max module item selections must be greater than zero") - - val resultUpdated = result.map { - case module: ModuleCandidateWithDetails if pipelineScope.contains(module) => - // this applies to all candidates in a module, even if they are from a different - // candidate source which can happen if items are added to a module during selection - module.copy(candidates = - DropSelector.takeUntil(requestedMaxModuleItemSelections, module.candidates)) - case candidate => candidate - } - - SelectorResult(remainingCandidates = remainingCandidates, result = resultUpdated) - } -} - -object DropRequestedMaxModuleItemCandidates { - def apply( - candidatePipeline: CandidatePipelineIdentifier, - defaultRequestedMaxModuleItemResultsParam: Param[Int] - ) = - new DropRequestedMaxModuleItemCandidates( - SpecificPipeline(candidatePipeline), - defaultRequestedMaxModuleItemResultsParam) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropRequestedMaxResults.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropRequestedMaxResults.docx new file mode 100644 index 000000000..ebcabf2e8 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropRequestedMaxResults.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropRequestedMaxResults.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropRequestedMaxResults.scala deleted file mode 100644 index d5da8f584..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropRequestedMaxResults.scala +++ /dev/null @@ -1,55 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.core.functional_component.common.AllPipelines -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.BoundedParam - -/** - * Limit the number of results to min(PipelineQuery.requestedMaxResults, ServerMaxResultsParam) - * - * PipelineQuery.requestedMaxResults is optionally set in the pipelineQuery. - * If it is not set, then the default value of DefaultRequestedMaxResultsParam is used. - * - * ServerMaxResultsParam specifies the maximum number of results supported, irrespective of what is - * specified by the client in PipelineQuery.requestedMaxResults - * (or the DefaultRequestedMaxResultsParam default if not specified) - * - * For example, if ServerMaxResultsParam is 5, PipelineQuery.requestedMaxResults is 3, - * and the results contain 10 items, then these items will be reduced to the first 3 selected items. - * - * If PipelineQuery.requestedMaxResults is not set, DefaultRequestedMaxResultsParam is 3, - * ServerMaxResultsParam is 5 and the results contain 10 items, - * then these items will be reduced to the first 3 selected items. - * - * Another example, if ServerMaxResultsParam is 5, PipelineQuery.requestedMaxResults is 8, - * and the results contain 10 items, then these will be reduced to the first 5 selected items. - * - * The items inside the modules will not be affected by this selector. - */ -case class DropRequestedMaxResults( - defaultRequestedMaxResultsParam: BoundedParam[Int], - serverMaxResultsParam: BoundedParam[Int]) - extends Selector[PipelineQuery] { - - override val pipelineScope: CandidateScope = AllPipelines - - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val requestedMaxResults = query.maxResults(defaultRequestedMaxResultsParam) - val serverMaxResults = query.params(serverMaxResultsParam) - assert(requestedMaxResults > 0, "Requested max results must be greater than zero") - assert(serverMaxResults > 0, "Server max results must be greater than zero") - - val appliedMaxResults = Math.min(requestedMaxResults, serverMaxResults) - val resultUpdated = DropSelector.takeUntil(appliedMaxResults, result) - - SelectorResult(remainingCandidates = remainingCandidates, result = resultUpdated) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropSelector.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropSelector.docx new file mode 100644 index 000000000..bef7433ef Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropSelector.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropSelector.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropSelector.scala deleted file mode 100644 index b2a1a76e1..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropSelector.scala +++ /dev/null @@ -1,111 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.component_library.model.candidate.CursorCandidate -import com.twitter.product_mixer.core.functional_component.common.AllPipelines -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import scala.collection.mutable - -private[selector] object DropSelector { - - /** - * Identify and merge duplicates using the supplied key extraction and merger functions. By default - * this will keep only the first instance of a candidate in the `candidate` as determined by comparing - * the contained candidate ID and class type. Subsequent matching instances will be dropped. For - * more details, see DropSelector#dropDuplicates. - * - * @note [[com.twitter.product_mixer.component_library.model.candidate.CursorCandidate]] are ignored. - * @note [[com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails]] are ignored. - * - * @param candidates which may have elements to drop - * @param duplicationKey how to generate a key for a candidate for identifying duplicates - * @param mergeStrategy how to merge two candidates with the same key (by default pick the first one) - */ - def dropDuplicates[Candidate <: CandidateWithDetails, Key]( - pipelineScope: CandidateScope, - candidates: Seq[Candidate], - duplicationKey: DeduplicationKey[Key], - mergeStrategy: CandidateMergeStrategy - ): Seq[Candidate] = { - val seenCandidatePositions = mutable.HashMap[Key, Int]() - // We assume that, most of the time, most candidates aren't duplicates so the result Seq will be - // approximately the size of the candidates Seq. - val deduplicatedCandidates = new mutable.ArrayBuffer[Candidate](candidates.length) - - for (candidate <- candidates) { - candidate match { - - // candidate is from one of the Pipelines the selector applies to and is not a CursorCandidate - case item: ItemCandidateWithDetails - if pipelineScope.contains(item) && - !item.candidate.isInstanceOf[CursorCandidate] => - val key = duplicationKey(item) - - // Perform a merge if the candidate has been seen already - if (seenCandidatePositions.contains(key)) { - val candidateIndex = seenCandidatePositions(key) - - // Safe because only ItemCandidateWithDetails are added to seenCandidatePositions so - // seenCandidatePositions(key) *must* point to an ItemCandidateWithDetails - val originalCandidate = - deduplicatedCandidates(candidateIndex).asInstanceOf[ItemCandidateWithDetails] - - deduplicatedCandidates.update( - candidateIndex, - mergeStrategy(originalCandidate, item).asInstanceOf[Candidate]) - } else { - // Otherwise add a new entry to the list of kept candidates and update our map to track - // the new index - deduplicatedCandidates.append(item.asInstanceOf[Candidate]) - seenCandidatePositions.update(key, deduplicatedCandidates.length - 1) - } - case item => deduplicatedCandidates.append(item) - } - } - - deduplicatedCandidates - } - - /** - * Takes `candidates` from all [[CandidateWithDetails.source]]s but only `candidates` in the provided - * `pipelineScope` are counted towards the `max` non-cursor candidates are included. - * - * @param max the maximum number of non-cursor candidates from the provided `pipelineScope` to return - * @param candidates a sequence of candidates which may have elements dropped - * @param pipelineScope the scope of which `candidates` should count towards the `max` - */ - def takeUntil[Candidate <: CandidateWithDetails]( - max: Int, - candidates: Seq[Candidate], - pipelineScope: CandidateScope = AllPipelines - ): Seq[Candidate] = { - val resultsBuilder = Seq.newBuilder[Candidate] - resultsBuilder.sizeHint(candidates) - - candidates.foldLeft(0) { - case ( - count, - candidate @ ItemCandidateWithDetails(_: CursorCandidate, _, _) - ) => - // keep cursors, not included in the `count` - resultsBuilder += candidate.asInstanceOf[Candidate] - count - - case (count, candidate) if !pipelineScope.contains(candidate) => - // keep candidates that don't match the provided `pipelineScope`, not included in the `count` - resultsBuilder += candidate - count - - case (count, candidate) if count < max => - // keep candidates if theres space and increment the `count` - resultsBuilder += candidate - count + 1 - - case (dropCurrentCandidate, _) => - // drop non-cursor candidate because theres no space left - dropCurrentCandidate - } - resultsBuilder.result() - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropTooFewResults.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropTooFewResults.docx new file mode 100644 index 000000000..78d2185aa Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropTooFewResults.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropTooFewResults.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropTooFewResults.scala deleted file mode 100644 index ce8be7d6a..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DropTooFewResults.scala +++ /dev/null @@ -1,35 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.core.functional_component.common.AllPipelines -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.PipelineResult -import com.twitter.timelines.configapi.Param - -/** - * Drop all results if the minimum item threshold is not met. Some products would rather return - * nothing than, for example, a single tweet. This lets us leverage existing client logic for - * handling no results such as logic to not render the product at all. - */ -case class DropTooFewResults(minResultsParam: Param[Int]) extends Selector[PipelineQuery] { - - override val pipelineScope: CandidateScope = AllPipelines - - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val minResults = query.params(minResultsParam) - assert(minResults > 0, "Min results must be greater than zero") - - if (PipelineResult.resultSize(result) < minResults) { - SelectorResult(remainingCandidates = remainingCandidates, result = Seq.empty) - } else { - SelectorResult(remainingCandidates = remainingCandidates, result = result) - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DynamicPositionSelector.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DynamicPositionSelector.docx new file mode 100644 index 000000000..bf54ad634 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DynamicPositionSelector.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DynamicPositionSelector.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DynamicPositionSelector.scala deleted file mode 100644 index b0d3326fc..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/DynamicPositionSelector.scala +++ /dev/null @@ -1,124 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -private[selector] object DynamicPositionSelector { - - sealed trait IndexType - case object RelativeIndices extends IndexType - case object AbsoluteIndices extends IndexType - - /** - * Given an existing `result` seq, inserts candidates from `candidatesToInsertByIndex` into the `result` 1-by-1 with - * the provided index being the index relative to the `result` if given [[RelativeIndices]] or - * absolute index if given [[AbsoluteIndices]] (excluding duplicate insertions at an index, see below). - * - * Indices below 0 are added to the front and indices > the length are added to the end - * - * @note if multiple candidates exist with the same index, they are inserted in the order which they appear and only count - * as a single element with regards to the absolute index values, see the example below - * - * @example when using [[RelativeIndices]] {{{ - * mergeByIndexIntoResult( - * Seq( - * 0 -> "a", - * 0 -> "b", - * 0 -> "c", - * 1 -> "e", - * 2 -> "g", - * 2 -> "h"), - * Seq( - * "D", - * "F" - * ), - * RelativeIndices) == Seq( - * "a", - * "b", - * "c", - * "D", - * "e", - * "F", - * "g", - * "h" - * ) - * }}} - * - * @example when using [[AbsoluteIndices]] {{{ - * mergeByIndexIntoResult( - * Seq( - * 0 -> "a", - * 0 -> "b", - * 1 -> "c", - * 3 -> "e", - * 5 -> "g", - * 6 -> "h"), - * Seq( - * "D", - * "F" - * ), - * AbsoluteIndices) == Seq( - * "a", // index 0, "a" and "b" together only count as 1 element with regards to indexes because they have duplicate insertion points - * "b", // index 0 - * "c", // index 1 - * "D", // index 2 - * "e", // index 3 - * "F", // index 4 - * "g", // index 5 - * "h" // index 6 - * ) - * }}} - */ - def mergeByIndexIntoResult[T]( // generic on `T` to simplify unit testing - candidatesToInsertByIndex: Seq[(Int, T)], - result: Seq[T], - indexType: IndexType - ): Seq[T] = { - val positionAndCandidateList = candidatesToInsertByIndex.sortWith { - case ((indexLeft: Int, _), (indexRight: Int, _)) => - indexLeft < indexRight // order by desired absolute index ascending - } - - // Merge result and positionAndCandidateList into resultUpdated while making sure that the entries - // from the positionAndCandidateList are inserted at the right index. - val resultUpdated = Seq.newBuilder[T] - resultUpdated.sizeHint(result.size + positionAndCandidateList.size) - - var currentResultIndex = 0 - val inputResultIterator = result.iterator - val positionAndCandidateIterator = positionAndCandidateList.iterator.buffered - var previousInsertPosition: Option[Int] = None - - while (inputResultIterator.nonEmpty && positionAndCandidateIterator.nonEmpty) { - positionAndCandidateIterator.head match { - case (nextInsertionPosition, nextCandidateToInsert) - if previousInsertPosition.contains(nextInsertionPosition) => - // inserting multiple candidates at the same index - resultUpdated += nextCandidateToInsert - // do not increment any indices, but insert the candidate and advance to the next candidate - positionAndCandidateIterator.next() - - case (nextInsertionPosition, nextCandidateToInsert) - if currentResultIndex >= nextInsertionPosition => - // inserting a candidate at a new index - // add candidate to the results - resultUpdated += nextCandidateToInsert - // save the position of the inserted element to handle duplicate index insertions - previousInsertPosition = Some(nextInsertionPosition) - // advance to next candidate - positionAndCandidateIterator.next() - if (indexType == AbsoluteIndices) { - // if the indices are absolute, instead of relative to the original `result` we need to - // count the insertions of candidates into the results towards the `currentResultIndex` - currentResultIndex += 1 - } - case _ => - // no candidate to insert by index so use the candidates from the result and increment the index - resultUpdated += inputResultIterator.next() - currentResultIndex += 1 - } - } - // one of the iterators is empty, so append the remaining candidates in order to the end - resultUpdated ++= positionAndCandidateIterator.map { case (_, candidate) => candidate } - resultUpdated ++= inputResultIterator - - resultUpdated.result() - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendIntoModuleCandidates.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendIntoModuleCandidates.docx new file mode 100644 index 000000000..eb0f0a6da Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendIntoModuleCandidates.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendIntoModuleCandidates.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendIntoModuleCandidates.scala deleted file mode 100644 index 3096d9a50..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendIntoModuleCandidates.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.component_library.selector.InsertIntoModule.ModuleAndIndex -import com.twitter.product_mixer.component_library.selector.InsertIntoModule.ModuleWithItemsToAddAndOtherCandidates -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * Append all candidates from [[candidatePipeline]] into a module from [[targetModuleCandidatePipeline]]. - * If the results contain multiple modules from the target candidate pipeline, - * then the candidates will be inserted into the first module. - * - * @note this will throw an [[UnsupportedOperationException]] if the [[candidatePipeline]] contains any modules. - * - * @note this updates the module in the `remainingCandidates` - */ -case class InsertAppendIntoModuleCandidates( - candidatePipeline: CandidatePipelineIdentifier, - targetModuleCandidatePipeline: CandidatePipelineIdentifier) - extends Selector[PipelineQuery] { - - override val pipelineScope: CandidateScope = - SpecificPipelines(candidatePipeline, targetModuleCandidatePipeline) - - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - - val ModuleWithItemsToAddAndOtherCandidates( - moduleToUpdateAndIndex, - itemsToInsertIntoModule, - otherCandidates) = - InsertIntoModule.moduleToUpdate( - candidatePipeline, - targetModuleCandidatePipeline, - remainingCandidates) - - val updatedRemainingCandidates = moduleToUpdateAndIndex match { - case None => remainingCandidates - case _ if itemsToInsertIntoModule.isEmpty => remainingCandidates - case Some(ModuleAndIndex(moduleToUpdate, indexOfModuleInOtherCandidates)) => - val updatedModuleItems = moduleToUpdate.candidates ++ itemsToInsertIntoModule - val updatedModule = moduleToUpdate.copy(candidates = updatedModuleItems) - otherCandidates.updated(indexOfModuleInOtherCandidates, updatedModule) - } - - SelectorResult(remainingCandidates = updatedRemainingCandidates, result = result) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendPatternResults.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendPatternResults.docx new file mode 100644 index 000000000..3283b7ca7 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendPatternResults.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendPatternResults.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendPatternResults.scala deleted file mode 100644 index 79dda59f4..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendPatternResults.scala +++ /dev/null @@ -1,105 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import scala.collection.mutable - -/** - * Select candidates and add them according to the `pattern`. - * The pattern is repeated until all candidates contained in the pattern are added to the `result`. - * If the candidates for a specific [[Bucket]] in the pattern are exhausted, that [[Bucket]] will be - * skipped on subsequent iterations. - * If a candidate has a [[Bucket]] that isn't in the pattern it is added to the end of the `result`. - * The end result is all candidates from all [[candidatePipelines]]s provided will end up in the result. - * - * @example If there are no more candidates from a given `CandidatePipeline` then it is skipped, so - * with the pattern `Seq(A, A, B, C)`, if there are no more candidates from `B` then it is - * effectively the same as `Seq(A, A, C)`. The `result` will contain all candidates from all - * `CandidatePipeline`s who's `Bucket` is in the `pattern`. - * - * @example If the pattern is `Seq(A, A, B, C)` and the remaining candidates - * from the provided `candidatePipelines` are: - * - 5 `A`s - * - 2 `B`s - * - 1 `C` - * - 1 `D`s - * - * then the resulting output for each iteration over the pattern is - * - `Seq(A, A, B, C)` - * - `Seq(A, A, B)` since there's no more `C`s - * - `Seq(A)` since there are no more `B`s or `C`s - * - `Seq(D)` since it wasn't in the pattern but is from one of the provided - * `candidatePipelines`, it's appended at the end - * - * so the `result` that's returned would be `Seq(A, A, B, C, A, A, B, A, D)` - */ -case class InsertAppendPatternResults[-Query <: PipelineQuery, Bucket]( - candidatePipelines: Set[CandidatePipelineIdentifier], - bucketer: Bucketer[Bucket], - pattern: Seq[Bucket]) - extends Selector[Query] { - - require(pattern.nonEmpty, "`pattern` must be non-empty") - - override val pipelineScope: CandidateScope = SpecificPipelines(candidatePipelines) - - private sealed trait PatternResult - private case object NotASelectedCandidatePipeline extends PatternResult - private case object NotABucketInThePattern extends PatternResult - private case class Bucketed(bucket: Bucket) extends PatternResult - - private val allBucketsInPattern = pattern.toSet - - override def apply( - query: Query, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val groupedCandidates: Map[PatternResult, Seq[CandidateWithDetails]] = - remainingCandidates.groupBy { candidateWithDetails => - if (pipelineScope.contains(candidateWithDetails)) { - // if a candidate's Bucket doesnt appear in the pattern it's backfilled at the end - val bucket = bucketer(candidateWithDetails) - if (allBucketsInPattern.contains(bucket)) { - Bucketed(bucket) - } else { - NotABucketInThePattern - } - } else { - NotASelectedCandidatePipeline - } - } - - val otherCandidates = - groupedCandidates.getOrElse(NotASelectedCandidatePipeline, Seq.empty) - - val notABucketInThePattern = - groupedCandidates.getOrElse(NotABucketInThePattern, Seq.empty) - - // mutable so we can remove finished iterators to optimize when looping for large patterns - val groupedBucketsIterators = mutable.HashMap(groupedCandidates.collect { - case (Bucketed(bucket), candidatesWithDetails) => (bucket, candidatesWithDetails.iterator) - }.toSeq: _*) - - val patternIterator = Iterator.continually(pattern).flatten - - val newResult = new mutable.ArrayBuffer[CandidateWithDetails]() - while (groupedBucketsIterators.nonEmpty) { - val bucket = patternIterator.next() - groupedBucketsIterators.get(bucket) match { - case Some(iterator) if iterator.nonEmpty => newResult += iterator.next() - case Some(_) => groupedBucketsIterators.remove(bucket) - case None => - } - } - - SelectorResult( - remainingCandidates = otherCandidates, - result = result ++ newResult ++ notABucketInThePattern) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendRatioResults.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendRatioResults.docx new file mode 100644 index 000000000..c42ae06bf Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendRatioResults.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendRatioResults.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendRatioResults.scala deleted file mode 100644 index f60b0abbc..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendRatioResults.scala +++ /dev/null @@ -1,171 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.Param -import scala.annotation.tailrec -import scala.collection.mutable -import scala.util.Random - -/** - * Select candidates and add them according to the ratio assigned for each [[Bucket]] - * For instance, if given `Set((A, 0.8), (B, 0.2))` then candidates will randomly be added to the - * results with an 80% chance of any candidate being from `A` and 20% from`B`. If there are no more - * candidates from a given `CandidatePipeline` then it's simply skipped, so if we run out of `A` - * candidates the rest will be `B`. The end result is all candidates from all [[candidatePipelines]]s - * provided will end up in the result. - * - * For example, an output may look like `Seq(A, A, B, A, A)`, `Seq(A, A, A, A, B)`. If we eventually - * run out of `A` candidates then we would end up with the remaining candidates at the end, - * `Seq(A, A, B, A, A, A, B, A, A, A [run out of A], B, B, B, B, B, B)` - * - * @note the ratios provided are proportional to the sum of all ratios, so if you give 0.3 and 0.7, - * they will be function as to 30% and 70%, and the same for if you provided 3000 and 7000 for - * ratios. - * - * @note Its important to be sure to update all [[Param]]s when changing the ratio for 1 of them - * otherwise you may get unexpected results. For instance, of you have 0.3 and 0.7 which - * correspond to 30% and 70%, and you change `0.7 -> 0.9`, then the total sum of the ratios is - * now 1.2, so you have 25% and 75% when you intended to have 10% and 90%. To prevent this, - * be sure to update all [[Param]]s together, so `0.3 -> 0.1` and `0.7 -> 0.9` so the total - * remains the same. - */ -case class InsertAppendRatioResults[-Query <: PipelineQuery, Bucket]( - candidatePipelines: Set[CandidatePipelineIdentifier], - bucketer: Bucketer[Bucket], - ratios: Map[Bucket, Param[Double]], - random: Random = new Random(0)) - extends Selector[Query] { - - require(ratios.nonEmpty, "bucketRatios must be non-empty") - - override val pipelineScope: CandidateScope = SpecificPipelines(candidatePipelines) - - private sealed trait PatternResult - private case object NotASelectedCandidatePipeline extends PatternResult - private case object NotABucketInThePattern extends PatternResult - private case class Bucketed(bucket: Bucket) extends PatternResult - - override def apply( - query: Query, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - - val groupedCandidates: Map[PatternResult, Seq[CandidateWithDetails]] = - remainingCandidates.groupBy { candidateWithDetails => - if (pipelineScope.contains(candidateWithDetails)) { - // if a candidate's Bucket doesnt appear in the pattern it's backfilled at the end - val bucket = bucketer(candidateWithDetails) - if (ratios.contains(bucket)) { - Bucketed(bucket) - } else { - NotABucketInThePattern - } - } else { - NotASelectedCandidatePipeline - } - } - - val otherCandidates = - groupedCandidates.getOrElse(NotASelectedCandidatePipeline, Seq.empty) - - val notABucketInThePattern = - groupedCandidates.getOrElse(NotABucketInThePattern, Seq.empty) - - val groupedCandidatesIterators = groupedCandidates.collect { - case (Bucketed(bucket), candidatesWithDetails) => (bucket, candidatesWithDetails.iterator) - } - - // using a LinkedHashMap and sorting by descending ratio - // the highest ratios will always be checked first when iterating - // mutable so we can remove finished ratios when they are finished to optimize looping for large numbers of ratios - val currentBucketRatios: mutable.Map[Bucket, Double] = { - val bucketsAndRatiosSortedByRatio = - ratios.iterator - .map { - case (bucket, param) => - val ratio = query.params(param) - require( - ratio >= 0, - "The ratio for an InsertAppendRatioResults selector can not be negative") - (bucket, ratio) - }.toSeq - .sortBy { case (_, ratio) => ratio }(Ordering.Double.reverse) - mutable.LinkedHashMap(bucketsAndRatiosSortedByRatio: _*) - } - - // keep track of the sum of all ratios so we can look only at random values between 0 and that - var ratioSum = currentBucketRatios.valuesIterator.sum - - // add candidates to `newResults` until all remaining candidates are for a single bucket - val newResult = new mutable.ArrayBuffer[CandidateWithDetails]() - while (currentBucketRatios.size > 1) { - // random number between 0 and the sum of the ratios of all params - val randomValue = random.nextDouble() * ratioSum - - val currentBucketRatiosIterator: Iterator[(Bucket, Double)] = - currentBucketRatios.iterator - val (currentBucket, ratio) = currentBucketRatiosIterator.next() - - val componentToTakeFrom = findBucketToTakeFrom( - randomValue = randomValue, - cumulativeSumOfRatios = ratio, - bucket = currentBucket, - bucketRatiosIterator = currentBucketRatiosIterator - ) - - groupedCandidatesIterators.get(componentToTakeFrom) match { - case Some(iteratorForBucket) if iteratorForBucket.nonEmpty => - newResult += iteratorForBucket.next() - case _ => - ratioSum -= currentBucketRatios(componentToTakeFrom) - currentBucketRatios.remove(componentToTakeFrom) - } - } - // with only have 1 source remaining, we can skip all the above work and insert them in bulk - val remainingBucketInRatio = - currentBucketRatios.keysIterator.flatMap(groupedCandidatesIterators.get).flatten - - SelectorResult( - remainingCandidates = otherCandidates, - result = result ++ newResult ++ remainingBucketInRatio ++ notABucketInThePattern) - } - - /** - * iterates through the `bucketRatiosIterator` until it finds a the - * [[Bucket]] that corresponds with the current `randomValue`. - * - * This method expects that `0 <= randomValue <= sum of all ratios` - * - * @example If the given ratios are `Seq(A -> 0.2, B -> 0.35, C -> 0.45)` - * check if the given `randomValue` is - * - `< 0.45`, if not then check - * - `< 0.8` (0.45 + 0.35), if not then check - * - `< 1.0` (0.45 + 0.35 + 0.2) - * - * and return the corresponding [[Bucket]] - */ - @tailrec private def findBucketToTakeFrom( - randomValue: Double, - cumulativeSumOfRatios: Double, - bucket: Bucket, - bucketRatiosIterator: Iterator[(Bucket, Double)] - ): Bucket = { - if (randomValue < cumulativeSumOfRatios || bucketRatiosIterator.isEmpty) { - bucket - } else { - val (nextBucket, ratio) = bucketRatiosIterator.next() - findBucketToTakeFrom( - randomValue, - cumulativeSumOfRatios + ratio, - nextBucket, - bucketRatiosIterator) - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendResults.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendResults.docx new file mode 100644 index 000000000..27690348c Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendResults.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendResults.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendResults.scala deleted file mode 100644 index a3e8485c3..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendResults.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import CandidateScope.PartitionedCandidates -import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object InsertAppendResults { - def apply(candidatePipeline: CandidatePipelineIdentifier): InsertAppendResults[PipelineQuery] = - new InsertAppendResults(SpecificPipeline(candidatePipeline)) - - def apply( - candidatePipelines: Set[CandidatePipelineIdentifier] - ): InsertAppendResults[PipelineQuery] = new InsertAppendResults( - SpecificPipelines(candidatePipelines)) -} - -/** - * Select all candidates from candidate pipeline(s) and append to the end of the result. - * - * @note that if multiple candidate pipelines are specified, their candidates will be added - * to the result in the order in which they appear in the candidate pool. This ordering often - * reflects the order in which the candidate pipelines were listed in the mixer/recommendations - * pipeline, unless for example an UpdateSortCandidates selector was run prior to running - * this selector which could change this ordering. - * - * @note if inserting results from multiple candidate pipelines (see note above related to ordering), - * it is more performant to include all (or a subset) of the candidate pipelines in a single - * InsertAppendResults, as opposed to calling InsertAppendResults individually for each - * candidate pipeline because each selector does an O(n) pass on the candidate pool. - */ -case class InsertAppendResults[-Query <: PipelineQuery]( - override val pipelineScope: CandidateScope) - extends Selector[Query] { - - override def apply( - query: Query, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val PartitionedCandidates(selectedCandidates, otherCandidates) = - pipelineScope.partition(remainingCandidates) - - val resultUpdated = result ++ selectedCandidates - - SelectorResult(remainingCandidates = otherCandidates, result = resultUpdated) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendWeaveResults.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendWeaveResults.docx new file mode 100644 index 000000000..cfe521417 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendWeaveResults.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendWeaveResults.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendWeaveResults.scala deleted file mode 100644 index 73ba481b5..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendWeaveResults.scala +++ /dev/null @@ -1,120 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import scala.collection.mutable - -object InsertAppendWeaveResults { - def apply[Query <: PipelineQuery, Bucket]( - candidatePipelines: Set[CandidatePipelineIdentifier], - bucketer: Bucketer[Bucket], - ): InsertAppendWeaveResults[Query, Bucket] = - new InsertAppendWeaveResults(SpecificPipelines(candidatePipelines), bucketer) - - def apply[Query <: PipelineQuery, Bucket]( - candidatePipeline: CandidatePipelineIdentifier, - bucketer: Bucketer[Bucket], - ): InsertAppendWeaveResults[Query, Bucket] = - new InsertAppendWeaveResults(SpecificPipeline(candidatePipeline), bucketer) -} - -/** - * Select candidates weave them together according to their [[Bucket]]. - * - * Candidates are grouped according to [[Bucket]] and one candidate is added from each group until - * no candidates belonging to any group are left. - * - * Functionally similar to [[InsertAppendPatternResults]]. [[InsertAppendPatternResults]] is useful - * if you have more complex ordering requirements but it requires you to know all the buckets in - * advance. - * - * @note The order in which candidates are weaved together depends on the order in which the buckets - * were first seen on candidates. - * - * @example If the candidates are Seq(Tweet(10), Tweet(8), Tweet(3), Tweet(13)) and they are bucketed - * using an IsEven bucketing function, then the resulting buckets would be: - * - * - Seq(Tweet(10), Tweet(8)) - * - Seq(Tweet(3), Tweet(13)) - * - * The selector would then loop through these buckets and produce: - * - * - Tweet(10) - * - Tweet(3) - * - Tweet(8) - * - Tweet(13) - * - * Note that first bucket encountered was the 'even' bucket so weaving proceeds first with - * the even bucket then the odd bucket. Tweet(3) had been first then the opposite would be - * true. - */ -case class InsertAppendWeaveResults[-Query <: PipelineQuery, Bucket]( - override val pipelineScope: CandidateScope, - bucketer: Bucketer[Bucket]) - extends Selector[Query] { - - override def apply( - query: Query, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val (bucketableCandidates, otherCandidates) = - remainingCandidates.partition(pipelineScope.contains) - - val groupedCandidates = groupByBucket(bucketableCandidates) - - val candidateBucketQueues: mutable.Queue[mutable.Queue[CandidateWithDetails]] = - mutable.Queue() ++= groupedCandidates - val newResult = mutable.ArrayBuffer[CandidateWithDetails]() - - // Take the next group of candidates from the queue and attempt to add the first candidate from - // that group into the result. The loop will terminate when every queue is empty. - while (candidateBucketQueues.nonEmpty) { - val nextCandidateQueue = candidateBucketQueues.dequeue() - - if (nextCandidateQueue.nonEmpty) { - newResult += nextCandidateQueue.dequeue() - - // Re-queue this bucket of candidates if it's still non-empty - if (nextCandidateQueue.nonEmpty) { - candidateBucketQueues.enqueue(nextCandidateQueue) - } - } - } - - SelectorResult(remainingCandidates = otherCandidates, result = result ++ newResult) - } - - /** - * Similar to `groupBy` but respect the order in which individual bucket values are first seen. - * This is useful when the candidates have already been sorted prior to the selector running. - */ - private def groupByBucket( - candidates: Seq[CandidateWithDetails] - ): mutable.ArrayBuffer[mutable.Queue[CandidateWithDetails]] = { - val bucketToCandidateGroupIndex = mutable.Map.empty[Bucket, Int] - val candidateGroups = mutable.ArrayBuffer[mutable.Queue[CandidateWithDetails]]() - - candidates.foreach { candidate => - val bucket = bucketer(candidate) - - // Index points to the specific sub-group in candidateGroups where we want to insert the next - // candidate. If a bucket has already been seen then this value is known, otherwise we need - // to add a new entry for it. - if (!bucketToCandidateGroupIndex.contains(bucket)) { - candidateGroups.append(mutable.Queue()) - bucketToCandidateGroupIndex.put(bucket, candidateGroups.length - 1) - } - - candidateGroups(bucketToCandidateGroupIndex(bucket)).enqueue(candidate) - } - - candidateGroups - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendWithoutFeatureResults.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendWithoutFeatureResults.docx new file mode 100644 index 000000000..0c4bc1f64 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendWithoutFeatureResults.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendWithoutFeatureResults.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendWithoutFeatureResults.scala deleted file mode 100644 index 26767ffb7..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertAppendWithoutFeatureResults.scala +++ /dev/null @@ -1,35 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * A selector that appends all candidates missing a specific feature to the results pool and keeps - * the rest in the remaining candidates. This is useful for backfill scoring candidates without - * a score from a previous scorer. - * @param pipelineScope The pipeline scope to check - * @param missingFeature The missing feature to check for. - */ -case class InsertAppendWithoutFeatureResults( - override val pipelineScope: CandidateScope, - missingFeature: Feature[_, _]) - extends Selector[PipelineQuery] { - - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val (candidatesWithMissingFeature, candidatesWithFeature) = remainingCandidates.partition { - candidate => - pipelineScope.contains(candidate) && !candidate.features.getSuccessfulFeatures - .contains(missingFeature) - } - val updatedResults = result ++ candidatesWithMissingFeature - SelectorResult(remainingCandidates = candidatesWithFeature, result = updatedResults) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertDynamicPositionResults.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertDynamicPositionResults.docx new file mode 100644 index 000000000..ae9f3d238 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertDynamicPositionResults.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertDynamicPositionResults.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertDynamicPositionResults.scala deleted file mode 100644 index e0fb9bebb..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertDynamicPositionResults.scala +++ /dev/null @@ -1,69 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.selector._ -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object InsertDynamicPositionResults { - def apply[Query <: PipelineQuery]( - candidatePipeline: CandidatePipelineIdentifier, - dynamicInsertionPosition: DynamicInsertionPosition[Query], - ): InsertDynamicPositionResults[Query] = - new InsertDynamicPositionResults(SpecificPipeline(candidatePipeline), dynamicInsertionPosition) - - def apply[Query <: PipelineQuery]( - candidatePipelines: Set[CandidatePipelineIdentifier], - dynamicInsertionPosition: DynamicInsertionPosition[Query] - ): InsertDynamicPositionResults[Query] = - new InsertDynamicPositionResults( - SpecificPipelines(candidatePipelines), - dynamicInsertionPosition) -} - -/** - * Compute a position for inserting the candidates into result. If a `None` is returned, the - * Selector using this would not insert the candidates into the result. - */ -trait DynamicInsertionPosition[-Query <: PipelineQuery] { - def apply( - query: Query, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): Option[Int] -} - -/** - * Insert all candidates in a pipeline scope at a 0-indexed dynamic position computed - * using the provided [[DynamicInsertionPosition]] instance. If the current results are a shorter - * length than the computed position, then the candidates will be appended to the results. - * If the [[DynamicInsertionPosition]] returns a `None`, the candidates are not - * added to the result. - */ -case class InsertDynamicPositionResults[-Query <: PipelineQuery]( - override val pipelineScope: CandidateScope, - dynamicInsertionPosition: DynamicInsertionPosition[Query]) - extends Selector[Query] { - - override def apply( - query: Query, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - dynamicInsertionPosition(query, remainingCandidates, result) match { - case Some(position) => - InsertSelector.insertIntoResultsAtPosition( - position = position, - pipelineScope = pipelineScope, - remainingCandidates = remainingCandidates, - result = result) - case None => - // When a valid position is not provided, do not insert the candidates. - // Both the remainingCandidates and result are unchanged. - SelectorResult(remainingCandidates = remainingCandidates, result = result) - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertFixedPositionIntoModuleCandidates.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertFixedPositionIntoModuleCandidates.docx new file mode 100644 index 000000000..0796d03da Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertFixedPositionIntoModuleCandidates.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertFixedPositionIntoModuleCandidates.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertFixedPositionIntoModuleCandidates.scala deleted file mode 100644 index 1fb4d61be..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertFixedPositionIntoModuleCandidates.scala +++ /dev/null @@ -1,69 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.component_library.selector.InsertIntoModule.ModuleAndIndex -import com.twitter.product_mixer.component_library.selector.InsertIntoModule.ModuleWithItemsToAddAndOtherCandidates -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.Param - -/** - * Insert all candidates from [[candidatePipeline]] at a 0-indexed fixed position into a module from - * [[targetModuleCandidatePipeline]]. If the results contain multiple modules from the target candidate - * pipeline, then the candidates will be inserted into the first module. If the target module's - * items are a shorter length than the requested position, then the candidates will be appended - * to the results. - * - * @note this will throw an [[UnsupportedOperationException]] if the [[candidatePipeline]] contains any modules. - * - * @note this updates the module in the `remainingCandidates` - */ -case class InsertFixedPositionIntoModuleCandidates( - candidatePipeline: CandidatePipelineIdentifier, - targetModuleCandidatePipeline: CandidatePipelineIdentifier, - positionParam: Param[Int]) - extends Selector[PipelineQuery] { - - override val pipelineScope: CandidateScope = - SpecificPipelines(candidatePipeline, targetModuleCandidatePipeline) - - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - - val position = query.params(positionParam) - assert(position >= 0, "Position must be equal to or greater than zero") - - val ModuleWithItemsToAddAndOtherCandidates( - moduleToUpdateAndIndex, - itemsToInsertIntoModule, - otherCandidates) = - InsertIntoModule.moduleToUpdate( - candidatePipeline, - targetModuleCandidatePipeline, - remainingCandidates) - - val updatedRemainingCandidates = moduleToUpdateAndIndex match { - case None => remainingCandidates - case _ if itemsToInsertIntoModule.isEmpty => remainingCandidates - case Some(ModuleAndIndex(moduleToUpdate, indexOfModuleInOtherCandidates)) => - val updatedModuleItems = - if (position < moduleToUpdate.candidates.length) { - val (left, right) = moduleToUpdate.candidates.splitAt(position) - left ++ itemsToInsertIntoModule ++ right - } else { - moduleToUpdate.candidates ++ itemsToInsertIntoModule - } - val updatedModule = moduleToUpdate.copy(candidates = updatedModuleItems) - otherCandidates.updated(indexOfModuleInOtherCandidates, updatedModule) - } - - SelectorResult(remainingCandidates = updatedRemainingCandidates, result = result) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertFixedPositionResults.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertFixedPositionResults.docx new file mode 100644 index 000000000..b80c58428 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertFixedPositionResults.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertFixedPositionResults.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertFixedPositionResults.scala deleted file mode 100644 index ef2f392a1..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertFixedPositionResults.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.Param - -object InsertFixedPositionResults { - def apply( - candidatePipeline: CandidatePipelineIdentifier, - positionParam: Param[Int], - ): InsertFixedPositionResults = - new InsertFixedPositionResults(SpecificPipeline(candidatePipeline), positionParam) - - def apply( - candidatePipelines: Set[CandidatePipelineIdentifier], - positionParam: Param[Int] - ): InsertFixedPositionResults = - new InsertFixedPositionResults(SpecificPipelines(candidatePipelines), positionParam) -} - -/** - * Insert all candidates in a pipeline scope at a 0-indexed fixed position. If the current - * results are a shorter length than the requested position, then the candidates will be appended - * to the results. - */ -case class InsertFixedPositionResults( - override val pipelineScope: CandidateScope, - positionParam: Param[Int]) - extends Selector[PipelineQuery] { - - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = InsertSelector.insertIntoResultsAtPosition( - position = query.params(positionParam), - pipelineScope = pipelineScope, - remainingCandidates = remainingCandidates, - result = result) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertIntoModule.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertIntoModule.docx new file mode 100644 index 000000000..56f9cfe1f Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertIntoModule.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertIntoModule.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertIntoModule.scala deleted file mode 100644 index 53f3df75f..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertIntoModule.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails -import scala.collection.immutable.Queue - -private[selector] object InsertIntoModule { - case class ModuleAndIndex( - moduleToInsertInto: ModuleCandidateWithDetails, - indexOfModuleInOtherCandidates: Int) - - case class ModuleWithItemsToAddAndOtherCandidates( - moduleToUpdateAndIndex: Option[ModuleAndIndex], - itemsToInsertIntoModule: Queue[ItemCandidateWithDetails], - otherCandidates: Queue[CandidateWithDetails]) - - /** - * Given a Seq of `candidates`, returns the first module with it's index that matches the - * `targetModuleCandidatePipeline` with all the [[ItemCandidateWithDetails]] that match the - * `candidatePipeline` added to the `itemsToInsert` and the remaining candidates, including the - * module, in the `otherCandidates` - */ - def moduleToUpdate( - candidatePipeline: CandidatePipelineIdentifier, - targetModuleCandidatePipeline: CandidatePipelineIdentifier, - candidates: Seq[CandidateWithDetails] - ): ModuleWithItemsToAddAndOtherCandidates = { - candidates.foldLeft[ModuleWithItemsToAddAndOtherCandidates]( - ModuleWithItemsToAddAndOtherCandidates(None, Queue.empty, Queue.empty)) { - case ( - state @ ModuleWithItemsToAddAndOtherCandidates(_, itemsToInsertIntoModule, _), - selectedItem: ItemCandidateWithDetails) if selectedItem.source == candidatePipeline => - state.copy(itemsToInsertIntoModule = itemsToInsertIntoModule :+ selectedItem) - - case ( - state @ ModuleWithItemsToAddAndOtherCandidates(None, _, otherCandidates), - module: ModuleCandidateWithDetails) if module.source == targetModuleCandidatePipeline => - val insertionIndex = otherCandidates.length - val moduleAndIndex = Some( - ModuleAndIndex( - moduleToInsertInto = module, - indexOfModuleInOtherCandidates = insertionIndex)) - val otherCandidatesWithModuleAppended = otherCandidates :+ module - state.copy( - moduleToUpdateAndIndex = moduleAndIndex, - otherCandidates = otherCandidatesWithModuleAppended) - - case (_, invalidModule: ModuleCandidateWithDetails) - if invalidModule.source == candidatePipeline => - /** - * while not exactly an illegal state, its most likely an incorrectly configured candidate pipeline - * that returned a module instead of returning the candidates the module contains. Since you can't - * nest a module inside of a module, we can either throw or ignore it and we choose to ignore it - * to catch a potential bug a customer may do accidentally. - */ - throw new UnsupportedOperationException( - s"Expected the candidatePipeline $candidatePipeline to contain items to put into the module from the targetModuleCandidatePipeline $targetModuleCandidatePipeline, but not contain modules itself. " + - s"This can occur if your $candidatePipeline was incorrectly configured and returns a module when you intended to return the candidates the module contained." - ) - - case ( - state @ ModuleWithItemsToAddAndOtherCandidates(_, _, otherCandidates), - unselectedCandidate) => - state.copy(otherCandidates = otherCandidates :+ unselectedCandidate) - } - } - -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertPerCandidateDynamicPositionResults.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertPerCandidateDynamicPositionResults.docx new file mode 100644 index 000000000..8abd24f27 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertPerCandidateDynamicPositionResults.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertPerCandidateDynamicPositionResults.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertPerCandidateDynamicPositionResults.scala deleted file mode 100644 index 5c923ad9c..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertPerCandidateDynamicPositionResults.scala +++ /dev/null @@ -1,78 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object InsertPerCandidateDynamicPositionResults { - def apply[Query <: PipelineQuery]( - candidatePipeline: CandidatePipelineIdentifier, - candidatePositionInResults: CandidatePositionInResults[Query] - ): InsertPerCandidateDynamicPositionResults[Query] = - InsertPerCandidateDynamicPositionResults[Query]( - SpecificPipeline(candidatePipeline), - candidatePositionInResults) - - def apply[Query <: PipelineQuery]( - candidatePipelines: Set[CandidatePipelineIdentifier], - candidatePositionInResults: CandidatePositionInResults[Query] - ): InsertPerCandidateDynamicPositionResults[Query] = - InsertPerCandidateDynamicPositionResults[Query]( - SpecificPipelines(candidatePipelines), - candidatePositionInResults) -} - -/** - * Insert each candidate in the [[CandidateScope]] at the index relative to the original candidate in the `result` - * at that index using the provided [[CandidatePositionInResults]] instance. If the current results are shorter - * length than the computed position, then the candidate will be appended to the results. - * - * When the [[CandidatePositionInResults]] returns a `None`, that candidate is not - * added to the result. Negative position values are treated as 0 (front of the results). - * - * @example if [[CandidatePositionInResults]] results in a candidate mapping from index to candidate of - * `{0 -> a, 0 -> b, 0 -> c, 1 -> e, 2 -> g, 2 -> h} ` with original `results` = `[D, F]`, - * then the resulting output would look like `[a, b, c, D, e, F, g, h]` - */ -case class InsertPerCandidateDynamicPositionResults[-Query <: PipelineQuery]( - pipelineScope: CandidateScope, - candidatePositionInResults: CandidatePositionInResults[Query]) - extends Selector[Query] { - - override def apply( - query: Query, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val (candidatesToInsert, otherRemainingCandidatesTuples) = remainingCandidates - .map { candidate: CandidateWithDetails => - val position = - if (pipelineScope.contains(candidate)) - candidatePositionInResults(query, candidate, result) - else - None - (position, candidate) - }.partition { case (index, _) => index.isDefined } - - val otherRemainingCandidates = otherRemainingCandidatesTuples.map { - case (_, candidate) => candidate - } - - val positionAndCandidateList = candidatesToInsert.collect { - case (Some(position), candidate) => (position, candidate) - } - - val mergedResult = DynamicPositionSelector.mergeByIndexIntoResult( - positionAndCandidateList, - result, - DynamicPositionSelector.RelativeIndices - ) - - SelectorResult(remainingCandidates = otherRemainingCandidates, result = mergedResult) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertRandomPositionResults.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertRandomPositionResults.docx new file mode 100644 index 000000000..8d957a313 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertRandomPositionResults.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertRandomPositionResults.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertRandomPositionResults.scala deleted file mode 100644 index a6f4feae8..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertRandomPositionResults.scala +++ /dev/null @@ -1,139 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.component_library.selector.InsertRandomPositionResults.randomIndices -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.common.CandidateScope.PartitionedCandidates -import com.twitter.product_mixer.core.functional_component.configapi.StaticParam -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.Param - -import scala.collection.mutable -import scala.util.Random - -object InsertRandomPositionResults { - - /** - * Iterator containing random index between `startIndex` and `endIndex` + `n` - * where `n` is the number of times `next` has been called on the iterator - * without duplication - */ - private[selector] def randomIndices( - resultLength: Int, - startIndex: Int, - endIndex: Int, - random: Random - ): Iterator[Int] = { - - /** exclusive because [[Random.nextInt]]'s bound is exclusive */ - val indexUpperBound = Math.min(endIndex, resultLength) - - /** - * keep track of the available indices, `O(n)` space where `n` is `min(endIndex, resultLength) - max(startIndex, 0)` - * this ensures fairness which duplicate indices could otherwise skew - */ - val values = mutable.ArrayBuffer(Math.max(0, startIndex) to indexUpperBound: _*) - - /** - * Iterator that starts at 1 above the last valid index, [[indexUpperBound]] + 1, and increments monotonically - * representing the new highest index possible in the results for the next call - */ - Iterator - .from(indexUpperBound + 1) - .map { indexUpperBound => - /** - * pick a random index-to-insert-candidate-into-results from [[values]] replacing the value at - * the chosen index with the new highest index from [[indexUpperBound]], this results in - * constant time for picking the random index and adding the new highest valid index instead - * of removing the item from the middle and appending the new, which would be `O(n)` to shift - * all indices after the removal point - */ - val i = random.nextInt(values.length) - val randomIndexToUse = values(i) - // override the value at i with the new `upperBoundExclusive` to account for the new index value in the next iteration - values(i) = indexUpperBound - - randomIndexToUse - } - } -} - -sealed trait InsertedCandidateOrder - -/** - * Candidates from the `remainingCandidates` side will be inserted in a random order into the `result` - * - * @example if inserting `[ x, y, z ]` into the `result` then the relative positions of `x`, `y` and `z` - * to each other is random, e.g. `y` could come before `x` in the result. - */ -case object UnstableOrderingOfInsertedCandidates extends InsertedCandidateOrder - -/** - * Candidates from the `remainingCandidates` side will be inserted in their original order into the `result` - * - * @example if inserting `[ x, y, z ]` into the `result` then the relative positions of `x`, `y` and `z` - * to each other will remain the same, e.g. `x` is always before `y` is always before `z` in the final result - */ -case object StableOrderingOfInsertedCandidates extends InsertedCandidateOrder - -/** - * Insert `remainingCandidates` into a random position between the specified indices (inclusive) - * - * @example let `result` = `[ a, b, c, d ]` and we want to insert randomly `[ x, y, z ]` - * with `startIndex` = 1, `endIndex` = 2, and [[UnstableOrderingOfInsertedCandidates]]. - * We can expect a result that looks like `[ a, ... , d ]` where `...` is - * a random insertion of `x`, `y`, and `z` into `[ b, c ]`. So this could look like - * `[ a, y, b, x, c, z, d ]`, note that the inserted elements are randomly distributed - * among the elements that were originally between the specified indices. - * This functions like taking a slice of the original `result` between the indices, - * e.g. `[ b, c ]`, then randomly inserting into the slice, e.g. `[ y, b, x, c, z ]`, - * before reassembling the `result`, e.g. `[ a ] ++ [ y, b, x, c, z ] ++ [ d ]`. - * - * @example let `result` = `[ a, b, c, d ]` and we want to insert randomly `[ x, y, z ]` - * with `startIndex` = 1, `endIndex` = 2, and [[StableOrderingOfInsertedCandidates]]. - * We can expect a result that looks something like `[ a, x, b, y, c, z, d ]`, - * where `x` is before `y` which is before `z` - * - * @param startIndex an inclusive index which starts the range where the candidates will be inserted - * @param endIndex an inclusive index which ends the range where the candidates will be inserted - */ -case class InsertRandomPositionResults[-Query <: PipelineQuery]( - pipelineScope: CandidateScope, - remainingCandidateOrder: InsertedCandidateOrder, - startIndex: Param[Int] = StaticParam(0), - endIndex: Param[Int] = StaticParam(Int.MaxValue), - random: Random = new Random(0)) - extends Selector[Query] { - - override def apply( - query: Query, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - - val PartitionedCandidates(candidatesInScope, candidatesOutOfScope) = - pipelineScope.partition(remainingCandidates) - - val randomIndexIterator = { - val randomIndexIterator = - randomIndices(result.length, query.params(startIndex), query.params(endIndex), random) - - remainingCandidateOrder match { - case StableOrderingOfInsertedCandidates => - randomIndexIterator.take(candidatesInScope.length).toSeq.sorted.iterator - case UnstableOrderingOfInsertedCandidates => - randomIndexIterator - } - } - - val mergedResult = DynamicPositionSelector.mergeByIndexIntoResult( - candidatesToInsertByIndex = randomIndexIterator.zip(candidatesInScope.iterator).toSeq, - result = result, - DynamicPositionSelector.AbsoluteIndices - ) - - SelectorResult(remainingCandidates = candidatesOutOfScope, result = mergedResult) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertRelativePositionResults.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertRelativePositionResults.docx new file mode 100644 index 000000000..2033de6f1 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertRelativePositionResults.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertRelativePositionResults.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertRelativePositionResults.scala deleted file mode 100644 index e3212c58b..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertRelativePositionResults.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.selector.Selector -import CandidateScope.PartitionedCandidates -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.Param - -/** - * Insert all candidates from a candidate pipeline at a position below, relative to the last - * selection of the relative to candidate pipeline. If the relative to candidate pipeline does not - * contain candidates, then the candidates will be inserted with padding relative to position zero. - * If the current results are a shorter length than the requested padding, then the candidates will - * be appended to the results. - */ -case class InsertRelativePositionResults( - candidatePipeline: CandidatePipelineIdentifier, - relativeToCandidatePipeline: CandidatePipelineIdentifier, - paddingAboveParam: Param[Int]) - extends Selector[PipelineQuery] { - - override val pipelineScope: CandidateScope = SpecificPipelines(candidatePipeline) - - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - - val paddingAbove = query.params(paddingAboveParam) - assert(paddingAbove >= 0, "Padding above must be equal to or greater than zero") - - val PartitionedCandidates(selectedCandidates, otherCandidates) = - pipelineScope.partition(remainingCandidates) - - val resultUpdated = if (selectedCandidates.nonEmpty) { - // If the relativeToCandidatePipeline has zero candidates, lastIndexWhere will return -1 which - // will start padding from the zero position - val relativePosition = result.lastIndexWhere(_.source == relativeToCandidatePipeline) + 1 - val position = relativePosition + paddingAbove - - if (position < result.length) { - val (left, right) = result.splitAt(position) - left ++ selectedCandidates ++ right - } else { - result ++ selectedCandidates - } - } else { - result - } - - SelectorResult(remainingCandidates = otherCandidates, result = resultUpdated) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertSelector.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertSelector.docx new file mode 100644 index 000000000..d2016a944 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertSelector.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertSelector.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertSelector.scala deleted file mode 100644 index dd814bf72..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/InsertSelector.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import CandidateScope.PartitionedCandidates -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails - -private[selector] object InsertSelector { - - /** - * Insert all candidates from a candidate pipeline at a 0-indexed fixed position. If the current - * results are a shorter length than the requested position, then the candidates will be appended - * to the results. - */ - def insertIntoResultsAtPosition( - position: Int, - pipelineScope: CandidateScope, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - assert(position >= 0, "Position must be equal to or greater than zero") - - val PartitionedCandidates(selectedCandidates, otherCandidates) = - pipelineScope.partition(remainingCandidates) - - val resultUpdated = if (selectedCandidates.nonEmpty) { - if (position < result.length) { - val (left, right) = result.splitAt(position) - left ++ selectedCandidates ++ right - } else { - result ++ selectedCandidates - } - } else { - result - } - - SelectorResult(remainingCandidates = otherCandidates, result = resultUpdated) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/SelectConditionally.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/SelectConditionally.docx new file mode 100644 index 000000000..89363b048 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/SelectConditionally.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/SelectConditionally.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/SelectConditionally.scala deleted file mode 100644 index f4e8bb515..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/SelectConditionally.scala +++ /dev/null @@ -1,85 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.Param - -trait IncludeSelector[-Query <: PipelineQuery] { - def apply( - query: Query, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): Boolean -} - -/** - * Run [[selector]] if [[includeSelector]] resolves to true, else no-op the selector - */ -case class SelectConditionally[-Query <: PipelineQuery]( - selector: Selector[Query], - includeSelector: IncludeSelector[Query]) - extends Selector[Query] { - - override val pipelineScope: CandidateScope = selector.pipelineScope - - override def apply( - query: Query, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - if (includeSelector(query, remainingCandidates, result)) { - selector(query, remainingCandidates, result) - } else SelectorResult(remainingCandidates = remainingCandidates, result = result) - } -} - -object SelectConditionally { - - /** - * Wrap each [[Selector]] in `selectors` in an [[IncludeSelector]] with `includeSelector` as the [[SelectConditionally.includeSelector]] - */ - def apply[Query <: PipelineQuery]( - selectors: Seq[Selector[Query]], - includeSelector: IncludeSelector[Query] - ): Seq[Selector[Query]] = - selectors.map(SelectConditionally(_, includeSelector)) - - /** - * A [[SelectConditionally]] based on a [[Param]] - */ - def paramGated[Query <: PipelineQuery]( - selector: Selector[Query], - enabledParam: Param[Boolean], - ): SelectConditionally[Query] = - SelectConditionally(selector, (query, _, _) => query.params(enabledParam)) - - /** - * Wrap each [[Selector]] in `selectors` in a [[SelectConditionally]] based on a [[Param]] - */ - def paramGated[Query <: PipelineQuery]( - selectors: Seq[Selector[Query]], - enabledParam: Param[Boolean], - ): Seq[Selector[Query]] = - selectors.map(SelectConditionally.paramGated(_, enabledParam)) - - /** - * A [[SelectConditionally]] based on an inverted [[Param]] - */ - def paramNotGated[Query <: PipelineQuery]( - selector: Selector[Query], - enabledParamToInvert: Param[Boolean], - ): SelectConditionally[Query] = - SelectConditionally(selector, (query, _, _) => !query.params(enabledParamToInvert)) - - /** - * Wrap each [[Selector]] in `selectors` in a [[SelectConditionally]] based on an inverted [[Param]] - */ - def paramNotGated[Query <: PipelineQuery]( - selectors: Seq[Selector[Query]], - enabledParamToInvert: Param[Boolean], - ): Seq[Selector[Query]] = - selectors.map(SelectConditionally.paramNotGated(_, enabledParamToInvert)) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/SelectFromSubpoolCandidates.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/SelectFromSubpoolCandidates.docx new file mode 100644 index 000000000..455fa7d7c Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/SelectFromSubpoolCandidates.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/SelectFromSubpoolCandidates.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/SelectFromSubpoolCandidates.scala deleted file mode 100644 index 86ff297b7..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/SelectFromSubpoolCandidates.scala +++ /dev/null @@ -1,147 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import scala.reflect.ClassTag - -sealed trait SubpoolIncludeTypes - -trait IncludeInSubpool[-Query <: PipelineQuery] extends SubpoolIncludeTypes { - - /** - * Given the `query`, current `remainingCandidate`, and the `result`, - * returns whether the specific `remainingCandidate` should be passed into the - * [[SelectFromSubpoolCandidates]]'s [[SelectFromSubpoolCandidates.selector]] - * - * @note the `result` contains the [[SelectorResult.result]] that was passed into this selector, - * so each `remainingCandidate` will get the same `result` Seq. - */ - def apply( - query: Query, - remainingCandidate: CandidateWithDetails, - result: Seq[CandidateWithDetails] - ): Boolean -} - -case class IncludeCandidateTypeInSubpool[CandidateType <: UniversalNoun[_]]( -)( - implicit tag: ClassTag[CandidateType]) - extends IncludeInSubpool[PipelineQuery] { - - override def apply( - query: PipelineQuery, - remainingCandidate: CandidateWithDetails, - result: Seq[CandidateWithDetails] - ): Boolean = remainingCandidate.isCandidateType[CandidateType]() -} - -trait IncludeSetInSubpool[-Query <: PipelineQuery] extends SubpoolIncludeTypes { - - /** - * Given the `query`, all `remainingCandidates`` and `results`, - * returns a Set of which candidates should be included in the subpool. - * - * @note the returned set is only used to determine subpool membership. Mutating the candidates - * is invalid and won't work. The order of the candidates will be preserved from the current - * order of the remaining candidates sequence. - */ - def apply( - query: Query, - remainingCandidate: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): Set[CandidateWithDetails] -} - -sealed trait SubpoolRemainingCandidatesHandler - -/** - * Candidates remaining in the subpool after running the selector will be - * prepended to the beginning of the [[SelectorResult.remainingCandidates]] - */ -case object PrependToBeginningOfRemainingCandidates extends SubpoolRemainingCandidatesHandler - -/** - * Candidates remaining in the subpool after running the selector will be - * appended to the end of the [[SelectorResult.remainingCandidates]] - */ -case object AppendToEndOfRemainingCandidates extends SubpoolRemainingCandidatesHandler - -/** - * Creates a subpool of all `remainingCandidates` for which [[subpoolInclude]] resolves to true - * (in the same order as the original `remainingCandidates`) and runs the [[selector]] with the - * subpool passed in as the `remainingCandidates`. - * - * Most customers want to use a IncludeInSubpool that chooses if each candidate should be included - * in the subpool. - * Where necessary, IncludeSetInSubpool allows you to define them in bulk w/ a Set. - * - * @note any candidates in the subpool which are not added to the [[SelectorResult.result]] - * will be treated according to the [[SubpoolRemainingCandidatesHandler]] - */ -class SelectFromSubpoolCandidates[-Query <: PipelineQuery] private[selector] ( - val selector: Selector[Query], - subpoolInclude: SubpoolIncludeTypes, - subpoolRemainingCandidatesHandler: SubpoolRemainingCandidatesHandler = - AppendToEndOfRemainingCandidates) - extends Selector[Query] { - - override val pipelineScope: CandidateScope = selector.pipelineScope - - override def apply( - query: Query, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - - val (selectedCandidates, otherCandidates) = subpoolInclude match { - case includeInSubpool: IncludeInSubpool[Query] => - remainingCandidates.partition(candidate => - pipelineScope.contains(candidate) && includeInSubpool(query, candidate, result)) - case includeSetInSubpool: IncludeSetInSubpool[Query] => - val includeSet = - includeSetInSubpool(query, remainingCandidates.filter(pipelineScope.contains), result) - remainingCandidates.partition(candidate => includeSet.contains(candidate)) - } - - val underlyingSelectorResult = selector.apply(query, selectedCandidates, result) - val remainingCandidatesWithSubpoolRemainingCandidates = - subpoolRemainingCandidatesHandler match { - case AppendToEndOfRemainingCandidates => - otherCandidates ++ underlyingSelectorResult.remainingCandidates - case PrependToBeginningOfRemainingCandidates => - underlyingSelectorResult.remainingCandidates ++ otherCandidates - } - underlyingSelectorResult.copy(remainingCandidates = - remainingCandidatesWithSubpoolRemainingCandidates) - } - - override def toString: String = s"SelectFromSubpoolCandidates(${selector.toString}))" -} - -object SelectFromSubpoolCandidates { - def apply[Query <: PipelineQuery]( - selector: Selector[Query], - includeInSubpool: IncludeInSubpool[Query], - subpoolRemainingCandidatesHandler: SubpoolRemainingCandidatesHandler = - AppendToEndOfRemainingCandidates - ) = new SelectFromSubpoolCandidates[Query]( - selector, - includeInSubpool, - subpoolRemainingCandidatesHandler - ) - - def includeSet[Query <: PipelineQuery]( - selector: Selector[Query], - includeSetInSubpool: IncludeSetInSubpool[Query], - subpoolRemainingCandidatesHandler: SubpoolRemainingCandidatesHandler = - AppendToEndOfRemainingCandidates - ) = new SelectFromSubpoolCandidates[Query]( - selector, - includeSetInSubpool, - subpoolRemainingCandidatesHandler - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/UpdateSortCandidates.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/UpdateSortCandidates.docx new file mode 100644 index 000000000..b4eeb8ceb Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/UpdateSortCandidates.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/UpdateSortCandidates.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/UpdateSortCandidates.scala deleted file mode 100644 index 235bcf778..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/UpdateSortCandidates.scala +++ /dev/null @@ -1,86 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.component_library.selector.sorter.SorterFromOrdering -import com.twitter.product_mixer.component_library.selector.sorter.SorterProvider -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.common.CandidateScope.PartitionedCandidates -import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.selector._ -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object UpdateSortCandidates { - def apply( - candidatePipeline: CandidatePipelineIdentifier, - sorterProvider: SorterProvider, - ) = new UpdateSortCandidates(SpecificPipeline(candidatePipeline), sorterProvider) - - def apply( - candidatePipeline: CandidatePipelineIdentifier, - ordering: Ordering[CandidateWithDetails] - ) = - new UpdateSortCandidates(SpecificPipeline(candidatePipeline), SorterFromOrdering(ordering)) - - def apply( - candidatePipelines: Set[CandidatePipelineIdentifier], - ordering: Ordering[CandidateWithDetails] - ) = - new UpdateSortCandidates(SpecificPipelines(candidatePipelines), SorterFromOrdering(ordering)) - - def apply( - candidatePipelines: Set[CandidatePipelineIdentifier], - sorterProvider: SorterProvider, - ) = new UpdateSortCandidates(SpecificPipelines(candidatePipelines), sorterProvider) - - def apply( - pipelineScope: CandidateScope, - ordering: Ordering[CandidateWithDetails] - ) = new UpdateSortCandidates(pipelineScope, SorterFromOrdering(ordering)) -} - -/** - * Sort item and module (not items inside modules) candidates in a pipeline scope. - * Note that if sorting across multiple candidate sources, the candidates will be grouped together - * in sorted order, starting from the position of the first candidate. - * - * For example, we could specify the following ordering to sort by score descending: - * Ordering - * .by[CandidateWithDetails, Double](_.features.get(ScoreFeature) match { - * case Scored(score) => score - * case _ => Double.MinValue - * }).reverse - */ -case class UpdateSortCandidates( - override val pipelineScope: CandidateScope, - sorterProvider: SorterProvider) - extends Selector[PipelineQuery] { - - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val PartitionedCandidates(selectedCandidates, otherCandidates) = - pipelineScope.partition(remainingCandidates) - - val updatedRemainingCandidates = if (selectedCandidates.nonEmpty) { - // Safe .head due to nonEmpty check - val position = remainingCandidates.indexOf(selectedCandidates.head) - val orderedSelectedCandidates = - sorterProvider.sorter(query, remainingCandidates, result).sort(selectedCandidates) - - if (position < otherCandidates.length) { - val (left, right) = otherCandidates.splitAt(position) - left ++ orderedSelectedCandidates ++ right - } else { - otherCandidates ++ orderedSelectedCandidates - } - } else { - remainingCandidates - } - - SelectorResult(remainingCandidates = updatedRemainingCandidates, result = result) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/UpdateSortModuleItemCandidates.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/UpdateSortModuleItemCandidates.docx new file mode 100644 index 000000000..75e6a4810 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/UpdateSortModuleItemCandidates.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/UpdateSortModuleItemCandidates.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/UpdateSortModuleItemCandidates.scala deleted file mode 100644 index 33de20999..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/UpdateSortModuleItemCandidates.scala +++ /dev/null @@ -1,96 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.component_library.selector.sorter.SorterFromOrdering -import com.twitter.product_mixer.component_library.selector.sorter.SorterProvider -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object UpdateSortModuleItemCandidates { - def apply( - candidatePipeline: CandidatePipelineIdentifier, - ordering: Ordering[CandidateWithDetails] - ): UpdateSortModuleItemCandidates = - UpdateSortModuleItemCandidates( - SpecificPipeline(candidatePipeline), - SorterFromOrdering(ordering)) - - def apply( - candidatePipeline: CandidatePipelineIdentifier, - sorterProvider: SorterProvider - ): UpdateSortModuleItemCandidates = - UpdateSortModuleItemCandidates(SpecificPipeline(candidatePipeline), sorterProvider) - - def apply( - candidatePipelines: Set[CandidatePipelineIdentifier], - ordering: Ordering[CandidateWithDetails] - ): UpdateSortModuleItemCandidates = - UpdateSortModuleItemCandidates( - SpecificPipelines(candidatePipelines), - SorterFromOrdering(ordering)) - - def apply( - candidatePipelines: Set[CandidatePipelineIdentifier], - sorterProvider: SorterProvider - ): UpdateSortModuleItemCandidates = - UpdateSortModuleItemCandidates(SpecificPipelines(candidatePipelines), sorterProvider) -} - -/** - * Sort items inside a module from a candidate source and update the remainingCandidates. - * - * For example, we could specify the following ordering to sort by score descending: - * - * {{{ - * Ordering - * .by[CandidateWithDetails, Double](_.features.get(ScoreFeature) match { - * case Scored(score) => score - * case _ => Double.MinValue - * }).reverse - * - * // Before sorting: - * ModuleCandidateWithDetails( - * Seq( - * ItemCandidateWithLowScore, - * ItemCandidateWithMidScore, - * ItemCandidateWithHighScore), - * ... other params - * ) - * - * // After sorting: - * ModuleCandidateWithDetails( - * Seq( - * ItemCandidateWithHighScore, - * ItemCandidateWithMidScore, - * ItemCandidateWithLowScore), - * ... other params - * ) - * }}} - * - * @note this updates the modules in the `remainingCandidates` - */ -case class UpdateSortModuleItemCandidates( - override val pipelineScope: CandidateScope, - sorterProvider: SorterProvider) - extends Selector[PipelineQuery] { - - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val updatedCandidates = remainingCandidates.map { - case module: ModuleCandidateWithDetails if pipelineScope.contains(module) => - module.copy(candidates = - sorterProvider.sorter(query, remainingCandidates, result).sort(module.candidates)) - case candidate => candidate - } - SelectorResult(remainingCandidates = updatedCandidates, result = result) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/UpdateSortResults.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/UpdateSortResults.docx new file mode 100644 index 000000000..1750efa95 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/UpdateSortResults.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/UpdateSortResults.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/UpdateSortResults.scala deleted file mode 100644 index 76785780f..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/UpdateSortResults.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.twitter.product_mixer.component_library.selector - -import com.twitter.product_mixer.component_library.selector.sorter.SorterFromOrdering -import com.twitter.product_mixer.component_library.selector.sorter.SorterProvider -import com.twitter.product_mixer.core.functional_component.common.AllPipelines -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object UpdateSortResults { - def apply( - ordering: Ordering[CandidateWithDetails] - ) = - new UpdateSortResults((_, _, _) => SorterFromOrdering(ordering)) -} - -/** - * Sort item and module (not items inside modules) results. - * - * For example, we could specify the following ordering to sort by score descending: - * Ordering - * .by[CandidateWithDetails, Double](_.features.get(ScoreFeature) match { - * case Scored(score) => score - * case _ => Double.MinValue - * }).reverse - */ -case class UpdateSortResults( - sorterProvider: SorterProvider, - override val pipelineScope: CandidateScope = AllPipelines) - extends Selector[PipelineQuery] { - - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val updatedResult = sorterProvider.sorter(query, remainingCandidates, result).sort(result) - - SelectorResult(remainingCandidates = remainingCandidates, result = updatedResult) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads/AdsInjector.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads/AdsInjector.docx new file mode 100644 index 000000000..09d18ade2 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads/AdsInjector.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads/AdsInjector.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads/AdsInjector.scala deleted file mode 100644 index 21d811ee7..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads/AdsInjector.scala +++ /dev/null @@ -1,73 +0,0 @@ -package com.twitter.product_mixer.component_library.selector.ads - -import com.google.inject.Inject -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.goldfinch.adaptors.ads.productmixer.ProductMixerPromotedEntriesAdaptor -import com.twitter.goldfinch.adaptors.productmixer.ProductMixerNonPromotedEntriesAdaptor -import com.twitter.goldfinch.adaptors.productmixer.ProductMixerQueryConverter -import com.twitter.goldfinch.api.AdsInjectionRequestContextConverter -import com.twitter.goldfinch.api.AdsInjectionSurfaceAreas.SurfaceAreaName -import com.twitter.goldfinch.api.{AdsInjector => GoldfinchAdsInjector} -import com.twitter.goldfinch.api.NonPromotedEntriesAdaptor -import com.twitter.goldfinch.api.PromotedEntriesAdaptor -import com.twitter.goldfinch.impl.injector.AdsInjectorBuilder -import com.twitter.goldfinch.impl.injector.product_mixer.AdsInjectionSurfaceAreaAdjustersMap -import com.twitter.goldfinch.impl.injector.product_mixer.VerticalSizeAdjustmentConfigMap -import com.twitter.inject.Logging -import com.twitter.product_mixer.component_library.model.query.ads._ -import com.twitter.product_mixer.core.model.common.presentation._ -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import javax.inject.Singleton -import com.twitter.goldfinch.impl.core.DefaultFeatureSwitchResultsFactory -import com.twitter.goldfinch.impl.core.LocalDevelopmentFeatureSwitchResultsFactory -import com.twitter.inject.annotations.Flag -import com.twitter.product_mixer.core.module.product_mixer_flags.ProductMixerFlagModule.ConfigRepoLocalPath -import com.twitter.product_mixer.core.module.product_mixer_flags.ProductMixerFlagModule.ServiceLocal - -@Singleton -class AdsInjector @Inject() ( - statsReceiver: StatsReceiver, - @Flag(ConfigRepoLocalPath) localConfigRepoPath: String, - @Flag(ServiceLocal) isServiceLocal: Boolean) - extends Logging { - private val adsQueryRequestConverter: AdsInjectionRequestContextConverter[ - PipelineQuery with AdsQuery - ] = ProductMixerQueryConverter - - def forSurfaceArea( - surfaceAreaName: SurfaceAreaName - ): GoldfinchAdsInjector[ - PipelineQuery with AdsQuery, - CandidateWithDetails, - CandidateWithDetails - ] = { - - val scopedStatsReceiver: StatsReceiver = - statsReceiver.scope("goldfinch", surfaceAreaName.toString) - - val nonAdsAdaptor: NonPromotedEntriesAdaptor[CandidateWithDetails] = - ProductMixerNonPromotedEntriesAdaptor( - VerticalSizeAdjustmentConfigMap.configsBySurfaceArea(surfaceAreaName), - scopedStatsReceiver) - - val adsAdaptor: PromotedEntriesAdaptor[CandidateWithDetails] = - new ProductMixerPromotedEntriesAdaptor(scopedStatsReceiver) - - val featureSwitchFactory = if (isServiceLocal) { - new LocalDevelopmentFeatureSwitchResultsFactory( - surfaceAreaName.toString, - configRepoAbsPath = localConfigRepoPath) - } else new DefaultFeatureSwitchResultsFactory(surfaceAreaName.toString) - - new AdsInjectorBuilder[PipelineQuery with AdsQuery, CandidateWithDetails, CandidateWithDetails]( - requestAdapter = adsQueryRequestConverter, - nonPromotedEntriesAdaptor = nonAdsAdaptor, - promotedEntriesAdaptor = adsAdaptor, - adjusters = - AdsInjectionSurfaceAreaAdjustersMap.getAdjusters(surfaceAreaName, scopedStatsReceiver), - featureSwitchFactory = featureSwitchFactory, - statsReceiver = scopedStatsReceiver, - logger = logger - ).build() - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads/BUILD.bazel b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads/BUILD.bazel deleted file mode 100644 index 6536b7cf4..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads/BUILD.bazel +++ /dev/null @@ -1,30 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - scalac_plugins = ["no-roomba"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "ads-injection/lib/src/main/scala/com/twitter/goldfinch/adaptors/ads/productmixer", - "ads-injection/lib/src/main/scala/com/twitter/goldfinch/adaptors/ads/productmixer/util", - "ads-injection/lib/src/main/scala/com/twitter/goldfinch/adaptors/productmixer", - "ads-injection/lib/src/main/scala/com/twitter/goldfinch/api", - "ads-injection/lib/src/main/scala/com/twitter/goldfinch/impl/core", - "ads-injection/lib/src/main/scala/com/twitter/goldfinch/impl/injector", - "ads-injection/lib/src/main/scala/com/twitter/goldfinch/impl/injector/product_mixer", - "finatra/inject/inject-slf4j/src/main/scala/com/twitter/inject", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector", - ], - exports = [ - "ads-injection/lib/src/main/scala/com/twitter/goldfinch/impl/injector", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads/BUILD.docx new file mode 100644 index 000000000..386998d8e Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads/InsertAdResults.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads/InsertAdResults.docx new file mode 100644 index 000000000..0447349e3 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads/InsertAdResults.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads/InsertAdResults.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads/InsertAdResults.scala deleted file mode 100644 index 54eab9e69..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads/InsertAdResults.scala +++ /dev/null @@ -1,95 +0,0 @@ -package com.twitter.product_mixer.component_library.selector.ads - -import com.twitter.goldfinch.api.AdsInjectionSurfaceAreas.SurfaceAreaName -import com.twitter.goldfinch.api.AdsInjectorAdditionalRequestParams -import com.twitter.goldfinch.api.AdsInjectorOutput -import com.twitter.goldfinch.api.{AdsInjector => GoldfinchAdsInjector} -import com.twitter.product_mixer.component_library.model.query.ads._ -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.selector.Selector -import CandidateScope.PartitionedCandidates -import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * Injects the sequence of AdCandidates in the `result` in the - * sequence of the Other Candidates(which are not ads). - * - * Every SurfaceArea or DisplayLocation runs their own desired set of adjusters(set in pipeline) - * to inject ads and reposition the ads in the sequence of other candidates of `result` : - * which are fetched by AdsInjectionSurfaceAreaAdjustersMap - * Note: The original sequence of non_promoted entries(non-ads) is retained and the ads - * are inserted in the sequence using `goldfinch` library based on the 'insertion-position' - * hydrated in AdsCandidate by Adserver/Admixer. - * - * ***** Goldfinch recommends to run this selector as close to the marshalling of candidates to have - * more realistic view of served-timeline in Goldfinch-BQ-Logs and avoid any further updates on the - * timeline(sequence of entries) created. **** - * - * Any surface area like `search_tweets(surface_area)` can call - * InsertAdResults(surfaceArea = "TweetSearch", candidatePipeline = adsCandidatePipeline.identifier, - * ProductMixerAdsInjector = productMixerAdsInjector) - * where the pipeline config can call - * productMixerAdsInjector.forSurfaceArea("TweetSearch") to get AdsInjector Object - * - * @example - * `Seq(source1NonAd_Id1, source1NonAd_Id2, source2NonAd_Id1, source2NonAd_Id2,source1NonAd_Id3, source3NonAd_Id3,source3Ad_Id1_InsertionPos1, source3Ad_Id2_InsertionPos4)` - * then the output result can be - * `Seq(source1NonAd_Id1, source3Ad_Id1_InsertionPos1, source1NonAd_Id2, source2NonAd_Id1, source3Ad_Id2_InsertionPos4,source2NonAd_Id2, source1NonAd_Id3, source3NonAd_Id3)` - * depending on the insertion position of Ads and other adjusters shifting the ads - */ -case class InsertAdResults( - surfaceAreaName: SurfaceAreaName, - adsInjector: GoldfinchAdsInjector[ - PipelineQuery with AdsQuery, - CandidateWithDetails, - CandidateWithDetails - ], - adsCandidatePipeline: CandidatePipelineIdentifier) - extends Selector[PipelineQuery with AdsQuery] { - - override val pipelineScope: CandidateScope = SpecificPipeline(adsCandidatePipeline) - - override def apply( - query: PipelineQuery with AdsQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - // Read into ads and non-ads candidates. - val PartitionedCandidates(adCandidates, otherRemainingCandidates) = - pipelineScope.partition(remainingCandidates) - - // Create this param from Query/AdsCandidate based on surface_area, if required. - val adsInjectorAdditionalRequestParams = - AdsInjectorAdditionalRequestParams(budgetAwareExperimentId = None) - - val adsInjectorOutput: AdsInjectorOutput[CandidateWithDetails, CandidateWithDetails] = - adsInjector.applyForAllEntries( - query = query, - nonPromotedEntries = result, - promotedEntries = adCandidates, - adsInjectorAdditionalRequestParams = adsInjectorAdditionalRequestParams) - - val updatedRemainingCandidates = otherRemainingCandidates ++ - GoldfinchResults(adsInjectorOutput.unusedEntries).adapt - val mergedResults = GoldfinchResults(adsInjectorOutput.mergedEntries).adapt - SelectorResult(remainingCandidates = updatedRemainingCandidates, result = mergedResults) - } - - /** - * Goldfinch separates NonPromotedEntryType and PromotedEntryType models, while in ProMix both - * non-promoted and promoted entries are CandidateWithDetails. As such, we need to flatten the - * result back into a single Seq of CandidateWithDetails. See [[AdsInjectorOutput]] - */ - case class GoldfinchResults(results: Seq[Either[CandidateWithDetails, CandidateWithDetails]]) { - def adapt: Seq[CandidateWithDetails] = { - results.collect { - case Right(value) => value - case Left(value) => value - } - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/BUILD deleted file mode 100644 index a1c0b286f..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/BUILD +++ /dev/null @@ -1,20 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - scalac_plugins = ["no-roomba"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/BUILD.docx new file mode 100644 index 000000000..02db38cc3 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/FeatureValueSorter.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/FeatureValueSorter.docx new file mode 100644 index 000000000..9931905fa Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/FeatureValueSorter.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/FeatureValueSorter.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/FeatureValueSorter.scala deleted file mode 100644 index f1940f312..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/FeatureValueSorter.scala +++ /dev/null @@ -1,248 +0,0 @@ -package com.twitter.product_mixer.component_library.selector.sorter - -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import scala.reflect.runtime.universe._ - -object FeatureValueSorter { - - /** - * Sort by a feature value ascending. If the feature failed or is missing, use an inferred default - * based on the type of [[FeatureValue]]. For Numeric values this is the MinValue - * (e.g. Long.MinValue, Double.MinValue). - * - * @param feature feature with value to sort by - * @param dummyImplicit due to type erasure, implicit used to disambiguate `def ascending()` - * between def with param `feature: Feature[Candidate, FeatureValue]` - * from def with param `feature: Feature[Candidate, Option[FeatureValue]]` - * @param typeTag allows for inferring default value from the FeatureValue type. - * See [[featureValueSortDefaultValue]] - * @tparam Candidate candidate for the feature - * @tparam FeatureValue feature value with an [[Ordering]] context bound - */ - def ascending[Candidate <: UniversalNoun[Any], FeatureValue: Ordering]( - feature: Feature[Candidate, FeatureValue] - )( - implicit dummyImplicit: DummyImplicit, - typeTag: TypeTag[FeatureValue] - ): SorterProvider = { - val defaultFeatureValue: FeatureValue = featureValueSortDefaultValue(feature, Ascending) - - ascending(feature, defaultFeatureValue) - } - - /** - * Sort by a feature value ascending. If the feature failed or is missing, use the provided - * default. - * - * @param feature feature with value to sort by - * @param dummyImplicit due to type erasure, implicit used to disambiguate `def ascending()` - * between def with param `feature: Feature[Candidate, FeatureValue]` - * from def with param `feature: Feature[Candidate, Option[FeatureValue]]` - * @tparam Candidate candidate for the feature - * @tparam FeatureValue feature value with an [[Ordering]] context bound - */ - def ascending[Candidate <: UniversalNoun[Any], FeatureValue: Ordering]( - feature: Feature[Candidate, FeatureValue], - defaultFeatureValue: FeatureValue - )( - implicit dummyImplicit: DummyImplicit - ): SorterProvider = { - val ordering = Ordering.by[CandidateWithDetails, FeatureValue]( - _.features.getOrElse(feature, defaultFeatureValue)) - - SorterFromOrdering(ordering, Ascending) - } - - /** - * Sort by an optional feature value ascending. If the feature failed or is missing, use an - * inferred default based on the type of [[FeatureValue]]. For Numeric values this is the MinValue - * (e.g. Long.MinValue, Double.MinValue). - * - * @param feature feature with value to sort by - * @param typeTag allows for inferring default value from the FeatureValue type. - * See [[featureOptionalValueSortDefaultValue]] - * @tparam Candidate candidate for the feature - * @tparam FeatureValue feature value with an [[Ordering]] context bound - */ - def ascending[Candidate <: UniversalNoun[Any], FeatureValue: Ordering]( - feature: Feature[Candidate, Option[FeatureValue]] - )( - implicit typeTag: TypeTag[FeatureValue] - ): SorterProvider = { - val defaultFeatureValue: FeatureValue = featureOptionalValueSortDefaultValue(feature, Ascending) - - ascending(feature, defaultFeatureValue) - } - - /** - * Sort by an optional feature value ascending. If the feature failed or is missing, use the - * provided default. - * - * @param feature feature with value to sort by - * @tparam Candidate candidate for the feature - * @tparam FeatureValue feature value with an [[Ordering]] context bound - */ - def ascending[Candidate <: UniversalNoun[Any], FeatureValue: Ordering]( - feature: Feature[Candidate, Option[FeatureValue]], - defaultFeatureValue: FeatureValue - ): SorterProvider = { - val ordering = Ordering.by[CandidateWithDetails, FeatureValue]( - _.features.getOrElse(feature, None).getOrElse(defaultFeatureValue)) - - SorterFromOrdering(ordering, Ascending) - } - - /** - * Sort by a feature value descending. If the feature failed or is missing, use an inferred - * default based on the type of [[FeatureValue]]. For Numeric values this is the MaxValue - * (e.g. Long.MaxValue, Double.MaxValue). - * - * @param feature feature with value to sort by - * @param dummyImplicit due to type erasure, implicit used to disambiguate `def descending()` - * between def with param `feature: Feature[Candidate, FeatureValue]` - * from def with param `feature: Feature[Candidate, Option[FeatureValue]]` - * @param typeTag allows for inferring default value from the FeatureValue type. - * See [[featureValueSortDefaultValue]] - * @tparam Candidate candidate for the feature - * @tparam FeatureValue feature value with an [[Ordering]] context bound - */ - def descending[Candidate <: UniversalNoun[Any], FeatureValue: Ordering]( - feature: Feature[Candidate, FeatureValue] - )( - implicit dummyImplicit: DummyImplicit, - typeTag: TypeTag[FeatureValue] - ): SorterProvider = { - val defaultFeatureValue: FeatureValue = featureValueSortDefaultValue(feature, Descending) - - descending(feature, defaultFeatureValue) - } - - /** - * Sort by a feature value descending. If the feature failed or is missing, use the provided - * default. - * - * @param feature feature with value to sort by - * @param dummyImplicit due to type erasure, implicit used to disambiguate `def descending()` - * between def with param `feature: Feature[Candidate, FeatureValue]` - * from def with param `feature: Feature[Candidate, Option[FeatureValue]]` - * @tparam Candidate candidate for the feature - * @tparam FeatureValue feature value with an [[Ordering]] context bound - */ - def descending[Candidate <: UniversalNoun[Any], FeatureValue: Ordering]( - feature: Feature[Candidate, FeatureValue], - defaultFeatureValue: FeatureValue - )( - implicit dummyImplicit: DummyImplicit - ): SorterProvider = { - val ordering = Ordering.by[CandidateWithDetails, FeatureValue]( - _.features.getOrElse(feature, defaultFeatureValue)) - - SorterFromOrdering(ordering, Descending) - } - - /** - * Sort by an optional feature value descending. If the feature failed or is missing, use an - * inferred default based on the type of [[FeatureValue]]. For Numeric values this is the MaxValue - * (e.g. Long.MaxValue, Double.MaxValue). - * - * @param feature feature with value to sort by - * @param typeTag allows for inferring default value from the FeatureValue type. - * See [[featureOptionalValueSortDefaultValue]] - * @tparam Candidate candidate for the feature - * @tparam FeatureValue feature value with an [[Ordering]] context bound - */ - def descending[Candidate <: UniversalNoun[Any], FeatureValue: Ordering]( - feature: Feature[Candidate, Option[FeatureValue]] - )( - implicit typeTag: TypeTag[FeatureValue] - ): SorterProvider = { - val defaultFeatureValue: FeatureValue = - featureOptionalValueSortDefaultValue(feature, Descending) - - descending(feature, defaultFeatureValue) - } - - /** - * Sort by an optional feature value descending. If the feature failed or is missing, use the - * provided default. - * - * @param feature feature with value to sort by - * @tparam Candidate candidate for the feature - * @tparam FeatureValue feature value with an [[Ordering]] context bound - */ - def descending[Candidate <: UniversalNoun[Any], FeatureValue: Ordering]( - feature: Feature[Candidate, Option[FeatureValue]], - defaultFeatureValue: FeatureValue - ): SorterProvider = { - val ordering = Ordering.by[CandidateWithDetails, FeatureValue]( - _.features.getOrElse(feature, None).getOrElse(defaultFeatureValue)) - - SorterFromOrdering(ordering, Descending) - } - - private[sorter] def featureValueSortDefaultValue[FeatureValue: Ordering]( - feature: Feature[_, FeatureValue], - sortOrder: SortOrder - )( - implicit typeTag: TypeTag[FeatureValue] - ): FeatureValue = { - val defaultValue = sortOrder match { - case Descending => - typeOf[FeatureValue] match { - case t if t <:< typeOf[Short] => Short.MinValue - case t if t <:< typeOf[Int] => Int.MinValue - case t if t <:< typeOf[Long] => Long.MinValue - case t if t <:< typeOf[Double] => Double.MinValue - case t if t <:< typeOf[Float] => Float.MinValue - case _ => - throw new UnsupportedOperationException(s"Default value not supported for $feature") - } - case Ascending => - typeOf[FeatureValue] match { - case t if t <:< typeOf[Short] => Short.MaxValue - case t if t <:< typeOf[Int] => Int.MaxValue - case t if t <:< typeOf[Long] => Long.MaxValue - case t if t <:< typeOf[Double] => Double.MaxValue - case t if t <:< typeOf[Float] => Float.MaxValue - case _ => - throw new UnsupportedOperationException(s"Default value not supported for $feature") - } - } - - defaultValue.asInstanceOf[FeatureValue] - } - - private[sorter] def featureOptionalValueSortDefaultValue[FeatureValue: Ordering]( - feature: Feature[_, Option[FeatureValue]], - sortOrder: SortOrder - )( - implicit typeTag: TypeTag[FeatureValue] - ): FeatureValue = { - val defaultValue = sortOrder match { - case Descending => - typeOf[Option[FeatureValue]] match { - case t if t <:< typeOf[Option[Short]] => Short.MinValue - case t if t <:< typeOf[Option[Int]] => Int.MinValue - case t if t <:< typeOf[Option[Long]] => Long.MinValue - case t if t <:< typeOf[Option[Double]] => Double.MinValue - case t if t <:< typeOf[Option[Float]] => Float.MinValue - case _ => - throw new UnsupportedOperationException(s"Default value not supported for $feature") - } - case Ascending => - typeOf[Option[FeatureValue]] match { - case t if t <:< typeOf[Option[Short]] => Short.MaxValue - case t if t <:< typeOf[Option[Int]] => Int.MaxValue - case t if t <:< typeOf[Option[Long]] => Long.MaxValue - case t if t <:< typeOf[Option[Double]] => Double.MaxValue - case t if t <:< typeOf[Option[Float]] => Float.MaxValue - case _ => - throw new UnsupportedOperationException(s"Default value not supported for $feature") - } - } - - defaultValue.asInstanceOf[FeatureValue] - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/RandomShuffleSorter.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/RandomShuffleSorter.docx new file mode 100644 index 000000000..6ff14852f Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/RandomShuffleSorter.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/RandomShuffleSorter.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/RandomShuffleSorter.scala deleted file mode 100644 index 1bd9ee317..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/RandomShuffleSorter.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.product_mixer.component_library.selector.sorter - -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import scala.util.Random - -/** - * Randomly shuffles candidates using the provided [[random]] - * - * @example `UpdateSortResults(RandomShuffleSorter())` - * @param random used to set the seed and for ease of testing, in most cases leaving it as the default is fine. - */ -case class RandomShuffleSorter(random: Random = new Random(0)) extends SorterProvider with Sorter { - - override def sort[Candidate <: CandidateWithDetails](candidates: Seq[Candidate]): Seq[Candidate] = - random.shuffle(candidates) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/ReverseSorter.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/ReverseSorter.docx new file mode 100644 index 000000000..537f05548 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/ReverseSorter.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/ReverseSorter.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/ReverseSorter.scala deleted file mode 100644 index 418cf2982..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/ReverseSorter.scala +++ /dev/null @@ -1,14 +0,0 @@ -package com.twitter.product_mixer.component_library.selector.sorter - -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails - -/** - * Reverse candidates. - * - * @example `UpdateSortResults(ReverseSorter())` - */ -object ReverseSorter extends SorterProvider with Sorter { - - override def sort[Candidate <: CandidateWithDetails](candidates: Seq[Candidate]): Seq[Candidate] = - candidates.reverse -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/SortOrder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/SortOrder.docx new file mode 100644 index 000000000..26e597623 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/SortOrder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/SortOrder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/SortOrder.scala deleted file mode 100644 index 3ddd5fc84..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/SortOrder.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.product_mixer.component_library.selector.sorter - -sealed trait SortOrder -case object Ascending extends SortOrder -case object Descending extends SortOrder diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/SorterFromOrdering.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/SorterFromOrdering.docx new file mode 100644 index 000000000..ab98c399b Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/SorterFromOrdering.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/SorterFromOrdering.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/SorterFromOrdering.scala deleted file mode 100644 index c09792eb8..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/SorterFromOrdering.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.product_mixer.component_library.selector.sorter - -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails - -object SorterFromOrdering { - def apply(ordering: Ordering[CandidateWithDetails], sortOrder: SortOrder): SorterFromOrdering = - SorterFromOrdering(if (sortOrder == Descending) ordering.reverse else ordering) -} - -/** - * Sorts candidates based on the provided [[ordering]] - * - * @note the [[Ordering]] must be transitive, so if `A < B` and `B < C` then `A < C`. - * @note sorting randomly via `Ordering.by[CandidateWithDetails, Double](_ => Random.nextDouble())` - * is not safe and can fail at runtime since TimSort depends on stable sort values for - * pivoting. To sort randomly, use [[RandomShuffleSorter]] instead. - */ -case class SorterFromOrdering( - ordering: Ordering[CandidateWithDetails]) - extends SorterProvider - with Sorter { - - override def sort[Candidate <: CandidateWithDetails](candidates: Seq[Candidate]): Seq[Candidate] = - candidates.sorted(ordering) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/SorterProvider.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/SorterProvider.docx new file mode 100644 index 000000000..413258b18 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/SorterProvider.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/SorterProvider.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/SorterProvider.scala deleted file mode 100644 index d0c1a3eb2..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/SorterProvider.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.product_mixer.component_library.selector.sorter - -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * Makes a [[Sorter]] to run for the given input based on the - * [[PipelineQuery]], the `remainingCandidates`, and the `result`. - * - * @note this should be used to choose between different [[Sorter]]s, - * if you want to conditionally sort wrap your [[Sorter]] with - * [[com.twitter.product_mixer.component_library.selector.SelectConditionally]] instead. - */ -trait SorterProvider { - - /** Makes a [[Sorter]] for the given inputs */ - def sorter( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): Sorter -} - -/** - * Sorts the candidates - * - * All [[Sorter]]s also implement [[SorterProvider]] to provide themselves for convenience. - */ -trait Sorter { self: SorterProvider => - - /** Sorts the `candidates` */ - def sort[Candidate <: CandidateWithDetails](candidates: Seq[Candidate]): Seq[Candidate] - - /** Any [[Sorter]] can be used in place of a [[SorterProvider]] to provide itself */ - override final def sorter( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): Sorter = self -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/featurestorev1/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/featurestorev1/BUILD deleted file mode 100644 index a074c678b..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/featurestorev1/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - scalac_plugins = ["no-roomba"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/featurestorev1", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/featurestorev1", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/featurestorev1/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/featurestorev1/BUILD.docx new file mode 100644 index 000000000..95deeb1a9 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/featurestorev1/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/featurestorev1/FeatureStoreV1FeatureValueSorter.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/featurestorev1/FeatureStoreV1FeatureValueSorter.docx new file mode 100644 index 000000000..d1c330028 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/featurestorev1/FeatureStoreV1FeatureValueSorter.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/featurestorev1/FeatureStoreV1FeatureValueSorter.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/featurestorev1/FeatureStoreV1FeatureValueSorter.scala deleted file mode 100644 index b4819e6a8..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/sorter/featurestorev1/FeatureStoreV1FeatureValueSorter.scala +++ /dev/null @@ -1,98 +0,0 @@ -package com.twitter.product_mixer.component_library.selector.sorter.featurestorev1 - -import com.twitter.ml.featurestore.lib.EntityId -import com.twitter.product_mixer.component_library.selector.sorter.Ascending -import com.twitter.product_mixer.component_library.selector.sorter.Descending -import com.twitter.product_mixer.component_library.selector.sorter.FeatureValueSorter.featureValueSortDefaultValue -import com.twitter.product_mixer.component_library.selector.sorter.SorterFromOrdering -import com.twitter.product_mixer.component_library.selector.sorter.SorterProvider -import com.twitter.product_mixer.core.feature.featuremap.featurestorev1.FeatureStoreV1FeatureMap._ -import com.twitter.product_mixer.core.feature.featurestorev1.FeatureStoreV1CandidateFeature -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import scala.reflect.runtime.universe._ - -/** - * Feature Store v1 version of [[com.twitter.product_mixer.component_library.selector.sorter.FeatureValueSorter]] - */ -object FeatureStoreV1FeatureValueSorter { - - /** - * Sort by a Feature Store v1 feature value ascending. If the feature failed or is missing, use an - * inferred default based on the type of [[FeatureValue]]. For Numeric values this is the MinValue - * (e.g. Long.MinValue, Double.MinValue). - * - * @param feature Feature Store v1 feature with value to sort by - * @param typeTag allows for inferring default value from the FeatureValue type. - * See [[com.twitter.product_mixer.component_library.selector.sorter.FeatureValueSorter.featureValueSortDefaultValue]] - * @tparam Candidate candidate for the feature - * @tparam FeatureValue feature value with an [[Ordering]] context bound - */ - def ascending[Candidate <: UniversalNoun[Any], FeatureValue: Ordering]( - feature: FeatureStoreV1CandidateFeature[PipelineQuery, Candidate, _ <: EntityId, FeatureValue] - )( - implicit typeTag: TypeTag[FeatureValue] - ): SorterProvider = { - val defaultFeatureValue: FeatureValue = featureValueSortDefaultValue(feature, Ascending) - - ascending(feature, defaultFeatureValue) - } - - /** - * Sort by a Feature Store v1 feature value ascending. If the feature failed or is missing, use - * the provided default. - * - * @param feature Feature Store v1 feature with value to sort by - * @tparam Candidate candidate for the feature - * @tparam FeatureValue feature value with an [[Ordering]] context bound - */ - def ascending[Candidate <: UniversalNoun[Any], FeatureValue: Ordering]( - feature: FeatureStoreV1CandidateFeature[PipelineQuery, Candidate, _ <: EntityId, FeatureValue], - defaultFeatureValue: FeatureValue - ): SorterProvider = { - val ordering = Ordering.by[CandidateWithDetails, FeatureValue]( - _.features.getOrElseFeatureStoreV1CandidateFeature(feature, defaultFeatureValue)) - - SorterFromOrdering(ordering, Ascending) - } - - /** - * Sort by a Feature Store v1 feature value descending. If the feature failed or is missing, use - * an inferred default based on the type of [[FeatureValue]]. For Numeric values this is the - * MaxValue (e.g. Long.MaxValue, Double.MaxValue). - * - * @param feature Feature Store v1 feature with value to sort by - * @param typeTag allows for inferring default value from the FeatureValue type. - * See [[com.twitter.product_mixer.component_library.selector.sorter.FeatureValueSorter.featureValueSortDefaultValue]] - * @tparam Candidate candidate for the feature - * @tparam FeatureValue feature value with an [[Ordering]] context bound - */ - def descending[Candidate <: UniversalNoun[Any], FeatureValue: Ordering]( - feature: FeatureStoreV1CandidateFeature[PipelineQuery, Candidate, _ <: EntityId, FeatureValue] - )( - implicit typeTag: TypeTag[FeatureValue] - ): SorterProvider = { - val defaultFeatureValue: FeatureValue = featureValueSortDefaultValue(feature, Descending) - - descending(feature, defaultFeatureValue) - } - - /** - * Sort by a Feature Store v1 feature value descending. If the feature failed or is missing, use - * the provided default. - * - * @param feature Feature Store v1 feature with value to sort by - * @tparam Candidate candidate for the feature - * @tparam FeatureValue feature value with an [[Ordering]] context bound - */ - def descending[Candidate <: UniversalNoun[Any], FeatureValue: Ordering]( - feature: FeatureStoreV1CandidateFeature[PipelineQuery, Candidate, _ <: EntityId, FeatureValue], - defaultFeatureValue: FeatureValue - ): SorterProvider = { - val ordering = Ordering.by[CandidateWithDetails, FeatureValue]( - _.features.getOrElseFeatureStoreV1CandidateFeature(feature, defaultFeatureValue)) - - SorterFromOrdering(ordering, Descending) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/BUILD b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/BUILD deleted file mode 100644 index 9d3a2bc80..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/BUILD +++ /dev/null @@ -1,30 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/twitter/src/java/com/twitter/logpipeline/client:logpipeline-event-publisher-thin", - "abdecider/src/main/scala:abdeciderutils", - "decider/src/main/scala", - "finatra-internal/messaging/kafka/src/main/scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/side_effect", - "scribelib/marshallers/src/main/scala/com/twitter/scribelib/marshallers", - "scribelib/validators/src/main/scala/com/twitter/scribelib/validators", - "scrooge/scrooge-serializer/src/main/scala", - "src/thrift/com/twitter/clientapp/gen:clientapp-scala", - "stitch/stitch-core", - "strato/src/main/scala/com/twitter/strato/client", - "user_session_store/src/main/scala/com/twitter/user_session_store", - "util/util-core:util-core-util", - ], - exports = [ - "3rdparty/jvm/com/twitter/src/java/com/twitter/logpipeline/client:logpipeline-event-publisher-thin", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/side_effect", - "scrooge/scrooge-serializer/src/main/scala", - "src/thrift/com/twitter/clientapp/gen:clientapp-scala", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/BUILD.docx new file mode 100644 index 000000000..afa8b1c93 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/KafkaPublishingSideEffect.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/KafkaPublishingSideEffect.docx new file mode 100644 index 000000000..83ee67460 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/KafkaPublishingSideEffect.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/KafkaPublishingSideEffect.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/KafkaPublishingSideEffect.scala deleted file mode 100644 index 5bda25844..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/KafkaPublishingSideEffect.scala +++ /dev/null @@ -1,233 +0,0 @@ -package com.twitter.product_mixer.component_library.side_effect - -import com.twitter.conversions.DurationOps._ -import com.twitter.conversions.StorageUnitOps._ -import com.twitter.finatra.kafka.producers.FinagleKafkaProducerBuilder -import com.twitter.finatra.kafka.producers.KafkaProducerBase -import com.twitter.finatra.kafka.producers.TwitterKafkaProducerConfig -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.util.Duration -import com.twitter.util.StorageUnit -import org.apache.kafka.clients.producer.ProducerRecord -import org.apache.kafka.common.serialization.Serializer -import org.apache.kafka.common.record.CompressionType - -/** - * The Kafka publishing side effect. - * This class creates a Kafka producer with provided and default parameters. - * Note that callers may not provide arbitrary params as this class will do validity check on some - * params, e.g. maxBlock, to make sure it is safe for online services. - * - * PLEASE NOTE: caller needs to add the following to the Aurora file to successfully enable the TLS - * '-com.twitter.finatra.kafka.producers.principal={{role}}', - * - * @tparam K type of the key - * @tparam V type of the value - * @tparam Query pipeline query - */ -trait KafkaPublishingSideEffect[K, V, Query <: PipelineQuery, ResponseType <: HasMarshalling] - extends PipelineResultSideEffect[Query, ResponseType] { - - /** - * Kafka servers list. It is usually a WilyNs name at Twitter - */ - val bootstrapServer: String - - /** - * The serde of the key - */ - val keySerde: Serializer[K] - - /** - * The serde of the value - */ - val valueSerde: Serializer[V] - - /** - * An id string to pass to the server when making requests. - * The purpose of this is to be able to track the source of requests beyond just ip/port by - * allowing a logical application name to be included in server-side request logging. - */ - val clientId: String - - /** - * The configuration controls how long KafkaProducer.send() and - * KafkaProducer.partitionsFor() will block. - * These methods can be blocked either because the buffer is full or metadata unavailable. - * Blocking in the user-supplied serializers or partitioner will not be counted against this timeout. - * - * Set 200ms by default to not blocking the thread too long which is critical to most ProMixer - * powered services. Please note that there is a hard limit check of not greater than 1 second. - * - */ - val maxBlock: Duration = 200.milliseconds - - /** - * Retries due to broker failures, etc., may write duplicates of the retried message in the - * stream. Note that enabling idempotence requires - * MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION to be less than or equal to 5, - * RETRIES_CONFIG to be greater than 0 and ACKS_CONFIG - * must be 'all'. If these values are not explicitly set by the user, suitable values will be - * chosen. If incompatible values are set, a ConfigException will be thrown. - * - * false by default, setting to true may introduce issues to brokers since brokers will keep - * tracking all requests which is resource expensive. - */ - val idempotence: Boolean = false - - /** - * The producer will attempt to batch records together into fewer requests whenever multiple - * records are being sent to the same partition. This helps performance on both the client and - * the server. This configuration controls the default batch size in bytes. - * No attempt will be made to batch records larger than this size. - * Requests sent to brokers will contain multiple batches, one for each partition with data - * available to be sent. A small batch size will make batching less common and may reduce - * throughput (a batch size of zero will disable batching entirely). - * A very large batch size may use memory a bit more wastefully as we will always allocate a - * buffer of the specified batch size in anticipation of additional records. - * - * Default 16KB which comes from Kafka's default - */ - val batchSize: StorageUnit = 16.kilobytes - - /** - * The producer groups together any records that arrive in between request transmissions into - * a single batched request. "Normally this occurs only under load when records arrive faster - * than they can be sent out. However in some circumstances the client may want to reduce the - * number of requests even under moderate load. This setting accomplishes this by adding a - * small amount of artificial delay—that is, rather than immediately sending out a record - * the producer will wait for up to the given delay to allow other records to be sent so that - * the sends can be batched together. This can be thought of as analogous to Nagle's algorithm - * in TCP. This setting gives the upper bound on the delay for batching: once we get - * BATCH_SIZE_CONFIG worth of records for a partition it will be sent immediately regardless - * of this setting, however if we have fewer than this many bytes accumulated for this - * partition we will 'linger' for the specified time waiting for more records to show up. - * This setting defaults to 0 (i.e. no delay). Setting LINGER_MS_CONFIG=5, for example, - * would have the effect of reducing the number of requests sent but would add up to 5ms of - * latency to records sent in the absence of load. - * - * Default 0ms, which is Kafka's default. If the record size is much larger than the batchSize, - * you may consider to enlarge both batchSize and linger to have better compression (only when - * compression is enabled.) - */ - val linger: Duration = 0.milliseconds - - /** - * The total bytes of memory the producer can use to buffer records waiting to be sent to the - * server. If records are sent faster than they can be delivered to the server the producer - * will block for MAX_BLOCK_MS_CONFIG after which it will throw an exception. - * This setting should correspond roughly to the total memory the producer will use, but is not - * a hard bound since not all memory the producer uses is used for buffering. - * Some additional memory will be used for compression (if compression is enabled) as well as - * for maintaining in-flight requests. - * - * Default 32MB which is Kafka's default. Please consider to enlarge this value if the EPS and - * the per-record size is large (millions EPS with >1KB per-record size) in case the broker has - * issues (which fills the buffer pretty quickly.) - */ - val bufferMemorySize: StorageUnit = 32.megabytes - - /** - * Producer compression type - * - * Default LZ4 which is a good tradeoff between compression and efficiency. - * Please be careful of choosing ZSTD, which the compression rate is better it might introduce - * huge burden to brokers once the topic is consumed, which needs decompression at the broker side. - */ - val compressionType: CompressionType = CompressionType.LZ4 - - /** - * Setting a value greater than zero will cause the client to resend any request that fails - * with a potentially transient error - * - * Default set to 3, to intentionally reduce the retries. - */ - val retries: Int = 3 - - /** - * The amount of time to wait before attempting to retry a failed request to a given topic - * partition. This avoids repeatedly sending requests in a tight loop under some failure - * scenarios - */ - val retryBackoff: Duration = 1.second - - /** - * The configuration controls the maximum amount of time the client will wait - * for the response of a request. If the response is not received before the timeout - * elapses the client will resend the request if necessary or fail the request if - * retries are exhausted. - * - * Default 5 seconds which is intentionally low but not too low. - * Since Kafka's publishing is async this is in general safe (as long as the bufferMem is not full.) - */ - val requestTimeout: Duration = 5.seconds - - require( - maxBlock.inMilliseconds <= 1000, - "We intentionally set the maxBlock to be smaller than 1 second to not block the thread for too long!") - - lazy val kafkaProducer: KafkaProducerBase[K, V] = { - val jaasConfig = TwitterKafkaProducerConfig().configMap - val builder = FinagleKafkaProducerBuilder[K, V]() - .keySerializer(keySerde) - .valueSerializer(valueSerde) - .dest(bootstrapServer, 1.second) // NOTE: this method blocks! - .clientId(clientId) - .maxBlock(maxBlock) - .batchSize(batchSize) - .linger(linger) - .bufferMemorySize(bufferMemorySize) - .maxRequestSize(4.megabytes) - .compressionType(compressionType) - .enableIdempotence(idempotence) - .maxInFlightRequestsPerConnection(5) - .retries(retries) - .retryBackoff(retryBackoff) - .requestTimeout(requestTimeout) - .withConfig("acks", "all") - .withConfig("delivery.timeout.ms", requestTimeout + linger) - - builder.withConfig(jaasConfig).build() - } - - /** - * Build the record to be published to Kafka from query, selections and response - * @param query PipelineQuery - * @param selectedCandidates Result after Selectors are executed - * @param remainingCandidates Candidates which were not selected - * @param droppedCandidates Candidates dropped during selection - * @param response Result after Unmarshalling - * @return A sequence of to-be-published ProducerRecords - */ - def buildRecords( - query: Query, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: ResponseType - ): Seq[ProducerRecord[K, V]] - - final override def apply( - inputs: PipelineResultSideEffect.Inputs[Query, ResponseType] - ): Stitch[Unit] = { - val records = buildRecords( - query = inputs.query, - selectedCandidates = inputs.selectedCandidates, - remainingCandidates = inputs.remainingCandidates, - droppedCandidates = inputs.droppedCandidates, - response = inputs.response - ) - - Stitch - .collect( - records - .map { record => - Stitch.callFuture(kafkaProducer.send(record)) - } - ).unit - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/ParamGatedPipelineResultSideEffect.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/ParamGatedPipelineResultSideEffect.docx new file mode 100644 index 000000000..a4001f19e Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/ParamGatedPipelineResultSideEffect.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/ParamGatedPipelineResultSideEffect.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/ParamGatedPipelineResultSideEffect.scala deleted file mode 100644 index a84fa1ee3..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/ParamGatedPipelineResultSideEffect.scala +++ /dev/null @@ -1,76 +0,0 @@ -package com.twitter.product_mixer.component_library.side_effect - -import com.twitter.product_mixer.component_library.side_effect.ParamGatedPipelineResultSideEffect.IdentifierPrefix -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.side_effect.ExecuteSynchronously -import com.twitter.product_mixer.core.functional_component.side_effect.FailOpen -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.Conditionally -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.Param - -/** - * A [[PipelineResultSideEffect]] with [[Conditionally]] based on a [[Param]] - * - * @param enabledParam the param to turn this filter on and off - * @param sideEffect the underlying side effect to run when `enabledParam` is true - * @tparam Query The domain model for the query or request - */ -sealed case class ParamGatedPipelineResultSideEffect[ - -Query <: PipelineQuery, - ResultType <: HasMarshalling -] private ( - enabledParam: Param[Boolean], - sideEffect: PipelineResultSideEffect[Query, ResultType]) - extends PipelineResultSideEffect[Query, ResultType] - with PipelineResultSideEffect.Conditionally[Query, ResultType] { - override val identifier: SideEffectIdentifier = SideEffectIdentifier( - IdentifierPrefix + sideEffect.identifier.name) - override val alerts: Seq[Alert] = sideEffect.alerts - override def onlyIf( - query: Query, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: ResultType - ): Boolean = - Conditionally.and( - PipelineResultSideEffect - .Inputs(query, selectedCandidates, remainingCandidates, droppedCandidates, response), - sideEffect, - query.params(enabledParam)) - override def apply(inputs: PipelineResultSideEffect.Inputs[Query, ResultType]): Stitch[Unit] = - sideEffect.apply(inputs) -} - -object ParamGatedPipelineResultSideEffect { - - val IdentifierPrefix = "ParamGated" - - /** - * A [[PipelineResultSideEffect]] with [[Conditionally]] based on a [[Param]] - * - * @param enabledParam the param to turn this filter on and off - * @param sideEffect the underlying side effect to run when `enabledParam` is true - * @tparam Query The domain model for the query or request - */ - def apply[Query <: PipelineQuery, ResultType <: HasMarshalling]( - enabledParam: Param[Boolean], - sideEffect: PipelineResultSideEffect[Query, ResultType] - ): ParamGatedPipelineResultSideEffect[Query, ResultType] = { - sideEffect match { - case _: FailOpen => - new ParamGatedPipelineResultSideEffect(enabledParam, sideEffect) - with ExecuteSynchronously - with FailOpen - case _: ExecuteSynchronously => - new ParamGatedPipelineResultSideEffect(enabledParam, sideEffect) with ExecuteSynchronously - case _ => - new ParamGatedPipelineResultSideEffect(enabledParam, sideEffect) - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/ScribeClientEventSideEffect.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/ScribeClientEventSideEffect.docx new file mode 100644 index 000000000..6ec8dcd78 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/ScribeClientEventSideEffect.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/ScribeClientEventSideEffect.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/ScribeClientEventSideEffect.scala deleted file mode 100644 index bee6ec92e..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/ScribeClientEventSideEffect.scala +++ /dev/null @@ -1,118 +0,0 @@ -package com.twitter.product_mixer.component_library.side_effect - -import com.twitter.abdecider.ScribingABDeciderUtil -import com.twitter.clientapp.thriftscala.LogEvent -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.scribelib.marshallers -import com.twitter.scribelib.marshallers.ClientDataProvider -import com.twitter.scribelib.marshallers.LogEventMarshaller - -/** - * Side effect to log client events server-side. Create an implementation of this trait by - * defining the `buildClientEvents` method, and the `page` val. - * The ClientEvent will be automatically converted into a [[LogEvent]] and scribed. - */ -trait ScribeClientEventSideEffect[ - Query <: PipelineQuery, - UnmarshalledResponseType <: HasMarshalling] - extends ScribeLogEventSideEffect[LogEvent, Query, UnmarshalledResponseType] { - - /** - * The page which will be defined in the namespace. This is typically the service name that's scribing - */ - val page: String - - /** - * Build the client events from query, selections and response - * - * @param query PipelineQuery - * @param selectedCandidates Result after Selectors are executed - * @param remainingCandidates Candidates which were not selected - * @param droppedCandidates Candidates dropped during selection - * @param response Result after Unmarshalling - */ - def buildClientEvents( - query: Query, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: UnmarshalledResponseType - ): Seq[ScribeClientEventSideEffect.ClientEvent] - - final override def buildLogEvents( - query: Query, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: UnmarshalledResponseType - ): Seq[LogEvent] = { - buildClientEvents( - query = query, - selectedCandidates = selectedCandidates, - remainingCandidates = remainingCandidates, - droppedCandidates = droppedCandidates, - response = response).flatMap { event => - val clientData = clientContextToClientDataProvider(query) - - val clientName = ScribingABDeciderUtil.clientForAppId(clientData.clientApplicationId) - - val namespaceMap: Map[String, String] = Map( - "client" -> Some(clientName), - "page" -> Some(page), - "section" -> event.namespace.section, - "component" -> event.namespace.component, - "element" -> event.namespace.element, - "action" -> event.namespace.action - ).collect { case (k, Some(v)) => k -> v } - - val data: Map[Any, Any] = Seq( - event.eventValue.map("event_value" -> _), - event.latencyMs.map("latency_ms" -> _) - ).flatten.toMap - - val clientEventData = data + - ("event_namespace" -> namespaceMap) + - (marshallers.CategoryKey -> "client_event") - - LogEventMarshaller.marshal( - data = clientEventData, - clientData = clientData - ) - } - } - - /** - * Makes a [[ClientDataProvider]] from the [[PipelineQuery.clientContext]] from the [[query]] - */ - private def clientContextToClientDataProvider(query: Query): ClientDataProvider = { - new ClientDataProvider { - override val userId = query.clientContext.userId - override val guestId = query.clientContext.guestId - override val personalizationId = None - override val deviceId = query.clientContext.deviceId - override val clientApplicationId = query.clientContext.appId - override val parentApplicationId = None - override val countryCode = query.clientContext.countryCode - override val languageCode = query.clientContext.languageCode - override val userAgent = query.clientContext.userAgent - override val isSsl = None - override val referer = None - override val externalReferer = None - } - } -} - -object ScribeClientEventSideEffect { - case class EventNamespace( - section: Option[String] = None, - component: Option[String] = None, - element: Option[String] = None, - action: Option[String] = None) - - case class ClientEvent( - namespace: EventNamespace, - eventValue: Option[Long] = None, - latencyMs: Option[Long] = None) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/ScribeLogEventAsyncSideEffect.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/ScribeLogEventAsyncSideEffect.docx new file mode 100644 index 000000000..4d26ed73f Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/ScribeLogEventAsyncSideEffect.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/ScribeLogEventAsyncSideEffect.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/ScribeLogEventAsyncSideEffect.scala deleted file mode 100644 index f27e9190e..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/ScribeLogEventAsyncSideEffect.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.product_mixer.component_library.side_effect - -import com.twitter.logpipeline.client.common.EventPublisher -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.scrooge.ThriftStruct -import com.twitter.stitch.Stitch - -/** - * A [[PipelineResultSideEffect]] that logs [[Thrift]] data may not already be available to Scribe - */ -trait ScribeLogEventAsyncSideEffect[ - Thrift <: ThriftStruct, - Query <: PipelineQuery, - ResponseType <: HasMarshalling] - extends PipelineResultSideEffect[Query, ResponseType] { - - /** - * Build the log events from query, selections and response - * @param query PipelineQuery - * @param selectedCandidates Result after Selectors are executed - * @param remainingCandidates Candidates which were not selected - * @param droppedCandidates Candidates dropped during selection - * @param response Result after Unmarshalling - * @return LogEvent in thrift - */ - def buildLogEvents( - query: Query, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: ResponseType - ): Stitch[Seq[Thrift]] - - val logPipelinePublisher: EventPublisher[Thrift] - - final override def apply( - inputs: PipelineResultSideEffect.Inputs[Query, ResponseType] - ): Stitch[Unit] = { - val logEvents = buildLogEvents( - query = inputs.query, - selectedCandidates = inputs.selectedCandidates, - remainingCandidates = inputs.remainingCandidates, - droppedCandidates = inputs.droppedCandidates, - response = inputs.response - ) - - logEvents.flatMap { logEvents: Seq[Thrift] => - Stitch.collect { - logEvents.map { logEvent => - Stitch.callFuture(logPipelinePublisher.publish(logEvent)) - } - }.unit - } - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/ScribeLogEventSideEffect.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/ScribeLogEventSideEffect.docx new file mode 100644 index 000000000..91f309cc6 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/ScribeLogEventSideEffect.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/ScribeLogEventSideEffect.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/ScribeLogEventSideEffect.scala deleted file mode 100644 index df9551e15..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/ScribeLogEventSideEffect.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.product_mixer.component_library.side_effect - -import com.twitter.logpipeline.client.common.EventPublisher -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.scrooge.ThriftStruct -import com.twitter.stitch.Stitch - -/** - * A [[PipelineResultSideEffect]] that logs [[Thrift]] data that's already available to Scribe - */ -trait ScribeLogEventSideEffect[ - Thrift <: ThriftStruct, - Query <: PipelineQuery, - ResponseType <: HasMarshalling] - extends PipelineResultSideEffect[Query, ResponseType] { - - /** - * Build the log events from query, selections and response - * @param query PipelineQuery - * @param selectedCandidates Result after Selectors are executed - * @param remainingCandidates Candidates which were not selected - * @param droppedCandidates Candidates dropped during selection - * @param response Result after Unmarshalling - * @return LogEvent in thrift - */ - def buildLogEvents( - query: Query, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: ResponseType - ): Seq[Thrift] - - val logPipelinePublisher: EventPublisher[Thrift] - - final override def apply( - inputs: PipelineResultSideEffect.Inputs[Query, ResponseType] - ): Stitch[Unit] = { - val logEvents = buildLogEvents( - query = inputs.query, - selectedCandidates = inputs.selectedCandidates, - remainingCandidates = inputs.remainingCandidates, - droppedCandidates = inputs.droppedCandidates, - response = inputs.response - ) - - Stitch - .collect( - logEvents - .map { logEvent => - Stitch.callFuture(logPipelinePublisher.publish(logEvent)) - } - ).unit - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/StratoInsertSideEffect.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/StratoInsertSideEffect.docx new file mode 100644 index 000000000..8b441bde0 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/StratoInsertSideEffect.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/StratoInsertSideEffect.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/StratoInsertSideEffect.scala deleted file mode 100644 index 77d9b815c..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/StratoInsertSideEffect.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.twitter.product_mixer.component_library.side_effect - -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Inserter - -/** - * Side effect that writes to Strato column's Insert Op. Create an implementation of this trait by - * defining the `buildEvents` method and providing a Strato Column inserter of type - * (StratoKeyarg, StratoValue) -> Any. - * See https://docbird.twitter.biz/strato/ColumnCatalog.html#insert for information about - * the Insert operation in Strato. - * - * @tparam StratoKeyarg Argument used as a key for Strato column. Could be Unit for common use-cases. - * @tparam StratoValue Value that is inserted at the Strato column. - * @tparam Query PipelineQuery - * @tparam DomainResponseType Timeline response that is marshalled to domain model (e.g. URT, Slice etc). - */ -trait StratoInsertSideEffect[ - StratoKeyarg, - StratoValue, - Query <: PipelineQuery, - DomainResponseType <: HasMarshalling] - extends PipelineResultSideEffect[Query, DomainResponseType] { - - /** - * Inserter for the InsertOp on a StratoColumn. In Strato, the InsertOp is represented as - * (Keyarg, Value) => Key, where Key represents the result returned by the Insert operation. - * For the side-effect behavior, we do not need the return value and use Any instead. - */ - val stratoInserter: Inserter[StratoKeyarg, StratoValue, Any] - - /** - * Builds the events that are inserted to the Strato column. This method supports generating - * multiple events for a single side-effect invocation. - * - * @param query PipelineQuery - * @param selectedCandidates Result after Selectors are executed - * @param remainingCandidates Candidates which were not selected - * @param droppedCandidates Candidates dropped during selection - * @param response Timeline response that is marshalled to domain model (e.g. URT, Slice etc). - * @return Tuples of (StratoKeyArg, StratoValue) that are used to call the stratoInserter. - */ - def buildEvents( - query: Query, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: DomainResponseType - ): Seq[(StratoKeyarg, StratoValue)] - - final override def apply( - inputs: PipelineResultSideEffect.Inputs[Query, DomainResponseType] - ): Stitch[Unit] = { - val events = buildEvents( - query = inputs.query, - selectedCandidates = inputs.selectedCandidates, - remainingCandidates = inputs.remainingCandidates, - droppedCandidates = inputs.droppedCandidates, - response = inputs.response - ) - - Stitch - .traverse(events) { case (keyarg, value) => stratoInserter.insert(keyarg, value) } - .unit - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/UserSessionStoreUpdateSideEffect.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/UserSessionStoreUpdateSideEffect.docx new file mode 100644 index 000000000..0172caa7f Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/UserSessionStoreUpdateSideEffect.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/UserSessionStoreUpdateSideEffect.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/UserSessionStoreUpdateSideEffect.scala deleted file mode 100644 index 6199794d6..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/UserSessionStoreUpdateSideEffect.scala +++ /dev/null @@ -1,35 +0,0 @@ -package com.twitter.product_mixer.component_library.side_effect - -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.user_session_store.ReadWriteUserSessionStore -import com.twitter.user_session_store.WriteRequest - -/** - * A [[PipelineResultSideEffect]] that writes to a [[ReadWriteUserSessionStore]] - */ -trait UserSessionStoreUpdateSideEffect[ - Request <: WriteRequest, - Query <: PipelineQuery, - ResponseType <: HasMarshalling] - extends PipelineResultSideEffect[Query, ResponseType] { - - /** - * Build the write request from the query - * @param query PipelineQuery - * @return WriteRequest - */ - def buildWriteRequest(query: Query): Option[Request] - - val userSessionStore: ReadWriteUserSessionStore - - final override def apply( - inputs: PipelineResultSideEffect.Inputs[Query, ResponseType] - ): Stitch[Unit] = { - buildWriteRequest(inputs.query) - .map(userSessionStore.write) - .getOrElse(Stitch.Unit) - } -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/metrics/BUILD.bazel b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/metrics/BUILD.bazel deleted file mode 100644 index 7f8229524..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/metrics/BUILD.bazel +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt", - "util/util-core:util-core-util", - "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", - ], -) diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/metrics/BUILD.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/metrics/BUILD.docx new file mode 100644 index 000000000..2b1883038 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/metrics/BUILD.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/metrics/CandidateMetricFunction.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/metrics/CandidateMetricFunction.docx new file mode 100644 index 000000000..6db54b645 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/metrics/CandidateMetricFunction.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/metrics/CandidateMetricFunction.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/metrics/CandidateMetricFunction.scala deleted file mode 100644 index 1dfcd8865..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/metrics/CandidateMetricFunction.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.twitter.product_mixer.component_library.side_effect.metrics - -import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate -import com.twitter.product_mixer.component_library.model.candidate.BaseUserCandidate -import com.twitter.product_mixer.component_library.side_effect.metrics.CandidateMetricFunction.getCountForType -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails - -/** - * Function to extract numerical metric value from [[CandidateWithDetails]]. - * This CandidateMetricFunction will be applied on all [[CandidateWithDetails]] instances in the - * candidateSelection from the RecommendationPipeline. - */ -trait CandidateMetricFunction { - def apply(candidateWithDetails: CandidateWithDetails): Long -} - -object CandidateMetricFunction { - - private val defaultCountOnePf: PartialFunction[CandidateWithDetails, Long] = { - case _ => 0L - } - - /** - * Count the occurrences of a certain candidate type from [[CandidateWithDetails]]. - */ - def getCountForType( - candidateWithDetails: CandidateWithDetails, - countOnePf: PartialFunction[CandidateWithDetails, Long] - ): Long = { - (countOnePf orElse defaultCountOnePf)(candidateWithDetails) - } -} - -object DefaultServedTweetsSumFunction extends CandidateMetricFunction { - override def apply(candidateWithDetails: CandidateWithDetails): Long = - getCountForType( - candidateWithDetails, - { - case item: ItemCandidateWithDetails => - item.candidate match { - case _: BaseTweetCandidate => 1L - case _ => 0L - } - }) -} - -object DefaultServedUsersSumFunction extends CandidateMetricFunction { - override def apply(candidateWithDetails: CandidateWithDetails): Long = - getCountForType( - candidateWithDetails, - { - case item: ItemCandidateWithDetails => - item.candidate match { - case _: BaseUserCandidate => 1L - case _ => 0L - } - }) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/metrics/ScribeClientEventMetricsSideEffect.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/metrics/ScribeClientEventMetricsSideEffect.docx new file mode 100644 index 000000000..fc1d65395 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/metrics/ScribeClientEventMetricsSideEffect.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/metrics/ScribeClientEventMetricsSideEffect.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/metrics/ScribeClientEventMetricsSideEffect.scala deleted file mode 100644 index 1c3d2e817..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/metrics/ScribeClientEventMetricsSideEffect.scala +++ /dev/null @@ -1,74 +0,0 @@ -package com.twitter.product_mixer.component_library.side_effect.metrics - -import com.twitter.clientapp.thriftscala.LogEvent -import com.twitter.logpipeline.client.common.EventPublisher -import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect -import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.EventNamespace -import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.ClientEvent -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * Config of a client event to be scribed under certain namespace. - * @param eventNamespaceOverride overrides the default eventNamespace in the side effect. - * Note that its fields (section/component/element/action) will - * override the default namespace fields only if the fields are not - * None. i.e. if you want to override the "section" field in the - * default namespace with an empty section, you must specify - * section = Some("") - * in the override instead of - * section = None - * - * @param metricFunction the function that extracts the metric value from a candidate. - */ -case class EventConfig( - eventNamespaceOverride: EventNamespace, - metricFunction: CandidateMetricFunction) - -/** - * Side effect to log client events server-side and to build metrics in the metric center. - * By default will return "requests" event config. - */ -class ScribeClientEventMetricsSideEffect[ - Query <: PipelineQuery, - UnmarshalledResponseType <: HasMarshalling -]( - override val identifier: SideEffectIdentifier, - override val logPipelinePublisher: EventPublisher[LogEvent], - override val page: String, - defaultEventNamespace: EventNamespace, - eventConfigs: Seq[EventConfig]) - extends ScribeClientEventSideEffect[Query, UnmarshalledResponseType] { - - override def buildClientEvents( - query: Query, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: UnmarshalledResponseType - ): Seq[ScribeClientEventSideEffect.ClientEvent] = { - // count the number of client events of type "requests" - val requestClientEvent = ClientEvent( - namespace = buildEventNamespace(EventNamespace(action = Some("requests"))) - ) - - eventConfigs - .map { config => - ClientEvent( - namespace = buildEventNamespace(config.eventNamespaceOverride), - eventValue = Some(selectedCandidates.map(config.metricFunction(_)).sum)) - } - // scribe client event only when the metric sum is non-zero - .filter(clientEvent => clientEvent.eventValue.exists(_ > 0L)) :+ requestClientEvent - } - - private def buildEventNamespace(eventNamespaceOverride: EventNamespace): EventNamespace = - EventNamespace( - section = eventNamespaceOverride.section.orElse(defaultEventNamespace.section), - component = eventNamespaceOverride.component.orElse(defaultEventNamespace.component), - element = eventNamespaceOverride.element.orElse(defaultEventNamespace.element), - action = eventNamespaceOverride.action.orElse(defaultEventNamespace.action) - ) -} diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/metrics/ScribeClientEventMetricsSideEffectBuilder.docx b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/metrics/ScribeClientEventMetricsSideEffectBuilder.docx new file mode 100644 index 000000000..7c8c88838 Binary files /dev/null and b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/metrics/ScribeClientEventMetricsSideEffectBuilder.docx differ diff --git a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/metrics/ScribeClientEventMetricsSideEffectBuilder.scala b/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/metrics/ScribeClientEventMetricsSideEffectBuilder.scala deleted file mode 100644 index ae3c69353..000000000 --- a/product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect/metrics/ScribeClientEventMetricsSideEffectBuilder.scala +++ /dev/null @@ -1,84 +0,0 @@ -package com.twitter.product_mixer.component_library.side_effect.metrics - -import com.twitter.clientapp.thriftscala.LogEvent -import com.twitter.logpipeline.client.common.EventPublisher -import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.EventNamespace -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * Build [[ScribeClientEventMetricsSideEffect]] with extra [[EventConfig]] - */ -case class ScribeClientEventMetricsSideEffectBuilder( - eventConfigs: Seq[EventConfig] = Seq.empty) { - - /** - * Append extra [[EventConfig]] to [[ScribeClientEventMetricsSideEffectBuilder]] - */ - def withEventConfig( - eventConfig: EventConfig - ): ScribeClientEventMetricsSideEffectBuilder = - this.copy(eventConfigs = this.eventConfigs :+ eventConfig) - - /** - * Build [[EventConfig]] with customized [[EventNamespace]] and customized [[CandidateMetricFunction]] - * @param eventNamespaceOverride Override the default event namespace in [[ScribeClientEventMetricsSideEffect]] - * @param metricFunction [[CandidateMetricFunction]] - */ - def withEventConfig( - eventNamespaceOverride: EventNamespace, - metricFunction: CandidateMetricFunction - ): ScribeClientEventMetricsSideEffectBuilder = - withEventConfig(EventConfig(eventNamespaceOverride, metricFunction)) - - /** - * Log served tweets events server side and build metrics in the metric center. - * Default event name space action is "served_tweets", default metric function is [[DefaultServedTweetsSumFunction]] - * @param eventNamespaceOverride Override the default event namespace in [[ScribeClientEventMetricsSideEffect]] - * @param metricFunction [[CandidateMetricFunction]] - */ - def withServedTweets( - eventNamespaceOverride: EventNamespace = EventNamespace(action = Some("served_tweets")), - metricFunction: CandidateMetricFunction = DefaultServedTweetsSumFunction - ): ScribeClientEventMetricsSideEffectBuilder = withEventConfig( - eventNamespaceOverride = eventNamespaceOverride, - metricFunction = metricFunction) - - /** - * Log served users events server side and build metrics in the metric center. - * Default event name space action is "served_users", default metric function is [[DefaultServedUsersSumFunction]] - * @param eventNamespaceOverride Override the default event namespace in [[ScribeClientEventMetricsSideEffect]] - * @param metricFunction [[CandidateMetricFunction]] - */ - def withServedUsers( - eventNamespaceOverride: EventNamespace = EventNamespace(action = Some("served_users")), - metricFunction: CandidateMetricFunction = DefaultServedUsersSumFunction - ): ScribeClientEventMetricsSideEffectBuilder = withEventConfig( - eventNamespaceOverride = eventNamespaceOverride, - metricFunction = metricFunction) - - /** - * Build [[ScribeClientEventMetricsSideEffect]] - * @param identifier unique identifier of the side effect - * @param defaultEventNamespace default event namespace to log - * @param logPipelinePublisher [[EventPublisher]] to publish events - * @param page The page which will be defined in the namespace. This is typically the service name that's scribing - * @tparam Query [[PipelineQuery]] - * @tparam UnmarshalledResponseType [[HasMarshalling]] - * @return [[ScribeClientEventMetricsSideEffect]] - */ - def build[Query <: PipelineQuery, UnmarshalledResponseType <: HasMarshalling]( - identifier: SideEffectIdentifier, - defaultEventNamespace: EventNamespace, - logPipelinePublisher: EventPublisher[LogEvent], - page: String - ): ScribeClientEventMetricsSideEffect[Query, UnmarshalledResponseType] = { - new ScribeClientEventMetricsSideEffect[Query, UnmarshalledResponseType]( - identifier = identifier, - logPipelinePublisher = logPipelinePublisher, - defaultEventNamespace = defaultEventNamespace, - page = page, - eventConfigs = eventConfigs) - } -} diff --git a/product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope/BUILD b/product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope/BUILD deleted file mode 100644 index d6cf0b82c..000000000 --- a/product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope/BUILD +++ /dev/null @@ -1,8 +0,0 @@ -java_library( - sources = ["*.java"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - ], -) diff --git a/product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope/BUILD.docx b/product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope/BUILD.docx new file mode 100644 index 000000000..1b0b53d7a Binary files /dev/null and b/product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope/BUILD.docx differ diff --git a/product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope/ProductScoped.docx b/product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope/ProductScoped.docx new file mode 100644 index 000000000..1bc783eae Binary files /dev/null and b/product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope/ProductScoped.docx differ diff --git a/product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope/ProductScoped.java b/product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope/ProductScoped.java deleted file mode 100644 index dffc5cd7f..000000000 --- a/product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope/ProductScoped.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.twitter.product_mixer.core.product.guice.scope; - -import static java.lang.annotation.ElementType.METHOD; -import static java.lang.annotation.ElementType.TYPE; -import java.lang.annotation.Retention; -import static java.lang.annotation.RetentionPolicy.RUNTIME; -import java.lang.annotation.Target; - -import com.google.inject.ScopeAnnotation; - -@Target({ TYPE, METHOD }) -@Retention(RUNTIME) -@ScopeAnnotation -public @interface ProductScoped {} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/AlertConfig.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/AlertConfig.docx new file mode 100644 index 000000000..83bf9cb43 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/AlertConfig.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/AlertConfig.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/AlertConfig.scala deleted file mode 100644 index 05240a66a..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/AlertConfig.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.product_mixer.core.controllers - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup -import com.twitter.product_mixer.core.functional_component.common.alert.Source - -/** - * Simple representation for an [[Alert]] used for Product Mixer's JSON API, which in turn is - * consumed by our monitoring script generation job and Turntable. - * - * @note not all mixers will upgrade at the same time so new fields should be added with backwards - * compatibility in mind. - */ -@JsonIgnoreProperties(ignoreUnknown = true) -private[core] case class AlertConfig( - source: Source, - metricType: String, - notificationGroup: NotificationGroup, - warnPredicate: PredicateConfig, - criticalPredicate: PredicateConfig, - runbookLink: Option[String], - metricSuffix: Option[String]) - -private[core] object AlertConfig { - - /** Represent this [[Alert]] as an [[AlertConfig]] case class */ - private[core] def apply(alert: Alert): AlertConfig = - AlertConfig( - alert.source, - alert.alertType.metricType, - alert.notificationGroup, - PredicateConfig(alert.warnPredicate), - PredicateConfig(alert.criticalPredicate), - alert.runbookLink, - alert.metricSuffix - ) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/BUILD deleted file mode 100644 index 879a1f49d..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/BUILD +++ /dev/null @@ -1,39 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/twitter/bijection:core", - "3rdparty/jvm/javax/inject:javax.inject", - "finagle/finagle-thriftmux/src/main/scala", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/http-server/src/main/scala/com/twitter/finatra/http", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/product_mixer_flags", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/component_registry", - "scrooge-internal/scrooge-schema/src/main/scala/com/twitter/scrooge/schema", - "scrooge-internal/scrooge-schema/src/main/scala/com/twitter/scrooge/schema/json", - "scrooge-internal/scrooge-schema/src/main/scala/com/twitter/scrooge/schema/scrooge/scala", - "scrooge-internal/scrooge-schema/src/main/scala/com/twitter/scrooge/schema/serialization/thrift", - "scrooge-internal/scrooge-schema/src/main/scala/com/twitter/scrooge/schema/tree", - "scrooge/scrooge-core/src/main/scala", - "scrooge/scrooge-generator/src/main/scala", - "scrooge/scrooge-serializer", - "src/thrift/com/twitter/context:twitter-context-scala", - "src/thrift/com/twitter/scrooge_internal/schema:thrift-scala", - "twitter-context/src/main/scala", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "src/thrift/com/twitter/context:twitter-context-scala", - "twitter-context/src/main/scala", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/BUILD.docx new file mode 100644 index 000000000..ad3abbb67 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/DebugTwitterContext.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/DebugTwitterContext.docx new file mode 100644 index 000000000..43335537d Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/DebugTwitterContext.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/DebugTwitterContext.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/DebugTwitterContext.scala deleted file mode 100644 index 29d1a904c..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/DebugTwitterContext.scala +++ /dev/null @@ -1,55 +0,0 @@ -package com.twitter.product_mixer.core.controllers - -import com.twitter.context.TwitterContext -import com.twitter.context.thriftscala.Viewer -import com.twitter.product_mixer.TwitterContextPermit -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext - -/** - * Mixes in support to forge the UserIds in TwitterContext for debug purposes. - * - * A thrift controller can extend DebugTwitterContext and wrap it's execution logic: - * - * {{{ - * withDebugTwitterContext(request.clientContext) { - * Stitch.run(...) - * } - * }}} - */ -trait DebugTwitterContext { - - private val ctx = TwitterContext(TwitterContextPermit) - - /** - * Wrap some function in a debug TwitterContext with hardcoded userIds - * to the ClientContext.userId. - * - * @param clientContext - A product mixer request client context - * @param f The function to wrap - */ - def withDebugTwitterContext[T](clientContext: ClientContext)(f: => T): T = { - ctx.let( - forgeTwitterContext( - clientContext.userId - .getOrElse(throw new IllegalArgumentException("missing required field: user id"))) - )(f) - } - - // Generate a fake Twitter Context for debug usage. - // Generally the TwitterContext is created by the API service, and Strato uses it for permission control. - // When we use our debug endpoint, we instead create our own context so that Strato finds something useful. - // We enforce ACLs directly via Thrift Web Forms' permission system. - private def forgeTwitterContext(userId: Long): Viewer = { - Viewer( - auditIp = None, - ipTags = Set.empty, - userId = Some(userId), - guestId = None, - clientApplicationId = None, - userAgent = None, - locationToken = None, - authenticatedUserId = Some(userId), - guestToken = None - ) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/GetComponentRegistryHandler.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/GetComponentRegistryHandler.docx new file mode 100644 index 000000000..9cb27c03f Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/GetComponentRegistryHandler.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/GetComponentRegistryHandler.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/GetComponentRegistryHandler.scala deleted file mode 100644 index 543888093..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/GetComponentRegistryHandler.scala +++ /dev/null @@ -1,114 +0,0 @@ -package com.twitter.product_mixer.core.controllers - -import com.twitter.finagle.http.Request -import com.twitter.inject.Injector -import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy -import com.twitter.product_mixer.core.functional_component.common.access_policy.WithDebugAccessPolicies -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.pipeline.Pipeline -import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig -import com.twitter.product_mixer.core.pipeline.product.ProductPipelineConfig -import com.twitter.product_mixer.core.pipeline.recommendation.RecommendationPipelineConfig -import com.twitter.product_mixer.core.quality_factor.QualityFactorConfig -import com.twitter.product_mixer.core.service.component_registry -import com.twitter.product_mixer.core.service.component_registry.ComponentRegistry -import com.twitter.product_mixer.core.service.component_registry.ComponentRegistrySnapshot -import com.twitter.util.Future - -case class GetComponentRegistryHandler(injector: Injector) { - lazy val componentRegistry: ComponentRegistry = injector.instance[ComponentRegistry] - - def apply(request: Request): Future[ComponentRegistryResponse] = { - componentRegistry.get.map { currentComponentRegistry: ComponentRegistrySnapshot => - val registeredComponents = currentComponentRegistry.getAllRegisteredComponents.map { - registeredComponent => - val componentIdentifier = registeredComponent.identifier - val childComponents = currentComponentRegistry - .getChildComponents(componentIdentifier) - .map { childComponent => - ChildComponent( - componentType = childComponent.componentType, - name = childComponent.name, - relativeScopes = componentIdentifier.toScopes ++ childComponent.toScopes, - qualityFactorMonitoringConfig = - buildQualityFactoringMonitoringConfig(registeredComponent, childComponent) - ) - } - - RegisteredComponent( - componentType = componentIdentifier.componentType, - name = componentIdentifier.name, - scopes = componentIdentifier.toScopes, - children = childComponents, - alertConfig = Some(registeredComponent.component.alerts.map(AlertConfig.apply)), - sourceFile = Some(registeredComponent.sourceFile), - debugAccessPolicies = Some(registeredComponent.component match { - case withDebugAccessPolicies: WithDebugAccessPolicies => - withDebugAccessPolicies.debugAccessPolicies - case _ => Set.empty - }) - ) - } - - ComponentRegistryResponse(registeredComponents) - } - } - - private def buildQualityFactoringMonitoringConfig( - parent: component_registry.RegisteredComponent, - child: ComponentIdentifier - ): Option[QualityFactorMonitoringConfig] = { - val qualityFactorConfigs: Option[Map[ComponentIdentifier, QualityFactorConfig]] = - parent.component match { - case pipeline: Pipeline[_, _] => - pipeline.config match { - case config: RecommendationPipelineConfig[_, _, _, _] => - Some(config.qualityFactorConfigs) - case config: MixerPipelineConfig[_, _, _] => - Some( - config.qualityFactorConfigs - .asInstanceOf[Map[ComponentIdentifier, QualityFactorConfig]]) - case config: ProductPipelineConfig[_, _, _] => - Some(config.qualityFactorConfigs) - case _ => None - } - case _ => None - } - - val qfConfigForChild: Option[QualityFactorConfig] = qualityFactorConfigs.flatMap(_.get(child)) - - qfConfigForChild.map { qfConfig => - QualityFactorMonitoringConfig( - boundMin = qfConfig.qualityFactorBounds.bounds.minInclusive, - boundMax = qfConfig.qualityFactorBounds.bounds.maxInclusive - ) - } - } -} - -case class RegisteredComponent( - componentType: String, - name: String, - scopes: Seq[String], - children: Seq[ChildComponent], - alertConfig: Option[Seq[AlertConfig]], - sourceFile: Option[String], - debugAccessPolicies: Option[Set[AccessPolicy]]) - -case class ChildComponent( - componentType: String, - name: String, - relativeScopes: Seq[String], - qualityFactorMonitoringConfig: Option[QualityFactorMonitoringConfig]) - -/** - * The shape of the data returned to callers after hitting the `component-registry` endpoint - * - * @note changes to [[ComponentRegistryResponse]] or contained types should be reflected - * in dashboard generation code in the `monitoring-configs/product_mixer` directory. - */ -case class ComponentRegistryResponse( - registeredComponents: Seq[RegisteredComponent]) - -case class ProductPipeline(identifier: String) -case class ProductPipelinesResponse(productPipelines: Seq[ProductPipeline]) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/GetDebugConfigurationHandler.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/GetDebugConfigurationHandler.docx new file mode 100644 index 000000000..1808db196 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/GetDebugConfigurationHandler.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/GetDebugConfigurationHandler.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/GetDebugConfigurationHandler.scala deleted file mode 100644 index 7814100dd..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/GetDebugConfigurationHandler.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.product_mixer.core.controllers - -import com.twitter.finagle.http.Request -import com.twitter.scrooge.BinaryThriftStructSerializer -import com.twitter.scrooge.ThriftMethod -import com.twitter.scrooge.schema.ThriftDefinitions -import com.twitter.scrooge.schema.scrooge.scala.CompiledScroogeDefBuilder -import com.twitter.scrooge.schema.serialization.thrift.ReferenceResolver -import com.twitter.scrooge.schema.serialization.thrift.ThriftDefinitionsSerializer -import com.twitter.scrooge.schema.{thriftscala => THRIFT} - -/** - * Endpoint to expose a Mixer's expected query configuration, including the request schema. - * - * @param debugEndpoint the debug Thrift endpoint. Passing [[None]] disables the query debugging - * feature. - * @tparam ServiceIface a thrift service containing the [[debugEndpoint]] - */ -case class GetDebugConfigurationHandler[ServiceIface]( - thriftMethod: ThriftMethod -)( - implicit val serviceIFace: Manifest[ServiceIface]) { - - // We need to binary encode the service def because the underlying Thrift isn't sufficiently - // annotated to be serialized/deserialized by Jackson - private val serviceDef = { - val fullServiceDefinition: ThriftDefinitions.ServiceDef = CompiledScroogeDefBuilder - .build(serviceIFace).asInstanceOf[ThriftDefinitions.ServiceDef] - - val endpointDefinition: ThriftDefinitions.ServiceEndpointDef = - fullServiceDefinition.endpointsByName(thriftMethod.name) - - // Create a service definition which just contains the debug endpoint. At a bare minimum, we need - // to give callers a way to identify the debug endpoint. Sending back all the endpoints is - // redundant. - val serviceDefinition: ThriftDefinitions.ServiceDef = - fullServiceDefinition.copy(endpoints = Seq(endpointDefinition)) - - val thriftDefinitionsSerializer = { - // We don't make use of references but a reference resolver is required by the Scrooge API - val noopReferenceResolver: ReferenceResolver = - (_: THRIFT.ReferenceDef) => throw new Exception("no references") - - new ThriftDefinitionsSerializer(noopReferenceResolver, enableReferences = false) - } - - val thriftBinarySerializer = BinaryThriftStructSerializer.apply(THRIFT.Definition) - - val serializedServiceDef = thriftDefinitionsSerializer(serviceDefinition) - - thriftBinarySerializer.toBytes(serializedServiceDef) - } - - def apply(request: Request): DebugConfigurationResponse = - DebugConfigurationResponse(thriftMethod.name, serviceDef) -} - -case class DebugConfigurationResponse( - debugEndpointName: String, - serviceDefinition: Array[Byte]) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/PredicateConfig.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/PredicateConfig.docx new file mode 100644 index 000000000..635c8801e Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/PredicateConfig.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/PredicateConfig.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/PredicateConfig.scala deleted file mode 100644 index c2f217400..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/PredicateConfig.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.product_mixer.core.controllers - -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.Predicate - -/** Simple representation for a [[Predicate]] used for dashboard generation */ -private[core] case class PredicateConfig( - operator: String, - threshold: Double, - datapointsPastThreshold: Int, - duration: Int, - metricGranularity: String) - -private[core] object PredicateConfig { - - /** Convert this [[Predicate]] into a [[PredicateConfig]] */ - def apply(predicate: Predicate): PredicateConfig = PredicateConfig( - predicate.operator.toString, - predicate.threshold, - predicate.datapointsPastThreshold, - predicate.duration, - predicate.metricGranularity.unit) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/ProductMixerController.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/ProductMixerController.docx new file mode 100644 index 000000000..b02d1f83f Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/ProductMixerController.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/ProductMixerController.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/ProductMixerController.scala deleted file mode 100644 index e7f761825..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/ProductMixerController.scala +++ /dev/null @@ -1,79 +0,0 @@ -package com.twitter.product_mixer.core.controllers - -import com.twitter.finagle.http.Request -import com.twitter.finagle.http.Response -import com.twitter.finagle.http.Status -import com.twitter.finagle.http.RouteIndex -import com.twitter.finatra.http.Controller -import com.twitter.scrooge.ThriftMethod -import com.twitter.inject.Injector -import com.twitter.inject.annotations.Flags -import com.twitter.product_mixer.core.model.common.identifier.ProductIdentifier -import com.twitter.product_mixer.core.module.product_mixer_flags.ProductMixerFlagModule.ServiceLocal -import com.twitter.product_mixer.core.service.component_registry.ComponentRegistry -import com.twitter.product_mixer.core.service.component_registry.{ - RegisteredComponent => ComponentRegistryRegisteredComponent -} -import com.twitter.util.Future -import java.net.URLEncoder - -/** - * Register endpoints necessary for enabling Product Mixer tooling such as alerts, dashboard - * generation and Turntable. - * - * @param debugEndpoint a debug endpoint to run queries against. This feature is experimental and we - * do not recommend that teams use it yet. Providing [[None]] will disable - * debug queries. - * @tparam ServiceIface a thrift service containing the [[debugEndpoint]] - */ -case class ProductMixerController[ServiceIface]( - injector: Injector, - debugEndpoint: ThriftMethod, -)( - implicit val serviceIFace: Manifest[ServiceIface]) - extends Controller { - - val isLocal: Boolean = injector.instance[Boolean](Flags.named(ServiceLocal)) - - if (!isLocal) { - prefix("/admin/product-mixer") { - val productNamesFut: Future[Seq[String]] = - injector.instance[ComponentRegistry].get.map { componentRegistry => - componentRegistry.getAllRegisteredComponents.collect { - case ComponentRegistryRegisteredComponent(identifier: ProductIdentifier, _, _) => - identifier.name - } - } - - productNamesFut.map { productNames => - productNames.foreach { productName => - get( - route = "/debug-query/" + productName, - admin = true, - index = Some(RouteIndex(alias = "Query " + productName, group = "Feeds/Debug Query")) - ) { _: Request => - val auroraPath = - URLEncoder.encode(System.getProperty("aurora.instanceKey", ""), "UTF-8") - - // Extract service name from clientId since there isn't a specific flag for that - val serviceName = injector - .instance[String](Flags.named("thrift.clientId")) - .split("\\.")(0) - - val redirectUrl = - s"https://feeds.twitter.biz/dtab/$serviceName/$productName?servicePath=$auroraPath" - - val response = Response().status(Status.Found) - response.location = redirectUrl - response - } - } - } - } - } - - prefix("/product-mixer") { - get(route = "/component-registry")(GetComponentRegistryHandler(injector).apply) - get(route = "/debug-configuration")(GetDebugConfigurationHandler(debugEndpoint).apply) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/QualityFactorMonitoringConfig.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/QualityFactorMonitoringConfig.docx new file mode 100644 index 000000000..fa0f9c18a Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/QualityFactorMonitoringConfig.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/QualityFactorMonitoringConfig.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/QualityFactorMonitoringConfig.scala deleted file mode 100644 index 12bbe4850..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers/QualityFactorMonitoringConfig.scala +++ /dev/null @@ -1,6 +0,0 @@ -package com.twitter.product_mixer.core.controllers - -// Bounds here are inclusive -case class QualityFactorMonitoringConfig( - boundMin: Double, - boundMax: Double) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/BUILD deleted file mode 100644 index cd054940f..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/BUILD +++ /dev/null @@ -1,10 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "util/util-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/BUILD.docx new file mode 100644 index 000000000..3aedde6b2 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/Feature.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/Feature.docx new file mode 100644 index 000000000..cea26f2ee Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/Feature.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/Feature.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/Feature.scala deleted file mode 100644 index 820abe25c..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/Feature.scala +++ /dev/null @@ -1,74 +0,0 @@ -package com.twitter.product_mixer.core.feature - -/** - * A [[Feature]] is a single measurable or computable property of an entity. - * - * @note If a [[Feature]] is optional then the [[Value]] should be `Option[Value]` - * - * @note If a [[Feature]] is populated with a [[com.twitter.product_mixer.core.functional_component.feature_hydrator.FeatureHydrator]] - * and the hydration fails, a failure will be stored for the [[Feature]]. - * If that [[Feature]] is accessed with [[com.twitter.product_mixer.core.feature.featuremap.FeatureMap.get]] - * then the stored exception will be thrown, essentially failing-closed. - * You can use [[FeatureWithDefaultOnFailure]] or [[com.twitter.product_mixer.core.feature.featuremap.FeatureMap.getOrElse]] - * instead to avoid these issues and instead fail-open. - * If correctly hydrating a Feature's value is essential to the request being correct, - * then you should fail-closed on failure to hydrate it by extending [[Feature]] directly. - * - * This does not apply to [[Feature]]s from [[com.twitter.product_mixer.core.functional_component.transformer.FeatureTransformer]] - * which throw in the calling Pipeline instead of storing their failures. - * - * @tparam Entity The type of entity that this feature works with. This could be a User, Tweet, - * Query, etc. - * @tparam Value The type of the value of this feature. - */ -trait Feature[-Entity, Value] { self => - override def toString: String = { - Feature.getSimpleName(self.getClass) - } -} - -/** - * With a [[Feature]], if the [[com.twitter.product_mixer.core.functional_component.feature_hydrator.FeatureHydrator]] fails, - * the failure will be caught by the platform and stored in the [[com.twitter.product_mixer.core.feature.featuremap.FeatureMap]]. - * Accessing a failed feature via [[com.twitter.product_mixer.core.feature.featuremap.FeatureMap.get()]] - * will throw the exception that was caught while attempting to hydrate the feature. If there's a - * reasonable default for a [[Feature]] to fail-open with, then throwing the exception at read time - * can be prevented by defining a `defaultValue` via [[FeatureWithDefaultOnFailure]]. When accessing - * a failed feature via [[com.twitter.product_mixer.core.feature.featuremap.FeatureMap.get()]] - * for a [[FeatureWithDefaultOnFailure]], the `defaultValue` will be returned. - * - * - * @note [[com.twitter.product_mixer.core.feature.featuremap.FeatureMap.getOrElse()]] can also be used - * to access a failed feature without throwing the exception, by defining the default via the - * [[com.twitter.product_mixer.core.feature.featuremap.FeatureMap.getOrElse()]] method call - * instead of as part of the feature declaration. - * @note This does not apply to [[FeatureWithDefaultOnFailure]]s from [[com.twitter.product_mixer.core.functional_component.transformer.FeatureTransformer]] - * which throw in the calling Pipeline instead of storing their failures. - * - * @tparam Entity The type of entity that this feature works with. This could be a User, Tweet, - * Query, etc. - * @tparam Value The type of the value of this feature. - */ -trait FeatureWithDefaultOnFailure[Entity, Value] extends Feature[Entity, Value] { - - /** The default value a feature should return should it fail to be hydrated */ - def defaultValue: Value -} - -trait ModelFeatureName { self: Feature[_, _] => - def featureName: String -} - -object Feature { - - /** - * Avoid `malformed class name` exceptions due to the presence of the `$` character - * Also strip off trailing $ signs for readability - */ - def getSimpleName[T](c: Class[T]): String = { - c.getName.stripSuffix("$").lastIndexOf("$") match { - case -1 => c.getSimpleName.stripSuffix("$") - case index => c.getName.substring(index + 1).stripSuffix("$") - } - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord/BUILD deleted file mode 100644 index c94c4bec6..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "src/java/com/twitter/ml/api:api-base", - "src/scala/com/twitter/ml/api/util:datarecord", - "src/thrift/com/twitter/dal/personal_data:personal_data-java", - "src/thrift/com/twitter/ml/api:data-java", - "src/thrift/com/twitter/ml/api:data-scala", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "src/thrift/com/twitter/dal/personal_data:personal_data-java", - "src/thrift/com/twitter/ml/api:data-java", - "src/thrift/com/twitter/ml/api:data-scala", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord/BUILD.docx new file mode 100644 index 000000000..c80e2ccc3 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord/DataRecordCompatible.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord/DataRecordCompatible.docx new file mode 100644 index 000000000..3813aa772 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord/DataRecordCompatible.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord/DataRecordCompatible.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord/DataRecordCompatible.scala deleted file mode 100644 index 1a69e9a7f..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord/DataRecordCompatible.scala +++ /dev/null @@ -1,316 +0,0 @@ -package com.twitter.product_mixer.core.feature.datarecord - -import com.twitter.dal.personal_data.thriftjava.PersonalDataType -import com.twitter.ml.api.Feature -import com.twitter.ml.api.DataType -import com.twitter.ml.api.thriftscala.GeneralTensor -import com.twitter.ml.api.thriftscala.StringTensor -import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions -import com.twitter.ml.api.{GeneralTensor => JGeneralTensor} -import com.twitter.ml.api.{RawTypedTensor => JRawTypedTensor} -import com.twitter.ml.api.{Feature => MlFeature} -import java.nio.ByteBuffer -import java.nio.ByteOrder -import java.util.{Map => JMap} -import java.util.{Set => JSet} -import java.lang.{Long => JLong} -import java.lang.{Boolean => JBoolean} -import java.lang.{Double => JDouble} -import scala.collection.JavaConverters._ - -/** - * Defines a conversion function for customers to mix-in when constructing a DataRecord supported - * feature. We do this because the ML Feature representation is written in Java and uses Java types. - * Furthermore, allowing customers to construct their own ML Feature directly can leave room - * for mistyping errors, such as using a Double ML Feature on a String Product Mixer feature. - * This mix in enforces that the customer only uses the right types, while making it easier - * to setup a DataRecord Feature with nothing but a feature name and personal data types. - * @tparam FeatureValueType The type of the underlying Product Mixer feature value. - */ -sealed trait DataRecordCompatible[FeatureValueType] { - // The feature value type in ProMix. - final type FeatureType = FeatureValueType - // The underlying DataRecord value type, sometimes this differs from the Feature Store and ProMix type. - type DataRecordType - - def featureName: String - def personalDataTypes: Set[PersonalDataType] - - private[product_mixer] def mlFeature: MlFeature[DataRecordType] - - /** - * To & from Data Record value converters. In most cases, this is one to one when the types match - * but in some cases, certain features are modeled as different types in Data Record. For example, - * some features that are Long (e.g, such as TweepCred) are sometimes stored as Doubles. - */ - private[product_mixer] def toDataRecordFeatureValue(featureValue: FeatureType): DataRecordType - private[product_mixer] def fromDataRecordFeatureValue(featureValue: DataRecordType): FeatureType - -} - -/** - * Converter for going from String feature value to String ML Feature. - */ -trait StringDataRecordCompatible extends DataRecordCompatible[String] { - override type DataRecordType = String - - final override lazy val mlFeature: MlFeature[String] = - new MlFeature.Text(featureName, personalDataTypes.asJava) - - override private[product_mixer] def fromDataRecordFeatureValue( - featureValue: String - ): String = featureValue - - override private[product_mixer] def toDataRecordFeatureValue( - featureValue: String - ): String = featureValue -} - -/** - * Converter for going from Long feature value to Discrete/Long ML Feature. - */ -trait LongDiscreteDataRecordCompatible extends DataRecordCompatible[Long] { - override type DataRecordType = JLong - - final override lazy val mlFeature: MlFeature[JLong] = - new Feature.Discrete(featureName, personalDataTypes.asJava) - - override private[product_mixer] def fromDataRecordFeatureValue( - featureValue: JLong - ): Long = featureValue - - override private[product_mixer] def toDataRecordFeatureValue( - featureValue: Long - ): JLong = featureValue -} - -/** - * Converter for going from Long feature value to Continuous/Double ML Feature. - */ -trait LongContinuousDataRecordCompatible extends DataRecordCompatible[Long] { - override type DataRecordType = JDouble - - final override lazy val mlFeature: MlFeature[JDouble] = - new Feature.Continuous(featureName, personalDataTypes.asJava) - - override private[product_mixer] def toDataRecordFeatureValue( - featureValue: FeatureType - ): JDouble = featureValue.toDouble - - override private[product_mixer] def fromDataRecordFeatureValue( - featureValue: JDouble - ): Long = featureValue.longValue() -} - -/** - * Converter for going from an Integer feature value to Long/Discrete ML Feature. - */ -trait IntDiscreteDataRecordCompatible extends DataRecordCompatible[Int] { - override type DataRecordType = JLong - - final override lazy val mlFeature: MlFeature[JLong] = - new MlFeature.Discrete(featureName, personalDataTypes.asJava) - - override private[product_mixer] def fromDataRecordFeatureValue( - featureValue: JLong - ): Int = featureValue.toInt - - override private[product_mixer] def toDataRecordFeatureValue( - featureValue: Int - ): JLong = featureValue.toLong -} - -/** - * Converter for going from Integer feature value to Continuous/Double ML Feature. - */ -trait IntContinuousDataRecordCompatible extends DataRecordCompatible[Int] { - override type DataRecordType = JDouble - - final override lazy val mlFeature: MlFeature[JDouble] = - new Feature.Continuous(featureName, personalDataTypes.asJava) - - override private[product_mixer] def toDataRecordFeatureValue( - featureValue: Int - ): JDouble = featureValue.toDouble - - override private[product_mixer] def fromDataRecordFeatureValue( - featureValue: JDouble - ): Int = featureValue.toInt -} - -/** - * Converter for going from Double feature value to Continuous/Double ML Feature. - */ -trait DoubleDataRecordCompatible extends DataRecordCompatible[Double] { - override type DataRecordType = JDouble - - final override lazy val mlFeature: MlFeature[JDouble] = - new MlFeature.Continuous(featureName, personalDataTypes.asJava) - - override private[product_mixer] def fromDataRecordFeatureValue( - featureValue: JDouble - ): Double = featureValue - - override private[product_mixer] def toDataRecordFeatureValue( - featureValue: Double - ): JDouble = featureValue -} - -/** - * Converter for going from Boolean feature value to Boolean ML Feature. - */ -trait BoolDataRecordCompatible extends DataRecordCompatible[Boolean] { - override type DataRecordType = JBoolean - - final override lazy val mlFeature: MlFeature[JBoolean] = - new MlFeature.Binary(featureName, personalDataTypes.asJava) - - override private[product_mixer] def fromDataRecordFeatureValue( - featureValue: JBoolean - ): Boolean = featureValue - - override private[product_mixer] def toDataRecordFeatureValue( - featureValue: Boolean - ): JBoolean = featureValue -} - -/** - * Converter for going from a ByteBuffer feature value to ByteBuffer ML Feature. - */ -trait BlobDataRecordCompatible extends DataRecordCompatible[ByteBuffer] { - override type DataRecordType = ByteBuffer - - final override lazy val mlFeature: MlFeature[ByteBuffer] = - new Feature.Blob(featureName, personalDataTypes.asJava) - - override private[product_mixer] def fromDataRecordFeatureValue( - featureValue: ByteBuffer - ): ByteBuffer = featureValue - - override private[product_mixer] def toDataRecordFeatureValue( - featureValue: ByteBuffer - ): ByteBuffer = featureValue -} - -/** - * Converter for going from a Map[String, Double] feature value to Sparse Double/Continious ML Feature. - */ -trait SparseContinuousDataRecordCompatible extends DataRecordCompatible[Map[String, Double]] { - override type DataRecordType = JMap[String, JDouble] - - final override lazy val mlFeature: MlFeature[JMap[String, JDouble]] = - new Feature.SparseContinuous(featureName, personalDataTypes.asJava) - - override private[product_mixer] def toDataRecordFeatureValue( - featureValue: Map[String, Double] - ): JMap[String, JDouble] = - featureValue.mapValues(_.asInstanceOf[JDouble]).asJava - - override private[product_mixer] def fromDataRecordFeatureValue( - featureValue: JMap[String, JDouble] - ) = featureValue.asScala.toMap.mapValues(_.doubleValue()) -} - -/** - * Converter for going from a Set[String] feature value to SparseBinary/String Set ML Feature. - */ -trait SparseBinaryDataRecordCompatible extends DataRecordCompatible[Set[String]] { - override type DataRecordType = JSet[String] - - final override lazy val mlFeature: MlFeature[JSet[String]] = - new Feature.SparseBinary(featureName, personalDataTypes.asJava) - - override private[product_mixer] def fromDataRecordFeatureValue( - featureValue: JSet[String] - ) = featureValue.asScala.toSet - - override private[product_mixer] def toDataRecordFeatureValue( - featureValue: FeatureType - ): JSet[String] = featureValue.asJava -} - -/** - * Marker trait for any feature value to Tensor ML Feature. Not directly usable. - */ -sealed trait TensorDataRecordCompatible[FeatureV] extends DataRecordCompatible[FeatureV] { - override type DataRecordType = JGeneralTensor - override def mlFeature: MlFeature[JGeneralTensor] -} - -/** - * Converter for a double to a Tensor feature encoded as float encoded RawTypedTensor - */ -trait RawTensorFloatDoubleDataRecordCompatible extends TensorDataRecordCompatible[Double] { - final override lazy val mlFeature: MlFeature[JGeneralTensor] = - new Feature.Tensor( - featureName, - DataType.FLOAT, - List.empty[JLong].asJava, - personalDataTypes.asJava) - - override private[product_mixer] def toDataRecordFeatureValue( - featureValue: FeatureType - ) = { - val byteBuffer: ByteBuffer = - ByteBuffer - .allocate(4).order(ByteOrder.LITTLE_ENDIAN).putFloat(featureValue.toFloat) - byteBuffer.flip() - val tensor = new JGeneralTensor() - tensor.setRawTypedTensor(new JRawTypedTensor(DataType.FLOAT, byteBuffer)) - tensor - } - - override private[product_mixer] def fromDataRecordFeatureValue( - featureValue: JGeneralTensor - ) = { - val tensor = Option(featureValue.getRawTypedTensor) - .getOrElse(throw new UnexpectedTensorException(featureValue)) - tensor.content.order(ByteOrder.LITTLE_ENDIAN).getFloat().toDouble - } -} - -/** - * Converter for a scala general tensor to java general tensor ML feature. - */ -trait GeneralTensorDataRecordCompatible extends TensorDataRecordCompatible[GeneralTensor] { - - def dataType: DataType - final override lazy val mlFeature: MlFeature[JGeneralTensor] = - new Feature.Tensor(featureName, dataType, List.empty[JLong].asJava, personalDataTypes.asJava) - - override private[product_mixer] def toDataRecordFeatureValue( - featureValue: FeatureType - ) = ScalaToJavaDataRecordConversions.scalaTensor2Java(featureValue) - - override private[product_mixer] def fromDataRecordFeatureValue( - featureValue: JGeneralTensor - ) = ScalaToJavaDataRecordConversions.javaTensor2Scala(featureValue) -} - -/** - * Converter for a scala string tensor to java general tensor ML feature. - */ -trait StringTensorDataRecordCompatible extends TensorDataRecordCompatible[StringTensor] { - final override lazy val mlFeature: MlFeature[JGeneralTensor] = - new Feature.Tensor( - featureName, - DataType.STRING, - List.empty[JLong].asJava, - personalDataTypes.asJava) - - override private[product_mixer] def fromDataRecordFeatureValue( - featureValue: JGeneralTensor - ) = { - ScalaToJavaDataRecordConversions.javaTensor2Scala(featureValue) match { - case GeneralTensor.StringTensor(stringTensor) => stringTensor - case _ => throw new UnexpectedTensorException(featureValue) - } - } - - override private[product_mixer] def toDataRecordFeatureValue( - featureValue: FeatureType - ) = ScalaToJavaDataRecordConversions.scalaTensor2Java(GeneralTensor.StringTensor(featureValue)) -} - -class UnexpectedTensorException(tensor: JGeneralTensor) - extends Exception(s"Unexpected Tensor: $tensor") diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord/DataRecordFeature.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord/DataRecordFeature.docx new file mode 100644 index 000000000..69b850e7d Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord/DataRecordFeature.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord/DataRecordFeature.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord/DataRecordFeature.scala deleted file mode 100644 index a718f9644..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord/DataRecordFeature.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.product_mixer.core.feature.datarecord - -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.core.feature.Feature - -/** - * A DataRecord supported feature mixin for enabling conversions from Product Mixer Features - * to DataRecords. When using Feature Store features, this is pre-configured for the customer - * under the hood. For non-Feature Store features, customers must mix in either [[DataRecordFeature]] - * for required features, or [[DataRecordOptionalFeature]] for optional features, as well as mixing - * in a corresponding [[DataRecordCompatible]] for their feature type. - * @tparam Entity The type of entity that this feature works with. This could be a User, Tweet, - * Query, etc. - * @tparam Value The type of the value of this feature. - */ -sealed trait BaseDataRecordFeature[-Entity, Value] extends Feature[Entity, Value] - -private[product_mixer] abstract class FeatureStoreDataRecordFeature[-Entity, Value] - extends BaseDataRecordFeature[Entity, Value] - -/** - * Feature in a DataRecord for a required feature value; the corresponding feature will always be - * available in the built DataRecord. - */ -trait DataRecordFeature[-Entity, Value] extends BaseDataRecordFeature[Entity, Value] { - self: DataRecordCompatible[Value] => -} - -/** - * Feature in a DataRecord for an optional feature value; the corresponding feature will only - * ever be set in a DataRecord if the value in the feature map is defined (Some(V)). - */ -trait DataRecordOptionalFeature[-Entity, Value] - extends BaseDataRecordFeature[Entity, Option[Value]] { - self: DataRecordCompatible[Value] => -} - -/** - * An entire DataRecord as a feature. This is useful when there is an existing DataRecord that - * should be used as a whole instead of as individual [[DataRecordFeature]]s for example. - */ -trait DataRecordInAFeature[-Entity] extends BaseDataRecordFeature[Entity, DataRecord] diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/BUILD deleted file mode 100644 index ea22972ca..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/featurevalue", - "util/util-core", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "util/util-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/BUILD.docx new file mode 100644 index 000000000..9a53263ae Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/FeatureMap.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/FeatureMap.docx new file mode 100644 index 000000000..6c7a6ad14 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/FeatureMap.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/FeatureMap.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/FeatureMap.scala deleted file mode 100644 index 870f8ad9f..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/FeatureMap.scala +++ /dev/null @@ -1,195 +0,0 @@ -package com.twitter.product_mixer.core.feature.featuremap - -import com.fasterxml.jackson.databind.annotation.JsonSerialize -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.featurestorev1.featurevalue.FeatureStoreV1Response -import com.twitter.product_mixer.core.feature.featurestorev1.featurevalue.{ - FeatureStoreV1ResponseFeature => FSv1Feature -} -import com.twitter.util.Return -import com.twitter.util.Throw -import com.twitter.util.Try - -/** - * A set of features and their values. Associated with a specific instance of an Entity, though - * that association is maintained by the framework. - * - * [[FeatureMapBuilder]] is used to build new FeatureMap instances - */ -@JsonSerialize(using = classOf[FeatureMapSerializer]) -case class FeatureMap private[feature] ( - private[core] val underlyingMap: Map[Feature[_, _], Try[_]]) { - - /** - * Returns the [[Value]] associated with the Feature - * - * If the Feature is missing from the feature map, it throws a [[MissingFeatureException]]. - * If the Feature failed and isn't a [[FeatureWithDefaultOnFailure]] this will throw the underlying exception - * that the feature failed with during hydration. - */ - def get[Value](feature: Feature[_, Value]): Value = - getOrElse(feature, throw MissingFeatureException(feature), None) - - /** - * Returns the [[Value]] associated with the Feature with the same semantics as - * [[FeatureMap.get()]], but the underlying [[Try]] is returned to allow for checking the success - * or error state of a feature hydration. This is helpful for implementing fall-back behavior in - * case the feature is missing or hydration failed without a [[FeatureWithDefaultOnFailure]] set. - * - * @note The [[FeatureMap.getOrElse()]] logic is duplicated here to avoid unpacking and repacking - * the [[Try]] that is already available in the [[underlyingMap]] - */ - def getTry[Value](feature: Feature[_, Value]): Try[Value] = - underlyingMap.get(feature) match { - case None => Throw(MissingFeatureException(feature)) - case Some(value @ Return(_)) => value.asInstanceOf[Return[Value]] - case Some(value @ Throw(_)) => - feature match { - case f: FeatureWithDefaultOnFailure[_, Value] @unchecked => Return(f.defaultValue) - case _ => value.asInstanceOf[Throw[Value]] - } - } - - /** - * Returns the [[Value]] associated with the feature or a default if the key is not contained in the map - * `default` can also be used to throw an exception. - * - * e.g. `.getOrElse(feature, throw new MyCustomException())` - * - * @note for [[FeatureWithDefaultOnFailure]], the [[FeatureWithDefaultOnFailure.defaultValue]] - * will be returned if the [[Feature]] failed, but if it is missing/never hydrated, - * then the `default` provided here will be used. - */ - def getOrElse[Value](feature: Feature[_, Value], default: => Value): Value = { - getOrElse(feature, default, Some(default)) - } - - /** - * Private helper for getting features from the feature map, allowing us to define a default - * when the feature is missing from the feature map, vs when its in the feature map but failed. - * In the case of failed features, if the feature is a [FeatureWithDefaultOnFailure], it will - * prioritize that default. - * @param feature The feature to retrieve - * @param missingDefault The default value to use when the feature is missing from the map. - * @param failureDefault The default value to use when the feature is present but failed. - * @tparam Value The value type of the feature. - * @return The value stored in the map. - */ - private def getOrElse[Value]( - feature: Feature[_, Value], - missingDefault: => Value, - failureDefault: => Option[Value] - ): Value = - underlyingMap.get(feature) match { - case None => missingDefault - case Some(Return(value)) => value.asInstanceOf[Value] - case Some(Throw(err)) => - feature match { - case f: FeatureWithDefaultOnFailure[_, Value] @unchecked => f.defaultValue - case _ => failureDefault.getOrElse(throw err) - } - } - - /** - * returns a new FeatureMap with - * - the new Feature and Value pair if the Feature was not present - * - overriding the previous Value if that Feature was previously present - */ - def +[V](key: Feature[_, V], value: V): FeatureMap = - new FeatureMap(underlyingMap + (key -> Return(value))) - - /** - * returns a new FeatureMap with all the elements of current FeatureMap and `other`. - * - * @note if a [[Feature]] exists in both maps, the Value from `other` takes precedence - */ - def ++(other: FeatureMap): FeatureMap = { - if (other.isEmpty) { - this - } else if (isEmpty) { - other - } else if (this.getFeatures.contains(FSv1Feature) && other.getFeatures.contains(FSv1Feature)) { - val mergedResponse = - FeatureStoreV1Response.merge(this.get(FSv1Feature), other.get(FSv1Feature)) - val mergedResponseFeatureMap = FeatureMapBuilder().add(FSv1Feature, mergedResponse).build() - new FeatureMap(underlyingMap ++ other.underlyingMap ++ mergedResponseFeatureMap.underlyingMap) - } else { - new FeatureMap(underlyingMap ++ other.underlyingMap) - } - } - - /** returns the keySet of Features in the map */ - def getFeatures: Set[Feature[_, _]] = underlyingMap.keySet - - /** - * The Set of Features in the FeatureMap that have a successfully returned value. Failed features - * will obviously not be here. - */ - def getSuccessfulFeatures: Set[Feature[_, _]] = underlyingMap.collect { - case (feature, Return(_)) => feature - }.toSet - - def isEmpty: Boolean = underlyingMap.isEmpty - - override def toString: String = s"FeatureMap(${underlyingMap.toString})" -} - -object FeatureMap { - // Restrict access to the apply method. - // This shouldn't be required after scala 2.13.2 (https://github.com/scala/scala/pull/7702) - private[feature] def apply(underlyingMap: Map[Feature[_, _], Try[_]]): FeatureMap = - FeatureMap(underlyingMap) - - /** Merges an arbitrary number of [[FeatureMap]]s from left-to-right */ - def merge(featureMaps: TraversableOnce[FeatureMap]): FeatureMap = { - val builder = FeatureMapBuilder() - - /** - * merge the current [[FSv1Feature]] with the existing accumulated one - * and add the rest of the [[FeatureMap]]'s [[Feature]]s to the `builder` - */ - def mergeInternal( - featureMap: FeatureMap, - accumulatedFsV1Response: Option[FeatureStoreV1Response] - ): Option[FeatureStoreV1Response] = { - if (featureMap.isEmpty) { - accumulatedFsV1Response - } else { - - val currentFsV1Response = - if (featureMap.getFeatures.contains(FSv1Feature)) - Some(featureMap.get(FSv1Feature)) - else - None - - val mergedFsV1Response = (accumulatedFsV1Response, currentFsV1Response) match { - case (Some(merged), Some(current)) => - // both present so merge them - Some(FeatureStoreV1Response.merge(merged, current)) - case (merged, current) => - // one or both are missing so use whichever is available - merged.orElse(current) - } - - featureMap.underlyingMap.foreach { - case (FSv1Feature, _) => // FSv1Feature is only added once at the very end - case (feature, value) => builder.addWithoutValidation(feature, value) - } - mergedFsV1Response - } - } - - featureMaps - .foldLeft[Option[FeatureStoreV1Response]](None) { - case (fsV1Response, featureMap) => mergeInternal(featureMap, fsV1Response) - }.foreach( - // add merged `FSv1Feature` to the `builder` at the end - builder.add(FSv1Feature, _) - ) - - builder.build() - } - - val empty = new FeatureMap(Map.empty) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/FeatureMapBuilder.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/FeatureMapBuilder.docx new file mode 100644 index 000000000..2a9f8d41d Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/FeatureMapBuilder.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/FeatureMapBuilder.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/FeatureMapBuilder.scala deleted file mode 100644 index 91fe54d44..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/FeatureMapBuilder.scala +++ /dev/null @@ -1,110 +0,0 @@ -package com.twitter.product_mixer.core.feature.featuremap - -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.util.Return -import com.twitter.util.Throw -import com.twitter.util.Try -import scala.collection.mutable - -/** - * [[FeatureMapBuilder]] is a typesafe way (it checks types vs the [[Feature]]s on `.add`) to build a [[FeatureMap]]. - * - * Throws a [[DuplicateFeatureException]] if you try to add the same [[Feature]] more than once. - * - * These builders are __not__ reusable. - */ - -class FeatureMapBuilder { - private val underlying = Map.newBuilder[Feature[_, _], Try[Any]] - private val keys = mutable.HashSet.empty[Feature[_, _]] - private var built = false - - /** - * Add a [[Try]] of a [[Feature]] `value` to the map, - * handling both the [[Return]] and [[Throw]] cases. - * - * Throws a [[DuplicateFeatureException]] if it's already present. - * - * @note If you have a [[Feature]] with a non-optional value type `Feature[_, V]` - * but have an `Option[V]` you can use [[Try.orThrow]] to convert the [[Option]] - * to a [[Try]], which will store the successful or failed [[Feature]] in the map. - */ - def add[V](feature: Feature[_, V], value: Try[V]): FeatureMapBuilder = addTry(feature, value) - - /** - * Add a successful [[Feature]] `value` to the map - * - * Throws a [[DuplicateFeatureException]] if it's already present. - * - * @note If you have a [[Feature]] with a non-optional value type `Feature[_, V]` - * but have an `Option[V]` you can use [[Option.get]] or [[Option.getOrElse]] - * to convert the [[Option]] to extract the underlying value, - * which will throw immediately if it's [[None]] or add the successful [[Feature]] in the map. - */ - def add[V](feature: Feature[_, V], value: V): FeatureMapBuilder = - addTry(feature, Return(value)) - - /** - * Add a failed [[Feature]] `value` to the map - * - * Throws a [[DuplicateFeatureException]] if it's already present. - */ - def addFailure(feature: Feature[_, _], throwable: Throwable): FeatureMapBuilder = - addTry(feature, Throw(throwable)) - - /** - * [[add]] but for when the [[Feature]] types aren't known - * - * Add a [[Try]] of a [[Feature]] `value` to the map, - * handling both the [[Return]] and [[Throw]] cases. - * - * Throws a [[DuplicateFeatureException]] if it's already present. - * - * @note If you have a [[Feature]] with a non-optional value type `Feature[_, V]` - * but have an `Option[V]` you can use [[Try.orThrow]] to convert the [[Option]] - * to a [[Try]], which will store the successful or failed [[Feature]] in the map. - */ - def addTry(feature: Feature[_, _], value: Try[_]): FeatureMapBuilder = { - if (keys.contains(feature)) { - throw new DuplicateFeatureException(feature) - } - addWithoutValidation(feature, value) - } - - /** - * [[addTry]] but without a [[DuplicateFeatureException]] check - * - * @note Only for use internally within [[FeatureMap.merge]] - */ - private[featuremap] def addWithoutValidation( - feature: Feature[_, _], - value: Try[_] - ): FeatureMapBuilder = { - keys += feature - underlying += ((feature, value)) - this - } - - /** Builds the FeatureMap */ - def build(): FeatureMap = { - if (built) { - throw ReusedFeatureMapBuilderException - } - - built = true - new FeatureMap(underlying.result()) - } -} - -object FeatureMapBuilder { - - /** Returns a new [[FeatureMapBuilder]] for making [[FeatureMap]]s */ - def apply(): FeatureMapBuilder = new FeatureMapBuilder -} - -class DuplicateFeatureException(feature: Feature[_, _]) - extends UnsupportedOperationException(s"Feature $feature already exists in FeatureMap") - -object ReusedFeatureMapBuilderException - extends UnsupportedOperationException( - "build() cannot be called more than once since FeatureMapBuilders are not reusable") diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/FeatureMapException.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/FeatureMapException.docx new file mode 100644 index 000000000..ebefd4d03 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/FeatureMapException.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/FeatureMapException.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/FeatureMapException.scala deleted file mode 100644 index bbd57cfb6..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/FeatureMapException.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.product_mixer.core.feature.featuremap - -import com.twitter.product_mixer.core.feature.Feature - -case class MissingFeatureException(feature: Feature[_, _]) - extends Exception("Missing value for " + feature) { - override def toString: String = getMessage -} - -class InvalidPredictionRecordMergeException - extends Exception( - "Use FeatureMap.plusPlusOptimized instead of FeatureMap.++ when the FeatureMaps on both sides of the merge contain PredictionRecords") diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/FeatureMapSerializer.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/FeatureMapSerializer.docx new file mode 100644 index 000000000..a2fd89d17 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/FeatureMapSerializer.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/FeatureMapSerializer.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/FeatureMapSerializer.scala deleted file mode 100644 index 746f178fb..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/FeatureMapSerializer.scala +++ /dev/null @@ -1,63 +0,0 @@ -package com.twitter.product_mixer.core.feature.featuremap - -import com.fasterxml.jackson.core.JsonGenerator -import com.fasterxml.jackson.databind.JsonSerializer -import com.fasterxml.jackson.databind.SerializerProvider -import com.twitter.product_mixer.core.feature.featurestorev1.featurevalue.FeatureStoreV1Response -import com.twitter.product_mixer.core.feature.featurestorev1.featurevalue.FeatureStoreV1ResponseFeature -import com.twitter.util.Return - -/** - * Rendering feature maps is dangerous because we don't control all the data that's stored in them. - * This can result failed requests, as we might try to render a recursive structure, very large - * structure, etc. Create a simple map using toString, this mostly works and is better than failing - * the request. - * - * @note changes to serialization logic can have serious performance implications given how hot the - * serialization path is. Consider benchmarking changes with [[com.twitter.product_mixer.core.benchmark.CandidatePipelineResultSerializationBenchmark]] - */ -private[featuremap] class FeatureMapSerializer() extends JsonSerializer[FeatureMap] { - override def serialize( - featureMap: FeatureMap, - gen: JsonGenerator, - serializers: SerializerProvider - ): Unit = { - gen.writeStartObject() - - featureMap.underlyingMap.foreach { - case (FeatureStoreV1ResponseFeature, Return(value)) => - // We know that value has to be [[FeatureStoreV1Response]] but its type has been erased, - // preventing us from pattern-matching. - val featureStoreResponse = value.asInstanceOf[FeatureStoreV1Response] - - val featuresIterator = featureStoreResponse.richDataRecord.allFeaturesIterator() - while (featuresIterator.moveNext()) { - gen.writeStringField( - featuresIterator.getFeature.getFeatureName, - s"${featuresIterator.getFeatureType.name}(${truncateString( - featuresIterator.getFeatureValue.toString)})") - } - - featureStoreResponse.failedFeatures.foreach { - case (failedFeature, failureReasons) => - gen.writeStringField( - failedFeature.toString, - s"Failed(${truncateString(failureReasons.toString)})") - } - case (name, Return(value)) => - gen.writeStringField(name.toString, truncateString(value.toString)) - case (name, error) => - // Note: we don't match on Throw(error) because we want to keep it for the toString - gen.writeStringField(name.toString, truncateString(error.toString)) - - } - - gen.writeEndObject() - } - - // Some features can be very large when stringified, for example when a dependant candidate - // pipeline is used, the entire previous candidate pipeline result is serialized into a feature. - // This causes significant performance issues when the result is later sent over the wire. - private def truncateString(input: String): String = - if (input.length > 1000) input.take(1000) + "..." else input -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap/AsyncFeatureMap.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap/AsyncFeatureMap.docx new file mode 100644 index 000000000..d43e824eb Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap/AsyncFeatureMap.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap/AsyncFeatureMap.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap/AsyncFeatureMap.scala deleted file mode 100644 index 49d2d44eb..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap/AsyncFeatureMap.scala +++ /dev/null @@ -1,134 +0,0 @@ -package com.twitter.product_mixer.core.feature.featuremap.asyncfeaturemap - -import com.fasterxml.jackson.databind.annotation.JsonSerialize -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier -import com.twitter.stitch.Stitch - -import scala.collection.immutable.Queue - -/** - * An internal representation of an async [[FeatureMap]] containing [[Stitch]]s of [[FeatureMap]]s - * which are already running in the background. - * - * Async features are added by providing the [[PipelineStepIdentifier]] of the [[com.twitter.product_mixer.core.pipeline.PipelineBuilder.Step Step]] - * before which the async [[Feature]]s are needed, and a [[Stitch]] of the async [[FeatureMap]]. - * It's expected that the [[Stitch]] has already been started and is running in the background. - * - * While not essential to it's core behavior, [[AsyncFeatureMap]] also keeps track of the [[FeatureHydratorIdentifier]] - * and the Set of [[Feature]]s which will be hydrated for each [[Stitch]] of a [[FeatureMap]] it's given. - * - * @param asyncFeatureMaps the [[FeatureMap]]s for [[PipelineStepIdentifier]]s which have not been reached yet - * - * @note [[PipelineStepIdentifier]]s must only refer to [[com.twitter.product_mixer.core.pipeline.PipelineBuilder.Step Step]]s - * in the current [[com.twitter.product_mixer.core.pipeline.Pipeline Pipeline]]. - * Only plain [[FeatureMap]]s are passed into underlying [[com.twitter.product_mixer.core.model.common.Component Component]]s and - * [[com.twitter.product_mixer.core.pipeline.Pipeline Pipeline]]s so [[AsyncFeatureMap]]s are scoped - * for a specific [[com.twitter.product_mixer.core.pipeline.Pipeline Pipeline]] only. - */ -@JsonSerialize(using = classOf[AsyncFeatureMapSerializer]) -private[core] case class AsyncFeatureMap( - asyncFeatureMaps: Map[PipelineStepIdentifier, Queue[ - (FeatureHydratorIdentifier, Set[Feature[_, _]], Stitch[FeatureMap]) - ]]) { - - def ++(right: AsyncFeatureMap): AsyncFeatureMap = { - val map = Map.newBuilder[ - PipelineStepIdentifier, - Queue[(FeatureHydratorIdentifier, Set[Feature[_, _]], Stitch[FeatureMap])]] - (asyncFeatureMaps.keysIterator ++ right.asyncFeatureMaps.keysIterator).foreach { key => - val currentThenRightAsyncFeatureMaps = - asyncFeatureMaps.getOrElse(key, Queue.empty) ++ - right.asyncFeatureMaps.getOrElse(key, Queue.empty) - map += (key -> currentThenRightAsyncFeatureMaps) - } - AsyncFeatureMap(map.result()) - } - - /** - * Returns a new [[AsyncFeatureMap]] which now keeps track of the provided `features` - * and will make them available when calling [[hydrate]] with `hydrateBefore`. - * - * @param featureHydratorIdentifier the [[FeatureHydratorIdentifier]] of the [[com.twitter.product_mixer.core.functional_component.feature_hydrator.FeatureHydrator FeatureHydrator]] - * which these [[Feature]]s are from - * @param hydrateBefore the [[PipelineStepIdentifier]] before which the [[Feature]]s need to be hydrated - * @param featuresToHydrate a Set of the [[Feature]]s which will be hydrated - * @param features a [[Stitch]] of the [[FeatureMap]] - */ - def addAsyncFeatures( - featureHydratorIdentifier: FeatureHydratorIdentifier, - hydrateBefore: PipelineStepIdentifier, - featuresToHydrate: Set[Feature[_, _]], - features: Stitch[FeatureMap] - ): AsyncFeatureMap = { - val featureMapList = - asyncFeatureMaps.getOrElse(hydrateBefore, Queue.empty) :+ - ((featureHydratorIdentifier, featuresToHydrate, features)) - AsyncFeatureMap(asyncFeatureMaps + (hydrateBefore -> featureMapList)) - } - - /** - * The current state of the [[AsyncFeatureMap]] excluding the [[Stitch]]s. - */ - def features: Map[PipelineStepIdentifier, Seq[(FeatureHydratorIdentifier, Set[Feature[_, _]])]] = - asyncFeatureMaps.mapValues(_.map { - case (featureHydratorIdentifier, features, _) => (featureHydratorIdentifier, features) - }) - - /** - * Returns a [[Some]] containing a [[Stitch]] with a [[FeatureMap]] holding the [[Feature]]s that are - * supposed to be hydrated at `identifier` if there are [[Feature]]s to hydrate at `identifier` - * - * Returns [[None]] if there are no [[Feature]]s to hydrate at the provided `identifier`, - * this allows for determining if there is work to do without running a [[Stitch]]. - * - * @note this only hydrates the [[Feature]]s for the specific `identifier`, it does not hydrate - * [[Feature]]s for earlier Steps. - * @param identifier the [[PipelineStepIdentifier]] to hydrate [[Feature]]s for - */ - def hydrate( - identifier: PipelineStepIdentifier - ): Option[Stitch[FeatureMap]] = - asyncFeatureMaps.get(identifier) match { - case Some(Queue((_, _, featureMap))) => - // if there is only 1 `FeatureMap` we dont need to do a collect so just return that Stitch - Some(featureMap) - case Some(featureMapList) => - // if there are multiple `FeatureMap`s we need to collect and merge them together - Some( - Stitch - .collect(featureMapList.map { case (_, _, featureMap) => featureMap }) - .map { featureMapList => FeatureMap.merge(featureMapList) }) - case None => - // No results for the provided `identifier` so return `None` - None - } -} - -private[core] object AsyncFeatureMap { - val empty: AsyncFeatureMap = AsyncFeatureMap(Map.empty) - - /** - * Builds the an [[AsyncFeatureMap]] from a Seq of [[Stitch]] of [[FeatureMap]] - * tupled with the relevant metadata we use to build the necessary state. - * - * This is primarily for convenience, since in most cases an [[AsyncFeatureMap]] - * will be built from the result of individual [[com.twitter.product_mixer.core.functional_component.feature_hydrator.FeatureHydrator FeatureHydrator]]s - * and combining them into the correct internal state. - */ - def fromFeatureMaps( - asyncFeatureMaps: Seq[ - (FeatureHydratorIdentifier, PipelineStepIdentifier, Set[Feature[_, _]], Stitch[FeatureMap]) - ] - ): AsyncFeatureMap = - AsyncFeatureMap( - asyncFeatureMaps - .groupBy { case (_, hydrateBefore, _, _) => hydrateBefore } - .mapValues(featureMaps => - Queue(featureMaps.map { - case (hydratorIdentifier, _, featuresToHydrate, stitch) => - (hydratorIdentifier, featuresToHydrate, stitch) - }: _*))) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap/AsyncFeatureMapSerializer.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap/AsyncFeatureMapSerializer.docx new file mode 100644 index 000000000..dfcba8186 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap/AsyncFeatureMapSerializer.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap/AsyncFeatureMapSerializer.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap/AsyncFeatureMapSerializer.scala deleted file mode 100644 index 2ed4c6735..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap/AsyncFeatureMapSerializer.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.product_mixer.core.feature.featuremap.asyncfeaturemap - -import com.fasterxml.jackson.core.JsonGenerator -import com.fasterxml.jackson.databind.JsonSerializer -import com.fasterxml.jackson.databind.SerializerProvider - -/** - * Since an [[AsyncFeatureMap]] is typically incomplete, and by the time it's serialized, all the [[com.twitter.product_mixer.core.feature.Feature]]s - * it will typically be completed and part of the Query or Candidate's individual [[com.twitter.product_mixer.core.feature.Feature]]s - * we instead opt to provide a summary of the Features which would be hydrated using [[AsyncFeatureMap.features]] - * - * This indicates which [[com.twitter.product_mixer.core.feature.Feature]]s will be ready at which Steps - * and which [[com.twitter.product_mixer.core.functional_component.feature_hydrator.FeatureHydrator]] - * are responsible for those [[com.twitter.product_mixer.core.feature.Feature]] - * - * @note changes to serialization logic can have serious performance implications given how hot the - * serialization path is. Consider benchmarking changes with [[com.twitter.product_mixer.core.benchmark.AsyncQueryFeatureMapSerializationBenchmark]] - */ -private[asyncfeaturemap] class AsyncFeatureMapSerializer() extends JsonSerializer[AsyncFeatureMap] { - override def serialize( - asyncFeatureMap: AsyncFeatureMap, - gen: JsonGenerator, - serializers: SerializerProvider - ): Unit = { - gen.writeStartObject() - - asyncFeatureMap.features.foreach { - case (stepIdentifier, featureHydrators) => - gen.writeObjectFieldStart(stepIdentifier.toString) - - featureHydrators.foreach { - case (hydratorIdentifier, featuresFromHydrator) => - gen.writeArrayFieldStart(hydratorIdentifier.toString) - - featuresFromHydrator.foreach(feature => gen.writeString(feature.toString)) - - gen.writeEndArray() - } - - gen.writeEndObject() - } - - gen.writeEndObject() - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap/BUILD deleted file mode 100644 index 2d193a3b8..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "stitch/stitch-core", - "util/util-jackson/src/main/scala/com/twitter/util/jackson", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap/BUILD.docx new file mode 100644 index 000000000..c2f9d657f Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/asyncfeaturemap/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord/BUILD deleted file mode 100644 index 8511d47bc..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/featurevalue", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "src/java/com/twitter/ml/api:api-base", - "src/scala/com/twitter/ml/api/util:datarecord", - "util/util-core", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "util/util-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord/BUILD.docx new file mode 100644 index 000000000..9ac99fffe Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord/DataRecordConverter.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord/DataRecordConverter.docx new file mode 100644 index 000000000..b7d137b1f Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord/DataRecordConverter.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord/DataRecordConverter.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord/DataRecordConverter.scala deleted file mode 100644 index 56533bd88..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord/DataRecordConverter.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.product_mixer.core.feature.featuremap.datarecord - -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.DataRecordMerger -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.datarecord._ -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap - -object DataRecordConverter { - val merger = new DataRecordMerger -} - -/** - * Constructs a FeatureMap from a DataRecord, given a predefined set of features from a FeaturesScope. - * - * @param featuresScope scope of predefined set of BaseDataRecordFeatures that should be included in the output FeatureMap. - */ -class DataRecordConverter[DRFeature <: BaseDataRecordFeature[_, _]]( - featuresScope: FeaturesScope[DRFeature]) { - import DataRecordConverter._ - - def toDataRecord(featureMap: FeatureMap): DataRecord = { - // Initialize a DataRecord with the Feature Store features in it and then add all the - // non-Feature Store features that support DataRecords to DataRecord. We don't - // need to add Feature Store features because they're already in the initial DataRecord. - // If there are any pre-built DataRecords, we merge those in. - val richDataRecord = featuresScope.getFeatureStoreFeaturesDataRecord(featureMap) - val features = featuresScope.getNonFeatureStoreDataRecordFeatures(featureMap) - features.foreach { - case _: FeatureStoreDataRecordFeature[_, _] => - case requiredFeature: DataRecordFeature[_, _] with DataRecordCompatible[_] => - richDataRecord.setFeatureValue( - requiredFeature.mlFeature, - requiredFeature.toDataRecordFeatureValue( - featureMap.get(requiredFeature).asInstanceOf[requiredFeature.FeatureType])) - case optionalFeature: DataRecordOptionalFeature[_, _] with DataRecordCompatible[_] => - featureMap - .get( - optionalFeature.asInstanceOf[Feature[_, Option[optionalFeature.FeatureType]]]).foreach { - value => - richDataRecord - .setFeatureValue( - optionalFeature.mlFeature, - optionalFeature.toDataRecordFeatureValue(value)) - } - case dataRecordInAFeature: DataRecordInAFeature[_] => - merger.merge(richDataRecord.getRecord, featureMap.get(dataRecordInAFeature)) - } - richDataRecord.getRecord - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord/DataRecordExtractor.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord/DataRecordExtractor.docx new file mode 100644 index 000000000..805570559 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord/DataRecordExtractor.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord/DataRecordExtractor.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord/DataRecordExtractor.scala deleted file mode 100644 index a91b01d22..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord/DataRecordExtractor.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.product_mixer.core.feature.featuremap.datarecord - -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.util.SRichDataRecord -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.datarecord._ -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.pipeline.pipeline_failure.IllegalStateFailure -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import scala.collection.JavaConverters._ - -/** - * Constructs a DataRecord from a FeatureMap, given a predefined set of features. - * - * @param features predefined set of BaseDataRecordFeatures that should be included in the output DataRecord. - */ -class DataRecordExtractor[DRFeature <: BaseDataRecordFeature[_, _]]( - features: Set[DRFeature]) { - - private val featureContext = new FeatureContext(features.collect { - case dataRecordCompatible: DataRecordCompatible[_] => dataRecordCompatible.mlFeature - }.asJava) - - def fromDataRecord(dataRecord: DataRecord): FeatureMap = { - val featureMapBuilder = FeatureMapBuilder() - val richDataRecord = SRichDataRecord(dataRecord, featureContext) - features.foreach { - // FeatureStoreDataRecordFeature is currently not supported - case _: FeatureStoreDataRecordFeature[_, _] => - throw new UnsupportedOperationException( - "FeatureStoreDataRecordFeature cannot be extracted from a DataRecord") - case feature: DataRecordFeature[_, _] with DataRecordCompatible[_] => - // Java API will return null, so use Option to convert it to Scala Option which is None when null. - richDataRecord.getFeatureValueOpt(feature.mlFeature)( - feature.fromDataRecordFeatureValue) match { - case Some(value) => - featureMapBuilder.add(feature.asInstanceOf[Feature[_, feature.FeatureType]], value) - case None => - featureMapBuilder.addFailure( - feature, - PipelineFailure( - IllegalStateFailure, - s"Required DataRecord feature is missing: ${feature.mlFeature.getFeatureName}") - ) - } - case feature: DataRecordOptionalFeature[_, _] with DataRecordCompatible[_] => - val featureValue = - richDataRecord.getFeatureValueOpt(feature.mlFeature)(feature.fromDataRecordFeatureValue) - featureMapBuilder - .add(feature.asInstanceOf[Feature[_, Option[feature.FeatureType]]], featureValue) - // DataRecordInAFeature is currently not supported - case _: DataRecordInAFeature[_] => - throw new UnsupportedOperationException( - "DataRecordInAFeature cannot be extracted from a DataRecord") - } - featureMapBuilder.build() - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord/FeaturesScope.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord/FeaturesScope.docx new file mode 100644 index 000000000..67b02c5cb Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord/FeaturesScope.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord/FeaturesScope.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord/FeaturesScope.scala deleted file mode 100644 index 8ef1898a9..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord/FeaturesScope.scala +++ /dev/null @@ -1,157 +0,0 @@ -package com.twitter.product_mixer.core.feature.featuremap.datarecord - -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.util.SRichDataRecord -import scala.collection.JavaConverters._ -import com.twitter.product_mixer.core.feature.datarecord.BaseDataRecordFeature -import com.twitter.product_mixer.core.feature.datarecord.DataRecordCompatible -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featurestorev1.FeatureStoreV1Feature -import com.twitter.product_mixer.core.feature.featurestorev1.featurevalue.FeatureStoreV1ResponseFeature - -/** - * FeaturesScope for defining what features should be included in a DataRecord from a FeatureMap. - * Where possible, prefer [[SpecificFeatures]]. It fails loudly on missing features which can help - * identify programmer error, but can be complex to manage for multi-phase hydrators. - */ -sealed trait FeaturesScope[+DRFeature <: BaseDataRecordFeature[_, _]] { - def getNonFeatureStoreDataRecordFeatures(featureMap: FeatureMap): Seq[DRFeature] - - /** - * Because Feature Store features aren't direct features in the FeatureMap and instead live - * aggregated in a DataRecord in our Feature Map, we need to interface with the underlying Data - * Record instead. e.g. for the `AllFeatures` case, we won't know what all FStore ProMix Features - * we have in a FeatureMap just by looping through features & need to just return the DataRecord. - */ - def getFeatureStoreFeaturesDataRecord(featureMap: FeatureMap): SRichDataRecord -} - -/** - * Use all DataRecord features on a FeatureMap to output a DataRecord. - */ -case class AllFeatures[-Entity]() extends FeaturesScope[BaseDataRecordFeature[Entity, _]] { - override def getNonFeatureStoreDataRecordFeatures( - featureMap: FeatureMap - ): Seq[BaseDataRecordFeature[Entity, _]] = { - - /** - * See [[com.twitter.product_mixer.core.benchmark.FeatureMapBenchmark]] - * - * `toSeq`` is a no-op, `view`` makes later compositions lazy. Currently we only perform a `forEach` - * on the result but `view` here has no performance impact but protects us if we accidentally add - * more compositions in the middle. - * - * Feature Store features aren't in the FeatureMap so this will only ever return the non-FStore Features. - */ - featureMap.getFeatures.toSeq.view.collect { - case feature: BaseDataRecordFeature[Entity, _] => feature - } - } - - // Get the entire underlying DataRecord if available. - override def getFeatureStoreFeaturesDataRecord( - featureMap: FeatureMap - ): SRichDataRecord = if (featureMap.getFeatures.contains(FeatureStoreV1ResponseFeature)) { - // Note, we do not copy over the feature context because JRichDataRecord will enforce that - // all features are in the FeatureContext which we do not know at init time, and it's pricey - // to compute at run time. - SRichDataRecord(featureMap.get(FeatureStoreV1ResponseFeature).richDataRecord.getRecord) - } else { - SRichDataRecord(new DataRecord()) - } -} - -/** - * Build a DataRecord with only the given features from the FeatureMap used. Missing features - * will fail loudly. - * @param features the specific features to include in the DataRecord. - */ -case class SpecificFeatures[DRFeature <: BaseDataRecordFeature[_, _]]( - features: Set[DRFeature]) - extends FeaturesScope[DRFeature] { - - private val featuresForContext = features.collect { - case featureStoreFeatures: FeatureStoreV1Feature[_, _, _, _] => - featureStoreFeatures.boundFeature.mlApiFeature - case dataRecordCompatible: DataRecordCompatible[_] => dataRecordCompatible.mlFeature - }.asJava - - private val featureContext = new FeatureContext(featuresForContext) - - private val fsFeatures = features - .collect { - case featureStoreV1Feature: FeatureStoreV1Feature[_, _, _, _] => - featureStoreV1Feature - } - - // Since it's possible a customer will pass feature store features in the DR Feature list, let's - // partition them out to only return non-FS ones in getFeatures. See [[FeaturesScope]] comment. - private val nonFsFeatures: Seq[DRFeature] = features.flatMap { - case _: FeatureStoreV1Feature[_, _, _, _] => - None - case otherFeature => Some(otherFeature) - }.toSeq - - override def getNonFeatureStoreDataRecordFeatures( - featureMap: FeatureMap - ): Seq[DRFeature] = nonFsFeatures - - override def getFeatureStoreFeaturesDataRecord( - featureMap: FeatureMap - ): SRichDataRecord = - if (fsFeatures.nonEmpty && featureMap.getFeatures.contains(FeatureStoreV1ResponseFeature)) { - // Return a DataRecord only with the explicitly requested features set. - val richDataRecord = SRichDataRecord(new DataRecord(), featureContext) - val existingDataRecord = featureMap.get(FeatureStoreV1ResponseFeature).richDataRecord - fsFeatures.foreach { feature => - richDataRecord.setFeatureValue( - feature.boundFeature.mlApiFeature, - existingDataRecord.getFeatureValue(feature.boundFeature.mlApiFeature)) - } - richDataRecord - } else { - SRichDataRecord(new DataRecord()) - } -} - -/** - * Build a DataRecord with every feature available in a FeatureMap except for the ones provided. - * @param featuresToExclude the features to be excluded in the DataRecord. - */ -case class AllExceptFeatures( - featuresToExclude: Set[BaseDataRecordFeature[_, _]]) - extends FeaturesScope[BaseDataRecordFeature[_, _]] { - - private val fsFeatures = featuresToExclude - .collect { - case featureStoreV1Feature: FeatureStoreV1Feature[_, _, _, _] => - featureStoreV1Feature - } - - override def getNonFeatureStoreDataRecordFeatures( - featureMap: FeatureMap - ): Seq[BaseDataRecordFeature[_, _]] = - featureMap.getFeatures - .collect { - case feature: BaseDataRecordFeature[_, _] => feature - }.filterNot(featuresToExclude.contains).toSeq - - override def getFeatureStoreFeaturesDataRecord( - featureMap: FeatureMap - ): SRichDataRecord = if (featureMap.getFeatures.contains(FeatureStoreV1ResponseFeature)) { - // Return a data record only with the explicitly requested features set. Do this by copying - // the existing one and removing the features in the denylist. - // Note, we do not copy over the feature context because JRichDataRecord will enforce that - // all features are in the FeatureContext which we do not know at init time, and it's pricey - // to compute at run time. - val richDataRecord = SRichDataRecord( - featureMap.get(FeatureStoreV1ResponseFeature).richDataRecord.getRecord.deepCopy()) - fsFeatures.foreach { feature => - richDataRecord.clearFeature(feature.boundFeature.mlApiFeature) - } - richDataRecord - } else { - SRichDataRecord(new DataRecord()) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/featurestorev1/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/featurestorev1/BUILD deleted file mode 100644 index b34c59913..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/featurestorev1/BUILD +++ /dev/null @@ -1,17 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/featurevalue", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/featurevalue", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/featurestorev1/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/featurestorev1/BUILD.docx new file mode 100644 index 000000000..9b853b0e6 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/featurestorev1/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/featurestorev1/FeatureStoreV1FeatureMap.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/featurestorev1/FeatureStoreV1FeatureMap.docx new file mode 100644 index 000000000..46a738eaf Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/featurestorev1/FeatureStoreV1FeatureMap.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/featurestorev1/FeatureStoreV1FeatureMap.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/featurestorev1/FeatureStoreV1FeatureMap.scala deleted file mode 100644 index 54abfc05e..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/featurestorev1/FeatureStoreV1FeatureMap.scala +++ /dev/null @@ -1,191 +0,0 @@ -package com.twitter.product_mixer.core.feature.featuremap.featurestorev1 - -import com.twitter.ml.api.DataRecord -import com.twitter.ml.featurestore.lib.EntityId -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.MissingFeatureException -import com.twitter.product_mixer.core.feature.featurestorev1.FeatureStoreV1CandidateFeature -import com.twitter.product_mixer.core.feature.featurestorev1.FeatureStoreV1CandidateFeatureGroup -import com.twitter.product_mixer.core.feature.featurestorev1.FeatureStoreV1QueryFeature -import com.twitter.product_mixer.core.feature.featurestorev1.FeatureStoreV1QueryFeatureGroup -import com.twitter.product_mixer.core.feature.featurestorev1.featurevalue.FeatureStoreV1ResponseFeature -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.util.Try - -object FeatureStoreV1FeatureMap { - - /** - * Implicitly add convenience accessors for FeatureStoreV1 features in [[FeatureMap]]. Note that - * we cannot add these methods directly to [[FeatureMap]] because it would introduce a circular - * dependency ([[PipelineQuery]] depends on [[FeatureMap]], and the methods below depend on - * [[PipelineQuery]]) - * - * @param featureMap the featureMap we are wrapping - * @note The FeatureStoreV1Feature::defaultValue set on the BoundFeature is only used and set - * during PredictionRecord to DataRecord conversion. Therefore, the default will not be set - * on the PredictionRecord value if reading from it directly, and as such for convenience - * the defaultValue is manually returned during retrieval from PredictionRecord. - * @note the Value generic type on the methods below cannot be passed to - * FeatureStoreV1QueryFeature's Value generic type. While this is actually the same type, - * (note the explicit type cast back to Value), we must instead use an existential on - * FeatureStoreV1QueryFeature since it is constructed with an existential for the Value - * generic (see [[FeatureStoreV1QueryFeature]] and [[FeatureStoreV1CandidateFeature]]) - */ - implicit class FeatureStoreV1FeatureMapAccessors(private val featureMap: FeatureMap) { - - def getFeatureStoreV1QueryFeature[Query <: PipelineQuery, Value]( - feature: FeatureStoreV1QueryFeature[Query, _ <: EntityId, Value] - ): Value = - getOrElseFeatureStoreV1QueryFeature( - feature, - feature.defaultValue.getOrElse { - throw MissingFeatureException(feature) - }) - - def getFeatureStoreV1QueryFeatureTry[Query <: PipelineQuery, Value]( - feature: FeatureStoreV1QueryFeature[Query, _ <: EntityId, Value] - ): Try[Value] = - Try(getFeatureStoreV1QueryFeature(feature)) - - def getOrElseFeatureStoreV1QueryFeature[Query <: PipelineQuery, Value]( - feature: FeatureStoreV1QueryFeature[Query, _ <: EntityId, Value], - default: => Value - ): Value = { - - /** - * FeatureStoreV1ResponseFeature should never be missing from the FeatureMap as FSv1 is - * guaranteed to return a prediction record per feature store request. However, this may be - * called on candidates that never hydrated FSv1 features. For example by - * [[com.twitter.product_mixer.component_library.selector.sorter.featurestorev1.FeatureStoreV1FeatureValueSorter]] - */ - val featureStoreV1FeatureValueOpt = featureMap.getTry(FeatureStoreV1ResponseFeature).toOption - - val dataRecordValue: Option[Value] = featureStoreV1FeatureValueOpt.flatMap { - featureStoreV1FeatureValue => - featureStoreV1FeatureValue.richDataRecord.getFeatureValueOpt( - feature.boundFeature.mlApiFeature)(feature.fromDataRecordValue) - } - - dataRecordValue.getOrElse(default) - } - - def getFeatureStoreV1CandidateFeature[ - Query <: PipelineQuery, - Candidate <: UniversalNoun[Any], - Value - ]( - feature: FeatureStoreV1CandidateFeature[Query, Candidate, _ <: EntityId, Value] - ): Value = - getOrElseFeatureStoreV1CandidateFeature( - feature, - feature.defaultValue.getOrElse { - throw MissingFeatureException(feature) - }) - - def getFeatureStoreV1CandidateFeatureTry[ - Query <: PipelineQuery, - Candidate <: UniversalNoun[Any], - Value - ]( - feature: FeatureStoreV1CandidateFeature[Query, Candidate, _ <: EntityId, Value] - ): Try[Value] = - Try(getFeatureStoreV1CandidateFeature(feature)) - - def getOrElseFeatureStoreV1CandidateFeature[ - Query <: PipelineQuery, - Candidate <: UniversalNoun[Any], - Value - ]( - feature: FeatureStoreV1CandidateFeature[Query, Candidate, _ <: EntityId, Value], - default: => Value - ): Value = { - - /** - * FeatureStoreV1ResponseFeature should never be missing from the FeatureMap as FSv1 is - * guaranteed to return a prediction record per feature store request. However, this may be - * called on candidates that never hydrated FSv1 features. For example by - * [[com.twitter.product_mixer.component_library.selector.sorter.featurestorev1.FeatureStoreV1FeatureValueSorter]] - */ - val featureStoreV1FeatureValueOpt = featureMap.getTry(FeatureStoreV1ResponseFeature).toOption - - val dataRecordValue: Option[Value] = featureStoreV1FeatureValueOpt.flatMap { - featureStoreV1FeatureValue => - featureStoreV1FeatureValue.richDataRecord.getFeatureValueOpt( - feature.boundFeature.mlApiFeature)(feature.fromDataRecordValue) - } - - dataRecordValue.getOrElse(default) - } - - /** - * Get queryFeatureGroup, which is store in the featureMap as a DataRecordInAFeature - * It doesn't have the mlApiFeature as other regular FeatureStoreV1 features - * Please refer to [[com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature]] scaladoc for more details - */ - def getFeatureStoreV1QueryFeatureGroup[Query <: PipelineQuery]( - featureGroup: FeatureStoreV1QueryFeatureGroup[Query, _ <: EntityId] - ): DataRecord = - getOrElseFeatureStoreV1QueryFeatureGroup( - featureGroup, - throw MissingFeatureException(featureGroup) - ) - - def getFeatureStoreV1CandidateFeatureGroupTry[Query <: PipelineQuery]( - featureGroup: FeatureStoreV1QueryFeatureGroup[Query, _ <: EntityId] - ): Try[DataRecord] = - Try(getFeatureStoreV1QueryFeatureGroup(featureGroup)) - - def getOrElseFeatureStoreV1QueryFeatureGroup[Query <: PipelineQuery]( - featureGroup: FeatureStoreV1QueryFeatureGroup[Query, _ <: EntityId], - default: => DataRecord - ): DataRecord = { - featureMap.getTry(featureGroup).toOption.getOrElse(default) - } - - /** - * Get candidateFeatureGroup, which is store in the featureMap as a DataRecordInAFeature - * It doesn't have the mlApiFeature as other regular FeatureStoreV1 features - * Please refer to [[com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature]] scaladoc for more details - */ - def getFeatureStoreV1CandidateFeatureGroup[ - Query <: PipelineQuery, - Candidate <: UniversalNoun[Any] - ]( - featureGroup: FeatureStoreV1CandidateFeatureGroup[Query, Candidate, _ <: EntityId] - ): DataRecord = - getOrElseFeatureStoreV1CandidateFeatureGroup( - featureGroup, - throw MissingFeatureException(featureGroup) - ) - - def getFeatureStoreV1CandidateFeatureGroupTry[ - Query <: PipelineQuery, - Candidate <: UniversalNoun[Any] - ]( - featureGroup: FeatureStoreV1CandidateFeatureGroup[Query, Candidate, _ <: EntityId] - ): Try[DataRecord] = - Try(getFeatureStoreV1CandidateFeatureGroup(featureGroup)) - - def getOrElseFeatureStoreV1CandidateFeatureGroup[ - Query <: PipelineQuery, - Candidate <: UniversalNoun[Any] - ]( - featureGroup: FeatureStoreV1CandidateFeatureGroup[Query, Candidate, _ <: EntityId], - default: => DataRecord - ): DataRecord = { - featureMap.getTry(featureGroup).toOption.getOrElse(default) - } - - def getOrElseFeatureStoreV1FeatureDataRecord( - default: => DataRecord - ) = { - val featureStoreV1FeatureValueOpt = featureMap.getTry(FeatureStoreV1ResponseFeature).toOption - - featureStoreV1FeatureValueOpt - .map { featureStoreV1FeatureValue => - featureStoreV1FeatureValue.richDataRecord.getRecord - }.getOrElse(default) - } - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/BUILD deleted file mode 100644 index 408a2b569..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/BUILD +++ /dev/null @@ -1,29 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/scala/com/twitter/ml/featurestore/catalog/entities/core", - "src/scala/com/twitter/ml/featurestore/lib/data", - "src/scala/com/twitter/ml/featurestore/lib/dynamic", - "src/scala/com/twitter/ml/featurestore/lib/feature", - "src/scala/com/twitter/ml/featurestore/lib/feature:aggregates-feature-group", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/scala/com/twitter/ml/featurestore/catalog/entities/core", - "src/scala/com/twitter/ml/featurestore/lib/data", - "src/scala/com/twitter/ml/featurestore/lib/dynamic", - "src/scala/com/twitter/ml/featurestore/lib/feature", - "src/scala/com/twitter/ml/featurestore/lib/feature:aggregates-feature-group", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/BUILD.docx new file mode 100644 index 000000000..daec26180 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/FeatureStoreV1Entity.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/FeatureStoreV1Entity.docx new file mode 100644 index 000000000..70834a81b Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/FeatureStoreV1Entity.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/FeatureStoreV1Entity.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/FeatureStoreV1Entity.scala deleted file mode 100644 index bd47a18c4..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/FeatureStoreV1Entity.scala +++ /dev/null @@ -1,35 +0,0 @@ -package com.twitter.product_mixer.core.feature.featurestorev1 - -import com.twitter.ml.featurestore.lib.EntityId -import com.twitter.ml.featurestore.lib.entity.Entity -import com.twitter.ml.featurestore.lib.entity.EntityWithId -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -sealed trait FeatureStoreV1Entity[ - -Query <: PipelineQuery, - -Input, - FeatureStoreEntityId <: EntityId] { - - val entity: Entity[FeatureStoreEntityId] -} - -trait FeatureStoreV1QueryEntity[-Query <: PipelineQuery, FeatureStoreEntityId <: EntityId] - extends FeatureStoreV1Entity[Query, Query, FeatureStoreEntityId] { - - def entityWithId(query: Query): EntityWithId[FeatureStoreEntityId] -} - -trait FeatureStoreV1CandidateEntity[ - -Query <: PipelineQuery, - -Input <: UniversalNoun[Any], - FeatureStoreEntityId <: EntityId] - extends FeatureStoreV1Entity[Query, Input, FeatureStoreEntityId] { - - def entityWithId( - query: Query, - input: Input, - existingFeatures: FeatureMap - ): EntityWithId[FeatureStoreEntityId] -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/FeatureStoreV1Feature.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/FeatureStoreV1Feature.docx new file mode 100644 index 000000000..9f2869433 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/FeatureStoreV1Feature.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/FeatureStoreV1Feature.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/FeatureStoreV1Feature.scala deleted file mode 100644 index 0dc64e19e..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/FeatureStoreV1Feature.scala +++ /dev/null @@ -1,312 +0,0 @@ -package com.twitter.product_mixer.core.feature.featurestorev1 - -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.transform.FeatureRenameTransform -import com.twitter.ml.featurestore.lib.EntityId -import com.twitter.ml.featurestore.lib.dynamic.BaseGatedFeatures -import com.twitter.ml.featurestore.lib.feature.BoundFeature -import com.twitter.ml.featurestore.lib.feature.BoundFeatureSet -import com.twitter.ml.featurestore.lib.feature.TimelinesAggregationFrameworkFeatureGroup -import com.twitter.ml.featurestore.lib.feature.{Feature => FSv1Feature} -import com.twitter.product_mixer.core.feature.ModelFeatureName -import com.twitter.product_mixer.core.feature.datarecord.FeatureStoreDataRecordFeature -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.util.{Gate => ServoGate} -import com.twitter.timelines.configapi.FSParam -import scala.reflect.ClassTag - -/** - * The base trait for all feature store features on ProMix. This should not be constructed directly - * and should instead be used through the other implementations below - * @tparam Query Product Mixer Query Type - * @tparam Input The input type the feature should be keyed on, this is same as Query for query - * features and - * @tparam FeatureStoreEntityId Feature Store Entity Type - * @tparam Value The type of the value of this feature. - */ -sealed trait BaseFeatureStoreV1Feature[ - -Query <: PipelineQuery, - -Input, - FeatureStoreEntityId <: EntityId, - Value] - extends FeatureStoreDataRecordFeature[Input, Value] - with BaseGatedFeatures[Query] { - val fsv1Feature: FSv1Feature[FeatureStoreEntityId, Value] - - val entity: FeatureStoreV1Entity[Query, Input, FeatureStoreEntityId] - - val enabledParam: Option[FSParam[Boolean]] - - override final lazy val gate: ServoGate[Query] = enabledParam - .map { param => - new ServoGate[PipelineQuery] { - override def apply[U](query: U)(implicit asT: <:<[U, PipelineQuery]): Boolean = { - query.params(param) - } - } - }.getOrElse(ServoGate.True) - - override final lazy val boundFeatureSet: BoundFeatureSet = new BoundFeatureSet(Set(boundFeature)) - - val boundFeature: BoundFeature[FeatureStoreEntityId, Value] - - /** - * Since this trait is normally constructed inline, avoid the anonymous toString and use the bounded feature name. - */ - override lazy val toString: String = boundFeature.name -} - -/** - * A unitary (non-aggregate group) feature store feature in ProMix. This should be constructed using - * [[FeatureStoreV1CandidateFeature]] or [[FeatureStoreV1QueryFeature]]. - * @tparam Query Product Mixer Query Type - * @tparam Input The input type the feature should be keyed on, this is same as Query for query - * features and - * @tparam FeatureStoreEntityId Feature Store Entity Type - * @tparam Value The type of the value of this feature. - */ -sealed trait FeatureStoreV1Feature[ - -Query <: PipelineQuery, - -Input, - FeatureStoreEntityId <: EntityId, - Value] - extends BaseFeatureStoreV1Feature[Query, Input, FeatureStoreEntityId, Value] - with ModelFeatureName { - - val legacyName: Option[String] - val defaultValue: Option[Value] - - override lazy val featureName: String = boundFeature.name - - override final lazy val boundFeature = (legacyName, defaultValue) match { - case (Some(legacyName), Some(defaultValue)) => - fsv1Feature.bind(entity.entity).withLegacyName(legacyName).withDefault(defaultValue) - case (Some(legacyName), _) => - fsv1Feature.bind(entity.entity).withLegacyName(legacyName) - case (_, Some(defaultValue)) => - fsv1Feature.bind(entity.entity).withDefault(defaultValue) - case _ => - fsv1Feature.bind(entity.entity) - } - - def fromDataRecordValue(recordValue: boundFeature.feature.mfc.V): Value = - boundFeature.feature.mfc.fromDataRecordValue(recordValue) -} - -/** - * A feature store aggregated group feature in ProMix. This should be constructed using - * [[FeatureStoreV1CandidateFeatureGroup]] or [[FeatureStoreV1QueryFeatureGroup]]. - * - * @tparam Query Product Mixer Query Type - * @tparam Input The input type the feature should be keyed on, this is same as Query for query - * features and - * @tparam FeatureStoreEntityId Feature Store Entity Type - */ -abstract class FeatureStoreV1FeatureGroup[ - -Query <: PipelineQuery, - -Input, - FeatureStoreEntityId <: EntityId: ClassTag] - extends BaseFeatureStoreV1Feature[Query, Input, FeatureStoreEntityId, DataRecord] { - val keepLegacyNames: Boolean - val featureNameTransform: Option[FeatureRenameTransform] - - val featureGroup: TimelinesAggregationFrameworkFeatureGroup[FeatureStoreEntityId] - - override lazy val fsv1Feature: FSv1Feature[FeatureStoreEntityId, DataRecord] = - featureGroup.FeaturesAsDataRecord - - override final lazy val boundFeature = (keepLegacyNames, featureNameTransform) match { - case (_, Some(transform)) => - fsv1Feature.bind(entity.entity).withLegacyIndividualFeatureNames(transform) - case (true, _) => - fsv1Feature.bind(entity.entity).keepLegacyNames - case _ => - fsv1Feature.bind(entity.entity) - } -} - -sealed trait BaseFeatureStoreV1QueryFeature[ - -Query <: PipelineQuery, - FeatureStoreEntityId <: EntityId, - Value] - extends BaseFeatureStoreV1Feature[Query, Query, FeatureStoreEntityId, Value] { - - override val entity: FeatureStoreV1QueryEntity[Query, FeatureStoreEntityId] -} - -trait FeatureStoreV1QueryFeature[-Query <: PipelineQuery, FeatureStoreEntityId <: EntityId, Value] - extends FeatureStoreV1Feature[Query, Query, FeatureStoreEntityId, Value] - with BaseFeatureStoreV1QueryFeature[Query, FeatureStoreEntityId, Value] - -trait FeatureStoreV1QueryFeatureGroup[-Query <: PipelineQuery, FeatureStoreEntityId <: EntityId] - extends FeatureStoreV1FeatureGroup[Query, Query, FeatureStoreEntityId] - with BaseFeatureStoreV1QueryFeature[Query, FeatureStoreEntityId, DataRecord] - -object FeatureStoreV1QueryFeature { - - /** - * Query-based Feature Store backed feature - * @param feature The underling feature store feature this represents. - * @param _entity The entity for binding the Feature Store features - * @param _legacyName Feature Store legacy name if required - * @param _defaultValue The default value to return for this feature if not hydrated. - * @param _enabledParam The Feature Switch Param to gate this feature, always enabled if none. - * @tparam Query The Product Mixer query type this feature is keyed on. - * @tparam FeatureStoreEntityId Feature Store Entity ID - * @tparam Value The type of the value this feature contains. - * @return Product Mixer Feature - */ - def apply[Query <: PipelineQuery, FeatureStoreEntityId <: EntityId, Value]( - feature: FSv1Feature[FeatureStoreEntityId, Value], - _entity: FeatureStoreV1QueryEntity[Query, FeatureStoreEntityId], - _legacyName: Option[String] = None, - _defaultValue: Option[Value] = None, - _enabledParam: Option[FSParam[Boolean]] = None - ): FeatureStoreV1QueryFeature[Query, FeatureStoreEntityId, Value] = - new FeatureStoreV1QueryFeature[Query, FeatureStoreEntityId, Value] { - override val fsv1Feature: FSv1Feature[FeatureStoreEntityId, Value] = feature - override val entity: FeatureStoreV1QueryEntity[Query, FeatureStoreEntityId] = _entity - override val legacyName: Option[String] = _legacyName - override val defaultValue: Option[Value] = _defaultValue - override val enabledParam: Option[FSParam[Boolean]] = _enabledParam - } -} - -object FeatureStoreV1QueryFeatureGroup { - - /** - * Query-based Feature Store Aggregated group backed feature - * - * @param featureGroup The underling aggregation group feature this represents. - * @param _entity The entity for binding the Feature Store features - * @param _enabledParam The Feature Switch Param to gate this feature, always enabled if none. - * @param _keepLegacyNames Whether to keep the legacy names as is for the entire group - * @param _featureNameTransform Rename the entire group's legacy names using the [[FeatureRenameTransform]] - * @tparam Query The Product Mixer query type this feature is keyed on. - * @tparam FeatureStoreEntityId Feature Store Entity ID - * - * @return Product Mixer Feature - */ - def apply[Query <: PipelineQuery, FeatureStoreEntityId <: EntityId: ClassTag]( - _featureGroup: TimelinesAggregationFrameworkFeatureGroup[FeatureStoreEntityId], - _entity: FeatureStoreV1QueryEntity[Query, FeatureStoreEntityId], - _enabledParam: Option[FSParam[Boolean]] = None, - _keepLegacyNames: Boolean = false, - _featureNameTransform: Option[FeatureRenameTransform] = None - ): FeatureStoreV1QueryFeatureGroup[Query, FeatureStoreEntityId] = - new FeatureStoreV1QueryFeatureGroup[Query, FeatureStoreEntityId] { - override val entity: FeatureStoreV1QueryEntity[Query, FeatureStoreEntityId] = _entity - override val featureGroup: TimelinesAggregationFrameworkFeatureGroup[ - FeatureStoreEntityId - ] = _featureGroup - - override val enabledParam: Option[FSParam[Boolean]] = _enabledParam - - override val keepLegacyNames: Boolean = _keepLegacyNames - override val featureNameTransform: Option[FeatureRenameTransform] = _featureNameTransform - } -} - -sealed trait BaseFeatureStoreV1CandidateFeature[ - -Query <: PipelineQuery, - -Input <: UniversalNoun[Any], - FeatureStoreEntityId <: EntityId, - Value] - extends BaseFeatureStoreV1Feature[Query, Input, FeatureStoreEntityId, Value] { - - override val entity: FeatureStoreV1CandidateEntity[Query, Input, FeatureStoreEntityId] -} - -trait FeatureStoreV1CandidateFeature[ - -Query <: PipelineQuery, - -Input <: UniversalNoun[Any], - FeatureStoreEntityId <: EntityId, - Value] - extends FeatureStoreV1Feature[Query, Input, FeatureStoreEntityId, Value] - with BaseFeatureStoreV1CandidateFeature[Query, Input, FeatureStoreEntityId, Value] - -trait FeatureStoreV1CandidateFeatureGroup[ - -Query <: PipelineQuery, - -Input <: UniversalNoun[Any], - FeatureStoreEntityId <: EntityId] - extends FeatureStoreV1FeatureGroup[Query, Input, FeatureStoreEntityId] - with BaseFeatureStoreV1CandidateFeature[Query, Input, FeatureStoreEntityId, DataRecord] - -object FeatureStoreV1CandidateFeature { - - /** - * Candidate-based Feature Store backed feature - * @param feature The underling feature store feature this represents. - * @param _entity The entity for binding the Feature Store features - * @param _legacyName Feature Store legacy name if required - * @param _defaultValue The default value to return for this feature if not hydrated. - * @param _enabledParam The Feature Switch Param to gate this feature, always enabled if none. - * @tparam Query The Product Mixer query type this feature is keyed on. - * @tparam FeatureStoreEntityId The feature store entity type - * @tparam Input The type of the candidate this feature is keyed on - * @tparam Value The type of value this feature contains. - * @return Product Mixer Feature - */ - def apply[ - Query <: PipelineQuery, - Input <: UniversalNoun[Any], - FeatureStoreEntityId <: EntityId, - Value - ]( - feature: FSv1Feature[FeatureStoreEntityId, Value], - _entity: FeatureStoreV1CandidateEntity[Query, Input, FeatureStoreEntityId], - _legacyName: Option[String] = None, - _defaultValue: Option[Value] = None, - _enabledParam: Option[FSParam[Boolean]] = None - ): FeatureStoreV1CandidateFeature[Query, Input, FeatureStoreEntityId, Value] = - new FeatureStoreV1CandidateFeature[Query, Input, FeatureStoreEntityId, Value] { - override val fsv1Feature: FSv1Feature[FeatureStoreEntityId, Value] = feature - override val entity: FeatureStoreV1CandidateEntity[Query, Input, FeatureStoreEntityId] = - _entity - override val legacyName: Option[String] = _legacyName - override val defaultValue: Option[Value] = _defaultValue - override val enabledParam: Option[FSParam[Boolean]] = _enabledParam - } -} - -object FeatureStoreV1CandidateFeatureGroup { - - /** - * Candidate-based Feature Store Aggregated group backed feature - * - * @param featureGroup The underling aggregation group feature this represents. - * @param _entity The entity for binding the Feature Store features - * @param _enabledParam The Feature Switch Param to gate this feature, always enabled if none. - * @param _keepLegacyNames Whether to keep the legacy names as is for the entire group - * @param _featureNameTransform Rename the entire group's legacy names using the [[FeatureRenameTransform]] - * @tparam Query The Product Mixer query type this feature is keyed on. - * @tparam Input The type of the candidate this feature is keyed on - * @tparam FeatureStoreEntityId Feature Store Entity ID - * - * @return Product Mixer Feature - */ - def apply[ - Query <: PipelineQuery, - Input <: UniversalNoun[Any], - FeatureStoreEntityId <: EntityId: ClassTag, - ]( - _featureGroup: TimelinesAggregationFrameworkFeatureGroup[FeatureStoreEntityId], - _entity: FeatureStoreV1CandidateEntity[Query, Input, FeatureStoreEntityId], - _enabledParam: Option[FSParam[Boolean]] = None, - _keepLegacyNames: Boolean = false, - _featureNameTransform: Option[FeatureRenameTransform] = None - ): FeatureStoreV1CandidateFeatureGroup[Query, Input, FeatureStoreEntityId] = - new FeatureStoreV1CandidateFeatureGroup[Query, Input, FeatureStoreEntityId] { - override val entity: FeatureStoreV1CandidateEntity[Query, Input, FeatureStoreEntityId] = - _entity - override val featureGroup: TimelinesAggregationFrameworkFeatureGroup[ - FeatureStoreEntityId - ] = _featureGroup - - override val enabledParam: Option[FSParam[Boolean]] = _enabledParam - - override val keepLegacyNames: Boolean = _keepLegacyNames - override val featureNameTransform: Option[FeatureRenameTransform] = _featureNameTransform - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/featurevalue/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/featurevalue/BUILD deleted file mode 100644 index 931ea442a..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/featurevalue/BUILD +++ /dev/null @@ -1,19 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "src/scala/com/twitter/ml/api/util:datarecord", - "src/scala/com/twitter/ml/featurestore/lib/data", - "src/scala/com/twitter/ml/featurestore/lib/dynamic", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "src/scala/com/twitter/ml/api/util:datarecord", - "src/scala/com/twitter/ml/featurestore/lib/data", - "src/scala/com/twitter/ml/featurestore/lib/dynamic", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/featurevalue/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/featurevalue/BUILD.docx new file mode 100644 index 000000000..e0e3a1dee Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/featurevalue/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/featurevalue/FeatureStoreV1Response.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/featurevalue/FeatureStoreV1Response.docx new file mode 100644 index 000000000..1028bfe1a Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/featurevalue/FeatureStoreV1Response.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/featurevalue/FeatureStoreV1Response.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/featurevalue/FeatureStoreV1Response.scala deleted file mode 100644 index e1f495346..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1/featurevalue/FeatureStoreV1Response.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.twitter.product_mixer.core.feature.featurestorev1.featurevalue - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties -import com.fasterxml.jackson.annotation.JsonProperty -import com.twitter.ml.api.DataRecordMerger -import com.twitter.ml.api.util.SRichDataRecord -import com.twitter.ml.featurestore.lib.data.HydrationError -import com.twitter.product_mixer.core.feature.Feature - -private[product_mixer] object FeatureStoreV1ResponseFeature - extends Feature[Any, FeatureStoreV1Response] - -@JsonIgnoreProperties(Array("richDataRecord", "failedFeatures")) -private[product_mixer] case class FeatureStoreV1Response( - @JsonProperty("richDataRecord") richDataRecord: SRichDataRecord, - @JsonProperty("failedFeatures") failedFeatures: Map[_ <: Feature[_, _], Set[HydrationError]]) { - // Since RichDataRecord is Java, we need to override this. - override def equals(obj: Any): Boolean = obj match { - case that: FeatureStoreV1Response => - failedFeatures == that.failedFeatures && richDataRecord.getRecord.equals( - that.richDataRecord.getRecord) - case _ => false - } -} - -private[product_mixer] object FeatureStoreV1Response { - val dataRecordMerger = new DataRecordMerger - def merge( - left: FeatureStoreV1Response, - right: FeatureStoreV1Response - ): FeatureStoreV1Response = { - val newDataRecord = left.richDataRecord.getRecord.deepCopy() - dataRecordMerger.merge(newDataRecord, right.richDataRecord.getRecord) - FeatureStoreV1Response( - richDataRecord = SRichDataRecord(newDataRecord), - left.failedFeatures ++ right.failedFeatures - ) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/access_policy/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/access_policy/BUILD deleted file mode 100644 index cb0e59b7c..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/access_policy/BUILD +++ /dev/null @@ -1,9 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/fasterxml/jackson/core:jackson-annotations", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/access_policy/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/access_policy/BUILD.docx new file mode 100644 index 000000000..eb49d9b28 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/access_policy/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/BUILD deleted file mode 100644 index fe3e5b6bb..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "stitch/stitch-core", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/BUILD.docx new file mode 100644 index 000000000..f22be929c Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/CandidateSource.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/CandidateSource.docx new file mode 100644 index 000000000..ede24e8d8 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/CandidateSource.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/CandidateSource.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/CandidateSource.scala deleted file mode 100644 index 2d93ab91a..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/CandidateSource.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.candidate_source - -import com.twitter.product_mixer.core.model.common.Component -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch - -sealed trait BaseCandidateSource[-Request, +Candidate] extends Component { - - /** @see [[CandidateSourceIdentifier]] */ - val identifier: CandidateSourceIdentifier -} - -/** - * A [[CandidateSource]] returns a Seq of ''potential'' content - * - * @note [[CandidateSource]]s that return a single value need to transform - * it into a Seq, either by doing `Seq(value)` or extracting - * candidates from the value. - * - * @tparam Request arguments to get the potential content - * @tparam Candidate the potential content - */ -trait CandidateSource[-Request, +Candidate] extends BaseCandidateSource[Request, Candidate] { - - /** returns a Seq of ''potential'' content */ - def apply(request: Request): Stitch[Seq[Candidate]] -} - -/** - * A [[CandidateSourceWithExtractedFeatures]] returns a result containing both a Seq of - * ''potential'' candidates as well as an extracted feature map that will later be appended - * to the pipeline's [[com.twitter.product_mixer.core.pipeline.PipelineQuery]] feature map. This is - * useful for candidate sources that return features that might be useful later on without needing - * to re-hydrate them. - * - * @note [[CandidateSource]]s that return a single value need to transform - * it into a Seq, either by doing `Seq(value)` or extracting - * candidates from the value. - * - * @tparam Request arguments to get the potential content - * @tparam Candidate the potential content - */ -trait CandidateSourceWithExtractedFeatures[-Request, +Candidate] - extends BaseCandidateSource[Request, Candidate] { - - /** returns a result containing a seq of ''potential'' content and extracted features - * from the candidate source. - * */ - def apply(request: Request): Stitch[CandidatesWithSourceFeatures[Candidate]] -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/CandidatesWithSourceFeatures.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/CandidatesWithSourceFeatures.docx new file mode 100644 index 000000000..b66754690 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/CandidatesWithSourceFeatures.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/CandidatesWithSourceFeatures.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/CandidatesWithSourceFeatures.scala deleted file mode 100644 index 124f9a044..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/CandidatesWithSourceFeatures.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.candidate_source - -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap - -/** - * Results from a candidate source, optionally carrying extracted query level features to add - * to the query's feature map (e.g, extracting reusable features from the thrift response of thrift - * call). - * @param candidates The candidates returned from the underlying CandidateSoure - * @param features [[FeatureMap]] containing the features from the candidate source - * to merge back into the PipelineQuery FeatureMap. - * @tparam Candidate The type of result - */ -case class CandidatesWithSourceFeatures[+Candidate]( - candidates: Seq[Candidate], - features: FeatureMap) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/PassthroughCandidateSource.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/PassthroughCandidateSource.docx new file mode 100644 index 000000000..74784520c Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/PassthroughCandidateSource.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/PassthroughCandidateSource.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/PassthroughCandidateSource.scala deleted file mode 100644 index ff51cace1..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/PassthroughCandidateSource.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.candidate_source - -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -/** - * Retrieve Candidates from the Query - */ -trait CandidateExtractor[-Request, +Candidate] { - - def apply(query: Request): Seq[Candidate] -} - -/** - * Identity extractor for returning the Request as a Seq of candidates - */ -case class IdentityCandidateExtractor[Request]() extends CandidateExtractor[Request, Request] { - - def apply(candidate: Request): Seq[Request] = Seq(candidate) -} - -/** - * Retrieve Candidates from a [[Feature]] on the [[PipelineQuery]]'s FeatureMap. This extractor - * supports a transform if the Feature value and the Seq of [[Candidate]] types do not match - */ -trait QueryFeatureCandidateExtractor[-Query <: PipelineQuery, FeatureValue, +Candidate] - extends CandidateExtractor[Query, Candidate] { - - def feature: Feature[Query, FeatureValue] - - override def apply(query: Query): Seq[Candidate] = - query.features.map(featureMap => transform(featureMap.get(feature))).getOrElse(Seq.empty) - - def transform(featureValue: FeatureValue): Seq[Candidate] -} - -/** - * Retrieve Candidates from a [[Feature]] on the [[PipelineQuery]]'s FeatureMap. This extractor can - * be used with a single [[Feature]] if the Feature value and the Seq of [[Candidate]] types match. - */ -case class CandidateQueryFeatureCandidateExtractor[-Query <: PipelineQuery, Candidate]( - override val feature: Feature[Query, Seq[Candidate]]) - extends QueryFeatureCandidateExtractor[Query, Seq[Candidate], Candidate] { - - override def transform(featureValue: Seq[Candidate]): Seq[Candidate] = featureValue -} - -/** - * A [[CandidateSource]] that retrieves candidates from the Request via a [[CandidateExtractor]] - */ -case class PassthroughCandidateSource[-Request, +Candidate]( - override val identifier: CandidateSourceIdentifier, - candidateExtractor: CandidateExtractor[Request, Candidate]) - extends CandidateSource[Request, Candidate] { - - def apply(query: Request): Stitch[Seq[Candidate]] = Stitch.value(candidateExtractor(query)) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/StaticCandidateSource.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/StaticCandidateSource.docx new file mode 100644 index 000000000..516f48f52 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/StaticCandidateSource.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/StaticCandidateSource.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/StaticCandidateSource.scala deleted file mode 100644 index 7a1f3d4dd..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/StaticCandidateSource.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.candidate_source - -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch - -/** - * A [[CandidateSource]] that always returns [[result]] regardless of the input - */ -case class StaticCandidateSource[Candidate]( - override val identifier: CandidateSourceIdentifier, - result: Seq[Candidate]) - extends CandidateSource[Any, Candidate] { - - def apply(request: Any): Stitch[Seq[Candidate]] = Stitch.value(result) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/product_pipeline/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/product_pipeline/BUILD deleted file mode 100644 index f0cc41a88..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/product_pipeline/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry", - "stitch/stitch-core", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry", - "stitch/stitch-core", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/product_pipeline/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/product_pipeline/BUILD.docx new file mode 100644 index 000000000..96ab8fc8d Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/product_pipeline/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/product_pipeline/ProductPipelineCandidateSource.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/product_pipeline/ProductPipelineCandidateSource.docx new file mode 100644 index 000000000..f808e9147 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/product_pipeline/ProductPipelineCandidateSource.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/product_pipeline/ProductPipelineCandidateSource.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/product_pipeline/ProductPipelineCandidateSource.scala deleted file mode 100644 index 646b09928..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/product_pipeline/ProductPipelineCandidateSource.scala +++ /dev/null @@ -1,71 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.candidate_source.product_pipeline - -import com.google.inject.Provider -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder -import com.twitter.product_mixer.core.model.marshalling.request.Request -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.product.ProductPipelineRequest -import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry -import com.twitter.stitch.Stitch -import scala.reflect.runtime.universe._ - -/** - * A [[CandidateSource]] for getting candidates from a different - * [[com.twitter.product_mixer.core.model.marshalling.request.Product]] within the same Product - * Mixer-based service. This is useful when calling a RecommendationPipeline-based Product from a - * MixerPipeline-based Product. In this scenario, the two Products can remain - * independent and encapsulated within the Product Mixer service, which provides future optionality - * for migrating one of the two products into a new Product Mixer-based service based on the - * scaling needs. - * - * @tparam Query [[PipelineQuery]] from the originating Product - * @tparam MixerRequest the [[Request]] domain model for the Product Mixer service. Adds a Context - * bound (syntactic sugar) to add TypeTag to implicit scope for - * [[ProductPipelineRegistry.getProductPipeline()]]. Note that `trait` does not - * support context bounds, so this abstraction is expressed as an - * `abstract class`. - * @tparam ProductPipelineResult the return type of the candidate source Product. Adds a Context - * bound (syntactic sugar) to add TypeTag to implicit scope for - * [[ProductPipelineRegistry.getProductPipeline()]] - * @tparam Candidate the type of candidate returned by this candidate source, which is typically - * extracted from within the ProductPipelineResult type - */ -abstract class ProductPipelineCandidateSource[ - -Query <: PipelineQuery, - MixerRequest <: Request: TypeTag, - ProductPipelineResult: TypeTag, - +Candidate] - extends CandidateSource[Query, Candidate] { - - /** - * @note Define as a Guice [[Provider]] in order to break the circular injection dependency - */ - val productPipelineRegistry: Provider[ProductPipelineRegistry] - - /** - * @note Define as a Guice [[Provider]] in order to break the circular injection dependency - */ - val paramsBuilder: Provider[ParamsBuilder] - - def pipelineRequestTransformer(currentPipelineQuery: Query): MixerRequest - - def productPipelineResultTransformer(productPipelineResult: ProductPipelineResult): Seq[Candidate] - - override def apply(query: Query): Stitch[Seq[Candidate]] = { - val request = pipelineRequestTransformer(query) - - val params = paramsBuilder - .get().build( - clientContext = request.clientContext, - product = request.product, - featureOverrides = request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty) - ) - - productPipelineRegistry - .get() - .getProductPipeline[MixerRequest, ProductPipelineResult](request.product) - .process(ProductPipelineRequest(request, params)) - .map(productPipelineResultTransformer) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/BUILD deleted file mode 100644 index 24dea9785..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - "stitch/stitch-core", - "strato/src/main/scala/com/twitter/strato/catalog", - "strato/src/main/scala/com/twitter/strato/client", - "strato/src/main/scala/com/twitter/strato/response", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "stitch/stitch-core", - "strato/src/main/scala/com/twitter/strato/catalog", - "strato/src/main/scala/com/twitter/strato/client", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/BUILD.docx new file mode 100644 index 000000000..bfa3e1bf2 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoErrCategorizer.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoErrCategorizer.docx new file mode 100644 index 000000000..6ebababa0 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoErrCategorizer.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoErrCategorizer.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoErrCategorizer.scala deleted file mode 100644 index 4e72f1c44..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoErrCategorizer.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.candidate_source.strato - -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.pipeline_failure.Unauthorized -import com.twitter.stitch.Stitch -import com.twitter.strato.response.Err - -/** - * Categorize Strato's Err messages to our PipelineFailures. - * - * This should be used by all strato-based candidate source, and we can - * add more cases here as they're useful. - */ -object StratoErrCategorizer { - val CategorizeStratoException: PartialFunction[Throwable, Stitch[Nothing]] = { - case err @ Err(Err.Authorization, reason, context) => - Stitch.exception( - PipelineFailure(Unauthorized, s"$reason [${context.toString}]", underlying = Some(err)) - ) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyFetcherSeqSource.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyFetcherSeqSource.docx new file mode 100644 index 000000000..c16017b80 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyFetcherSeqSource.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyFetcherSeqSource.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyFetcherSeqSource.scala deleted file mode 100644 index 19c8a6dba..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyFetcherSeqSource.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.candidate_source.strato - -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher - -/** - * A [[CandidateSource]] for getting Candidates from Strato where the - * Strato column's View is [[Unit]] and the Value is a Seq of [[StratoResult]] - * - * @tparam StratoKey the column's Key type - * @tparam StratoResult the column's Value's Seq type - */ -trait StratoKeyFetcherSeqSource[StratoKey, StratoResult] - extends CandidateSource[StratoKey, StratoResult] { - - val fetcher: Fetcher[StratoKey, Unit, Seq[StratoResult]] - - override def apply(key: StratoKey): Stitch[Seq[StratoResult]] = { - fetcher - .fetch(key) - .map { result => - result.v - .getOrElse(Seq.empty) - }.rescue(StratoErrCategorizer.CategorizeStratoException) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyFetcherSource.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyFetcherSource.docx new file mode 100644 index 000000000..a81654d19 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyFetcherSource.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyFetcherSource.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyFetcherSource.scala deleted file mode 100644 index 9d3f0e113..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyFetcherSource.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.candidate_source.strato - -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher - -/** - * A [[CandidateSource]] for getting Candidates from Strato where the - * Strato column's View is [[Unit]] and the Value is a [[StratoValue]] - * - * A `stratoResultTransformer` must be defined to convert the [[StratoValue]] into a Seq of [[Candidate]] - * - * If you need to extract features from the [[StratoValue]] (like a cursor), - * use [[StratoKeyFetcherWithSourceFeaturesSource]] instead. - * - * @tparam StratoKey the column's Key type - * @tparam StratoValue the column's Value type - */ -trait StratoKeyFetcherSource[StratoKey, StratoValue, Candidate] - extends CandidateSource[StratoKey, Candidate] { - - val fetcher: Fetcher[StratoKey, Unit, StratoValue] - - /** - * Transforms the value type returned by Strato into a Seq[Candidate]. - * - * This might be as simple as `Seq(stratoResult)` if you're always returning a single candidate. - * - * Often, it just extracts a Seq from within a larger wrapper object. - * - * If there is global metadata that you need to include, you can zip it with the candidates, - * returning something like Seq((candiate, metadata), (candidate, metadata)) etc. - */ - protected def stratoResultTransformer(stratoResult: StratoValue): Seq[Candidate] - - override def apply(key: StratoKey): Stitch[Seq[Candidate]] = { - fetcher - .fetch(key) - .map { result => - result.v - .map(stratoResultTransformer) - .getOrElse(Seq.empty) - }.rescue(StratoErrCategorizer.CategorizeStratoException) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyFetcherWithSourceFeaturesSource.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyFetcherWithSourceFeaturesSource.docx new file mode 100644 index 000000000..23c7202b0 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyFetcherWithSourceFeaturesSource.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyFetcherWithSourceFeaturesSource.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyFetcherWithSourceFeaturesSource.scala deleted file mode 100644 index d868d2862..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyFetcherWithSourceFeaturesSource.scala +++ /dev/null @@ -1,65 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.candidate_source.strato - -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSourceWithExtractedFeatures -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidatesWithSourceFeatures -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher - -/** - * A [[CandidateSource]] for getting Candidates from Strato where the - * Strato column's View is [[Unit]] and the Value is a [[StratoValue]] - * - * A [[stratoResultTransformer]] must be defined to convert the - * [[StratoValue]] into a Seq of [[Candidate]] - * - * A [[extractFeaturesFromStratoResult]] must be defined to extract a - * [[FeatureMap]] from the [[StratoValue]]. If you don't need to do that, - * use a [[StratoKeyFetcherSource]] instead. - * - * @tparam StratoKey the column's Key type - * @tparam StratoValue the column's Value type - */ -trait StratoKeyFetcherWithSourceFeaturesSource[StratoKey, StratoValue, Candidate] - extends CandidateSourceWithExtractedFeatures[StratoKey, Candidate] { - - val fetcher: Fetcher[StratoKey, Unit, StratoValue] - - /** - * Transforms the value type returned by Strato into a Seq[Candidate]. - * - * This might be as simple as `Seq(stratoResult)` if you're always returning a single candidate. - * - * Often, it just extracts a Seq from within a larger wrapper object. - * - * If there is global metadata that you need to include, see [[extractFeaturesFromStratoResult]] - * below to put that into a Feature. - */ - protected def stratoResultTransformer(stratoResult: StratoValue): Seq[Candidate] - - /*** - * Transforms the value type returned by Strato into a FeatureMap. - * - * Override this to extract global metadata like cursors and place the results - * into a Feature. - * - * For example, a cursor. - */ - protected def extractFeaturesFromStratoResult(stratoResult: StratoValue): FeatureMap - - override def apply(key: StratoKey): Stitch[CandidatesWithSourceFeatures[Candidate]] = { - fetcher - .fetch(key) - .map { result => - val candidates = result.v - .map(stratoResultTransformer) - .getOrElse(Seq.empty) - - val features = result.v - .map(extractFeaturesFromStratoResult) - .getOrElse(FeatureMap.empty) - - CandidatesWithSourceFeatures(candidates, features) - }.rescue(StratoErrCategorizer.CategorizeStratoException) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyView.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyView.docx new file mode 100644 index 000000000..b316eb36b Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyView.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyView.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyView.scala deleted file mode 100644 index 564ea253c..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyView.scala +++ /dev/null @@ -1,4 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.candidate_source.strato - -/** Represents a Strato column's Key and View arguments */ -case class StratoKeyView[StratoKey, StratoView](key: StratoKey, view: StratoView) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyViewFetcherSeqSource.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyViewFetcherSeqSource.docx new file mode 100644 index 000000000..d86bb1643 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyViewFetcherSeqSource.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyViewFetcherSeqSource.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyViewFetcherSeqSource.scala deleted file mode 100644 index f1bcddbd3..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyViewFetcherSeqSource.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.candidate_source.strato - -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher - -/** - * A [[CandidateSource]] for getting Candidates from Strato where the - * Strato column's View is [[StratoView]] and the Value is a Seq of [[StratoResult]] - * - * @tparam StratoKey the column's Key type - * @tparam StratoView the column's View type - * @tparam StratoResult the column's Value's Seq type - */ -trait StratoKeyViewFetcherSeqSource[StratoKey, StratoView, StratoResult] - extends CandidateSource[StratoKeyView[StratoKey, StratoView], StratoResult] { - - val fetcher: Fetcher[StratoKey, StratoView, Seq[StratoResult]] - - override def apply( - request: StratoKeyView[StratoKey, StratoView] - ): Stitch[Seq[StratoResult]] = { - fetcher - .fetch(request.key, request.view) - .map { result => - result.v - .getOrElse(Seq.empty) - }.rescue(StratoErrCategorizer.CategorizeStratoException) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyViewFetcherSource.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyViewFetcherSource.docx new file mode 100644 index 000000000..76ba62781 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyViewFetcherSource.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyViewFetcherSource.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyViewFetcherSource.scala deleted file mode 100644 index 718dc632a..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyViewFetcherSource.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.candidate_source.strato - -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher - -/** - * A [[CandidateSource]] for getting Candidates from Strato where the - * Strato column's View is [[StratoView]] and the Value is a [[StratoValue]] - * - * A `stratoResultTransformer` must be defined to convert the [[StratoValue]] into a Seq of [[Candidate]] - * - * If you need to extract features from the [[StratoValue]] (like a cursor), - * use [[StratoKeyViewFetcherWithSourceFeaturesSource]] instead. - * - * @tparam StratoKey the column's Key type - * @tparam StratoView the column's View type - * @tparam StratoValue the column's Value type - */ -trait StratoKeyViewFetcherSource[StratoKey, StratoView, StratoValue, Candidate] - extends CandidateSource[StratoKeyView[StratoKey, StratoView], Candidate] { - - val fetcher: Fetcher[StratoKey, StratoView, StratoValue] - - /** - * Transforms the value type returned by Strato into a Seq[Candidate]. - * - * This might be as simple as `Seq(stratoResult)` if you're always returning a single candidate. - * - * Often, it just extracts a Seq from within a larger wrapper object. - * - * If there is global metadata that you need to include, you can zip it with the candidates, - * returning something like Seq((candiate, metadata), (candidate, metadata)) etc. - */ - protected def stratoResultTransformer( - stratoKey: StratoKey, - stratoResult: StratoValue - ): Seq[Candidate] - - override def apply( - request: StratoKeyView[StratoKey, StratoView] - ): Stitch[Seq[Candidate]] = { - fetcher - .fetch(request.key, request.view) - .map { result => - result.v - .map((stratoResult: StratoValue) => stratoResultTransformer(request.key, stratoResult)) - .getOrElse(Seq.empty) - }.rescue(StratoErrCategorizer.CategorizeStratoException) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyViewFetcherWithSourceFeaturesSource.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyViewFetcherWithSourceFeaturesSource.docx new file mode 100644 index 000000000..83e81e51a Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyViewFetcherWithSourceFeaturesSource.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyViewFetcherWithSourceFeaturesSource.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyViewFetcherWithSourceFeaturesSource.scala deleted file mode 100644 index 65678a14c..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato/StratoKeyViewFetcherWithSourceFeaturesSource.scala +++ /dev/null @@ -1,75 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.candidate_source.strato - -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSourceWithExtractedFeatures -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidatesWithSourceFeatures -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher - -/** - * A [[CandidateSource]] for getting Candidates from Strato where the - * Strato column's View is [[StratoView]] and the Value is a [[StratoValue]] - * - * A [[stratoResultTransformer]] must be defined to convert the - * [[StratoValue]] into a Seq of [[Candidate]] - * - * [[extractFeaturesFromStratoResult]] must be defined to extract a - * [[FeatureMap]] from the [[StratoValue]]. If you don't need to do that, - * use a [[StratoKeyViewFetcherSource]] instead. - * - * @tparam StratoKey the column's Key type - * @tparam StratoView the column's View type - * @tparam StratoValue the column's Value type - */ -trait StratoKeyViewFetcherWithSourceFeaturesSource[StratoKey, StratoView, StratoValue, Candidate] - extends CandidateSourceWithExtractedFeatures[StratoKeyView[StratoKey, StratoView], Candidate] { - - val fetcher: Fetcher[StratoKey, StratoView, StratoValue] - - /** - * Transforms the value type returned by Strato into a Seq[Candidate]. - * - * This might be as simple as `Seq(stratoResult)` if you're always returning a single candidate. - * - * Often, it just extracts a Seq from within a larger wrapper object. - * - * If there is global metadata that you need to include, see [[extractFeaturesFromStratoResult]] - * below to put that into a Feature. - */ - protected def stratoResultTransformer( - stratoKey: StratoKey, - stratoResult: StratoValue - ): Seq[Candidate] - - /** - * Transforms the value type returned by Strato into a FeatureMap. - * - * Override this to extract global metadata like cursors and place the results - * into a Feature. - * - * For example, a cursor. - */ - protected def extractFeaturesFromStratoResult( - stratoKey: StratoKey, - stratoResult: StratoValue - ): FeatureMap - - override def apply( - request: StratoKeyView[StratoKey, StratoView] - ): Stitch[CandidatesWithSourceFeatures[Candidate]] = { - fetcher - .fetch(request.key, request.view) - .map { result => - val candidates = result.v - .map((stratoResult: StratoValue) => stratoResultTransformer(request.key, stratoResult)) - .getOrElse(Seq.empty) - - val features = result.v - .map((stratoResult: StratoValue) => - extractFeaturesFromStratoResult(request.key, stratoResult)) - .getOrElse(FeatureMap.empty) - - CandidatesWithSourceFeatures(candidates, features) - }.rescue(StratoErrCategorizer.CategorizeStratoException) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/BUILD deleted file mode 100644 index c0bae6c88..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/BUILD +++ /dev/null @@ -1,15 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/BUILD.docx new file mode 100644 index 000000000..66f8f3e8c Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/CandidateScope.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/CandidateScope.docx new file mode 100644 index 000000000..304830a70 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/CandidateScope.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/CandidateScope.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/CandidateScope.scala deleted file mode 100644 index b01d0207b..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/CandidateScope.scala +++ /dev/null @@ -1,98 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.common - -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.CandidatePipelines - -/** - * Specifies whether a function component (e.g, [[Gate]] or [[Selector]]) - * should apply to a given [[CandidateWithDetails]] - */ -sealed trait CandidateScope { - - /** - * returns True if the provided `candidate` is in scope - */ - def contains(candidate: CandidateWithDetails): Boolean - - /** partitions `candidates` into those that this scope [[contains]] and those it does not */ - final def partition( - candidates: Seq[CandidateWithDetails] - ): CandidateScope.PartitionedCandidates = { - val (candidatesInScope, candidatesOutOfScope) = candidates.partition(contains) - CandidateScope.PartitionedCandidates(candidatesInScope, candidatesOutOfScope) - } -} - -object CandidateScope { - case class PartitionedCandidates( - candidatesInScope: Seq[CandidateWithDetails], - candidatesOutOfScope: Seq[CandidateWithDetails]) -} - -/** - * A [[CandidateScope]] that applies the given functional component - * to all candidates regardless of which pipeline is their [[com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails.source]]. - */ -case object AllPipelines extends CandidateScope { - override def contains(candidate: CandidateWithDetails): Boolean = true -} - -/** - * A [[CandidateScope]] that applies the given [[com.twitter.product_mixer.core.functional_component.selector.Selector]] - * only to candidates whose [[com.twitter.product_mixer.core.model.common.presentation.CandidatePipelines]] - * has an Identifier in the [[pipelines]] Set. - * In most cases where candidates are not pre-merged, the Set contains the candidate pipeline identifier the candidate - * came from. In the case where a candidate's feature maps were merged using [[CombineFeatureMapsCandidateMerger]], the - * set contains all candidate pipelines the merged candidate came from and this scope will include the candidate if any - * of the pipelines match. - */ -case class SpecificPipelines(pipelines: Set[CandidatePipelineIdentifier]) extends CandidateScope { - - require( - pipelines.nonEmpty, - "Expected `SpecificPipelines` have a non-empty Set of CandidatePipelineIdentifiers.") - - override def contains(candidate: CandidateWithDetails): Boolean = { - candidate.features.get(CandidatePipelines).exists(pipelines.contains) - } -} - -/** - * A [[CandidateScope]] that applies the given [[com.twitter.product_mixer.core.functional_component.selector.Selector]] - * only to candidates whose [[com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails.source]] - * is [[pipeline]]. - */ -case class SpecificPipeline(pipeline: CandidatePipelineIdentifier) extends CandidateScope { - - override def contains(candidate: CandidateWithDetails): Boolean = candidate.features - .get(CandidatePipelines).contains(pipeline) -} - -object SpecificPipelines { - def apply( - pipeline: CandidatePipelineIdentifier, - pipelines: CandidatePipelineIdentifier* - ): CandidateScope = { - if (pipelines.isEmpty) - SpecificPipeline(pipeline) - else - SpecificPipelines((pipeline +: pipelines).toSet) - } -} - -/** - * A [[CandidateScope]] that applies the given [[com.twitter.product_mixer.core.functional_component.selector.Selector]] - * to all candidates except for the candidates whose [[com.twitter.product_mixer.core.model.common.presentation.CandidatePipelines]] - * has an Identifier in the [[pipelinesToExclude]] Set. - * In most cases where candidates are not pre-merged, the Set contains the candidate pipeline identifier the candidate - * came from. In the case where a candidate's feature maps were merged using [[CombineFeatureMapsCandidateMerger]], the - * set contains all candidate pipelines the merged candidate came from and this scope will include the candidate if any - * of the pipelines match. - */ -case class AllExceptPipelines( - pipelinesToExclude: Set[CandidatePipelineIdentifier]) - extends CandidateScope { - override def contains(candidate: CandidateWithDetails): Boolean = !candidate.features - .get(CandidatePipelines).exists(pipelinesToExclude.contains) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy/AccessPolicy.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy/AccessPolicy.docx new file mode 100644 index 000000000..6ba74336b Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy/AccessPolicy.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy/AccessPolicy.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy/AccessPolicy.scala deleted file mode 100644 index 4a1d9d11c..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy/AccessPolicy.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.common.access_policy - -import com.fasterxml.jackson.annotation.JsonSubTypes -import com.fasterxml.jackson.annotation.JsonTypeInfo - -/** - * The Access Policy to set for gating querying in the turntable tool. - * - * @note implementations must be simple case classes with unique structures for serialization - */ -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY) -@JsonSubTypes( - Array( - new JsonSubTypes.Type(value = classOf[AllowedLdapGroups], name = "AllowedLdapGroups"), - new JsonSubTypes.Type(value = classOf[BlockEverything], name = "BlockEverything") - ) -) -sealed trait AccessPolicy - -/** - * Users must be in *at least* one of the provided LDAP groups in order to make a query. - * - * @param groups LDAP groups allowed to access this product - */ -case class AllowedLdapGroups(groups: Set[String]) extends AccessPolicy - -object AllowedLdapGroups { - def apply(group: String): AllowedLdapGroups = AllowedLdapGroups(Set(group)) -} - -/** - * Block all requests to a product. - * - * @note this needs to be a case class rather than an object because classOf doesn't work on objects - * and JsonSubTypes requires the annotation argument to be a constant (ruling out .getClass). - * This issue may be resolved in Scala 2.13: https://github.com/scala/scala/pull/9279 - */ -case class BlockEverything() extends AccessPolicy diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy/AccessPolicyEvaluator.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy/AccessPolicyEvaluator.docx new file mode 100644 index 000000000..b4d8f093f Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy/AccessPolicyEvaluator.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy/AccessPolicyEvaluator.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy/AccessPolicyEvaluator.scala deleted file mode 100644 index eaaa24fac..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy/AccessPolicyEvaluator.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.common.access_policy - -/** - * Controls how access policies are applied to allow/reject a request - */ -object AccessPolicyEvaluator { - def evaluate(productAccessPolicies: Set[AccessPolicy], userLdapGroups: Set[String]): Boolean = - productAccessPolicies.exists { - case AllowedLdapGroups(allowedGroups) => allowedGroups.exists(userLdapGroups.contains) - case _: BlockEverything => false - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy/BUILD deleted file mode 100644 index 0c605fff9..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy/BUILD +++ /dev/null @@ -1,15 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/fasterxml/jackson/core:jackson-annotations", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - ], - exports = [ - "3rdparty/jvm/com/fasterxml/jackson/core:jackson-annotations", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy/BUILD.docx new file mode 100644 index 000000000..521ff6f8c Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy/WithDebugAccessPolicies.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy/WithDebugAccessPolicies.docx new file mode 100644 index 000000000..41a0699d5 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy/WithDebugAccessPolicies.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy/WithDebugAccessPolicies.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy/WithDebugAccessPolicies.scala deleted file mode 100644 index 52322802c..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/access_policy/WithDebugAccessPolicies.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.common.access_policy - -import com.twitter.product_mixer.core.model.common.Component - -private[core] trait WithDebugAccessPolicies { self: Component => - - /** The [[AccessPolicy]]s that will be used for this component in turntable & other debug tooling - * to execute arbitrary queries against the component */ - val debugAccessPolicies: Set[AccessPolicy] = Set.empty -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/Alert.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/Alert.docx new file mode 100644 index 000000000..8b3329200 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/Alert.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/Alert.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/Alert.scala deleted file mode 100644 index b22a566a6..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/Alert.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.common.alert - -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.Predicate - -/** - * [[Alert]]s will trigger notifications to their [[NotificationGroup]] - * when the [[Predicate]]s are triggered. - */ -trait Alert { - - /** A group of alert levels and where the alerts for those levels should be sent */ - val notificationGroup: NotificationGroup - - /** Predicate indicating that the component is in a degraded state */ - val warnPredicate: Predicate - - /** Predicate indicating that the component is not functioning correctly */ - val criticalPredicate: Predicate - - /** An optional link to the runbook detailing how to respond to this alert */ - val runbookLink: Option[String] - - /** Indicates which metrics this [[Alert]] is for */ - val alertType: AlertType - - /** Where the metrics are from, @see [[Source]] */ - val source: Source = Server() - - /** A suffix to add to the end of the metric, this is often a [[Percentile]] */ - val metricSuffix: Option[String] = None -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/AlertType.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/AlertType.docx new file mode 100644 index 000000000..8ad615bb7 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/AlertType.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/AlertType.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/AlertType.scala deleted file mode 100644 index 58ae57b8a..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/AlertType.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.common.alert - -/** - * [[AlertType]] is used to indicate which metric an alert is for - * - * @note adding new [[AlertType]]s requires updating the dashboard generation code - */ -sealed trait AlertType { val metricType: String } - -/** Monitors the latency */ -case object Latency extends AlertType { override val metricType: String = "Latency" } - -/** Monitors the success rate __excluding__ client failures */ -case object SuccessRate extends AlertType { override val metricType: String = "SuccessRate" } - -/** Monitors the throughput */ -case object Throughput extends AlertType { override val metricType: String = "Throughput" } - -/** Monitors the empty response rate */ -case object EmptyResponseRate extends AlertType { - override val metricType: String = "EmptyResponseRate" -} - -/** Monitors the empty response size */ -case object ResponseSize extends AlertType { override val metricType: String = "ResponseSize" } diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/BUILD deleted file mode 100644 index bfdea7743..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/fasterxml/jackson/core:jackson-annotations", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/jvm/javax/mail", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/predicate", - "strato/src/main/scala/com/twitter/strato/catalog", - "util/util-core:scala", - ], - exports = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/predicate", - "strato/src/main/scala/com/twitter/strato/catalog", - "util/util-core:scala", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/BUILD.docx new file mode 100644 index 000000000..7ec6e0264 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/EmptyResponseRateAlert.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/EmptyResponseRateAlert.docx new file mode 100644 index 000000000..a08c6c3bc Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/EmptyResponseRateAlert.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/EmptyResponseRateAlert.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/EmptyResponseRateAlert.scala deleted file mode 100644 index d937abd80..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/EmptyResponseRateAlert.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.common.alert - -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfAbove - -/** - * [[EmptyResponseRateAlert]] triggers when the percentage of requests with empty responses (defined - * as the number of items returned excluding cursors) rises above the [[TriggerIfAbove]] threshold - * for a configured amount of time. - * - * @note EmptyResponseRate thresholds must be between 0 and 100% - */ -case class EmptyResponseRateAlert( - override val notificationGroup: NotificationGroup, - override val warnPredicate: TriggerIfAbove, - override val criticalPredicate: TriggerIfAbove, - override val runbookLink: Option[String] = None) - extends Alert { - override val alertType: AlertType = EmptyResponseRate - require( - warnPredicate.threshold > 0 && warnPredicate.threshold <= 100, - s"EmptyResponseRateAlert predicates must be between 0 and 100 but got warnPredicate = ${warnPredicate.threshold}" - ) - require( - criticalPredicate.threshold > 0 && criticalPredicate.threshold <= 100, - s"EmptyResponseRateAlert predicates must be between 0 and 100 but got criticalPredicate = ${criticalPredicate.threshold}" - ) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/GenericClientLatencyAlert.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/GenericClientLatencyAlert.docx new file mode 100644 index 000000000..5c2dc9042 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/GenericClientLatencyAlert.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/GenericClientLatencyAlert.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/GenericClientLatencyAlert.scala deleted file mode 100644 index a8a7c91ba..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/GenericClientLatencyAlert.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.common.alert - -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfLatencyAbove - -/** - * Similar to [[LatencyAlert]] but intended for use with an external client calling Product Mixer. - * - * [[GenericClientLatencyAlert]] triggers when the Latency for the specified client - * rises above the [[TriggerIfLatencyAbove]] threshold for the configured amount of time. - */ -case class GenericClientLatencyAlert( - override val source: GenericClient, - override val notificationGroup: NotificationGroup, - override val warnPredicate: TriggerIfLatencyAbove, - override val criticalPredicate: TriggerIfLatencyAbove, - override val runbookLink: Option[String] = None) - extends Alert { - override val alertType: AlertType = Latency -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/GenericClientSuccessRateAlert.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/GenericClientSuccessRateAlert.docx new file mode 100644 index 000000000..b40b4e88b Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/GenericClientSuccessRateAlert.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/GenericClientSuccessRateAlert.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/GenericClientSuccessRateAlert.scala deleted file mode 100644 index 27c03a2f9..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/GenericClientSuccessRateAlert.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.common.alert - -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfBelow - -/** - * Similar to [[SuccessRateAlert]] but intended for use with an external client calling Product Mixer - * - * [[GenericClientSuccessRateAlert]] triggers when the Success Rate for the external client - * drops below the [[TriggerIfBelow]] threshold for the configured amount of time - * - * @note SuccessRate thresholds must be between 0 and 100% - */ -case class GenericClientSuccessRateAlert( - override val source: GenericClient, - override val notificationGroup: NotificationGroup, - override val warnPredicate: TriggerIfBelow, - override val criticalPredicate: TriggerIfBelow, - override val runbookLink: Option[String] = None) - extends Alert { - override val alertType: AlertType = SuccessRate - require( - warnPredicate.threshold > 0 && warnPredicate.threshold <= 100, - s"SuccessRateAlert predicates must be between 0 and 100 but got warnPredicate = ${warnPredicate.threshold}" - ) - require( - criticalPredicate.threshold > 0 && criticalPredicate.threshold <= 100, - s"SuccessRateAlert predicates must be between 0 and 100 but got criticalPredicate = ${criticalPredicate.threshold}" - ) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/GenericClientThroughputAlert.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/GenericClientThroughputAlert.docx new file mode 100644 index 000000000..3ba2d7174 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/GenericClientThroughputAlert.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/GenericClientThroughputAlert.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/GenericClientThroughputAlert.scala deleted file mode 100644 index e69b226d1..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/GenericClientThroughputAlert.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.common.alert - -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.ThroughputPredicate - -/** - * Similar to [[ThroughputAlert]] but intended for an external client calling Product Mixer. - * - * [[GenericClientThroughputAlert]] triggers when the requests/sec for the external client - * is outside of the predicate set by a [[ThroughputPredicate]] for the configured amount of time - */ -case class GenericClientThroughputAlert( - override val source: GenericClient, - override val notificationGroup: NotificationGroup, - override val warnPredicate: ThroughputPredicate, - override val criticalPredicate: ThroughputPredicate, - override val runbookLink: Option[String] = None) - extends Alert { - override val alertType: AlertType = Throughput - require( - warnPredicate.threshold >= 0, - s"ThroughputAlert predicates must be >= 0 but got warnPredicate = ${warnPredicate.threshold}") - require( - criticalPredicate.threshold >= 0, - s"ThroughputAlert predicates must be >= 0 but got criticalPredicate = ${criticalPredicate.threshold}") -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/IsObservableFromStrato.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/IsObservableFromStrato.docx new file mode 100644 index 000000000..5682b0001 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/IsObservableFromStrato.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/IsObservableFromStrato.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/IsObservableFromStrato.scala deleted file mode 100644 index 08fe3f459..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/IsObservableFromStrato.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.common.alert - -/** - * Indicates that an [[Alert]] can be passed to [[StratoColumnAlert]]. Not all [[Alert]]s can be - * Strato alerts since our ability to observe from Strato's perspective is limited by the available - * metrics. - * - * @note can only be used in conjunction with [[Alert]] - */ -trait IsObservableFromStrato { _: Alert => } diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/LatencyAlert.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/LatencyAlert.docx new file mode 100644 index 000000000..6050bc826 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/LatencyAlert.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/LatencyAlert.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/LatencyAlert.scala deleted file mode 100644 index 47819f2d0..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/LatencyAlert.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.common.alert - -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfLatencyAbove - -/** - * [[GenericClientLatencyAlert]] triggers when the Latency for the component this is used with - * rises above the [[TriggerIfLatencyAbove]] threshold for the configured amount of time - */ -case class LatencyAlert( - override val notificationGroup: NotificationGroup, - percentile: Percentile, - override val warnPredicate: TriggerIfLatencyAbove, - override val criticalPredicate: TriggerIfLatencyAbove, - override val runbookLink: Option[String] = None) - extends Alert - with IsObservableFromStrato { - override val alertType: AlertType = Latency - - override val metricSuffix: Option[String] = Some(percentile.metricSuffix) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/NotificationGroup.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/NotificationGroup.docx new file mode 100644 index 000000000..594b71db6 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/NotificationGroup.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/NotificationGroup.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/NotificationGroup.scala deleted file mode 100644 index 400c122c8..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/NotificationGroup.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.common.alert - -import com.twitter.util.Try -import javax.mail.internet.InternetAddress - -/** - * Destination represents the place to which alerts will be sent. Often you will only need one field - * populated (either a Pager Duty key or a list of emails). - * - * See the Monitoring 2.0 docs for more information on [[https://docbird.twitter.biz/mon/how-to-guides.html?highlight=notificationgroup#set-up-email-pagerduty-and-slack-notifications setting up a Pager Duty rotation]] - */ -case class Destination( - pagerDutyKey: Option[String] = None, - emails: Seq[String] = Seq.empty) { - - require( - pagerDutyKey.forall(_.length == 32), - s"Expected `pagerDutyKey` to be 32 characters long but got `$pagerDutyKey`") - emails.foreach { email => - require( - Try(new InternetAddress(email).validate()).isReturn, - s"Expected only valid email addresses but got an invalid email address: `$email`") - } - require( - pagerDutyKey.nonEmpty || emails.nonEmpty, - s"Expected a `pagerDutyKey` or at least 1 email address but got neither") -} - -/** - * NotificationGroup maps alert levels to destinations. Having different destinations based on the - * urgency of the alert can sometimes be useful. For example, you could have a daytime on-call for - * warn alerts and a 24 on-call for critical alerts. - */ -case class NotificationGroup( - critical: Destination, - warn: Destination) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/Percentile.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/Percentile.docx new file mode 100644 index 000000000..699ca23e9 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/Percentile.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/Percentile.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/Percentile.scala deleted file mode 100644 index f2961b8bc..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/Percentile.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.common.alert - -/** - * [[Percentile]] is the specific metric that should be monitored. - * Some metrics such as Latency are recorded using [[https://twitter.github.io/util/docs/com/twitter/finagle/stats/Stat.html Stats]] - * the stats are recorded as various percentiles such as `my/stat.p95` or `my/stat.min`. - */ -sealed trait Percentile { val metricSuffix: String } -case object Min extends Percentile { override val metricSuffix: String = ".min" } -case object Avg extends Percentile { override val metricSuffix: String = ".avg" } -case object P50 extends Percentile { override val metricSuffix: String = ".p50" } -case object P90 extends Percentile { override val metricSuffix: String = ".p90" } -case object P95 extends Percentile { override val metricSuffix: String = ".p95" } -case object P99 extends Percentile { override val metricSuffix: String = ".p99" } -case object P999 extends Percentile { override val metricSuffix: String = ".p9990" } -case object P9999 extends Percentile { override val metricSuffix: String = ".p9999" } -case object Max extends Percentile { override val metricSuffix: String = ".max" } diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/ResponseSizeAlert.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/ResponseSizeAlert.docx new file mode 100644 index 000000000..5eef58541 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/ResponseSizeAlert.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/ResponseSizeAlert.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/ResponseSizeAlert.scala deleted file mode 100644 index 263354c24..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/ResponseSizeAlert.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.common.alert - -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.ThroughputPredicate - -/** - * [[ResponseSizeAlert]] triggers when the specified percentile of requests with empty responses (defined - * as the number of items returned excluding cursors) is beyond the [[ThroughputPredicate]] threshold - * for a configured amount of time. - */ -case class ResponseSizeAlert( - override val notificationGroup: NotificationGroup, - percentile: Percentile, - override val warnPredicate: ThroughputPredicate, - override val criticalPredicate: ThroughputPredicate, - override val runbookLink: Option[String] = None) - extends Alert { - override val metricSuffix: Option[String] = Some(percentile.metricSuffix) - override val alertType: AlertType = ResponseSize - require( - warnPredicate.threshold >= 0, - s"ResponseSizeAlert predicates must be >= 0 but got warnPredicate = ${warnPredicate.threshold}" - ) - require( - criticalPredicate.threshold >= 0, - s"ResponseSizeAlert predicates must be >= 0 but got criticalPredicate = ${criticalPredicate.threshold}" - ) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/Source.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/Source.docx new file mode 100644 index 000000000..7e652bf4b Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/Source.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/Source.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/Source.scala deleted file mode 100644 index 3e961c561..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/Source.scala +++ /dev/null @@ -1,47 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.common.alert - -import com.fasterxml.jackson.annotation.JsonSubTypes -import com.fasterxml.jackson.annotation.JsonTypeInfo - -/** - * where the metric originates from, such as from the server or from a client - * - * @note implementations must be simple case classes with unique structures for serialization - */ -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY) -@JsonSubTypes( - Array( - new JsonSubTypes.Type(value = classOf[Server], name = "Server"), - new JsonSubTypes.Type(value = classOf[Strato], name = "Strato"), - new JsonSubTypes.Type(value = classOf[GenericClient], name = "GenericClient") - ) -) -sealed trait Source - -/** metrics for the Product Mixer server */ -case class Server() extends Source - -/** metrics from the perspective of a Strato column */ -case class Strato(stratoColumnPath: String, stratoColumnOp: String) extends Source - -/** - * metrics from the perspective of a generic client - * - * @param displayName human readable name for the client - * @param service service referenced in the query, of the form .. - * @param metricSource the source of the metric query, usually of the form sd... - * @param failureMetric the name of the metric indicating a client failure - * @param requestMetric the name of the metric indicating a request has been made - * @param latencyMetric the name of the metric measuring a request's latency - * - * @note We strongly recommend the use of [[Strato]] where possible. [[GenericClient]] is provided as a - * catch-all source for teams that have unusual legacy call paths (such as Macaw). - */ -case class GenericClient( - displayName: String, - service: String, - metricSource: String, - failureMetric: String, - requestMetric: String, - latencyMetric: String) - extends Source diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/StratoColumnAlert.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/StratoColumnAlert.docx new file mode 100644 index 000000000..0b9e6e2a8 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/StratoColumnAlert.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/StratoColumnAlert.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/StratoColumnAlert.scala deleted file mode 100644 index 7f000e039..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/StratoColumnAlert.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.common.alert - -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.Predicate -import com.twitter.strato.catalog.OpTag - -/** - * triggers when the a Strato column's is outside of the predicate set by the provided [[Alert]] - * - * @note the [[Alert]] passed into a [[StratoColumnAlert]] - * can not be a [[StratoColumnAlert]] - */ -case class StratoColumnAlert(column: String, op: OpTag, alert: Alert with IsObservableFromStrato) - extends Alert { - - override val source: Source = Strato(column, op.tag) - override val notificationGroup: NotificationGroup = alert.notificationGroup - override val warnPredicate: Predicate = alert.warnPredicate - override val criticalPredicate: Predicate = alert.criticalPredicate - override val runbookLink: Option[String] = alert.runbookLink - override val alertType: AlertType = alert.alertType - override val metricSuffix: Option[String] = alert.metricSuffix -} - -object StratoColumnAlerts { - - /** Make a seq of Alerts for the provided Strato column */ - def apply( - column: String, - op: OpTag, - alerts: Seq[Alert with IsObservableFromStrato] - ): Seq[Alert] = { - alerts.map(StratoColumnAlert(column, op, _)) - } -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/SuccessRateAlert.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/SuccessRateAlert.docx new file mode 100644 index 000000000..88a040f25 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/SuccessRateAlert.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/SuccessRateAlert.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/SuccessRateAlert.scala deleted file mode 100644 index f8ccbf8a2..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/SuccessRateAlert.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.common.alert - -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfBelow - -/** - * [[SuccessRateAlert]] triggers when the Success Rate for the component this is used - * with drops below the [[TriggerIfBelow]] threshold for the configured amount of time - * - * @note SuccessRate thresholds must be between 0 and 100% - */ -case class SuccessRateAlert( - override val notificationGroup: NotificationGroup, - override val warnPredicate: TriggerIfBelow, - override val criticalPredicate: TriggerIfBelow, - override val runbookLink: Option[String] = None) - extends Alert - with IsObservableFromStrato { - override val alertType: AlertType = SuccessRate - require( - warnPredicate.threshold > 0 && warnPredicate.threshold <= 100, - s"SuccessRateAlert predicates must be between 0 and 100 but got warnPredicate = ${warnPredicate.threshold}" - ) - require( - criticalPredicate.threshold > 0 && criticalPredicate.threshold <= 100, - s"SuccessRateAlert predicates must be between 0 and 100 but got criticalPredicate = ${criticalPredicate.threshold}" - ) -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/ThroughputAlert.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/ThroughputAlert.docx new file mode 100644 index 000000000..01eee0e29 Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/ThroughputAlert.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/ThroughputAlert.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/ThroughputAlert.scala deleted file mode 100644 index d8a947ce1..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/ThroughputAlert.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.common.alert - -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.ThroughputPredicate - -/** - * [[ThroughputAlert]] triggers when the requests/sec for the component this is used - * with is outside of the predicate set by a [[ThroughputPredicate]] for - * the configured amount of time - */ -case class ThroughputAlert( - override val notificationGroup: NotificationGroup, - override val warnPredicate: ThroughputPredicate, - override val criticalPredicate: ThroughputPredicate, - override val runbookLink: Option[String] = None) - extends Alert - with IsObservableFromStrato { - override val alertType: AlertType = Throughput - require( - warnPredicate.threshold >= 0, - s"ThroughputAlert predicates must be >= 0 but got warnPredicate = ${warnPredicate.threshold}") - require( - criticalPredicate.threshold >= 0, - s"ThroughputAlert predicates must be >= 0 but got criticalPredicate = ${criticalPredicate.threshold}") -} diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/predicate/BUILD b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/predicate/BUILD deleted file mode 100644 index 1aee67e8e..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/predicate/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "util/util-core:scala", - ], - exports = [ - "util/util-core:scala", - ], -) diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/predicate/BUILD.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/predicate/BUILD.docx new file mode 100644 index 000000000..89503139f Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/predicate/BUILD.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/predicate/MetricGranularity.docx b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/predicate/MetricGranularity.docx new file mode 100644 index 000000000..b6a4a61aa Binary files /dev/null and b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/predicate/MetricGranularity.docx differ diff --git a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/predicate/MetricGranularity.scala b/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/predicate/MetricGranularity.scala deleted file mode 100644 index c0a6d2a56..000000000 --- a/product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert/predicate/MetricGranularity.scala +++ /dev/null @@ -1,35 +0,0 @@ -package com.twitter.product_mixer.core.functional_component.common.alert.predicate - -/** - * Specifies the metric granularity - * - * @see [[https://docbird.twitter.biz/mon/reference.html#predicate DURATION]] - */ -sealed trait MetricGranularity { val unit: String } - -/** - * Use minutely metrics and have alert durations in terms of minutes - * - * i.e. for a [[Predicate]] if [[Predicate.datapointsPastThreshold]] = 5 and [[Predicate.duration]] = 10 - * then the alert will trigger if there are at least 5 '''minutely''' metric points that are past the threshold - * in any 10 '''minute''' period - */ -case object Minutes extends MetricGranularity { override val unit: String = "m" } - -/** - * Use hourly metrics and have alert durations in terms of hours - * - * i.e. for a [[Predicate]] if [[Predicate.datapointsPastThreshold]] = 5 and [[Predicate.duration]] = 10 - * then the alert will trigger if there are at least 5 '''hourly''' metric points that are past the threshold - * in any 10 '''hour''' period - */ -case object Hours extends MetricGranularity { override val unit: String = "h" } - -/** - * Use daily metrics and have alert durations in terms of days - * - * i.e. for a [[Predicate]] if [[Predicate.datapointsPastThreshold]] = 5 and [[Predicate.duration]] = 10 - * then the alert will trigger if there are at least 5 '''daily''' metric points that are past the threshold - * in any 10 '''day''' period - */ -case object Days extends MetricGranularity { override val unit: String = "d" }