Mirrors/the-algorithm
2
0
Fork 0
mirror of https://github.com/twitter/the-algorithm.git synced 2024-11-16 08:29:21 +01:00
Code Issues Packages Projects Releases Wiki Activity

[docx] split commit for file 2400

Browse Source 
Signed-off-by: Ari Archer <ari.web.xyz@gmail.com>
This commit is contained in:
Ari Archer 2024-01-23 19:10:06 +02:00
parent b471ac86b4
commit 53c2f869f3
No known key found for this signature in database
GPG Key ID: A50D5B4B599AF8A2
400 changed files with 0 additions and 10865 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ClearCacheInstructionBuilder.docx Normal file
View File

Binary file not shown.

16
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ClearCacheInstructionBuilder.scala
View File

@ -1,16 +0,0 @@
package com.twitter.product_mixer.component_library.premarshaller.urt.builder
import com.twitter.product_mixer.core.model.marshalling.response.urt.ClearCacheTimelineInstruction
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry
import com.twitter.product_mixer.core.pipeline.PipelineQuery
case class ClearCacheInstructionBuilder[Query <: PipelineQuery](
override val includeInstruction: IncludeInstruction[Query] = AlwaysInclude)
extends UrtInstructionBuilder[Query, ClearCacheTimelineInstruction] {
override def build(
query: Query,
entries: Seq[TimelineEntry]
): Seq[ClearCacheTimelineInstruction] =
if (includeInstruction(query, entries)) Seq(ClearCacheTimelineInstruction()) else Seq.empty
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/FeaturePassThroughCursorBuilder.docx Normal file
View File

Binary file not shown.

33
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/FeaturePassThroughCursorBuilder.scala
View File

@ -1,33 +0,0 @@
package com.twitter.product_mixer.component_library.premarshaller.urt.builder
import com.twitter.product_mixer.component_library.model.cursor.UrtPassThroughCursor
import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorType
import com.twitter.product_mixer.core.pipeline.HasPipelineCursor
import com.twitter.product_mixer.core.pipeline.PipelineQuery
case class PassThroughCursorBuilder[
-Query <: PipelineQuery with HasPipelineCursor[UrtPassThroughCursor]
](
cursorFeature: Feature[Query, String],
override val cursorType: CursorType)
extends UrtCursorBuilder[Query] {
override val includeOperation: IncludeInstruction[Query] = { (query, _) =>
query.features.exists(_.getOrElse(cursorFeature, "").nonEmpty)
}
override def cursorValue(
query: Query,
entries: Seq[TimelineEntry]
): String =
UrtCursorSerializer.serializeCursor(
UrtPassThroughCursor(
cursorSortIndex(query, entries),
query.features.map(_.get(cursorFeature)).getOrElse(""),
cursorType = Some(cursorType)
)
)
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/IncludeInstruction.docx Normal file
View File

Binary file not shown.

31
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/IncludeInstruction.scala
View File

@ -1,31 +0,0 @@
package com.twitter.product_mixer.component_library.premarshaller.urt.builder
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry
import com.twitter.product_mixer.core.pipeline.HasPipelineCursor
import com.twitter.product_mixer.core.pipeline.PipelineQuery
trait IncludeInstruction[-Query <: PipelineQuery] { self =>
def apply(query: Query, entries: Seq[TimelineEntry]): Boolean
def inverse(): IncludeInstruction[Query] = new IncludeInstruction[Query] {
def apply(query: Query, entries: Seq[TimelineEntry]): Boolean = !self.apply(query, entries)
}
}
object AlwaysInclude extends IncludeInstruction[PipelineQuery] {
override def apply(query: PipelineQuery, entries: Seq[TimelineEntry]): Boolean = true
}
object IncludeOnFirstPage extends IncludeInstruction[PipelineQuery with HasPipelineCursor[_]] {
override def apply(
query: PipelineQuery with HasPipelineCursor[_],
entries: Seq[TimelineEntry]
): Boolean = query.isFirstPage
}
object IncludeAfterFirstPage extends IncludeInstruction[PipelineQuery with HasPipelineCursor[_]] {
override def apply(
query: PipelineQuery with HasPipelineCursor[_],
entries: Seq[TimelineEntry]
): Boolean = !query.isFirstPage
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/MarkUnreadInstructionBuilder.docx Normal file
View File

Binary file not shown.

32
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/MarkUnreadInstructionBuilder.scala
View File

@ -1,32 +0,0 @@
package com.twitter.product_mixer.component_library.premarshaller.urt.builder
import com.twitter.product_mixer.core.model.marshalling.response.urt.MarkEntriesUnreadInstruction
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.MarkUnreadableEntry
/**
* Build a MarkUnreadEntries instruction
*
* Note that this implementation currently supports top-level entries, but not module item entries.
*/
case class MarkUnreadInstructionBuilder[Query <: PipelineQuery](
override val includeInstruction: IncludeInstruction[Query] = AlwaysInclude)
extends UrtInstructionBuilder[Query, MarkEntriesUnreadInstruction] {
override def build(
query: Query,
entries: Seq[TimelineEntry]
): Seq[MarkEntriesUnreadInstruction] = {
if (includeInstruction(query, entries)) {
val filteredEntries = entries.collect {
case entry: MarkUnreadableEntry if entry.isMarkUnread.contains(true) =>
entry.entryIdentifier
}
if (filteredEntries.nonEmpty) Seq(MarkEntriesUnreadInstruction(filteredEntries))
else Seq.empty
} else {
Seq.empty
}
}
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/OrderedBottomCursorBuilder.docx Normal file
View File

Binary file not shown.

45
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/OrderedBottomCursorBuilder.scala
View File

@ -1,45 +0,0 @@
package com.twitter.product_mixer.component_library.premarshaller.urt.builder
import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor
import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.BottomCursor
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorType
import com.twitter.product_mixer.core.pipeline.HasPipelineCursor
import com.twitter.product_mixer.core.pipeline.PipelineCursorSerializer
import com.twitter.product_mixer.core.pipeline.PipelineQuery
/**
* Builds [[UrtOrderedCursor]] in the Bottom position
*
* @param idSelector Specifies the entry from which to derive the `id` field
* @param includeOperation Logic to determine whether or not to build the bottom cursor, which only
* applies if gap cursors are required (e.g. Home Latest). When applicable,
* this logic should always be the inverse of the logic used to decide
* whether or not to build the gap cursor via [[OrderedGapCursorBuilder]],
* since either the gap or the bottom cursor must always be returned.
* @param serializer Converts the cursor to an encoded string
*/
case class OrderedBottomCursorBuilder[
-Query <: PipelineQuery with HasPipelineCursor[UrtOrderedCursor]
](
idSelector: PartialFunction[TimelineEntry, Long],
override val includeOperation: IncludeInstruction[Query] = AlwaysInclude,
serializer: PipelineCursorSerializer[UrtOrderedCursor] = UrtCursorSerializer)
extends UrtCursorBuilder[Query] {
override val cursorType: CursorType = BottomCursor
override def cursorValue(query: Query, timelineEntries: Seq[TimelineEntry]): String = {
val bottomId = timelineEntries.reverseIterator.collectFirst(idSelector)
val id = bottomId.orElse(query.pipelineCursor.flatMap(_.id))
val cursor = UrtOrderedCursor(
initialSortIndex = nextBottomInitialSortIndex(query, timelineEntries),
id = id,
cursorType = Some(cursorType)
)
serializer.serializeCursor(cursor)
}
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/OrderedGapCursorBuilder.docx Normal file
View File

Binary file not shown.

54
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/OrderedGapCursorBuilder.scala
View File

@ -1,54 +0,0 @@
package com.twitter.product_mixer.component_library.premarshaller.urt.builder
import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor
import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorType
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor
import com.twitter.product_mixer.core.pipeline.HasPipelineCursor
import com.twitter.product_mixer.core.pipeline.PipelineCursorSerializer
import com.twitter.product_mixer.core.pipeline.PipelineQuery
/**
* Builds [[UrtOrderedCursor]] in the Bottom position as a Gap cursor.
*
* @param idSelector Specifies the entry from which to derive the `id` field
* @param includeOperation Logic to determine whether or not to build the gap cursor, which should
* always be the inverse of the logic used to decide whether or not to build
* the bottom cursor via [[OrderedBottomCursorBuilder]], since either the
* gap or the bottom cursor must always be returned.
* @param serializer Converts the cursor to an encoded string
*/
case class OrderedGapCursorBuilder[
-Query <: PipelineQuery with HasPipelineCursor[UrtOrderedCursor]
](
idSelector: PartialFunction[TimelineEntry, Long],
override val includeOperation: IncludeInstruction[Query],
serializer: PipelineCursorSerializer[UrtOrderedCursor] = UrtCursorSerializer)
extends UrtCursorBuilder[Query] {
override val cursorType: CursorType = GapCursor
override def cursorValue(
query: Query,
timelineEntries: Seq[TimelineEntry]
): String = {
// To determine the gap boundary, use any existing cursor gap boundary id (i.e. if submitted
// from a previous gap cursor, else use the existing cursor id (i.e. from a previous top cursor)
val gapBoundaryId = query.pipelineCursor.flatMap(_.gapBoundaryId).orElse {
query.pipelineCursor.flatMap(_.id)
}
val bottomId = timelineEntries.reverseIterator.collectFirst(idSelector)
val id = bottomId.orElse(gapBoundaryId)
val cursor = UrtOrderedCursor(
initialSortIndex = nextBottomInitialSortIndex(query, timelineEntries),
id = id,
cursorType = Some(cursorType),
gapBoundaryId = gapBoundaryId
)
serializer.serializeCursor(cursor)
}
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/OrderedTopCursorBuilder.docx Normal file
View File

Binary file not shown.

52
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/OrderedTopCursorBuilder.scala
View File

@ -1,52 +0,0 @@
package com.twitter.product_mixer.component_library.premarshaller.urt.builder
import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor
import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer
import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedTopCursorBuilder.TopCursorOffset
import com.twitter.product_mixer.core.model.common.UniversalNoun
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorType
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor
import com.twitter.product_mixer.core.pipeline.HasPipelineCursor
import com.twitter.product_mixer.core.pipeline.PipelineCursorSerializer
import com.twitter.product_mixer.core.pipeline.PipelineQuery
case object OrderedTopCursorBuilder {
// Ensure that the next initial sort index is at least 10000 entries away from top cursor's
// current sort index. This is to ensure that the contents of the next page can be populated
// without being assigned sort indices which conflict with that of the current page. This assumes
// that each page will have fewer than 10000 entries.
val TopCursorOffset = 10000L
}
/**
* Builds [[UrtOrderedCursor]] in the Top position
*
* @param idSelector Specifies the entry from which to derive the `id` field
* @param serializer Converts the cursor to an encoded string
*/
case class OrderedTopCursorBuilder(
idSelector: PartialFunction[UniversalNoun[_], Long],
serializer: PipelineCursorSerializer[UrtOrderedCursor] = UrtCursorSerializer)
extends UrtCursorBuilder[
PipelineQuery with HasPipelineCursor[UrtOrderedCursor]
] {
override val cursorType: CursorType = TopCursor
override def cursorValue(
query: PipelineQuery with HasPipelineCursor[UrtOrderedCursor],
timelineEntries: Seq[TimelineEntry]
): String = {
val topId = timelineEntries.collectFirst(idSelector)
val id = topId.orElse(query.pipelineCursor.flatMap(_.id))
val cursor = UrtOrderedCursor(
initialSortIndex = cursorSortIndex(query, timelineEntries) + TopCursorOffset,
id = id,
cursorType = Some(cursorType)
)
serializer.serializeCursor(cursor)
}
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/PinEntryInstructionBuilder.docx Normal file
View File

Binary file not shown.

23
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/PinEntryInstructionBuilder.scala
View File

@ -1,23 +0,0 @@
package com.twitter.product_mixer.component_library.premarshaller.urt.builder
import com.twitter.product_mixer.core.model.marshalling.response.urt.PinEntryTimelineInstruction
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.PinnableEntry
case class PinEntryInstructionBuilder()
extends UrtInstructionBuilder[PipelineQuery, PinEntryTimelineInstruction] {
override def build(
query: PipelineQuery,
entries: Seq[TimelineEntry]
): Seq[PinEntryTimelineInstruction] = {
// Only one entry can be pinned and the desirable behavior is to pick the entry with the highest
// sort index in the event that multiple pinned items exist. Since the entries are already
// sorted we can accomplish this by picking the first one.
entries.collectFirst {
case entry: PinnableEntry if entry.isPinned.getOrElse(false) =>
PinEntryTimelineInstruction(entry)
}.toSeq
}
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/PlaceholderTopCursorBuilder.docx Normal file
View File

Binary file not shown.

34
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/PlaceholderTopCursorBuilder.scala
View File

@ -1,34 +0,0 @@
package com.twitter.product_mixer.component_library.premarshaller.urt.builder
import com.twitter.product_mixer.component_library.model.cursor.UrtPlaceholderCursor
import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer
import com.twitter.product_mixer.component_library.premarshaller.urt.builder.PlaceholderTopCursorBuilder.DefaultPlaceholderCursor
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorType
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor
import com.twitter.product_mixer.core.pipeline.HasPipelineCursor
import com.twitter.product_mixer.core.pipeline.PipelineCursorSerializer
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.pipeline.UrtPipelineCursor
object PlaceholderTopCursorBuilder {
val DefaultPlaceholderCursor = UrtPlaceholderCursor()
}
/**
* Top cursor builder that can be used when the Product does not support paging up. The URT spec
* requires that both bottom and top cursors always be present on each page. Therefore, if the
* product does not support paging up, then we can use a cursor value that is not deserializable.
* This way if the client submits a TopCursor, the backend will treat the the request as if no
* cursor was submitted.
*/
case class PlaceholderTopCursorBuilder(
serializer: PipelineCursorSerializer[UrtPipelineCursor] = UrtCursorSerializer)
extends UrtCursorBuilder[PipelineQuery with HasPipelineCursor[UrtPipelineCursor]] {
override val cursorType: CursorType = TopCursor
override def cursorValue(
query: PipelineQuery with HasPipelineCursor[UrtPipelineCursor],
timelineEntries: Seq[TimelineEntry]
): String = serializer.serializeCursor(DefaultPlaceholderCursor)
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ReplaceEntryInstructionBuilder.docx Normal file
View File

Binary file not shown.

63
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ReplaceEntryInstructionBuilder.scala
View File

@ -1,63 +0,0 @@
package com.twitter.product_mixer.component_library.premarshaller.urt.builder
import com.twitter.product_mixer.core.model.marshalling.response.urt.ReplaceEntryTimelineInstruction
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorOperation
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorType
import com.twitter.product_mixer.core.pipeline.PipelineQuery
/**
* Selects one or more [[TimelineEntry]] instance from the input timeline entries.
*
* @tparam Query The domain model for the [[PipelineQuery]] used as input.
*/
trait EntriesToReplace[-Query <: PipelineQuery] {
def apply(query: Query, entries: Seq[TimelineEntry]): Seq[TimelineEntry]
}
/**
* Selects all entries with a non-empty valid entryIdToReplace.
*
* @note this will result in multiple [[ReplaceEntryTimelineInstruction]]s
*/
case object ReplaceAllEntries extends EntriesToReplace[PipelineQuery] {
def apply(query: PipelineQuery, entries: Seq[TimelineEntry]): Seq[TimelineEntry] =
entries.filter(_.entryIdToReplace.isDefined)
}
/**
* Selects a replaceable URT [[CursorOperation]] from the timeline entries, that matches the
* input cursorType.
*/
case class ReplaceUrtCursor(cursorType: CursorType) extends EntriesToReplace[PipelineQuery] {
override def apply(query: PipelineQuery, entries: Seq[TimelineEntry]): Seq[TimelineEntry] =
entries.collectFirst {
case cursorOperation: CursorOperation
if cursorOperation.cursorType == cursorType && cursorOperation.entryIdToReplace.isDefined =>
cursorOperation
}.toSeq
}
/**
* Create a ReplaceEntry instruction
*
* @param entriesToReplace each replace instruction can contain only one entry. Users specify which
* entry to replace using [[EntriesToReplace]]. If multiple entries are
* specified, multiple [[ReplaceEntryTimelineInstruction]]s will be created.
* @param includeInstruction whether the instruction should be included in the response
*/
case class ReplaceEntryInstructionBuilder[Query <: PipelineQuery](
entriesToReplace: EntriesToReplace[Query],
override val includeInstruction: IncludeInstruction[Query] = AlwaysInclude)
extends UrtInstructionBuilder[Query, ReplaceEntryTimelineInstruction] {
override def build(
query: Query,
entries: Seq[TimelineEntry]
): Seq[ReplaceEntryTimelineInstruction] = {
if (includeInstruction(query, entries))
entriesToReplace(query, entries).map(ReplaceEntryTimelineInstruction)
else
Seq.empty
}
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ShowAlertInstructionBuilder.docx Normal file
View File

Binary file not shown.

23
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ShowAlertInstructionBuilder.scala
View File

@ -1,23 +0,0 @@
package com.twitter.product_mixer.component_library.premarshaller.urt.builder
import com.twitter.product_mixer.core.model.marshalling.response.urt.ShowAlert
import com.twitter.product_mixer.core.model.marshalling.response.urt.ShowAlertInstruction
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry
import com.twitter.product_mixer.core.pipeline.PipelineQuery
case class ShowAlertInstructionBuilder[Query <: PipelineQuery](
override val includeInstruction: IncludeInstruction[Query] = AlwaysInclude)
extends UrtInstructionBuilder[Query, ShowAlertInstruction] {
override def build(
query: Query,
entries: Seq[TimelineEntry]
): Seq[ShowAlertInstruction] = {
if (includeInstruction(query, entries)) {
// Currently only one Alert is supported per response
entries.collectFirst {
case alertEntry: ShowAlert => ShowAlertInstruction(alertEntry)
}.toSeq
} else Seq.empty
}
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ShowCoverInstructionBuilder.docx Normal file
View File

Binary file not shown.

24
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/ShowCoverInstructionBuilder.scala
View File

@ -1,24 +0,0 @@
package com.twitter.product_mixer.component_library.premarshaller.urt.builder
import com.twitter.product_mixer.core.model.marshalling.response.urt.ShowCoverInstruction
import com.twitter.product_mixer.core.model.marshalling.response.urt.Cover
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry
import com.twitter.product_mixer.core.pipeline.PipelineQuery
case class ShowCoverInstructionBuilder[Query <: PipelineQuery](
override val includeInstruction: IncludeInstruction[Query] = AlwaysInclude)
extends UrtInstructionBuilder[Query, ShowCoverInstruction] {
override def build(
query: Query,
entries: Seq[TimelineEntry]
): Seq[ShowCoverInstruction] = {
if (includeInstruction(query, entries)) {
// Currently only one cover is supported per response
entries.collectFirst {
case coverEntry: Cover => ShowCoverInstruction(coverEntry)
}.toSeq
} else {
Seq.empty
}
}
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/StaticTimelineScribeConfigBuilder.docx Normal file
View File

Binary file not shown.

15
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/StaticTimelineScribeConfigBuilder.scala
View File

@ -1,15 +0,0 @@
package com.twitter.product_mixer.component_library.premarshaller.urt.builder
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig
import com.twitter.product_mixer.core.pipeline.PipelineQuery
case class StaticTimelineScribeConfigBuilder(
timelineScribeConfig: TimelineScribeConfig)
extends TimelineScribeConfigBuilder[PipelineQuery] {
def build(
query: PipelineQuery,
entries: Seq[TimelineEntry]
): Option[TimelineScribeConfig] = Some(timelineScribeConfig)
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/TerminateInstructionBuilder.docx Normal file
View File

Binary file not shown.

44
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/TerminateInstructionBuilder.scala
View File

@ -1,44 +0,0 @@
package com.twitter.product_mixer.component_library.premarshaller.urt.builder
import com.twitter.product_mixer.core.model.marshalling.response.urt.BottomTermination
import com.twitter.product_mixer.core.model.marshalling.response.urt.TerminateTimelineInstruction
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineTerminationDirection
import com.twitter.product_mixer.core.model.marshalling.response.urt.TopAndBottomTermination
import com.twitter.product_mixer.core.model.marshalling.response.urt.TopTermination
import com.twitter.product_mixer.core.pipeline.PipelineQuery
sealed trait TerminateInstructionBuilder[Query <: PipelineQuery]
extends UrtInstructionBuilder[Query, TerminateTimelineInstruction] {
def direction: TimelineTerminationDirection
override def build(
query: Query,
entries: Seq[TimelineEntry]
): Seq[TerminateTimelineInstruction] =
if (includeInstruction(query, entries))
Seq(TerminateTimelineInstruction(terminateTimelineDirection = direction))
else Seq.empty
}
case class TerminateTopInstructionBuilder[Query <: PipelineQuery](
override val includeInstruction: IncludeInstruction[Query] = AlwaysInclude)
extends TerminateInstructionBuilder[Query] {
override val direction = TopTermination
}
case class TerminateBottomInstructionBuilder[Query <: PipelineQuery](
override val includeInstruction: IncludeInstruction[Query] = AlwaysInclude)
extends TerminateInstructionBuilder[Query] {
override val direction = BottomTermination
}
case class TerminateTopAndBottomInstructionBuilder[Query <: PipelineQuery](
override val includeInstruction: IncludeInstruction[Query] = AlwaysInclude)
extends TerminateInstructionBuilder[Query] {
override val direction = TopAndBottomTermination
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/TimelineScribeConfigBuilder.docx Normal file
View File

Binary file not shown.

18
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/TimelineScribeConfigBuilder.scala
View File

@ -1,18 +0,0 @@
package com.twitter.product_mixer.component_library.premarshaller.urt.builder
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig
import com.twitter.product_mixer.core.pipeline.PipelineQuery
/**
* Trait for our builder which given a query and entries will return an `Option[TimelineScribeConfig]`
*
* @tparam Query
*/
trait TimelineScribeConfigBuilder[-Query <: PipelineQuery] {
def build(
query: Query,
entries: Seq[TimelineEntry]
): Option[TimelineScribeConfig]
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UnorderedBloomFilterBottomCursorBuilder.docx Normal file
View File

Binary file not shown.

43
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UnorderedBloomFilterBottomCursorBuilder.scala
View File

@ -1,43 +0,0 @@
package com.twitter.product_mixer.component_library.premarshaller.urt.builder
import com.twitter.product_mixer.component_library.model.cursor.UrtUnorderedBloomFilterCursor
import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer
import com.twitter.product_mixer.core.model.common.UniversalNoun
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.BottomCursor
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorType
import com.twitter.product_mixer.core.pipeline.HasPipelineCursor
import com.twitter.product_mixer.core.pipeline.PipelineCursorSerializer
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.search.common.util.bloomfilter.AdaptiveLongIntBloomFilterBuilder
/**
* Builds [[UrtUnorderedBloomFilterCursor]] in the Bottom position
*
* @param idSelector Specifies the entry from which to derive the `id` field
* @param serializer Converts the cursor to an encoded string
*/
case class UnorderedBloomFilterBottomCursorBuilder(
idSelector: PartialFunction[UniversalNoun[_], Long],
serializer: PipelineCursorSerializer[UrtUnorderedBloomFilterCursor] = UrtCursorSerializer)
extends UrtCursorBuilder[
PipelineQuery with HasPipelineCursor[UrtUnorderedBloomFilterCursor]
] {
override val cursorType: CursorType = BottomCursor
override def cursorValue(
query: PipelineQuery with HasPipelineCursor[UrtUnorderedBloomFilterCursor],
entries: Seq[TimelineEntry]
): String = {
val bloomFilter = query.pipelineCursor.map(_.longIntBloomFilter)
val ids = entries.collect(idSelector)
val cursor = UrtUnorderedBloomFilterCursor(
initialSortIndex = nextBottomInitialSortIndex(query, entries),
longIntBloomFilter = AdaptiveLongIntBloomFilterBuilder.build(ids, bloomFilter)
)
serializer.serializeCursor(cursor)
}
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UnorderedExcludeIdsBottomCursorBuilder.docx Normal file
View File

Binary file not shown.

26
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UnorderedExcludeIdsBottomCursorBuilder.scala
View File

@ -1,26 +0,0 @@
package com.twitter.product_mixer.component_library.premarshaller.urt.builder
import com.twitter.product_mixer.component_library.model.cursor.UrtUnorderedExcludeIdsCursor
import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer
import com.twitter.product_mixer.core.model.common.UniversalNoun
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry
import com.twitter.product_mixer.core.pipeline.PipelineCursorSerializer
import com.twitter.timelines.configapi.Param
/**
* Builds [[UrtUnorderedExcludeIdsCursor]] in the Bottom position
*
* @param excludedIdsMaxLengthParam The maximum length of the cursor
* @param excludeIdsSelector Specifies the entry Ids to populate on the `excludedIds` field
* @param serializer Converts the cursor to an encoded string
*/
case class UnorderedExcludeIdsBottomCursorBuilder(
override val excludedIdsMaxLengthParam: Param[Int],
excludeIdsSelector: PartialFunction[UniversalNoun[_], Long],
override val serializer: PipelineCursorSerializer[UrtUnorderedExcludeIdsCursor] =
UrtCursorSerializer)
extends BaseUnorderedExcludeIdsBottomCursorBuilder {
override def excludeEntriesCollector(entries: Seq[TimelineEntry]): Seq[Long] =
entries.collect(excludeIdsSelector)
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UnorderedExcludeIdsSeqBottomCursorBuilder.docx Normal file
View File

Binary file not shown.

30
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UnorderedExcludeIdsSeqBottomCursorBuilder.scala
View File

@ -1,30 +0,0 @@
package com.twitter.product_mixer.component_library.premarshaller.urt.builder
import com.twitter.product_mixer.component_library.model.cursor.UrtUnorderedExcludeIdsCursor
import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer
import com.twitter.product_mixer.core.model.common.UniversalNoun
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry
import com.twitter.product_mixer.core.pipeline.PipelineCursorSerializer
import com.twitter.timelines.configapi.Param
/**
* Builds [[UrtUnorderedExcludeIdsCursor]] in the Bottom position when we want to also exclude ids
* of items inside a module. The reason we cannot use [[UnorderedExcludeIdsBottomCursorBuilder]] in
* such case is that the excludeIdsSelector of [[UnorderedExcludeIdsBottomCursorBuilder]] is doing a
* one to one mapping between entries and excluded ids, but in case of having a module, a module
* entry can result in excluding a sequence of entries.
*
* @param excludedIdsMaxLengthParam The maximum length of the cursor
* @param excludeIdsSelector Specifies the entry Ids to populate on the `excludedIds` field
* @param serializer Converts the cursor to an encoded string
*/
case class UnorderedExcludeIdsSeqBottomCursorBuilder(
override val excludedIdsMaxLengthParam: Param[Int],
excludeIdsSelector: PartialFunction[UniversalNoun[_], Seq[Long]],
override val serializer: PipelineCursorSerializer[UrtUnorderedExcludeIdsCursor] =
UrtCursorSerializer)
extends BaseUnorderedExcludeIdsBottomCursorBuilder {
override def excludeEntriesCollector(entries: Seq[TimelineEntry]): Seq[Long] =
entries.collect(excludeIdsSelector).flatten
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtBuilder.docx Normal file
View File

Binary file not shown.

94
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtBuilder.scala
View File

@ -1,94 +0,0 @@
package com.twitter.product_mixer.component_library.premarshaller.urt.builder
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorOperation
import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineInstruction
import com.twitter.product_mixer.core.pipeline.HasPipelineCursor
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.pipeline.UrtPipelineCursor
import com.twitter.product_mixer.core.util.SortIndexBuilder
trait UrtBuilder[-Query <: PipelineQuery, +Instruction <: TimelineInstruction] {
private val TimelineIdSuffix = "-Timeline"
def instructionBuilders: Seq[UrtInstructionBuilder[Query, Instruction]]
def cursorBuilders: Seq[UrtCursorBuilder[Query]]
def cursorUpdaters: Seq[UrtCursorUpdater[Query]]
def metadataBuilder: Option[BaseUrtMetadataBuilder[Query]]
// Timeline entry sort indexes will count down by this value. Values higher than 1 are useful to
// leave room in the sequence for dynamically injecting content in between existing entries.
def sortIndexStep: Int = 1
final def buildTimeline(
query: Query,
entries: Seq[TimelineEntry]
): Timeline = {
val initialSortIndex = getInitialSortIndex(query)
// Set the sort indexes of the entries before we pass them to the cursor builders, since many
// cursor implementations use the sort index of the first/last entry as part of the cursor value
val sortIndexedEntries = updateSortIndexes(initialSortIndex, entries)
// Iterate over the cursorUpdaters in the order they were defined. Note that each updater will
// be passed the timelineEntries updated by the previous cursorUpdater.
val updatedCursorEntries: Seq[TimelineEntry] =
cursorUpdaters.foldLeft(sortIndexedEntries) { (timelineEntries, cursorUpdater) =>
cursorUpdater.update(query, timelineEntries)
}
val allCursoredEntries =
updatedCursorEntries ++ cursorBuilders.flatMap(_.build(query, updatedCursorEntries))
val instructions: Seq[Instruction] =
instructionBuilders.flatMap(_.build(query, allCursoredEntries))
val metadata = metadataBuilder.map(_.build(query, allCursoredEntries))
Timeline(
id = query.product.identifier.toString + TimelineIdSuffix,
instructions = instructions,
metadata = metadata
)
}
final def getInitialSortIndex(query: Query): Long =
query match {
case cursorQuery: HasPipelineCursor[_] =>
UrtPipelineCursor
.getCursorInitialSortIndex(cursorQuery)
.getOrElse(SortIndexBuilder.timeToId(query.queryTime))
case _ => SortIndexBuilder.timeToId(query.queryTime)
}
/**
* Updates the sort indexes in the timeline entries starting from the given initial sort index
* value and decreasing by the value defined in the sort index step field
*
* @param initialSortIndex The initial value of the sort index
* @param timelineEntries Timeline entries to update
*/
final def updateSortIndexes(
initialSortIndex: Long,
timelineEntries: Seq[TimelineEntry]
): Seq[TimelineEntry] = {
val indexRange =
initialSortIndex to (initialSortIndex - (timelineEntries.size * sortIndexStep)) by -sortIndexStep
// Skip any existing cursors because their sort indexes will be managed by their cursor updater.
// If the cursors are not removed first, then the remaining entries would have a gap everywhere
// an existing cursor was present.
val (cursorEntries, nonCursorEntries) = timelineEntries.partition {
case _: CursorOperation => true
case _ => false
}
nonCursorEntries.zip(indexRange).map {
case (entry, index) =>
entry.withSortIndex(index)
} ++ cursorEntries
}
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtCursorBuilder.docx Normal file
View File

Binary file not shown.

134
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtCursorBuilder.scala
View File

@ -1,134 +0,0 @@
package com.twitter.product_mixer.component_library.premarshaller.urt.builder
import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtCursorBuilder.DefaultSortIndex
import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtCursorBuilder.NextPageTopCursorEntryOffset
import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtCursorBuilder.UrtEntryOffset
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.BottomCursor
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorItem
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorOperation
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorType
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor
import com.twitter.product_mixer.core.pipeline.HasPipelineCursor
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.pipeline.UrtPipelineCursor
import com.twitter.product_mixer.core.util.SortIndexBuilder
object UrtCursorBuilder {
val NextPageTopCursorEntryOffset = 1L
val UrtEntryOffset = 1L
val DefaultSortIndex = (query: PipelineQuery) => SortIndexBuilder.timeToId(query.queryTime)
}
trait UrtCursorBuilder[-Query <: PipelineQuery] {
val includeOperation: IncludeInstruction[Query] = AlwaysInclude
def cursorType: CursorType
def cursorValue(query: Query, entries: Seq[TimelineEntry]): String
/**
* Identifier of an *existing* timeline cursor that this new cursor would replace, if this cursor
* is returned in a `ReplaceEntry` timeline instruction.
*
* Note:
* - This id is used to populate the `entryIdToReplace` field on the URT TimelineEntry
* generated. More details at [[CursorOperation.entryIdToReplace]].
* - As a convention, we use the sortIndex of the cursor for its id/entryId fields. So the
* `idToReplace` should represent the sortIndex of the existing cursor to be replaced.
*/
def idToReplace(query: Query): Option[Long] = None
def cursorSortIndex(query: Query, entries: Seq[TimelineEntry]): Long =
(query, cursorType) match {
case (query: PipelineQuery with HasPipelineCursor[_], TopCursor) =>
topCursorSortIndex(query, entries)
case (query: PipelineQuery with HasPipelineCursor[_], BottomCursor | GapCursor) =>
bottomCursorSortIndex(query, entries)
case _ =>
throw new UnsupportedOperationException(
"Automatic sort index support limited to top and bottom cursors")
}
def build(query: Query, entries: Seq[TimelineEntry]): Option[CursorOperation] = {
if (includeOperation(query, entries)) {
val sortIndex = cursorSortIndex(query, entries)
val cursorOperation = CursorOperation(
id = sortIndex,
sortIndex = Some(sortIndex),
value = cursorValue(query, entries),
cursorType = cursorType,
displayTreatment = None,
idToReplace = idToReplace(query),
)
Some(cursorOperation)
} else None
}
/**
* Build the top cursor sort index which handles the following cases:
* 1. When there is at least one non-cursor entry, use the first entry's sort index + UrtEntryOffset
* 2. When there are no non-cursor entries, and initialSortIndex is not set which indicates that
* it is the first page, use DefaultSortIndex + UrtEntryOffset
* 3. When there are no non-cursor entries, and initialSortIndex is set which indicates that it is
* not the first page, use the query.initialSortIndex from the passed-in cursor + UrtEntryOffset
*/
protected def topCursorSortIndex(
query: PipelineQuery with HasPipelineCursor[_],
entries: Seq[TimelineEntry]
): Long = {
val nonCursorEntries = entries.filter {
case _: CursorOperation => false
case _: CursorItem => false
case _ => true
}
lazy val initialSortIndex =
UrtPipelineCursor.getCursorInitialSortIndex(query).getOrElse(DefaultSortIndex(query))
nonCursorEntries.headOption.flatMap(_.sortIndex).getOrElse(initialSortIndex) + UrtEntryOffset
}
/**
* Specifies the point at which the next page's entries' sort indices will start counting.
*
* Note that in the case of URT, the next page's entries' does not include the top cursor. As
* such, the value of initialSortIndex passed back in the cursor is typically the bottom cursor's
* sort index - 2. Subtracting 2 leaves room for the next page's top cursor, which will have a
* sort index of top entry + 1.
*/
protected def nextBottomInitialSortIndex(
query: PipelineQuery with HasPipelineCursor[_],
entries: Seq[TimelineEntry]
): Long = {
bottomCursorSortIndex(query, entries) - NextPageTopCursorEntryOffset - UrtEntryOffset
}
/**
* Build the bottom cursor sort index which handles the following cases:
* 1. When there is at least one non-cursor entry, use the last entry's sort index - UrtEntryOffset
* 2. When there are no non-cursor entries, and initialSortIndex is not set which indicates that
* it is the first page, use DefaultSortIndex
* 3. When there are no non-cursor entries, and initialSortIndex is set which indicates that it is
* not the first page, use the query.initialSortIndex from the passed-in cursor
*/
protected def bottomCursorSortIndex(
query: PipelineQuery with HasPipelineCursor[_],
entries: Seq[TimelineEntry]
): Long = {
val nonCursorEntries = entries.filter {
case _: CursorOperation => false
case _: CursorItem => false
case _ => true
}
lazy val initialSortIndex =
UrtPipelineCursor.getCursorInitialSortIndex(query).getOrElse(DefaultSortIndex(query))
nonCursorEntries.lastOption
.flatMap(_.sortIndex).map(_ - UrtEntryOffset).getOrElse(initialSortIndex)
}
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtCursorUpdater.docx Normal file
View File

Binary file not shown.

44
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtCursorUpdater.scala
View File

@ -1,44 +0,0 @@
package com.twitter.product_mixer.component_library.premarshaller.urt.builder
import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtCursorUpdater.getCursorByType
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorOperation
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.CursorType
import com.twitter.product_mixer.core.pipeline.PipelineQuery
object UrtCursorUpdater {
def getCursorByType(
entries: Seq[TimelineEntry],
cursorType: CursorType
): Option[CursorOperation] = {
entries.collectFirst {
case cursor: CursorOperation if cursor.cursorType == cursorType => cursor
}
}
}
// If a CursorCandidate is returned by a Candidate Source, use this trait to update that Cursor as
// necessary (as opposed to building a new cursor which is done with the UrtCursorBuilder)
trait UrtCursorUpdater[-Query <: PipelineQuery] extends UrtCursorBuilder[Query] { self =>
def getExistingCursor(entries: Seq[TimelineEntry]): Option[CursorOperation] = {
getCursorByType(entries, self.cursorType)
}
def update(query: Query, entries: Seq[TimelineEntry]): Seq[TimelineEntry] = {
if (includeOperation(query, entries)) {
getExistingCursor(entries)
.map { existingCursor =>
// Safe .get because includeOperation() is shared in this context
// build() method creates a new CursorOperation. We copy over the `idToReplace`
// from the existing cursor.
val newCursor =
build(query, entries).get
.copy(idToReplace = existingCursor.idToReplace)
entries.filterNot(_ == existingCursor) :+ newCursor
}.getOrElse(entries)
} else entries
}
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtInstructionBuilder.docx Normal file
View File

Binary file not shown.

15
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtInstructionBuilder.scala
View File

@ -1,15 +0,0 @@
package com.twitter.product_mixer.component_library.premarshaller.urt.builder
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineInstruction
import com.twitter.product_mixer.core.pipeline.PipelineQuery
trait UrtInstructionBuilder[-Query <: PipelineQuery, +Instruction <: TimelineInstruction] {
def includeInstruction: IncludeInstruction[Query] = AlwaysInclude
def build(
query: Query,
entries: Seq[TimelineEntry]
): Seq[Instruction]
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtMetadataBuilder.docx Normal file
View File

Binary file not shown.

43
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/UrtMetadataBuilder.scala
View File

@ -1,43 +0,0 @@
package com.twitter.product_mixer.component_library.premarshaller.urt.builder
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineMetadata
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stringcenter.client.StringCenter
import com.twitter.stringcenter.client.core.ExternalString
trait BaseUrtMetadataBuilder[-Query <: PipelineQuery] {
def build(
query: Query,
entries: Seq[TimelineEntry]
): TimelineMetadata
}
case class UrtMetadataBuilder(
title: Option[String] = None,
scribeConfigBuilder: Option[TimelineScribeConfigBuilder[PipelineQuery]])
extends BaseUrtMetadataBuilder[PipelineQuery] {
override def build(
query: PipelineQuery,
entries: Seq[TimelineEntry]
): TimelineMetadata = TimelineMetadata(
title = title,
scribeConfig = scribeConfigBuilder.flatMap(_.build(query, entries))
)
}
case class UrtMetadataStringCenterBuilder(
titleKey: ExternalString,
scribeConfigBuilder: Option[TimelineScribeConfigBuilder[PipelineQuery]],
stringCenter: StringCenter)
extends BaseUrtMetadataBuilder[PipelineQuery] {
override def build(
query: PipelineQuery,
entries: Seq[TimelineEntry]
): TimelineMetadata = TimelineMetadata(
title = Some(stringCenter.prepare(titleKey)),
scribeConfig = scribeConfigBuilder.flatMap(_.build(query, entries))
)
}

30
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/BUILD
View File

@ -1,30 +0,0 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"3rdparty/jvm/io/grpc:grpc-netty",
"3rdparty/jvm/io/netty:netty4-tcnative-boringssl-static",
"3rdparty/jvm/io/opil:tensorflow-serving-client",
"3rdparty/jvm/javax/inject:javax.inject",
"3rdparty/jvm/triton/inference:triton-grpc",
"finagle-internal/finagle-grpc/src/main/scala",
"finagle-internal/finagle-grpc/src/test/java",
"finagle-internal/finagle-grpc/src/test/proto",
"finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication",
"finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/client",
"finagle/finagle-http/src/main/scala",
"finatra-internal/mtls/src/main/scala",
"finatra/inject/inject-core/src/main/scala/com/twitter/inject",
"ml-serving/scala:kfserving-tfserving-converter",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
"stitch/stitch-core",
],
exports = [
"3rdparty/jvm/triton/inference:triton-grpc",
"finagle/finagle-http/src/main/scala",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
"stitch/stitch-core",
],
)

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/BUILD.docx Normal file
View File

Binary file not shown.

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/MLModelInferenceClient.docx Normal file
View File

Binary file not shown.

12
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/MLModelInferenceClient.scala
View File

@ -1,12 +0,0 @@
package com.twitter.product_mixer.component_library.scorer.common
import com.twitter.stitch.Stitch
import inference.GrpcService.ModelInferRequest
import inference.GrpcService.ModelInferResponse
/**
* MLModelInferenceClient for calling different Inference Service such as ManagedModelClient or NaviModelClient.
*/
trait MLModelInferenceClient {
def score(request: ModelInferRequest): Stitch[ModelInferResponse]
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/ManagedModelClient.docx Normal file
View File

Binary file not shown.

33
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/ManagedModelClient.scala
View File

@ -1,33 +0,0 @@
package com.twitter.product_mixer.component_library.scorer.common
import com.twitter.finagle.Http
import com.twitter.finagle.grpc.FinagleChannelBuilder
import com.twitter.finagle.grpc.FutureConverters
import com.twitter.stitch.Stitch
import inference.GRPCInferenceServiceGrpc
import inference.GrpcService.ModelInferRequest
import inference.GrpcService.ModelInferResponse
import io.grpc.ManagedChannel
/**
* Client wrapper for calling a Cortex Managed Inference Service (go/cmis) ML Model using GRPC.
* @param httpClient Finagle HTTP Client to use for connection.
* @param modelPath Wily path to the ML Model service (e.g. /cluster/local/role/service/instance).
*/
case class ManagedModelClient(
httpClient: Http.Client,
modelPath: String)
extends MLModelInferenceClient {
private val channel: ManagedChannel =
FinagleChannelBuilder.forTarget(modelPath).httpClient(httpClient).build()
private val inferenceServiceStub = GRPCInferenceServiceGrpc.newFutureStub(channel)
def score(request: ModelInferRequest): Stitch[ModelInferResponse] = {
Stitch
.callFuture(
FutureConverters
.RichListenableFuture(inferenceServiceStub.modelInfer(request)).toTwitter)
}
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/ModelSelector.docx Normal file
View File

Binary file not shown.

28
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/ModelSelector.scala
View File

@ -1,28 +0,0 @@
package com.twitter.product_mixer.component_library.scorer.common
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.timelines.configapi.Param
/**
* Selector for choosing which Model ID/Name to use when calling an underlying ML Model Service.
*/
trait ModelSelector[-Query <: PipelineQuery] {
def apply(query: Query): Option[String]
}
/**
* Simple Model ID Selector that chooses model based off of a Param object.
* @param param ConfigAPI Param that decides the model id.
*/
case class ParamModelSelector[Query <: PipelineQuery](param: Param[String])
extends ModelSelector[Query] {
override def apply(query: Query): Option[String] = Some(query.params(param))
}
/**
* Static Selector that chooses the same model name always
* @param modelName The model name to use.
*/
case class StaticModelSelector(modelName: String) extends ModelSelector[PipelineQuery] {
override def apply(query: PipelineQuery): Option[String] = Some(modelName)
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/NaviModelClient.docx Normal file
View File

Binary file not shown.

50
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common/NaviModelClient.scala
View File

@ -1,50 +0,0 @@
package com.twitter.product_mixer.component_library.scorer.common
import com.twitter.finagle.Http
import com.twitter.finagle.grpc.FinagleChannelBuilder
import com.twitter.finagle.grpc.FutureConverters
import com.twitter.mlserving.frontend.TFServingInferenceServiceImpl
import com.twitter.stitch.Stitch
import tensorflow.serving.PredictionServiceGrpc
import inference.GrpcService.ModelInferRequest
import inference.GrpcService.ModelInferResponse
import io.grpc.ManagedChannel
import io.grpc.Status
/**
* Client wrapper for calling a Navi Inference Service (go/navi).
* @param httpClient Finagle HTTP Client to use for connection.
* @param modelPath Wily path to the ML Model service (e.g. /s/role/service).
*/
case class NaviModelClient(
httpClient: Http.Client,
modelPath: String)
extends MLModelInferenceClient {
private val channel: ManagedChannel =
FinagleChannelBuilder
.forTarget(modelPath)
.httpClient(httpClient)
// Navi enforces an authority name.
.overrideAuthority("rustserving")
// certain GRPC errors need to be retried.
.enableRetryForStatus(Status.UNKNOWN)
.enableRetryForStatus(Status.RESOURCE_EXHAUSTED)
// this is required at channel level as mTLS is enabled at httpClient level
.usePlaintext()
.build()
private val inferenceServiceStub = PredictionServiceGrpc.newFutureStub(channel)
def score(request: ModelInferRequest): Stitch[ModelInferResponse] = {
val tfServingRequest = TFServingInferenceServiceImpl.adaptModelInferRequest(request)
Stitch
.callFuture(
FutureConverters
.RichListenableFuture(inferenceServiceStub.predict(tfServingRequest)).toTwitter
.map { response =>
TFServingInferenceServiceImpl.adaptModelInferResponse(response)
}
)
}
}

35
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/BUILD
View File

@ -1,35 +0,0 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"finagle/finagle-http/src/main/scala",
"finatra-internal/mtls/src/main/scala",
"finatra/inject/inject-core/src/main/scala/com/twitter/inject",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/module/http",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/scorer",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure",
"src/scala/com/twitter/ml/featurestore/lib",
"src/thrift/com/twitter/ml/prediction_service:prediction_service-java",
],
exports = [
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/module/http",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/scorer",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
],
)

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/BUILD.docx Normal file
View File

Binary file not shown.

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceDataRecordScorer.docx Normal file
View File

Binary file not shown.

137
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceDataRecordScorer.scala
View File

@ -1,137 +0,0 @@
package com.twitter.product_mixer.component_library.scorer.cortex
import com.google.protobuf.ByteString
import com.twitter.ml.prediction_service.BatchPredictionRequest
import com.twitter.ml.prediction_service.BatchPredictionResponse
import com.twitter.product_mixer.component_library.scorer.common.ManagedModelClient
import com.twitter.product_mixer.component_library.scorer.common.ModelSelector
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.datarecord.BaseDataRecordFeature
import com.twitter.product_mixer.core.feature.datarecord.TensorDataRecordCompatible
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordConverter
import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordExtractor
import com.twitter.product_mixer.core.feature.featuremap.datarecord.FeaturesScope
import com.twitter.product_mixer.core.functional_component.scorer.Scorer
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.UniversalNoun
import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.pipeline.pipeline_failure.IllegalStateFailure
import inference.GrpcService
import inference.GrpcService.ModelInferRequest
import inference.GrpcService.ModelInferResponse
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure
import com.twitter.stitch.Stitch
import org.apache.thrift.TDeserializer
import org.apache.thrift.TSerializer
import scala.collection.JavaConverters._
private[cortex] class CortexManagedDataRecordScorer[
Query <: PipelineQuery,
Candidate <: UniversalNoun[Any],
QueryFeatures <: BaseDataRecordFeature[Query, _],
CandidateFeatures <: BaseDataRecordFeature[Candidate, _],
ResultFeatures <: BaseDataRecordFeature[Candidate, _] with TensorDataRecordCompatible[_]
](
override val identifier: ScorerIdentifier,
modelSignature: String,
modelSelector: ModelSelector[Query],
modelClient: ManagedModelClient,
queryFeatures: FeaturesScope[QueryFeatures],
candidateFeatures: FeaturesScope[CandidateFeatures],
resultFeatures: Set[ResultFeatures])
extends Scorer[Query, Candidate] {
require(resultFeatures.nonEmpty, "Result features cannot be empty")
override val features: Set[Feature[_, _]] = resultFeatures.asInstanceOf[Set[Feature[_, _]]]
private val queryDataRecordAdapter = new DataRecordConverter(queryFeatures)
private val candidatesDataRecordAdapter = new DataRecordConverter(candidateFeatures)
private val resultDataRecordExtractor = new DataRecordExtractor(resultFeatures)
private val localTSerializer = new ThreadLocal[TSerializer] {
override protected def initialValue: TSerializer = new TSerializer()
}
private val localTDeserializer = new ThreadLocal[TDeserializer] {
override protected def initialValue: TDeserializer = new TDeserializer()
}
override def apply(
query: Query,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[Seq[FeatureMap]] = {
modelClient.score(buildRequest(query, candidates)).map(buildResponse(candidates, _))
}
/**
* Takes candidates to be scored and converts it to a ModelInferRequest that can be passed to the
* managed ML service
*/
private def buildRequest(
query: Query,
scorerCandidates: Seq[CandidateWithFeatures[Candidate]]
): ModelInferRequest = {
// Convert the feature maps to thrift data records and construct thrift request.
val thriftDataRecords = scorerCandidates.map { candidate =>
candidatesDataRecordAdapter.toDataRecord(candidate.features)
}
val batchRequest = new BatchPredictionRequest(thriftDataRecords.asJava)
query.features.foreach { featureMap =>
batchRequest.setCommonFeatures(queryDataRecordAdapter.toDataRecord(featureMap))
}
val serializedBatchRequest = localTSerializer.get().serialize(batchRequest)
// Build Tensor Request
val requestBuilder = ModelInferRequest
.newBuilder()
modelSelector.apply(query).foreach { modelName =>
requestBuilder.setModelName(modelName) // model name in the model config
}
val inputTensorBuilder = ModelInferRequest.InferInputTensor
.newBuilder()
.setName("request")
.setDatatype("UINT8")
.addShape(serializedBatchRequest.length)
val inferParameter = GrpcService.InferParameter
.newBuilder()
.setStringParam(modelSignature) // signature of exported tf function
.build()
requestBuilder
.addInputs(inputTensorBuilder)
.addRawInputContents(ByteString.copyFrom(serializedBatchRequest))
.putParameters("signature_name", inferParameter)
.build()
}
private def buildResponse(
scorerCandidates: Seq[CandidateWithFeatures[Candidate]],
response: ModelInferResponse
): Seq[FeatureMap] = {
val responseByteString = if (response.getRawOutputContentsList.isEmpty()) {
throw PipelineFailure(
IllegalStateFailure,
"Model inference response has empty raw outputContents")
} else {
response.getRawOutputContents(0)
}
val batchPredictionResponse: BatchPredictionResponse = new BatchPredictionResponse()
localTDeserializer.get().deserialize(batchPredictionResponse, responseByteString.toByteArray)
// get the prediction values from the batch prediction response
val resultScoreMaps =
batchPredictionResponse.predictions.asScala.map(resultDataRecordExtractor.fromDataRecord)
if (resultScoreMaps.size != scorerCandidates.size) {
throw PipelineFailure(IllegalStateFailure, "Result Size mismatched candidates size")
}
resultScoreMaps
}
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceDataRecordScorerBuilder.docx Normal file
View File

Binary file not shown.

67
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceDataRecordScorerBuilder.scala
View File

@ -1,67 +0,0 @@
package com.twitter.product_mixer.component_library.scorer.cortex
import com.twitter.finagle.Http
import com.twitter.product_mixer.component_library.module.http.FinagleHttpClientModule.FinagleHttpClientModule
import com.twitter.product_mixer.component_library.scorer.common.ManagedModelClient
import com.twitter.product_mixer.component_library.scorer.common.ModelSelector
import com.twitter.product_mixer.core.feature.datarecord.BaseDataRecordFeature
import com.twitter.product_mixer.core.feature.datarecord.TensorDataRecordCompatible
import com.twitter.product_mixer.core.feature.featuremap.datarecord.FeaturesScope
import com.twitter.product_mixer.core.functional_component.scorer.Scorer
import com.twitter.product_mixer.core.model.common.UniversalNoun
import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import javax.inject.Inject
import javax.inject.Named
import javax.inject.Singleton
@Singleton
class CortexManagedInferenceServiceDataRecordScorerBuilder @Inject() (
@Named(FinagleHttpClientModule) httpClient: Http.Client) {
/**
* Builds a configurable Scorer to call into your desired DataRecord-backed Cortex Managed ML Model Service.
*
* If your service does not bind an Http.Client implementation, add
* [[com.twitter.product_mixer.component_library.module.http.FinagleHttpClientModule]]
* to your server module list
*
* @param scorerIdentifier Unique identifier for the scorer
* @param modelPath MLS path to model
* @param modelSignature Model Signature Key
* @param modelSelector [[ModelSelector]] for choosing the model name, can be an anon function.
* @param candidateFeatures Desired candidate level feature store features to pass to the model.
* @param resultFeatures Desired candidate level feature store features to extract from the model.
* Since the Cortex Managed Platform always returns tensor values, the
* feature must use a [[TensorDataRecordCompatible]].
* @tparam Query Type of pipeline query.
* @tparam Candidate Type of candidates to score.
* @tparam QueryFeatures type of the query level features consumed by the scorer.
* @tparam CandidateFeatures type of the candidate level features consumed by the scorer.
* @tparam ResultFeatures type of the candidate level features returned by the scorer.
*/
def build[
Query <: PipelineQuery,
Candidate <: UniversalNoun[Any],
QueryFeatures <: BaseDataRecordFeature[Query, _],
CandidateFeatures <: BaseDataRecordFeature[Candidate, _],
ResultFeatures <: BaseDataRecordFeature[Candidate, _] with TensorDataRecordCompatible[_]
](
scorerIdentifier: ScorerIdentifier,
modelPath: String,
modelSignature: String,
modelSelector: ModelSelector[Query],
queryFeatures: FeaturesScope[QueryFeatures],
candidateFeatures: FeaturesScope[CandidateFeatures],
resultFeatures: Set[ResultFeatures]
): Scorer[Query, Candidate] =
new CortexManagedDataRecordScorer(
identifier = scorerIdentifier,
modelSignature = modelSignature,
modelSelector = modelSelector,
modelClient = ManagedModelClient(httpClient, modelPath),
queryFeatures = queryFeatures,
candidateFeatures = candidateFeatures,
resultFeatures = resultFeatures
)
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceTensorScorer.docx Normal file
View File

Binary file not shown.

97
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceTensorScorer.scala
View File

@ -1,97 +0,0 @@
package com.twitter.product_mixer.component_library.scorer.cortex
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.product_mixer.component_library.scorer.common.MLModelInferenceClient
import com.twitter.product_mixer.component_library.scorer.tensorbuilder.ModelInferRequestBuilder
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.scorer.Scorer
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.UniversalNoun
import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.pipeline.pipeline_failure.IllegalStateFailure
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure
import com.twitter.stitch.Stitch
import com.twitter.util.logging.Logging
import inference.GrpcService.ModelInferRequest
import inference.GrpcService.ModelInferResponse.InferOutputTensor
import scala.collection.convert.ImplicitConversions.`collection AsScalaIterable`
private[scorer] class CortexManagedInferenceServiceTensorScorer[
Query <: PipelineQuery,
Candidate <: UniversalNoun[Any]
](
override val identifier: ScorerIdentifier,
modelInferRequestBuilder: ModelInferRequestBuilder[
Query,
Candidate
],
resultFeatureExtractors: Seq[FeatureWithExtractor[Query, Candidate, _]],
client: MLModelInferenceClient,
statsReceiver: StatsReceiver)
extends Scorer[Query, Candidate]
with Logging {
require(resultFeatureExtractors.nonEmpty, "Result Extractors cannot be empty")
private val managedServiceRequestFailures = statsReceiver.counter("managedServiceRequestFailures")
override val features: Set[Feature[_, _]] =
resultFeatureExtractors.map(_.feature).toSet.asInstanceOf[Set[Feature[_, _]]]
override def apply(
query: Query,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[Seq[FeatureMap]] = {
val batchInferRequest: ModelInferRequest = modelInferRequestBuilder(query, candidates)
val managedServiceResponse: Stitch[Seq[InferOutputTensor]] =
client.score(batchInferRequest).map(_.getOutputsList.toSeq).onFailure { e =>
error(s"request to ML Managed Service Failed: $e")
managedServiceRequestFailures.incr()
}
managedServiceResponse.map { responses =>
extractResponse(query, candidates.map(_.candidate), responses)
}
}
def extractResponse(
query: Query,
candidates: Seq[Candidate],
tensorOutput: Seq[InferOutputTensor]
): Seq[FeatureMap] = {
val featureMapBuilders = candidates.map { _ => FeatureMapBuilder.apply() }
// Extract the feature for each candidate from the tensor outputs
resultFeatureExtractors.foreach {
case FeatureWithExtractor(feature, extractor) =>
val extractedValues = extractor.apply(query, tensorOutput)
if (candidates.size != extractedValues.size) {
throw PipelineFailure(
IllegalStateFailure,
s"Managed Service returned a different number of $feature than the number of candidates." +
s"Returned ${extractedValues.size} scores but there were ${candidates.size} candidates."
)
}
// Go through the extracted features list one by one and update the feature map result for each candidate.
featureMapBuilders.zip(extractedValues).foreach {
case (builder, value) =>
builder.add(feature, Some(value))
}
}
featureMapBuilders.map(_.build())
}
}
case class FeatureWithExtractor[
-Query <: PipelineQuery,
-Candidate <: UniversalNoun[Any],
ResultType
](
feature: Feature[Candidate, Option[ResultType]],
featureExtractor: ModelFeatureExtractor[Query, ResultType])
class UnexpectedFeatureTypeException(feature: Feature[_, _])
extends UnsupportedOperationException(s"Unsupported Feature type passed in $feature")

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceTensorScorerBuilder.docx Normal file
View File

Binary file not shown.

47
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/CortexManagedInferenceServiceTensorScorerBuilder.scala
View File

@ -1,47 +0,0 @@
package com.twitter.product_mixer.component_library.scorer.cortex
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.product_mixer.component_library.scorer.common.MLModelInferenceClient
import com.twitter.product_mixer.component_library.scorer.tensorbuilder.ModelInferRequestBuilder
import com.twitter.product_mixer.core.functional_component.scorer.Scorer
import com.twitter.product_mixer.core.model.common.UniversalNoun
import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class CortexManagedInferenceServiceTensorScorerBuilder @Inject() (
statsReceiver: StatsReceiver) {
/**
* Builds a configurable Scorer to call into your desired Cortex Managed ML Model Service.
*
* If your service does not bind an Http.Client implementation, add
* [[com.twitter.product_mixer.component_library.module.http.FinagleHttpClientModule]]
* to your server module list
*
* @param scorerIdentifier Unique identifier for the scorer
* @param resultFeatureExtractors The result features an their tensor extractors for each candidate.
* @tparam Query Type of pipeline query.
* @tparam Candidate Type of candidates to score.
* @tparam QueryFeatures type of the query level features consumed by the scorer.
* @tparam CandidateFeatures type of the candidate level features consumed by the scorer.
*/
def build[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]](
scorerIdentifier: ScorerIdentifier,
modelInferRequestBuilder: ModelInferRequestBuilder[
Query,
Candidate
],
resultFeatureExtractors: Seq[FeatureWithExtractor[Query, Candidate, _]],
client: MLModelInferenceClient
): Scorer[Query, Candidate] =
new CortexManagedInferenceServiceTensorScorer(
scorerIdentifier,
modelInferRequestBuilder,
resultFeatureExtractors,
client,
statsReceiver.scope(scorerIdentifier.name)
)
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/ModelFeatureExtractor.docx Normal file
View File

Binary file not shown.

15
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cortex/ModelFeatureExtractor.scala
View File

@ -1,15 +0,0 @@
package com.twitter.product_mixer.component_library.scorer.cortex
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import inference.GrpcService.ModelInferResponse.InferOutputTensor
/**
* Extractor defining how a Scorer should go from outputted tensors to the individual results
* for each candidate being scored.
*
* @tparam Result the type of the Value being returned.
* Users can pass in an anonymous function
*/
trait ModelFeatureExtractor[-Query <: PipelineQuery, Result] {
def apply(query: Query, tensorOutput: Seq[InferOutputTensor]): Seq[Result]
}

18
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cr_ml_ranker/BUILD.bazel
View File

@ -1,18 +0,0 @@
scala_library(
compiler_option_sets = ["fatal_warnings"],
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"3rdparty/jvm/javax/inject:javax.inject",
"cr-ml-ranker/thrift/src/main/thrift:thrift-scala",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/cr_ml_ranker",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/scorer",
],
exports = [
"cr-ml-ranker/thrift/src/main/thrift:thrift-scala",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/cr_ml_ranker",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/scorer",
],
)

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cr_ml_ranker/BUILD.docx Normal file
View File

Binary file not shown.

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cr_ml_ranker/CrMlRankerScorer.docx Normal file
View File

Binary file not shown.

52
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cr_ml_ranker/CrMlRankerScorer.scala
View File

@ -1,52 +0,0 @@
package com.twitter.product_mixer.component_library.scorer.cr_ml_ranker
import com.twitter.product_mixer.component_library.feature_hydrator.query.cr_ml_ranker.CrMlRankerCommonFeatures
import com.twitter.product_mixer.component_library.feature_hydrator.query.cr_ml_ranker.CrMlRankerRankingConfig
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.scorer.Scorer
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import javax.inject.Inject
import javax.inject.Singleton
object CrMlRankerScore extends Feature[TweetCandidate, Double]
/**
* Scorer that scores tweets using the Content Recommender ML Light Ranker: http://go/cr-ml-ranker
*/
@Singleton
class CrMlRankerScorer @Inject() (crMlRanker: CrMlRankerScoreStitchClient)
extends Scorer[PipelineQuery, TweetCandidate] {
override val identifier: ScorerIdentifier = ScorerIdentifier("CrMlRanker")
override val features: Set[Feature[_, _]] = Set(CrMlRankerScore)
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = {
val queryFeatureMap = query.features.getOrElse(FeatureMap.empty)
val rankingConfig = queryFeatureMap.get(CrMlRankerRankingConfig)
val commonFeatures = queryFeatureMap.get(CrMlRankerCommonFeatures)
val userId = query.getRequiredUserId
val scoresStitch = Stitch.collect(candidates.map { candidateWithFeatures =>
crMlRanker
.getScore(userId, candidateWithFeatures.candidate, rankingConfig, commonFeatures).map(
_.score)
})
scoresStitch.map { scores =>
scores.map { score =>
FeatureMapBuilder()
.add(CrMlRankerScore, score)
.build()
}
}
}
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cr_ml_ranker/CrMlRankerStitchClient.docx Normal file
View File

Binary file not shown.

79
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/cr_ml_ranker/CrMlRankerStitchClient.scala
View File

@ -1,79 +0,0 @@
package com.twitter.product_mixer.component_library.scorer.cr_ml_ranker
import com.twitter.cr_ml_ranker.{thriftscala => t}
import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate
import com.twitter.stitch.SeqGroup
import com.twitter.stitch.Stitch
import com.twitter.util.Future
import com.twitter.util.Return
import com.twitter.util.Try
case class CrMlRankerResult(
tweetId: Long,
score: Double)
class CrMlRankerScoreStitchClient(
crMLRanker: t.CrMLRanker.MethodPerEndpoint,
maxBatchSize: Int) {
def getScore(
userId: Long,
tweetCandidate: BaseTweetCandidate,
rankingConfig: t.RankingConfig,
commonFeatures: t.CommonFeatures
): Stitch[CrMlRankerResult] = {
Stitch.call(
tweetCandidate,
CrMlRankerGroup(
userId = userId,
rankingConfig = rankingConfig,
commonFeatures = commonFeatures
)
)
}
private case class CrMlRankerGroup(
userId: Long,
rankingConfig: t.RankingConfig,
commonFeatures: t.CommonFeatures)
extends SeqGroup[BaseTweetCandidate, CrMlRankerResult] {
override val maxSize: Int = maxBatchSize
override protected def run(
tweetCandidates: Seq[BaseTweetCandidate]
): Future[Seq[Try[CrMlRankerResult]]] = {
val crMlRankerCandidates =
tweetCandidates.map { tweetCandidate =>
t.RankingCandidate(
tweetId = tweetCandidate.id,
hydrationContext = Some(
t.FeatureHydrationContext.HomeHydrationContext(t
.HomeFeatureHydrationContext(tweetAuthor = None)))
)
}
val thriftResults = crMLRanker.getRankedResults(
t.RankingRequest(
requestContext = t.RankingRequestContext(
userId = userId,
config = rankingConfig
),
candidates = crMlRankerCandidates,
commonFeatures = commonFeatures.commonFeatures
)
)
thriftResults.map { response =>
response.scoredTweets.map { scoredTweet =>
Return(
CrMlRankerResult(
tweetId = scoredTweet.tweetId,
score = scoredTweet.score
)
)
}
}
}
}
}

42
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/BUILD
View File

@ -1,42 +0,0 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"3rdparty/jvm/javax/inject:javax.inject",
"cortex-deepbird/thrift/src/main/thrift:thrift-java",
"finagle/finagle-http/src/main/scala",
"finatra-internal/mtls/src/main/scala",
"finatra/inject/inject-core/src/main/scala/com/twitter/inject",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/featurestorev1",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/scorer",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure",
"scrooge/scrooge-serializer",
"src/java/com/twitter/ml/api:api-base",
"src/java/com/twitter/ml/common/base",
"src/java/com/twitter/ml/prediction/core",
"src/thrift/com/twitter/ml/prediction_service:prediction_service-java",
"src/thrift/com/twitter/ml/prediction_service:prediction_service-scala",
"twml/runtime/src/main/scala/com/twitter/deepbird/runtime/prediction_engine",
],
exports = [
"cortex-deepbird/thrift/src/main/thrift:thrift-java",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/scorer",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure",
"src/java/com/twitter/ml/prediction/core",
"src/thrift/com/twitter/ml/prediction_service:prediction_service-java",
"twml/runtime/src/main/scala/com/twitter/deepbird/runtime/prediction_engine",
],
)

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/BUILD.docx Normal file
View File

Binary file not shown.

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/BaseDeepbirdV2Scorer.docx Normal file
View File

Binary file not shown.

91
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/BaseDeepbirdV2Scorer.scala
View File

@ -1,91 +0,0 @@
package com.twitter.product_mixer.component_library.scorer.deepbird
import com.twitter.product_mixer.core.feature.datarecord.BaseDataRecordFeature
import com.twitter.ml.prediction_service.BatchPredictionRequest
import com.twitter.ml.prediction_service.BatchPredictionResponse
import com.twitter.cortex.deepbird.thriftjava.{ModelSelector => TModelSelector}
import com.twitter.ml.api.DataRecord
import com.twitter.product_mixer.component_library.scorer.common.ModelSelector
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordConverter
import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordExtractor
import com.twitter.product_mixer.core.feature.featuremap.datarecord.FeaturesScope
import com.twitter.product_mixer.core.functional_component.scorer.Scorer
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.UniversalNoun
import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier
import scala.collection.JavaConverters._
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.pipeline.pipeline_failure.IllegalStateFailure
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure
import com.twitter.stitch.Stitch
import com.twitter.util.Future
abstract class BaseDeepbirdV2Scorer[
Query <: PipelineQuery,
Candidate <: UniversalNoun[Any],
QueryFeatures <: BaseDataRecordFeature[Query, _],
CandidateFeatures <: BaseDataRecordFeature[Candidate, _],
ResultFeatures <: BaseDataRecordFeature[Candidate, _]
](
override val identifier: ScorerIdentifier,
modelIdSelector: ModelSelector[Query],
queryFeatures: FeaturesScope[QueryFeatures],
candidateFeatures: FeaturesScope[CandidateFeatures],
resultFeatures: Set[ResultFeatures])
extends Scorer[Query, Candidate] {
private val queryDataRecordConverter = new DataRecordConverter(queryFeatures)
private val candidateDataRecordConverter = new DataRecordConverter(candidateFeatures)
private val resultDataRecordExtractor = new DataRecordExtractor(resultFeatures)
require(resultFeatures.nonEmpty, "Result features cannot be empty")
override val features: Set[Feature[_, _]] = resultFeatures.asInstanceOf[Set[Feature[_, _]]]
def getBatchPredictions(
request: BatchPredictionRequest,
modelSelector: TModelSelector
): Future[BatchPredictionResponse]
override def apply(
query: Query,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[Seq[FeatureMap]] = {
// Convert all candidate feature maps to java datarecords then to scala datarecords.
val thriftCandidateDataRecords = candidates.map { candidate =>
candidateDataRecordConverter.toDataRecord(candidate.features)
}
val request = new BatchPredictionRequest(thriftCandidateDataRecords.asJava)
// Convert the query feature map to data record if available.
query.features.foreach { featureMap =>
request.setCommonFeatures(queryDataRecordConverter.toDataRecord(featureMap))
}
val modelSelector = modelIdSelector
.apply(query).map { id =>
val selector = new TModelSelector()
selector.setId(id)
selector
}.orNull
Stitch.callFuture(getBatchPredictions(request, modelSelector)).map { response =>
val dataRecords = Option(response.predictions).map(_.asScala).getOrElse(Seq.empty)
buildResults(candidates, dataRecords)
}
}
private def buildResults(
candidates: Seq[CandidateWithFeatures[Candidate]],
dataRecords: Seq[DataRecord]
): Seq[FeatureMap] = {
if (dataRecords.size != candidates.size) {
throw PipelineFailure(IllegalStateFailure, "Result Size mismatched candidates size")
}
dataRecords.map { resultDataRecord =>
resultDataRecordExtractor.fromDataRecord(resultDataRecord)
}
}
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/DeepbirdV2PredictionServerScorer.docx Normal file
View File

Binary file not shown.

55
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/DeepbirdV2PredictionServerScorer.scala
View File

@ -1,55 +0,0 @@
package com.twitter.product_mixer.component_library.scorer.deepbird
import com.twitter.cortex.deepbird.{thriftjava => t}
import com.twitter.ml.prediction_service.BatchPredictionRequest
import com.twitter.ml.prediction_service.BatchPredictionResponse
import com.twitter.product_mixer.component_library.scorer.common.ModelSelector
import com.twitter.product_mixer.core.feature.datarecord.BaseDataRecordFeature
import com.twitter.product_mixer.core.feature.featuremap.datarecord.FeaturesScope
import com.twitter.product_mixer.core.model.common.UniversalNoun
import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.util.Future
/**
* Configurable Scorer that calls any Deepbird Prediction Service thrift.
* @param identifier Unique identifier for the scorer
* @param predictionService The Prediction Thrift Service
* @param modelSelector Model ID Selector to decide which model to select, can also be represented
* as an anonymous function: { query: Query => Some("Ex") }
* @param queryFeatures The Query Features to convert and pass to the deepbird model.
* @param candidateFeatures The Candidate Features to convert and pass to the deepbird model.
* @param resultFeatures The Candidate features returned by the model.
* @tparam Query Type of pipeline query.
* @tparam Candidate Type of candidates to score.
* @tparam QueryFeatures type of the query level features consumed by the scorer.
* @tparam CandidateFeatures type of the candidate level features consumed by the scorer.
* @tparam ResultFeatures type of the candidate level features returned by the scorer.
*/
case class DeepbirdV2PredictionServerScorer[
Query <: PipelineQuery,
Candidate <: UniversalNoun[Any],
QueryFeatures <: BaseDataRecordFeature[Query, _],
CandidateFeatures <: BaseDataRecordFeature[Candidate, _],
ResultFeatures <: BaseDataRecordFeature[Candidate, _]
](
override val identifier: ScorerIdentifier,
predictionService: t.DeepbirdPredictionService.ServiceToClient,
modelSelector: ModelSelector[Query],
queryFeatures: FeaturesScope[QueryFeatures],
candidateFeatures: FeaturesScope[CandidateFeatures],
resultFeatures: Set[ResultFeatures])
extends BaseDeepbirdV2Scorer[
Query,
Candidate,
QueryFeatures,
CandidateFeatures,
ResultFeatures
](identifier, modelSelector, queryFeatures, candidateFeatures, resultFeatures) {
override def getBatchPredictions(
request: BatchPredictionRequest,
modelSelector: t.ModelSelector
): Future[BatchPredictionResponse] =
predictionService.batchPredictFromModel(request, modelSelector)
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/LollyPredictionEngineScorer.docx Normal file
View File

Binary file not shown.

61
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/LollyPredictionEngineScorer.scala
View File

@ -1,61 +0,0 @@
package com.twitter.product_mixer.component_library.scorer.deepbird
import com.twitter.ml.prediction.core.PredictionEngine
import com.twitter.ml.prediction_service.PredictionRequest
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.datarecord.BaseDataRecordFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordConverter
import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordExtractor
import com.twitter.product_mixer.core.feature.featuremap.datarecord.FeaturesScope
import com.twitter.product_mixer.core.functional_component.scorer.Scorer
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.UniversalNoun
import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
/**
* Scorer that locally loads a Deepbird model.
* @param identifier Unique identifier for the scorer
* @param predictionEngine Prediction Engine hosting the Deepbird model.
* @param candidateFeatures The Candidate Features to convert and pass to the deepbird model.
* @param resultFeatures The Candidate features returned by the model.
* @tparam Query Type of pipeline query.
* @tparam Candidate Type of candidates to score.
* @tparam QueryFeatures type of the query level features consumed by the scorer.
* @tparam CandidateFeatures type of the candidate level features consumed by the scorer.
* @tparam ResultFeatures type of the candidate level features returned by the scorer.
*/
class LollyPredictionEngineScorer[
Query <: PipelineQuery,
Candidate <: UniversalNoun[Any],
QueryFeatures <: BaseDataRecordFeature[Query, _],
CandidateFeatures <: BaseDataRecordFeature[Candidate, _],
ResultFeatures <: BaseDataRecordFeature[Candidate, _]
](
override val identifier: ScorerIdentifier,
predictionEngine: PredictionEngine,
candidateFeatures: FeaturesScope[CandidateFeatures],
resultFeatures: Set[ResultFeatures])
extends Scorer[Query, Candidate] {
private val dataRecordAdapter = new DataRecordConverter(candidateFeatures)
require(resultFeatures.nonEmpty, "Result features cannot be empty")
override val features: Set[Feature[_, _]] = resultFeatures.asInstanceOf[Set[Feature[_, _]]]
private val resultsDataRecordExtractor = new DataRecordExtractor(resultFeatures)
override def apply(
query: Query,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[Seq[FeatureMap]] = {
val featureMaps = candidates.map { candidateWithFeatures =>
val dataRecord = dataRecordAdapter.toDataRecord(candidateWithFeatures.features)
val predictionResponse = predictionEngine.apply(new PredictionRequest(dataRecord), true)
resultsDataRecordExtractor.fromDataRecord(predictionResponse.getPrediction)
}
Stitch.value(featureMaps)
}
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/TensorflowPredictionEngineScorer.docx Normal file
View File

Binary file not shown.

58
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/deepbird/TensorflowPredictionEngineScorer.scala
View File

@ -1,58 +0,0 @@
package com.twitter.product_mixer.component_library.scorer.deepbird
import com.twitter.cortex.deepbird.runtime.prediction_engine.TensorflowPredictionEngine
import com.twitter.cortex.deepbird.thriftjava.ModelSelector
import com.twitter.ml.prediction_service.BatchPredictionRequest
import com.twitter.ml.prediction_service.BatchPredictionResponse
import com.twitter.product_mixer.core.feature.datarecord.BaseDataRecordFeature
import com.twitter.product_mixer.core.feature.featuremap.datarecord.FeaturesScope
import com.twitter.product_mixer.core.model.common.UniversalNoun
import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.util.Future
/**
* Configurable Scorer that calls a TensorflowPredictionEngine.
* @param identifier Unique identifier for the scorer
* @param tensorflowPredictionEngine The TensorFlow Prediction Engine
* @param queryFeatures The Query Features to convert and pass to the deepbird model.
* @param candidateFeatures The Candidate Features to convert and pass to the deepbird model.
* @param resultFeatures The Candidate features returned by the model.
* @tparam Query Type of pipeline query.
* @tparam Candidate Type of candidates to score.
* @tparam QueryFeatures type of the query level features consumed by the scorer.
* @tparam CandidateFeatures type of the candidate level features consumed by the scorer.
* @tparam ResultFeatures type of the candidate level features returned by the scorer.
*/
class TensorflowPredictionEngineScorer[
Query <: PipelineQuery,
Candidate <: UniversalNoun[Any],
QueryFeatures <: BaseDataRecordFeature[Query, _],
CandidateFeatures <: BaseDataRecordFeature[Candidate, _],
ResultFeatures <: BaseDataRecordFeature[Candidate, _]
](
override val identifier: ScorerIdentifier,
tensorflowPredictionEngine: TensorflowPredictionEngine,
queryFeatures: FeaturesScope[QueryFeatures],
candidateFeatures: FeaturesScope[CandidateFeatures],
resultFeatures: Set[ResultFeatures])
extends BaseDeepbirdV2Scorer[
Query,
Candidate,
QueryFeatures,
CandidateFeatures,
ResultFeatures
](
identifier,
{ _: Query =>
None
},
queryFeatures,
candidateFeatures,
resultFeatures) {
override def getBatchPredictions(
request: BatchPredictionRequest,
modelSelector: ModelSelector
): Future[BatchPredictionResponse] = tensorflowPredictionEngine.getBatchPrediction(request)
}

16
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/param_gated/BUILD
View File

@ -1,16 +0,0 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/scorer",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
],
exports = [
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/scorer",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
],
)

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/param_gated/BUILD.docx Normal file
View File

Binary file not shown.

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/param_gated/ParamGatedScorer.docx Normal file
View File

Binary file not shown.

43
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/param_gated/ParamGatedScorer.scala
View File

@ -1,43 +0,0 @@
package com.twitter.product_mixer.component_library.scorer.param_gated
import com.twitter.product_mixer.component_library.scorer.param_gated.ParamGatedScorer.IdentifierPrefix
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.common.alert.Alert
import com.twitter.product_mixer.core.functional_component.scorer.Scorer
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.Conditionally
import com.twitter.product_mixer.core.model.common.UniversalNoun
import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import com.twitter.timelines.configapi.Param
/**
* A [[scorer]] with [[Conditionally]] based on a [[Param]]
*
* @param enabledParam the param to turn this [[scorer]] on and off
* @param scorer the underlying [[scorer]] to run when `enabledParam` is true
* @tparam Query The domain model for the query or request
* @tparam Result The type of the candidates
*/
case class ParamGatedScorer[-Query <: PipelineQuery, Result <: UniversalNoun[Any]](
enabledParam: Param[Boolean],
scorer: Scorer[Query, Result])
extends Scorer[Query, Result]
with Conditionally[Query] {
override val identifier: ScorerIdentifier = ScorerIdentifier(
IdentifierPrefix + scorer.identifier.name)
override val alerts: Seq[Alert] = scorer.alerts
override val features: Set[Feature[_, _]] = scorer.features
override def onlyIf(query: Query): Boolean =
Conditionally.and(query, scorer, query.params(enabledParam))
override def apply(
query: Query,
candidates: Seq[CandidateWithFeatures[Result]]
): Stitch[Seq[FeatureMap]] = scorer(query, candidates)
}
object ParamGatedScorer {
val IdentifierPrefix = "ParamGated"
}

16
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/qualityfactor_gated/BUILD
View File

@ -1,16 +0,0 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/scorer",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
],
exports = [
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/scorer",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
],
)

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/qualityfactor_gated/BUILD.docx Normal file
View File

Binary file not shown.

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/qualityfactor_gated/QualityFactorGatedScorer.docx Normal file
View File

Binary file not shown.

59
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/qualityfactor_gated/QualityFactorGatedScorer.scala
View File

@ -1,59 +0,0 @@
package com.twitter.product_mixer.component_library.scorer.qualityfactor_gated
import com.twitter.product_mixer.component_library.scorer.qualityfactor_gated.QualityFactorGatedScorer.IdentifierPrefix
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.common.alert.Alert
import com.twitter.product_mixer.core.functional_component.scorer.Scorer
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.Conditionally
import com.twitter.product_mixer.core.model.common.UniversalNoun
import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier
import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus
import com.twitter.stitch.Stitch
import com.twitter.timelines.configapi.Param
/**
* A [[scorer]] with [[Conditionally]] based on quality factor value and threshold
*
* @param qualityFactorThreshold quliaty factor threshold that turn off the scorer
* @param pipelineIdentifier identifier of the pipeline that quality factor is based on
* @param scorer the underlying [[scorer]] to run when `enabledParam` is true
* @tparam Query The domain model for the query or request
* @tparam Result The type of the candidates
*/
case class QualityFactorGatedScorer[
-Query <: PipelineQuery with HasQualityFactorStatus,
Result <: UniversalNoun[Any]
](
pipelineIdentifier: ComponentIdentifier,
qualityFactorThresholdParam: Param[Double],
scorer: Scorer[Query, Result])
extends Scorer[Query, Result]
with Conditionally[Query] {
override val identifier: ScorerIdentifier = ScorerIdentifier(
IdentifierPrefix + scorer.identifier.name)
override val alerts: Seq[Alert] = scorer.alerts
override val features: Set[Feature[_, _]] = scorer.features
override def onlyIf(query: Query): Boolean =
Conditionally.and(
query,
scorer,
query.getQualityFactorCurrentValue(pipelineIdentifier) >= query.params(
qualityFactorThresholdParam))
override def apply(
query: Query,
candidates: Seq[CandidateWithFeatures[Result]]
): Stitch[Seq[FeatureMap]] = scorer(query, candidates)
}
object QualityFactorGatedScorer {
val IdentifierPrefix = "QualityFactorGated"
}

21
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/BUILD
View File

@ -1,21 +0,0 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/featurestorev1",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featurestorev1",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
"src/thrift/com/twitter/ml/api:embedding-scala",
],
exports = [
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/common",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
],
)

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/BUILD.docx Normal file
View File

Binary file not shown.

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/BooleanInferInputTensorBuilder.docx Normal file
View File

Binary file not shown.

13
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/BooleanInferInputTensorBuilder.scala
View File

@ -1,13 +0,0 @@
package com.twitter.product_mixer.component_library.scorer.tensorbuilder
import inference.GrpcService.ModelInferRequest.InferInputTensor
case object BooleanInferInputTensorBuilder extends InferInputTensorBuilder[Boolean] {
def apply(
featureName: String,
featureValues: Seq[Boolean]
): Seq[InferInputTensor] = {
val tensorShape = Seq(featureValues.size, 1)
InferInputTensorBuilder.buildBoolInferInputTensor(featureName, featureValues, tensorShape)
}
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/BytesInferInputTensorBuilder.docx Normal file
View File

Binary file not shown.

13
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/BytesInferInputTensorBuilder.scala
View File

@ -1,13 +0,0 @@
package com.twitter.product_mixer.component_library.scorer.tensorbuilder
import inference.GrpcService.ModelInferRequest.InferInputTensor
case object BytesInferInputTensorBuilder extends InferInputTensorBuilder[String] {
def apply(
featureName: String,
featureValues: Seq[String]
): Seq[InferInputTensor] = {
val tensorShape = Seq(featureValues.size, 1)
InferInputTensorBuilder.buildBytesInferInputTensor(featureName, featureValues, tensorShape)
}
}

BIN
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/CandidateInferInputTensorBuilder.docx Normal file
View File

Binary file not shown.

70
product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tensorbuilder/CandidateInferInputTensorBuilder.scala
View File

@ -1,70 +0,0 @@
package com.twitter.product_mixer.component_library.scorer.tensorbuilder
import com.twitter.ml.api.thriftscala.FloatTensor
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.feature.ModelFeatureName
import com.twitter.product_mixer.core.feature.featuremap.featurestorev1.FeatureStoreV1FeatureMap._
import com.twitter.product_mixer.core.feature.featurestorev1.FeatureStoreV1CandidateFeature
import com.twitter.product_mixer.core.feature.featurestorev1.FeatureStoreV1QueryFeature
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.UniversalNoun
import inference.GrpcService.ModelInferRequest.InferInputTensor
class CandidateInferInputTensorBuilder[-Candidate <: UniversalNoun[Any], +Value](
builder: InferInputTensorBuilder[Value],
features: Set[_ <: Feature[Candidate, _] with ModelFeatureName]) {
def apply(
candidates: Seq[CandidateWithFeatures[Candidate]],
): Seq[InferInputTensor] = {
features.flatMap { feature =>
val featureValues: Seq[Value] = feature match {
case feature: FeatureStoreV1CandidateFeature[_, Candidate, _, Value] =>
candidates.map(_.features.getFeatureStoreV1CandidateFeature(feature))
case feature: FeatureStoreV1QueryFeature[_, _, _] =>
throw new UnexpectedFeatureTypeException(feature)
case feature: FeatureWithDefaultOnFailure[Candidate, Value] =>
candidates.map(_.features.getTry(feature).toOption.getOrElse(feature.defaultValue))
case feature: Feature[Candidate, Value] =>
candidates.map(_.features.get(feature))
}
builder.apply(feature.featureName, featureValues)
}.toSeq
}
}
case class CandidateBooleanInferInputTensorBuilder[-Candidate <: UniversalNoun[Any]](
features: Set[_ <: Feature[Candidate, Boolean] with ModelFeatureName])
extends CandidateInferInputTensorBuilder[Candidate, Boolean](
BooleanInferInputTensorBuilder,
features)
case class CandidateBytesInferInputTensorBuilder[-Candidate <: UniversalNoun[Any]](
features: Set[_ <: Feature[Candidate, String] with ModelFeatureName])
extends CandidateInferInputTensorBuilder[Candidate, String](
BytesInferInputTensorBuilder,
features)
case class CandidateFloat32InferInputTensorBuilder[-Candidate <: UniversalNoun[Any]](
features: Set[_ <: Feature[Candidate, _ <: AnyVal] with ModelFeatureName])
extends CandidateInferInputTensorBuilder[Candidate, AnyVal](
Float32InferInputTensorBuilder,
features)
case class CandidateFloatTensorInferInputTensorBuilder[-Candidate <: UniversalNoun[Any]](
features: Set[_ <: Feature[Candidate, FloatTensor] with ModelFeatureName])
extends CandidateInferInputTensorBuilder[Candidate, FloatTensor](
FloatTensorInferInputTensorBuilder,
features)
case class CandidateInt64InferInputTensorBuilder[-Candidate <: UniversalNoun[Any]](
features: Set[_ <: Feature[Candidate, _ <: AnyVal] with ModelFeatureName])
extends CandidateInferInputTensorBuilder[Candidate, AnyVal](
Int64InferInputTensorBuilder,
features)
case class CandidateSparseMapInferInputTensorBuilder[-Candidate <: UniversalNoun[Any]](
features: Set[_ <: Feature[Candidate, Option[Map[Int, Double]]] with ModelFeatureName])
extends CandidateInferInputTensorBuilder[Candidate, Option[Map[Int, Double]]](
SparseMapInferInputTensorBuilder,
features)

Some files were not shown because too many files have changed in this diff Show More