[docx] split commit for file 1400
Signed-off-by: Ari Archer <ari.web.xyz@gmail.com>
This commit is contained in:
Ari Archer
parent
c80f53f99d
commit
e27b2e31c3
Binary file not shown.
|
|
@ -1,31 +0,0 @@
|
|||
package com.twitter.graph_feature_service.server.modules
|
||||
|
||||
import com.twitter.inject.TwitterModule
|
||||
|
||||
object ServerFlagNames {
|
||||
final val NumWorkers = "service.num_workers"
|
||||
final val ServiceRole = "service.role"
|
||||
final val ServiceEnv = "service.env"
|
||||
|
||||
final val MemCacheClientName = "service.mem_cache_client_name"
|
||||
final val MemCachePath = "service.mem_cache_path"
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes references to the flag values defined in the aurora.deploy file.
|
||||
* To check what the flag values are initialized in runtime, search FlagsModule in stdout
|
||||
*/
|
||||
object ServerFlagsModule extends TwitterModule {
|
||||
|
||||
import ServerFlagNames._
|
||||
|
||||
flag[Int](NumWorkers, "Num of workers")
|
||||
|
||||
flag[String](ServiceRole, "Service Role")
|
||||
|
||||
flag[String](ServiceEnv, "Service Env")
|
||||
|
||||
flag[String](MemCacheClientName, "MemCache Client Name")
|
||||
|
||||
flag[String](MemCachePath, "MemCache Path")
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,16 +0,0 @@
|
|||
package com.twitter.graph_feature_service.server.stores
|
||||
|
||||
import com.twitter.graph_feature_service.common.Configs.RandomSeed
|
||||
import com.twitter.graph_feature_service.thriftscala.FeatureType
|
||||
import scala.util.hashing.MurmurHash3
|
||||
|
||||
object FeatureTypesEncoder {
|
||||
|
||||
def apply(featureTypes: Seq[FeatureType]): String = {
|
||||
val byteArray = featureTypes.flatMap { featureType =>
|
||||
Array(featureType.leftEdgeType.getValue.toByte, featureType.rightEdgeType.getValue.toByte)
|
||||
}.toArray
|
||||
(MurmurHash3.bytesHash(byteArray, RandomSeed) & 0x7fffffff).toString // keep positive
|
||||
}
|
||||
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,181 +0,0 @@
|
|||
package com.twitter.graph_feature_service.server.stores
|
||||
|
||||
import com.twitter.finagle.RequestTimeoutException
|
||||
import com.twitter.finagle.stats.{Stat, StatsReceiver}
|
||||
import com.twitter.graph_feature_service.server.handlers.ServerGetIntersectionHandler.GetIntersectionRequest
|
||||
import com.twitter.graph_feature_service.server.modules.GraphFeatureServiceWorkerClients
|
||||
import com.twitter.graph_feature_service.server.stores.GetIntersectionStore.GetIntersectionQuery
|
||||
import com.twitter.graph_feature_service.thriftscala._
|
||||
import com.twitter.inject.Logging
|
||||
import com.twitter.storehaus.ReadableStore
|
||||
import com.twitter.util.Future
|
||||
import javax.inject.Singleton
|
||||
import scala.collection.mutable.ArrayBuffer
|
||||
|
||||
@Singleton
|
||||
case class GetIntersectionStore(
|
||||
graphFeatureServiceWorkerClients: GraphFeatureServiceWorkerClients,
|
||||
statsReceiver: StatsReceiver)
|
||||
extends ReadableStore[GetIntersectionQuery, CachedIntersectionResult]
|
||||
with Logging {
|
||||
|
||||
import GetIntersectionStore._
|
||||
|
||||
private val stats = statsReceiver.scope("get_intersection_store")
|
||||
private val requestCount = stats.counter(name = "request_count")
|
||||
private val aggregatorLatency = stats.stat("aggregator_latency")
|
||||
private val timeOutCounter = stats.counter("worker_timeouts")
|
||||
private val unknownErrorCounter = stats.counter("unknown_errors")
|
||||
|
||||
override def multiGet[K1 <: GetIntersectionQuery](
|
||||
ks: Set[K1]
|
||||
): Map[K1, Future[Option[CachedIntersectionResult]]] = {
|
||||
if (ks.isEmpty) {
|
||||
Map.empty
|
||||
} else {
|
||||
requestCount.incr()
|
||||
|
||||
val head = ks.head
|
||||
// We assume all the GetIntersectionQuery use the same userId and featureTypes
|
||||
val userId = head.userId
|
||||
val featureTypes = head.featureTypes
|
||||
val presetFeatureTypes = head.presetFeatureTypes
|
||||
val calculatedFeatureTypes = head.calculatedFeatureTypes
|
||||
val intersectionIdLimit = head.intersectionIdLimit
|
||||
|
||||
val request = WorkerIntersectionRequest(
|
||||
userId,
|
||||
ks.map(_.candidateId).toArray,
|
||||
featureTypes,
|
||||
presetFeatureTypes,
|
||||
intersectionIdLimit
|
||||
)
|
||||
|
||||
val resultFuture = Future
|
||||
.collect(
|
||||
graphFeatureServiceWorkerClients.workers.map { worker =>
|
||||
worker
|
||||
.getIntersection(request)
|
||||
.rescue {
|
||||
case _: RequestTimeoutException =>
|
||||
timeOutCounter.incr()
|
||||
Future.value(DefaultWorkerIntersectionResponse)
|
||||
case e =>
|
||||
unknownErrorCounter.incr()
|
||||
logger.error("Failure to load result.", e)
|
||||
Future.value(DefaultWorkerIntersectionResponse)
|
||||
}
|
||||
}
|
||||
).map { responses =>
|
||||
Stat.time(aggregatorLatency) {
|
||||
gfsIntersectionResponseAggregator(
|
||||
responses,
|
||||
calculatedFeatureTypes,
|
||||
request.candidateUserIds,
|
||||
intersectionIdLimit
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
ks.map { query =>
|
||||
query -> resultFuture.map(_.get(query.candidateId))
|
||||
}.toMap
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Function to merge GfsIntersectionResponse from workers into one result.
|
||||
*/
|
||||
private def gfsIntersectionResponseAggregator(
|
||||
responseList: Seq[WorkerIntersectionResponse],
|
||||
features: Seq[FeatureType],
|
||||
candidates: Seq[Long],
|
||||
intersectionIdLimit: Int
|
||||
): Map[Long, CachedIntersectionResult] = {
|
||||
|
||||
// Map of (candidate -> features -> type -> value)
|
||||
val cube = Array.fill[Int](candidates.length, features.length, 3)(0)
|
||||
// Map of (candidate -> features -> intersectionIds)
|
||||
val ids = Array.fill[Option[ArrayBuffer[Long]]](candidates.length, features.length)(None)
|
||||
val notZero = intersectionIdLimit != 0
|
||||
|
||||
for {
|
||||
response <- responseList
|
||||
(features, candidateIndex) <- response.results.zipWithIndex
|
||||
(workerValue, featureIndex) <- features.zipWithIndex
|
||||
} {
|
||||
cube(candidateIndex)(featureIndex)(CountIndex) += workerValue.count
|
||||
cube(candidateIndex)(featureIndex)(LeftDegreeIndex) += workerValue.leftNodeDegree
|
||||
cube(candidateIndex)(featureIndex)(RightDegreeIndex) += workerValue.rightNodeDegree
|
||||
|
||||
if (notZero && workerValue.intersectionIds.nonEmpty) {
|
||||
val arrayBuffer = ids(candidateIndex)(featureIndex) match {
|
||||
case Some(buffer) => buffer
|
||||
case None =>
|
||||
val buffer = ArrayBuffer[Long]()
|
||||
ids(candidateIndex)(featureIndex) = Some(buffer)
|
||||
buffer
|
||||
}
|
||||
val intersectionIds = workerValue.intersectionIds
|
||||
|
||||
// Scan the intersectionId based on the Shard. The response order is consistent.
|
||||
if (arrayBuffer.size < intersectionIdLimit) {
|
||||
if (intersectionIds.size > intersectionIdLimit - arrayBuffer.size) {
|
||||
arrayBuffer ++= intersectionIds.slice(0, intersectionIdLimit - arrayBuffer.size)
|
||||
} else {
|
||||
arrayBuffer ++= intersectionIds
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
candidates.zipWithIndex.map {
|
||||
case (candidate, candidateIndex) =>
|
||||
candidate -> CachedIntersectionResult(features.indices.map { featureIndex =>
|
||||
WorkerIntersectionValue(
|
||||
cube(candidateIndex)(featureIndex)(CountIndex),
|
||||
cube(candidateIndex)(featureIndex)(LeftDegreeIndex),
|
||||
cube(candidateIndex)(featureIndex)(RightDegreeIndex),
|
||||
ids(candidateIndex)(featureIndex).getOrElse(Nil)
|
||||
)
|
||||
})
|
||||
}.toMap
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
object GetIntersectionStore {
|
||||
|
||||
private[graph_feature_service] case class GetIntersectionQuery(
|
||||
userId: Long,
|
||||
candidateId: Long,
|
||||
featureTypes: Seq[FeatureType],
|
||||
presetFeatureTypes: PresetFeatureTypes,
|
||||
featureTypesString: String,
|
||||
calculatedFeatureTypes: Seq[FeatureType],
|
||||
intersectionIdLimit: Int)
|
||||
|
||||
private[graph_feature_service] object GetIntersectionQuery {
|
||||
def buildQueries(request: GetIntersectionRequest): Set[GetIntersectionQuery] = {
|
||||
request.candidateUserIds.toSet.map { candidateId: Long =>
|
||||
GetIntersectionQuery(
|
||||
request.userId,
|
||||
candidateId,
|
||||
request.featureTypes,
|
||||
request.presetFeatureTypes,
|
||||
request.calculatedFeatureTypesString,
|
||||
request.calculatedFeatureTypes,
|
||||
request.intersectionIdLimit.getOrElse(DefaultIntersectionIdLimit)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Don't return the intersectionId for better performance
|
||||
private val DefaultIntersectionIdLimit = 0
|
||||
private val DefaultWorkerIntersectionResponse = WorkerIntersectionResponse()
|
||||
|
||||
private val CountIndex = 0
|
||||
private val LeftDegreeIndex = 1
|
||||
private val RightDegreeIndex = 2
|
||||
}
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
scala_library(
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"graph-feature-service/src/main/thrift/com/twitter/graph_feature_service:graph_feature_service_thrift-scala",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,58 +0,0 @@
|
|||
package com.twitter.graph_feature_service.util
|
||||
|
||||
import com.twitter.graph_feature_service.thriftscala.EdgeType._
|
||||
import com.twitter.graph_feature_service.thriftscala.{FeatureType, PresetFeatureTypes}
|
||||
|
||||
object FeatureTypesCalculator {
|
||||
|
||||
final val DefaultTwoHop = Seq(
|
||||
FeatureType(Following, FollowedBy),
|
||||
FeatureType(Following, FavoritedBy),
|
||||
FeatureType(Following, RetweetedBy),
|
||||
FeatureType(Following, MentionedBy),
|
||||
FeatureType(Following, MutualFollow),
|
||||
FeatureType(Favorite, FollowedBy),
|
||||
FeatureType(Favorite, FavoritedBy),
|
||||
FeatureType(Favorite, RetweetedBy),
|
||||
FeatureType(Favorite, MentionedBy),
|
||||
FeatureType(Favorite, MutualFollow),
|
||||
FeatureType(MutualFollow, FollowedBy),
|
||||
FeatureType(MutualFollow, FavoritedBy),
|
||||
FeatureType(MutualFollow, RetweetedBy),
|
||||
FeatureType(MutualFollow, MentionedBy),
|
||||
FeatureType(MutualFollow, MutualFollow)
|
||||
)
|
||||
|
||||
final val SocialProofTwoHop = Seq(FeatureType(Following, FollowedBy))
|
||||
|
||||
final val HtlTwoHop = DefaultTwoHop
|
||||
|
||||
final val WtfTwoHop = SocialProofTwoHop
|
||||
|
||||
final val SqTwoHop = DefaultTwoHop
|
||||
|
||||
final val RuxTwoHop = DefaultTwoHop
|
||||
|
||||
final val MRTwoHop = DefaultTwoHop
|
||||
|
||||
final val UserTypeaheadTwoHop = SocialProofTwoHop
|
||||
|
||||
final val presetFeatureTypes =
|
||||
(HtlTwoHop ++ WtfTwoHop ++ SqTwoHop ++ RuxTwoHop ++ MRTwoHop ++ UserTypeaheadTwoHop).toSet
|
||||
|
||||
def getFeatureTypes(
|
||||
presetFeatureTypes: PresetFeatureTypes,
|
||||
featureTypes: Seq[FeatureType]
|
||||
): Seq[FeatureType] = {
|
||||
presetFeatureTypes match {
|
||||
case PresetFeatureTypes.HtlTwoHop => HtlTwoHop
|
||||
case PresetFeatureTypes.WtfTwoHop => WtfTwoHop
|
||||
case PresetFeatureTypes.SqTwoHop => SqTwoHop
|
||||
case PresetFeatureTypes.RuxTwoHop => RuxTwoHop
|
||||
case PresetFeatureTypes.MrTwoHop => MRTwoHop
|
||||
case PresetFeatureTypes.UserTypeaheadTwoHop => UserTypeaheadTwoHop
|
||||
case _ => featureTypes
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,242 +0,0 @@
|
|||
package com.twitter.graph_feature_service.util
|
||||
|
||||
import com.twitter.graph_feature_service.thriftscala.{
|
||||
FeatureType,
|
||||
IntersectionValue,
|
||||
WorkerIntersectionValue
|
||||
}
|
||||
import java.nio.ByteBuffer
|
||||
import scala.collection.mutable.ArrayBuffer
|
||||
|
||||
/**
|
||||
* Functions for computing feature values based on the values returned by constantDB.
|
||||
*/
|
||||
object IntersectionValueCalculator {
|
||||
|
||||
/**
|
||||
* Compute the size of the array in a ByteBuffer.
|
||||
* Note that this function assumes the ByteBuffer is encoded using Injections.seqLong2ByteBuffer
|
||||
*/
|
||||
def computeArraySize(x: ByteBuffer): Int = {
|
||||
x.remaining() >> 3 // divide 8
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
def apply(x: ByteBuffer, y: ByteBuffer, intersectionIdLimit: Int): WorkerIntersectionValue = {
|
||||
|
||||
val xSize = computeArraySize(x)
|
||||
val ySize = computeArraySize(y)
|
||||
|
||||
val largerArray = if (xSize > ySize) x else y
|
||||
val smallerArray = if (xSize > ySize) y else x
|
||||
|
||||
if (intersectionIdLimit == 0) {
|
||||
val result = computeIntersectionUsingBinarySearchOnLargerByteBuffer(smallerArray, largerArray)
|
||||
WorkerIntersectionValue(result, xSize, ySize)
|
||||
} else {
|
||||
val (result, ids) = computeIntersectionWithIds(smallerArray, largerArray, intersectionIdLimit)
|
||||
WorkerIntersectionValue(result, xSize, ySize, ids)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Note that this function assumes the ByteBuffer is encoded using Injections.seqLong2ByteBuffer
|
||||
*
|
||||
*/
|
||||
def computeIntersectionUsingBinarySearchOnLargerByteBuffer(
|
||||
smallArray: ByteBuffer,
|
||||
largeArray: ByteBuffer
|
||||
): Int = {
|
||||
var res: Int = 0
|
||||
var i: Int = 0
|
||||
|
||||
while (i < smallArray.remaining()) {
|
||||
if (binarySearch(largeArray, smallArray.getLong(i)) >= 0) {
|
||||
res += 1
|
||||
}
|
||||
i += 8
|
||||
}
|
||||
res
|
||||
}
|
||||
|
||||
def computeIntersectionWithIds(
|
||||
smallArray: ByteBuffer,
|
||||
largeArray: ByteBuffer,
|
||||
intersectionLimit: Int
|
||||
): (Int, Seq[Long]) = {
|
||||
var res: Int = 0
|
||||
var i: Int = 0
|
||||
// Most of the intersectionLimit is smaller than default size: 16
|
||||
val idBuffer = ArrayBuffer[Long]()
|
||||
|
||||
while (i < smallArray.remaining()) {
|
||||
val value = smallArray.getLong(i)
|
||||
if (binarySearch(largeArray, value) >= 0) {
|
||||
res += 1
|
||||
// Always get the smaller ids
|
||||
if (idBuffer.size < intersectionLimit) {
|
||||
idBuffer += value
|
||||
}
|
||||
}
|
||||
i += 8
|
||||
}
|
||||
(res, idBuffer)
|
||||
}
|
||||
|
||||
/**
|
||||
* Note that this function assumes the ByteBuffer is encoded using Injections.seqLong2ByteBuffer
|
||||
*
|
||||
*/
|
||||
private[util] def binarySearch(arr: ByteBuffer, value: Long): Int = {
|
||||
var start = 0
|
||||
var end = arr.remaining()
|
||||
|
||||
while (start <= end && start < arr.remaining()) {
|
||||
val mid = ((start + end) >> 1) & ~7 // take mid - mid % 8
|
||||
if (arr.getLong(mid) == value) {
|
||||
return mid // return the index of the value
|
||||
} else if (arr.getLong(mid) < value) {
|
||||
start = mid + 8
|
||||
} else {
|
||||
end = mid - 1
|
||||
}
|
||||
}
|
||||
// if not existed, return -1
|
||||
-1
|
||||
}
|
||||
|
||||
/**
|
||||
* TODO: for now it only computes intersection size. Will add more feature types (e.g., dot
|
||||
* product, maximum value).
|
||||
*
|
||||
* NOTE that this function assumes both x and y are SORTED arrays.
|
||||
* In graph feature service, the sorting is done in the offline Scalding job.
|
||||
*
|
||||
* @param x source user's array
|
||||
* @param y candidate user's array
|
||||
* @param featureType feature type
|
||||
* @return
|
||||
*/
|
||||
def apply(x: Array[Long], y: Array[Long], featureType: FeatureType): IntersectionValue = {
|
||||
|
||||
val xSize = x.length
|
||||
val ySize = y.length
|
||||
|
||||
val intersection =
|
||||
if (xSize.min(ySize) * math.log(xSize.max(ySize)) < (xSize + ySize).toDouble) {
|
||||
if (xSize < ySize) {
|
||||
computeIntersectionUsingBinarySearchOnLargerArray(x, y)
|
||||
} else {
|
||||
computeIntersectionUsingBinarySearchOnLargerArray(y, x)
|
||||
}
|
||||
} else {
|
||||
computeIntersectionUsingListMerging(x, y)
|
||||
}
|
||||
|
||||
IntersectionValue(
|
||||
featureType,
|
||||
Some(intersection.toInt),
|
||||
None, // return None for now
|
||||
Some(xSize),
|
||||
Some(ySize)
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Function for computing the intersections of two SORTED arrays by list merging.
|
||||
*
|
||||
* @param x one array
|
||||
* @param y another array
|
||||
* @param ordering ordering function for comparing values of T
|
||||
* @tparam T type
|
||||
* @return The intersection size and the list of intersected elements
|
||||
*/
|
||||
private[util] def computeIntersectionUsingListMerging[T](
|
||||
x: Array[T],
|
||||
y: Array[T]
|
||||
)(
|
||||
implicit ordering: Ordering[T]
|
||||
): Int = {
|
||||
|
||||
var res: Int = 0
|
||||
var i: Int = 0
|
||||
var j: Int = 0
|
||||
|
||||
while (i < x.length && j < y.length) {
|
||||
val comp = ordering.compare(x(i), y(j))
|
||||
if (comp > 0) j += 1
|
||||
else if (comp < 0) i += 1
|
||||
else {
|
||||
res += 1
|
||||
i += 1
|
||||
j += 1
|
||||
}
|
||||
}
|
||||
res
|
||||
}
|
||||
|
||||
/**
|
||||
* Function for computing the intersections of two arrays by binary search on the larger array.
|
||||
* Note that the larger array MUST be SORTED.
|
||||
*
|
||||
* @param smallArray smaller array
|
||||
* @param largeArray larger array
|
||||
* @param ordering ordering function for comparing values of T
|
||||
* @tparam T type
|
||||
*
|
||||
* @return The intersection size and the list of intersected elements
|
||||
*/
|
||||
private[util] def computeIntersectionUsingBinarySearchOnLargerArray[T](
|
||||
smallArray: Array[T],
|
||||
largeArray: Array[T]
|
||||
)(
|
||||
implicit ordering: Ordering[T]
|
||||
): Int = {
|
||||
var res: Int = 0
|
||||
var i: Int = 0
|
||||
while (i < smallArray.length) {
|
||||
val currentValue: T = smallArray(i)
|
||||
if (binarySearch(largeArray, currentValue) >= 0) {
|
||||
res += 1
|
||||
}
|
||||
i += 1
|
||||
}
|
||||
res
|
||||
}
|
||||
|
||||
/**
|
||||
* Function for doing the binary search
|
||||
*
|
||||
* @param arr array
|
||||
* @param value the target value for searching
|
||||
* @param ordering ordering function
|
||||
* @tparam T type
|
||||
* @return the index of element in the larger array.
|
||||
* If there is no such element in the array, return -1.
|
||||
*/
|
||||
private[util] def binarySearch[T](
|
||||
arr: Array[T],
|
||||
value: T
|
||||
)(
|
||||
implicit ordering: Ordering[T]
|
||||
): Int = {
|
||||
var start = 0
|
||||
var end = arr.length - 1
|
||||
|
||||
while (start <= end) {
|
||||
val mid = (start + end) >> 1
|
||||
val comp = ordering.compare(arr(mid), value)
|
||||
if (comp == 0) {
|
||||
return mid // return the index of the value
|
||||
} else if (comp < 0) {
|
||||
start = mid + 1
|
||||
} else {
|
||||
end = mid - 1
|
||||
}
|
||||
}
|
||||
// if not existed, return -1
|
||||
-1
|
||||
}
|
||||
}
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["**/*.scala"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"3rdparty/jvm/net/codingwell:scala-guice",
|
||||
"discovery-common/src/main/scala/com/twitter/discovery/common/stats",
|
||||
"finatra-internal/decider/src/main/scala",
|
||||
"finatra-internal/gizmoduck/src/main/scala",
|
||||
"finatra-internal/mtls-thriftmux/src/main/scala",
|
||||
"finatra/inject/inject-app/src/main/scala",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"finatra/inject/inject-server/src/main/scala",
|
||||
"finatra/inject/inject-thrift-client/src/main/scala",
|
||||
"finatra/inject/inject-utils/src/main/scala",
|
||||
"frigate/frigate-common:constdb_util",
|
||||
"graph-feature-service/src/main/resources",
|
||||
"graph-feature-service/src/main/scala/com/twitter/graph_feature_service/common",
|
||||
"graph-feature-service/src/main/scala/com/twitter/graph_feature_service/util",
|
||||
"graph-feature-service/src/main/thrift/com/twitter/graph_feature_service:graph_feature_service_thrift-scala",
|
||||
"hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common",
|
||||
"servo/request/src/main/scala",
|
||||
"twitter-server-internal/src/main/scala",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"util/util-app/src/main/scala",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,58 +0,0 @@
|
|||
package com.twitter.graph_feature_service.worker
|
||||
|
||||
import com.google.inject.Module
|
||||
import com.twitter.finatra.decider.modules.DeciderModule
|
||||
import com.twitter.finatra.gizmoduck.modules.TimerModule
|
||||
import com.twitter.finatra.mtls.thriftmux.Mtls
|
||||
import com.twitter.finatra.thrift.ThriftServer
|
||||
import com.twitter.finatra.thrift.filters.{
|
||||
LoggingMDCFilter,
|
||||
StatsFilter,
|
||||
ThriftMDCFilter,
|
||||
TraceIdMDCFilter
|
||||
}
|
||||
import com.twitter.finatra.mtls.thriftmux.modules.MtlsThriftWebFormsModule
|
||||
import com.twitter.finatra.thrift.routing.ThriftRouter
|
||||
import com.twitter.graph_feature_service.thriftscala
|
||||
import com.twitter.graph_feature_service.worker.controllers.WorkerController
|
||||
import com.twitter.graph_feature_service.worker.handlers.WorkerWarmupHandler
|
||||
import com.twitter.graph_feature_service.worker.modules.{
|
||||
GraphContainerProviderModule,
|
||||
WorkerFlagModule
|
||||
}
|
||||
import com.twitter.graph_feature_service.worker.util.GraphContainer
|
||||
import com.twitter.inject.thrift.modules.ThriftClientIdModule
|
||||
import com.twitter.util.Await
|
||||
|
||||
object Main extends WorkerMain
|
||||
|
||||
class WorkerMain extends ThriftServer with Mtls {
|
||||
|
||||
override val name = "graph_feature_service-worker"
|
||||
|
||||
override val modules: Seq[Module] = {
|
||||
Seq(
|
||||
WorkerFlagModule,
|
||||
DeciderModule,
|
||||
TimerModule,
|
||||
ThriftClientIdModule,
|
||||
GraphContainerProviderModule,
|
||||
new MtlsThriftWebFormsModule[thriftscala.Worker.MethodPerEndpoint](this)
|
||||
)
|
||||
}
|
||||
|
||||
override def configureThrift(router: ThriftRouter): Unit = {
|
||||
router
|
||||
.filter[LoggingMDCFilter]
|
||||
.filter[TraceIdMDCFilter]
|
||||
.filter[ThriftMDCFilter]
|
||||
.filter[StatsFilter]
|
||||
.add[WorkerController]
|
||||
}
|
||||
|
||||
override protected def warmup(): Unit = {
|
||||
val graphContainer = injector.instance[GraphContainer]
|
||||
Await.result(graphContainer.warmup)
|
||||
handle[WorkerWarmupHandler]()
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,38 +0,0 @@
|
|||
package com.twitter.graph_feature_service.worker.controllers
|
||||
|
||||
import com.twitter.discovery.common.stats.DiscoveryStatsFilter
|
||||
import com.twitter.finagle.Service
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.finatra.thrift.Controller
|
||||
import com.twitter.graph_feature_service.thriftscala
|
||||
import com.twitter.graph_feature_service.thriftscala.Worker.GetIntersection
|
||||
import com.twitter.graph_feature_service.thriftscala._
|
||||
import com.twitter.graph_feature_service.worker.handlers._
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class WorkerController @Inject() (
|
||||
workerGetIntersectionHandler: WorkerGetIntersectionHandler
|
||||
)(
|
||||
implicit statsReceiver: StatsReceiver)
|
||||
extends Controller(thriftscala.Worker) {
|
||||
|
||||
// use DiscoveryStatsFilter to filter out exceptions out of our control
|
||||
private val getIntersectionService: Service[
|
||||
WorkerIntersectionRequest,
|
||||
WorkerIntersectionResponse
|
||||
] =
|
||||
new DiscoveryStatsFilter[WorkerIntersectionRequest, WorkerIntersectionResponse](
|
||||
statsReceiver.scope("srv").scope("get_intersection")
|
||||
).andThen(Service.mk(workerGetIntersectionHandler))
|
||||
|
||||
val getIntersection: Service[GetIntersection.Args, WorkerIntersectionResponse] = { args =>
|
||||
getIntersectionService(args.request).onFailure { throwable =>
|
||||
logger.error(s"Failure to get intersection for request $args.", throwable)
|
||||
}
|
||||
}
|
||||
|
||||
handle(GetIntersection) { getIntersection }
|
||||
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,105 +0,0 @@
|
|||
package com.twitter.graph_feature_service.worker.handlers
|
||||
|
||||
import com.twitter.finagle.stats.{Stat, StatsReceiver}
|
||||
import com.twitter.graph_feature_service.thriftscala.{
|
||||
WorkerIntersectionRequest,
|
||||
WorkerIntersectionResponse,
|
||||
WorkerIntersectionValue
|
||||
}
|
||||
import com.twitter.graph_feature_service.util.{FeatureTypesCalculator, IntersectionValueCalculator}
|
||||
import com.twitter.graph_feature_service.util.IntersectionValueCalculator._
|
||||
import com.twitter.graph_feature_service.worker.util.GraphContainer
|
||||
import com.twitter.servo.request.RequestHandler
|
||||
import com.twitter.util.Future
|
||||
import java.nio.ByteBuffer
|
||||
import javax.inject.{Inject, Singleton}
|
||||
|
||||
@Singleton
|
||||
class WorkerGetIntersectionHandler @Inject() (
|
||||
graphContainer: GraphContainer,
|
||||
statsReceiver: StatsReceiver)
|
||||
extends RequestHandler[WorkerIntersectionRequest, WorkerIntersectionResponse] {
|
||||
|
||||
import WorkerGetIntersectionHandler._
|
||||
|
||||
private val stats: StatsReceiver = statsReceiver.scope("srv/get_intersection")
|
||||
private val numCandidatesCount = stats.counter("total_num_candidates")
|
||||
private val toPartialGraphQueryStat = stats.stat("to_partial_graph_query_latency")
|
||||
private val fromPartialGraphQueryStat = stats.stat("from_partial_graph_query_latency")
|
||||
private val intersectionCalculationStat = stats.stat("computation_latency")
|
||||
|
||||
override def apply(request: WorkerIntersectionRequest): Future[WorkerIntersectionResponse] = {
|
||||
|
||||
numCandidatesCount.incr(request.candidateUserIds.length)
|
||||
|
||||
val userId = request.userId
|
||||
|
||||
// NOTE: do not change the order of candidates
|
||||
val candidateIds = request.candidateUserIds
|
||||
|
||||
// NOTE: do not change the order of features
|
||||
val featureTypes =
|
||||
FeatureTypesCalculator.getFeatureTypes(request.presetFeatureTypes, request.featureTypes)
|
||||
|
||||
val leftEdges = featureTypes.map(_.leftEdgeType).distinct
|
||||
val rightEdges = featureTypes.map(_.rightEdgeType).distinct
|
||||
|
||||
val rightEdgeMap = Stat.time(toPartialGraphQueryStat) {
|
||||
rightEdges.map { rightEdge =>
|
||||
val map = graphContainer.toPartialMap.get(rightEdge) match {
|
||||
case Some(graph) =>
|
||||
candidateIds.flatMap { candidateId =>
|
||||
graph.apply(candidateId).map(candidateId -> _)
|
||||
}.toMap
|
||||
case None =>
|
||||
Map.empty[Long, ByteBuffer]
|
||||
}
|
||||
rightEdge -> map
|
||||
}.toMap
|
||||
}
|
||||
|
||||
val leftEdgeMap = Stat.time(fromPartialGraphQueryStat) {
|
||||
leftEdges.flatMap { leftEdge =>
|
||||
graphContainer.toPartialMap.get(leftEdge).flatMap(_.apply(userId)).map(leftEdge -> _)
|
||||
}.toMap
|
||||
}
|
||||
|
||||
val res = Stat.time(intersectionCalculationStat) {
|
||||
WorkerIntersectionResponse(
|
||||
// NOTE that candidate ordering is important
|
||||
candidateIds.map { candidateId =>
|
||||
// NOTE that the featureTypes ordering is important
|
||||
featureTypes.map {
|
||||
featureType =>
|
||||
val leftNeighborsOpt = leftEdgeMap.get(featureType.leftEdgeType)
|
||||
val rightNeighborsOpt =
|
||||
rightEdgeMap.get(featureType.rightEdgeType).flatMap(_.get(candidateId))
|
||||
|
||||
if (leftNeighborsOpt.isEmpty && rightNeighborsOpt.isEmpty) {
|
||||
EmptyWorkerIntersectionValue
|
||||
} else if (rightNeighborsOpt.isEmpty) {
|
||||
EmptyWorkerIntersectionValue.copy(
|
||||
leftNodeDegree = computeArraySize(leftNeighborsOpt.get)
|
||||
)
|
||||
} else if (leftNeighborsOpt.isEmpty) {
|
||||
EmptyWorkerIntersectionValue.copy(
|
||||
rightNodeDegree = computeArraySize(rightNeighborsOpt.get)
|
||||
)
|
||||
} else {
|
||||
IntersectionValueCalculator(
|
||||
leftNeighborsOpt.get,
|
||||
rightNeighborsOpt.get,
|
||||
request.intersectionIdLimit)
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
Future.value(res)
|
||||
}
|
||||
}
|
||||
|
||||
object WorkerGetIntersectionHandler {
|
||||
val EmptyWorkerIntersectionValue: WorkerIntersectionValue = WorkerIntersectionValue(0, 0, 0, Nil)
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,14 +0,0 @@
|
|||
package com.twitter.graph_feature_service.worker.handlers
|
||||
|
||||
import com.twitter.finatra.thrift.routing.ThriftWarmup
|
||||
import com.twitter.inject.Logging
|
||||
import com.twitter.inject.utils.Handler
|
||||
import javax.inject.{Inject, Singleton}
|
||||
|
||||
@Singleton
|
||||
class WorkerWarmupHandler @Inject() (warmup: ThriftWarmup) extends Handler with Logging {
|
||||
|
||||
override def handle(): Unit = {
|
||||
info("Warmup Done!")
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,62 +0,0 @@
|
|||
package com.twitter.graph_feature_service.worker.modules
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.twitter.concurrent.AsyncSemaphore
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.graph_feature_service.common.Configs._
|
||||
import com.twitter.graph_feature_service.worker.util
|
||||
import com.twitter.graph_feature_service.worker.util.AutoUpdatingGraph
|
||||
import com.twitter.graph_feature_service.worker.util.FollowedByPartialValueGraph
|
||||
import com.twitter.graph_feature_service.worker.util.FollowingPartialValueGraph
|
||||
import com.twitter.graph_feature_service.worker.util.GraphContainer
|
||||
import com.twitter.graph_feature_service.worker.util.GraphKey
|
||||
import com.twitter.graph_feature_service.worker.util.MutualFollowPartialValueGraph
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.inject.annotations.Flag
|
||||
import com.twitter.util.Timer
|
||||
import javax.inject.Singleton
|
||||
|
||||
object GraphContainerProviderModule extends TwitterModule {
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
def provideAutoUpdatingGraphs(
|
||||
@Flag(WorkerFlagNames.HdfsCluster) hdfsCluster: String,
|
||||
@Flag(WorkerFlagNames.HdfsClusterUrl) hdfsClusterUrl: String,
|
||||
@Flag(WorkerFlagNames.ShardId) shardId: Int
|
||||
)(
|
||||
implicit statsReceiver: StatsReceiver,
|
||||
timer: Timer
|
||||
): GraphContainer = {
|
||||
|
||||
// NOTE that we do not load some the graphs for saving RAM at this moment.
|
||||
val enabledGraphPaths: Map[GraphKey, String] =
|
||||
Map(
|
||||
FollowingPartialValueGraph -> FollowOutValPath,
|
||||
FollowedByPartialValueGraph -> FollowInValPath
|
||||
)
|
||||
|
||||
// Only allow one graph to update at the same time.
|
||||
val sharedSemaphore = new AsyncSemaphore(1)
|
||||
|
||||
val graphs: Map[GraphKey, AutoUpdatingGraph] =
|
||||
enabledGraphPaths.map {
|
||||
case (graphKey, path) =>
|
||||
graphKey -> AutoUpdatingGraph(
|
||||
dataPath = getHdfsPath(path),
|
||||
hdfsCluster = hdfsCluster,
|
||||
hdfsClusterUrl = hdfsClusterUrl,
|
||||
shard = shardId,
|
||||
minimumSizeForCompleteGraph = 1e6.toLong,
|
||||
sharedSemaphore = Some(sharedSemaphore)
|
||||
)(
|
||||
statsReceiver
|
||||
.scope("graphs")
|
||||
.scope(graphKey.getClass.getSimpleName),
|
||||
timer
|
||||
)
|
||||
}
|
||||
|
||||
util.GraphContainer(graphs)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,33 +0,0 @@
|
|||
package com.twitter.graph_feature_service.worker.modules
|
||||
|
||||
import com.twitter.inject.TwitterModule
|
||||
|
||||
object WorkerFlagNames {
|
||||
final val ServiceRole = "service.role"
|
||||
final val ServiceEnv = "service.env"
|
||||
final val ShardId = "service.shardId"
|
||||
final val NumShards = "service.numShards"
|
||||
final val HdfsCluster = "service.hdfsCluster"
|
||||
final val HdfsClusterUrl = "service.hdfsClusterUrl"
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes references to the flag values defined in the aurora.deploy file.
|
||||
* To check what the flag values are initialized in runtime, search FlagsModule in stdout
|
||||
*/
|
||||
object WorkerFlagModule extends TwitterModule {
|
||||
|
||||
import WorkerFlagNames._
|
||||
|
||||
flag[Int](ShardId, "Shard Id")
|
||||
|
||||
flag[Int](NumShards, "Num of Graph Shards")
|
||||
|
||||
flag[String](ServiceRole, "Service Role")
|
||||
|
||||
flag[String](ServiceEnv, "Service Env")
|
||||
|
||||
flag[String](HdfsCluster, "Hdfs cluster to download graph files from")
|
||||
|
||||
flag[String](HdfsClusterUrl, "Hdfs cluster url to download graph files from")
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,69 +0,0 @@
|
|||
package com.twitter.graph_feature_service.worker.util
|
||||
|
||||
import com.twitter.bijection.Injection
|
||||
import com.twitter.concurrent.AsyncSemaphore
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.common.constdb_util.{
|
||||
AutoUpdatingReadOnlyGraph,
|
||||
ConstDBImporter,
|
||||
Injections
|
||||
}
|
||||
import com.twitter.graph_feature_service.common.Configs
|
||||
import com.twitter.util.{Duration, Future, Timer}
|
||||
import java.nio.ByteBuffer
|
||||
|
||||
/**
|
||||
* @param dataPath the path to the data on HDFS
|
||||
* @param hdfsCluster cluster where we check for updates and download graph files from
|
||||
* @param hdfsClusterUrl url to HDFS cluster
|
||||
* @param shard The shard of the graph to download
|
||||
* @param minimumSizeForCompleteGraph minimumSize for complete graph - otherwise we don't load it
|
||||
* @param updateIntervalMin The interval after which the first update is tried and the interval between such updates
|
||||
* @param updateIntervalMax the maximum time before an update is triggered
|
||||
* @param deleteInterval The interval after which older data is deleted from disk
|
||||
* @param sharedSemaphore The semaphore controls the number of graph loads at same time on the instance.
|
||||
*/
|
||||
case class AutoUpdatingGraph(
|
||||
dataPath: String,
|
||||
hdfsCluster: String,
|
||||
hdfsClusterUrl: String,
|
||||
shard: Int,
|
||||
minimumSizeForCompleteGraph: Long,
|
||||
updateIntervalMin: Duration = 1.hour,
|
||||
updateIntervalMax: Duration = 12.hours,
|
||||
deleteInterval: Duration = 2.seconds,
|
||||
sharedSemaphore: Option[AsyncSemaphore] = None
|
||||
)(
|
||||
implicit statsReceiver: StatsReceiver,
|
||||
timer: Timer)
|
||||
extends AutoUpdatingReadOnlyGraph[Long, ByteBuffer](
|
||||
hdfsCluster,
|
||||
hdfsClusterUrl,
|
||||
shard,
|
||||
minimumSizeForCompleteGraph,
|
||||
updateIntervalMin,
|
||||
updateIntervalMax,
|
||||
deleteInterval,
|
||||
sharedSemaphore
|
||||
)
|
||||
with ConstDBImporter[Long, ByteBuffer] {
|
||||
|
||||
override def numGraphShards: Int = Configs.NumGraphShards
|
||||
|
||||
override def basePath: String = dataPath
|
||||
|
||||
override val keyInj: Injection[Long, ByteBuffer] = Injections.long2Varint
|
||||
|
||||
override val valueInj: Injection[ByteBuffer, ByteBuffer] = Injection.identity
|
||||
|
||||
override def get(targetId: Long): Future[Option[ByteBuffer]] =
|
||||
super
|
||||
.get(targetId)
|
||||
.map { res =>
|
||||
res.foreach(r => arraySizeStat.add(r.remaining()))
|
||||
res
|
||||
}
|
||||
|
||||
private val arraySizeStat = stats.scope("get").stat("size")
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,14 +0,0 @@
|
|||
package com.twitter.graph_feature_service.worker.util
|
||||
|
||||
import com.twitter.graph_feature_service.thriftscala.EdgeType
|
||||
|
||||
sealed trait GfsQuery {
|
||||
def edgeType: EdgeType
|
||||
def userId: Long
|
||||
}
|
||||
|
||||
/**
|
||||
* Search for edges for any users to users in local partition.
|
||||
*/
|
||||
case class ToPartialQuery(edgeType: EdgeType, userId: Long) extends GfsQuery
|
||||
|
||||
Binary file not shown.
|
|
@ -1,19 +0,0 @@
|
|||
package com.twitter.graph_feature_service.worker.util
|
||||
|
||||
import com.twitter.graph_feature_service.thriftscala.EdgeType
|
||||
import com.twitter.util.Future
|
||||
|
||||
case class GraphContainer(
|
||||
graphs: Map[GraphKey, AutoUpdatingGraph]) {
|
||||
|
||||
final val toPartialMap: Map[EdgeType, AutoUpdatingGraph] =
|
||||
graphs.collect {
|
||||
case (partialValueGraph: PartialValueGraph, graph) =>
|
||||
partialValueGraph.edgeType -> graph
|
||||
}
|
||||
|
||||
// load all the graphs from constantDB format to memory
|
||||
def warmup: Future[Unit] = {
|
||||
Future.collect(graphs.mapValues(_.warmup())).unit
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,32 +0,0 @@
|
|||
package com.twitter.graph_feature_service.worker.util
|
||||
|
||||
import com.twitter.graph_feature_service.thriftscala.EdgeType
|
||||
import com.twitter.graph_feature_service.thriftscala.EdgeType._
|
||||
|
||||
sealed trait GraphKey {
|
||||
|
||||
def edgeType: EdgeType
|
||||
}
|
||||
|
||||
sealed trait PartialValueGraph extends GraphKey
|
||||
|
||||
/**
|
||||
* Follow Graphs
|
||||
*/
|
||||
object FollowingPartialValueGraph extends PartialValueGraph {
|
||||
|
||||
override def edgeType: EdgeType = Following
|
||||
}
|
||||
|
||||
object FollowedByPartialValueGraph extends PartialValueGraph {
|
||||
|
||||
override def edgeType: EdgeType = FollowedBy
|
||||
}
|
||||
|
||||
/**
|
||||
* Mutual Follow Graphs
|
||||
*/
|
||||
object MutualFollowPartialValueGraph extends PartialValueGraph {
|
||||
|
||||
override def edgeType: EdgeType = MutualFollow
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,16 +0,0 @@
|
|||
package com.twitter.graph_feature_service.worker.util
|
||||
|
||||
//These classes are to help the GraphContainer choose the right data structure to answer queries
|
||||
sealed trait GraphType
|
||||
|
||||
object FollowGraph extends GraphType
|
||||
|
||||
object FavoriteGraph extends GraphType
|
||||
|
||||
object RetweetGraph extends GraphType
|
||||
|
||||
object ReplyGraph extends GraphType
|
||||
|
||||
object MentionGraph extends GraphType
|
||||
|
||||
object MutualFollowGraph extends GraphType
|
||||
|
|
@ -1,66 +0,0 @@
|
|||
scala_library(
|
||||
platform = "java8",
|
||||
tags = [
|
||||
"bazel-compatible",
|
||||
"bazel-only",
|
||||
],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/twitter/bijection:core",
|
||||
"frigate/frigate-common/src/main/scala/com/twitter/frigate/common/constdb_util",
|
||||
"graph-feature-service/src/main/scala/com/twitter/graph_feature_service/common",
|
||||
"src/scala/com/twitter/interaction_graph/scio/agg_all:interaction_graph_history_aggregated_edge_snapshot-scala",
|
||||
"src/scala/com/twitter/interaction_graph/scio/ml/scores:real_graph_in_scores-scala",
|
||||
"src/scala/com/twitter/pluck/source/user_audits:user_audit_final-scala",
|
||||
"src/scala/com/twitter/scalding_internal/dalv2",
|
||||
"src/scala/com/twitter/scalding_internal/job",
|
||||
"src/scala/com/twitter/scalding_internal/job/analytics_batch",
|
||||
],
|
||||
)
|
||||
|
||||
scalding_job(
|
||||
name = "graph_feature_service_adhoc_job",
|
||||
main = "com.twitter.graph_feature_service.scalding.GraphFeatureServiceAdhocApp",
|
||||
args = [
|
||||
"--date 2022-10-24",
|
||||
],
|
||||
config = [
|
||||
("hadoop.map.jvm.total-memory", "3072m"),
|
||||
("hadoop.reduce.jvm.total-memory", "3072m"),
|
||||
("hadoop.submitter.jvm.total-memory", "5120m"),
|
||||
("submitter.tier", "preemptible"),
|
||||
],
|
||||
contact = "recos-platform-alerts@twitter.com",
|
||||
hadoop_cluster = "atla-proc",
|
||||
hadoop_properties = [("mapreduce.job.hdfs-servers", "/atla/proc/user/cassowary")],
|
||||
platform = "java8",
|
||||
role = "cassowary",
|
||||
runtime_platform = "java8",
|
||||
tags = [
|
||||
"bazel-compatible:migrated",
|
||||
"bazel-only",
|
||||
],
|
||||
dependencies = [":scalding"],
|
||||
)
|
||||
|
||||
scalding_job(
|
||||
name = "graph_feature_service_daily_job",
|
||||
main = "com.twitter.graph_feature_service.scalding.GraphFeatureServiceScheduledApp",
|
||||
config = [
|
||||
("hadoop.map.jvm.total-memory", "3072m"),
|
||||
("hadoop.reduce.jvm.total-memory", "3072m"),
|
||||
("hadoop.submitter.jvm.total-memory", "5120m"),
|
||||
("submitter.tier", "preemptible"),
|
||||
],
|
||||
contact = "recos-platform-alerts@twitter.com",
|
||||
cron = "01,31 * * * *",
|
||||
hadoop_cluster = "atla-proc",
|
||||
hadoop_properties = [("mapreduce.job.hdfs-servers", "/atla/proc/user/cassowary")],
|
||||
platform = "java8",
|
||||
role = "cassowary",
|
||||
runtime_platform = "java8",
|
||||
tags = [
|
||||
"bazel-compatible:migrated",
|
||||
"bazel-only",
|
||||
],
|
||||
dependencies = [":scalding"],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,9 +0,0 @@
|
|||
package com.twitter.graph_feature_service.scalding
|
||||
|
||||
case class EdgeFeature(
|
||||
realGraphScore: Float,
|
||||
followScore: Option[Float] = None,
|
||||
mutualFollowScore: Option[Float] = None,
|
||||
favoriteScore: Option[Float] = None,
|
||||
retweetScore: Option[Float] = None,
|
||||
mentionScore: Option[Float] = None)
|
||||
Binary file not shown.
|
|
@ -1,85 +0,0 @@
|
|||
package com.twitter.graph_feature_service.scalding
|
||||
|
||||
import com.twitter.scalding._
|
||||
import com.twitter.scalding_internal.job.TwitterExecutionApp
|
||||
import com.twitter.scalding_internal.job.analytics_batch.{
|
||||
AnalyticsBatchExecution,
|
||||
AnalyticsBatchExecutionArgs,
|
||||
BatchDescription,
|
||||
BatchFirstTime,
|
||||
BatchIncrement,
|
||||
TwitterScheduledExecutionApp
|
||||
}
|
||||
import java.util.TimeZone
|
||||
|
||||
/**
|
||||
* Each job only needs to implement this runOnDateRange() function. It makes it easier for testing.
|
||||
*/
|
||||
trait GraphFeatureServiceBaseJob {
|
||||
implicit val timeZone: TimeZone = DateOps.UTC
|
||||
implicit val dateParser: DateParser = DateParser.default
|
||||
|
||||
def runOnDateRange(
|
||||
enableValueGraphs: Option[Boolean] = None,
|
||||
enableKeyGraphs: Option[Boolean] = None
|
||||
)(
|
||||
implicit dateRange: DateRange,
|
||||
timeZone: TimeZone,
|
||||
uniqueID: UniqueID
|
||||
): Execution[Unit]
|
||||
|
||||
/**
|
||||
* Print customized counters in the log
|
||||
*/
|
||||
def printerCounters[T](execution: Execution[T]): Execution[Unit] = {
|
||||
execution.getCounters
|
||||
.flatMap {
|
||||
case (_, counters) =>
|
||||
counters.toMap.toSeq
|
||||
.sortBy(e => (e._1.group, e._1.counter))
|
||||
.foreach {
|
||||
case (statKey, value) =>
|
||||
println(s"${statKey.group}\t${statKey.counter}\t$value")
|
||||
}
|
||||
Execution.unit
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Trait that wraps things about adhoc jobs.
|
||||
*/
|
||||
trait GraphFeatureServiceAdhocBaseApp extends TwitterExecutionApp with GraphFeatureServiceBaseJob {
|
||||
override def job: Execution[Unit] = Execution.withId { implicit uniqueId =>
|
||||
Execution.getArgs.flatMap { args: Args =>
|
||||
implicit val dateRange: DateRange = DateRange.parse(args.list("date"))(timeZone, dateParser)
|
||||
printerCounters(runOnDateRange())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Trait that wraps things about scheduled jobs.
|
||||
*
|
||||
* A new daily app only needs to declare the starting date.
|
||||
*/
|
||||
trait GraphFeatureServiceScheduledBaseApp
|
||||
extends TwitterScheduledExecutionApp
|
||||
with GraphFeatureServiceBaseJob {
|
||||
|
||||
def firstTime: RichDate // for example: RichDate("2018-02-21")
|
||||
|
||||
def batchIncrement: Duration = Days(1)
|
||||
|
||||
override def scheduledJob: Execution[Unit] = Execution.withId { implicit uniqueId =>
|
||||
val analyticsArgs = AnalyticsBatchExecutionArgs(
|
||||
batchDesc = BatchDescription(getClass.getName),
|
||||
firstTime = BatchFirstTime(firstTime),
|
||||
batchIncrement = BatchIncrement(batchIncrement)
|
||||
)
|
||||
|
||||
AnalyticsBatchExecution(analyticsArgs) { implicit dateRange =>
|
||||
printerCounters(runOnDateRange())
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,52 +0,0 @@
|
|||
package com.twitter.graph_feature_service.scalding
|
||||
|
||||
import com.twitter.scalding.DateRange
|
||||
import com.twitter.scalding.Execution
|
||||
import com.twitter.scalding.RichDate
|
||||
import com.twitter.scalding.UniqueID
|
||||
import java.util.Calendar
|
||||
import java.util.TimeZone
|
||||
import sun.util.calendar.BaseCalendar
|
||||
|
||||
/**
|
||||
* To launch an adhoc run:
|
||||
*
|
||||
scalding remote run --target graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding:graph_feature_service_adhoc_job
|
||||
*/
|
||||
object GraphFeatureServiceAdhocApp
|
||||
extends GraphFeatureServiceMainJob
|
||||
with GraphFeatureServiceAdhocBaseApp {}
|
||||
|
||||
/**
|
||||
* To schedule the job, upload the workflows config (only required for the first time and subsequent config changes):
|
||||
* scalding workflow upload --jobs graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding:graph_feature_service_daily_job --autoplay --build-cron-schedule "20 23 1 * *"
|
||||
* You can then build from the UI by clicking "Build" and pasting in your remote branch, or leave it empty if you're redeploying from master.
|
||||
* The workflows config above should automatically trigger once each month.
|
||||
*/
|
||||
object GraphFeatureServiceScheduledApp
|
||||
extends GraphFeatureServiceMainJob
|
||||
with GraphFeatureServiceScheduledBaseApp {
|
||||
override def firstTime: RichDate = RichDate("2018-05-18")
|
||||
|
||||
override def runOnDateRange(
|
||||
enableValueGraphs: Option[Boolean],
|
||||
enableKeyGraphs: Option[Boolean]
|
||||
)(
|
||||
implicit dateRange: DateRange,
|
||||
timeZone: TimeZone,
|
||||
uniqueID: UniqueID
|
||||
): Execution[Unit] = {
|
||||
// Only run the value Graphs on Tuesday, Thursday, Saturday
|
||||
val overrideEnableValueGraphs = {
|
||||
val dayOfWeek = dateRange.start.toCalendar.get(Calendar.DAY_OF_WEEK)
|
||||
dayOfWeek == BaseCalendar.TUESDAY |
|
||||
dayOfWeek == BaseCalendar.THURSDAY |
|
||||
dayOfWeek == BaseCalendar.SATURDAY
|
||||
}
|
||||
|
||||
super.runOnDateRange(
|
||||
Some(true),
|
||||
Some(false) // disable key Graphs since we are not using them in production
|
||||
)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,297 +0,0 @@
|
|||
package com.twitter.graph_feature_service.scalding
|
||||
|
||||
import com.twitter.bijection.Injection
|
||||
import com.twitter.frigate.common.constdb_util.Injections
|
||||
import com.twitter.frigate.common.constdb_util.ScaldingUtil
|
||||
import com.twitter.graph_feature_service.common.Configs
|
||||
import com.twitter.graph_feature_service.common.Configs._
|
||||
import com.twitter.interaction_graph.scio.agg_all.InteractionGraphHistoryAggregatedEdgeSnapshotScalaDataset
|
||||
import com.twitter.interaction_graph.scio.ml.scores.RealGraphInScoresScalaDataset
|
||||
import com.twitter.interaction_graph.thriftscala.FeatureName
|
||||
import com.twitter.interaction_graph.thriftscala.{EdgeFeature => TEdgeFeature}
|
||||
import com.twitter.pluck.source.user_audits.UserAuditFinalScalaDataset
|
||||
import com.twitter.scalding.DateRange
|
||||
import com.twitter.scalding.Days
|
||||
import com.twitter.scalding.Execution
|
||||
import com.twitter.scalding.Stat
|
||||
import com.twitter.scalding.UniqueID
|
||||
import com.twitter.scalding.typed.TypedPipe
|
||||
import com.twitter.scalding_internal.dalv2.DAL
|
||||
import com.twitter.scalding_internal.dalv2.remote_access.AllowCrossClusterSameDC
|
||||
import com.twitter.scalding_internal.multiformat.format.keyval.KeyVal
|
||||
import com.twitter.util.Time
|
||||
import com.twitter.wtf.candidate.thriftscala.CandidateSeq
|
||||
import java.nio.ByteBuffer
|
||||
import java.util.TimeZone
|
||||
|
||||
trait GraphFeatureServiceMainJob extends GraphFeatureServiceBaseJob {
|
||||
|
||||
// keeping hdfsPath as a separate variable in order to override it in unit tests
|
||||
protected val hdfsPath: String = BaseHdfsPath
|
||||
|
||||
protected def getShardIdForUser(userId: Long): Int = shardForUser(userId)
|
||||
|
||||
protected implicit val keyInj: Injection[Long, ByteBuffer] = Injections.long2Varint
|
||||
|
||||
protected implicit val valueInj: Injection[Long, ByteBuffer] = Injections.long2ByteBuffer
|
||||
|
||||
protected val bufferSize: Int = 1 << 26
|
||||
|
||||
protected val maxNumKeys: Int = 1 << 24
|
||||
|
||||
protected val numReducers: Int = NumGraphShards
|
||||
|
||||
protected val outputStreamBufferSize: Int = 1 << 26
|
||||
|
||||
protected final val shardingByKey = { (k: Long, _: Long) =>
|
||||
getShardIdForUser(k)
|
||||
}
|
||||
|
||||
protected final val shardingByValue = { (_: Long, v: Long) =>
|
||||
getShardIdForUser(v)
|
||||
}
|
||||
|
||||
private def writeGraphToDB(
|
||||
graph: TypedPipe[(Long, Long)],
|
||||
shardingFunction: (Long, Long) => Int,
|
||||
path: String
|
||||
)(
|
||||
implicit dateRange: DateRange
|
||||
): Execution[TypedPipe[(Int, Unit)]] = {
|
||||
ScaldingUtil
|
||||
.writeConstDB[Long, Long](
|
||||
graph.withDescription(s"sharding $path"),
|
||||
shardingFunction,
|
||||
shardId =>
|
||||
getTimedHdfsShardPath(
|
||||
shardId,
|
||||
getHdfsPath(path, Some(hdfsPath)),
|
||||
Time.fromMilliseconds(dateRange.end.timestamp)
|
||||
),
|
||||
Int.MaxValue,
|
||||
bufferSize,
|
||||
maxNumKeys,
|
||||
numReducers,
|
||||
outputStreamBufferSize
|
||||
)(
|
||||
keyInj,
|
||||
valueInj,
|
||||
Ordering[(Long, Long)]
|
||||
)
|
||||
.forceToDiskExecution
|
||||
}
|
||||
|
||||
def extractFeature(
|
||||
featureList: Seq[TEdgeFeature],
|
||||
featureName: FeatureName
|
||||
): Option[Float] = {
|
||||
featureList
|
||||
.find(_.name == featureName)
|
||||
.map(_.tss.ewma.toFloat)
|
||||
.filter(_ > 0.0)
|
||||
}
|
||||
|
||||
/**
|
||||
* Function to extract a subgraph (e.g., follow graph) from real graph and take top K by real graph
|
||||
* weight.
|
||||
*
|
||||
* @param input input real graph
|
||||
* @param edgeFilter filter function to only get the edges needed (e.g., only follow edges)
|
||||
* @param counter counter
|
||||
* @return a subgroup that contains topK, e.g., follow graph for each user.
|
||||
*/
|
||||
private def getSubGraph(
|
||||
input: TypedPipe[(Long, Long, EdgeFeature)],
|
||||
edgeFilter: EdgeFeature => Boolean,
|
||||
counter: Stat
|
||||
): TypedPipe[(Long, Long)] = {
|
||||
input
|
||||
.filter(c => edgeFilter(c._3))
|
||||
.map {
|
||||
case (srcId, destId, features) =>
|
||||
(srcId, (destId, features.realGraphScore))
|
||||
}
|
||||
.group
|
||||
// auto reducer estimation only allocates 15 reducers, so setting an explicit number here
|
||||
.withReducers(2000)
|
||||
.sortedReverseTake(TopKRealGraph)(Ordering.by(_._2))
|
||||
.flatMap {
|
||||
case (srcId, topKNeighbors) =>
|
||||
counter.inc()
|
||||
topKNeighbors.map {
|
||||
case (destId, _) =>
|
||||
(srcId, destId)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def getMauIds()(implicit dateRange: DateRange, uniqueID: UniqueID): TypedPipe[Long] = {
|
||||
val numMAUs = Stat("NUM_MAUS")
|
||||
val uniqueMAUs = Stat("UNIQUE_MAUS")
|
||||
|
||||
DAL
|
||||
.read(UserAuditFinalScalaDataset)
|
||||
.withRemoteReadPolicy(AllowCrossClusterSameDC)
|
||||
.toTypedPipe
|
||||
.collect {
|
||||
case user_audit if user_audit.isValid =>
|
||||
numMAUs.inc()
|
||||
user_audit.userId
|
||||
}
|
||||
.distinct
|
||||
.map { u =>
|
||||
uniqueMAUs.inc()
|
||||
u
|
||||
}
|
||||
}
|
||||
|
||||
def getRealGraphWithMAUOnly(
|
||||
implicit dateRange: DateRange,
|
||||
timeZone: TimeZone,
|
||||
uniqueID: UniqueID
|
||||
): TypedPipe[(Long, Long, EdgeFeature)] = {
|
||||
val numMAUs = Stat("NUM_MAUS")
|
||||
val uniqueMAUs = Stat("UNIQUE_MAUS")
|
||||
|
||||
val monthlyActiveUsers = DAL
|
||||
.read(UserAuditFinalScalaDataset)
|
||||
.withRemoteReadPolicy(AllowCrossClusterSameDC)
|
||||
.toTypedPipe
|
||||
.collect {
|
||||
case user_audit if user_audit.isValid =>
|
||||
numMAUs.inc()
|
||||
user_audit.userId
|
||||
}
|
||||
.distinct
|
||||
.map { u =>
|
||||
uniqueMAUs.inc()
|
||||
u
|
||||
}
|
||||
.asKeys
|
||||
|
||||
val realGraphAggregates = DAL
|
||||
.readMostRecentSnapshot(
|
||||
InteractionGraphHistoryAggregatedEdgeSnapshotScalaDataset,
|
||||
dateRange.embiggen(Days(5)))
|
||||
.withRemoteReadPolicy(AllowCrossClusterSameDC)
|
||||
.toTypedPipe
|
||||
.map { edge =>
|
||||
val featureList = edge.features
|
||||
val edgeFeature = EdgeFeature(
|
||||
edge.weight.getOrElse(0.0).toFloat,
|
||||
extractFeature(featureList, FeatureName.NumMutualFollows),
|
||||
extractFeature(featureList, FeatureName.NumFavorites),
|
||||
extractFeature(featureList, FeatureName.NumRetweets),
|
||||
extractFeature(featureList, FeatureName.NumMentions)
|
||||
)
|
||||
(edge.sourceId, (edge.destinationId, edgeFeature))
|
||||
}
|
||||
.join(monthlyActiveUsers)
|
||||
.map {
|
||||
case (srcId, ((destId, feature), _)) =>
|
||||
(destId, (srcId, feature))
|
||||
}
|
||||
.join(monthlyActiveUsers)
|
||||
.map {
|
||||
case (destId, ((srcId, feature), _)) =>
|
||||
(srcId, destId, feature)
|
||||
}
|
||||
realGraphAggregates
|
||||
}
|
||||
|
||||
def getTopKFollowGraph(
|
||||
implicit dateRange: DateRange,
|
||||
timeZone: TimeZone,
|
||||
uniqueID: UniqueID
|
||||
): TypedPipe[(Long, Long)] = {
|
||||
val followGraphMauStat = Stat("NumFollowEdges_MAU")
|
||||
val mau: TypedPipe[Long] = getMauIds()
|
||||
DAL
|
||||
.readMostRecentSnapshot(RealGraphInScoresScalaDataset, dateRange.embiggen(Days(7)))
|
||||
.withRemoteReadPolicy(AllowCrossClusterSameDC)
|
||||
.toTypedPipe
|
||||
.groupBy(_.key)
|
||||
.join(mau.asKeys)
|
||||
.withDescription("filtering srcId by mau")
|
||||
.flatMap {
|
||||
case (_, (KeyVal(srcId, CandidateSeq(candidates)), _)) =>
|
||||
followGraphMauStat.inc()
|
||||
val topK = candidates.sortBy(-_.score).take(TopKRealGraph)
|
||||
topK.map { c => (srcId, c.userId) }
|
||||
}
|
||||
}
|
||||
|
||||
override def runOnDateRange(
|
||||
enableValueGraphs: Option[Boolean],
|
||||
enableKeyGraphs: Option[Boolean]
|
||||
)(
|
||||
implicit dateRange: DateRange,
|
||||
timeZone: TimeZone,
|
||||
uniqueID: UniqueID
|
||||
): Execution[Unit] = {
|
||||
|
||||
val processValueGraphs = enableValueGraphs.getOrElse(Configs.EnableValueGraphs)
|
||||
val processKeyGraphs = enableKeyGraphs.getOrElse(Configs.EnableKeyGraphs)
|
||||
|
||||
if (!processKeyGraphs && !processValueGraphs) {
|
||||
// Skip the batch job
|
||||
Execution.unit
|
||||
} else {
|
||||
// val favoriteGraphStat = Stat("NumFavoriteEdges")
|
||||
// val retweetGraphStat = Stat("NumRetweetEdges")
|
||||
// val mentionGraphStat = Stat("NumMentionEdges")
|
||||
|
||||
// val realGraphAggregates = getRealGraphWithMAUOnly
|
||||
|
||||
val followGraph = getTopKFollowGraph
|
||||
// val mutualFollowGraph = followGraph.asKeys.join(followGraph.swap.asKeys).keys
|
||||
|
||||
// val favoriteGraph =
|
||||
// getSubGraph(realGraphAggregates, _.favoriteScore.isDefined, favoriteGraphStat)
|
||||
|
||||
// val retweetGraph =
|
||||
// getSubGraph(realGraphAggregates, _.retweetScore.isDefined, retweetGraphStat)
|
||||
|
||||
// val mentionGraph =
|
||||
// getSubGraph(realGraphAggregates, _.mentionScore.isDefined, mentionGraphStat)
|
||||
|
||||
val writeValDataSetExecutions = if (processValueGraphs) {
|
||||
Seq(
|
||||
(followGraph, shardingByValue, FollowOutValPath),
|
||||
(followGraph.swap, shardingByValue, FollowInValPath)
|
||||
// (mutualFollowGraph, shardingByValue, MutualFollowValPath),
|
||||
// (favoriteGraph, shardingByValue, FavoriteOutValPath),
|
||||
// (favoriteGraph.swap, shardingByValue, FavoriteInValPath),
|
||||
// (retweetGraph, shardingByValue, RetweetOutValPath),
|
||||
// (retweetGraph.swap, shardingByValue, RetweetInValPath),
|
||||
// (mentionGraph, shardingByValue, MentionOutValPath),
|
||||
// (mentionGraph.swap, shardingByValue, MentionInValPath)
|
||||
)
|
||||
} else {
|
||||
Seq.empty
|
||||
}
|
||||
|
||||
val writeKeyDataSetExecutions = if (processKeyGraphs) {
|
||||
Seq(
|
||||
(followGraph, shardingByKey, FollowOutKeyPath),
|
||||
(followGraph.swap, shardingByKey, FollowInKeyPath)
|
||||
// (favoriteGraph, shardingByKey, FavoriteOutKeyPath),
|
||||
// (favoriteGraph.swap, shardingByKey, FavoriteInKeyPath),
|
||||
// (retweetGraph, shardingByKey, RetweetOutKeyPath),
|
||||
// (retweetGraph.swap, shardingByKey, RetweetInKeyPath),
|
||||
// (mentionGraph, shardingByKey, MentionOutKeyPath),
|
||||
// (mentionGraph.swap, shardingByKey, MentionInKeyPath),
|
||||
// (mutualFollowGraph, shardingByKey, MutualFollowKeyPath)
|
||||
)
|
||||
} else {
|
||||
Seq.empty
|
||||
}
|
||||
|
||||
Execution
|
||||
.sequence((writeValDataSetExecutions ++ writeKeyDataSetExecutions).map {
|
||||
case (graph, shardingMethod, path) =>
|
||||
writeGraphToDB(graph, shardingMethod, path)
|
||||
}).unit
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
scala_library(
|
||||
platform = "java8",
|
||||
tags = ["bazel-only"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/twitter/bijection:core",
|
||||
"3rdparty/jvm/com/twitter/bijection:scrooge",
|
||||
"frigate/frigate-common/src/main/scala/com/twitter/frigate/common/constdb_util",
|
||||
"src/java/com/twitter/ml/api:api-base",
|
||||
"src/scala/com/twitter/ml/api:api-base",
|
||||
"src/scala/com/twitter/scalding_internal/job",
|
||||
"src/scala/com/twitter/scalding_internal/job/analytics_batch",
|
||||
"src/thrift/com/twitter/ml/api:data-java",
|
||||
],
|
||||
)
|
||||
|
||||
hadoop_binary(
|
||||
name = "gfs_random_request-adhoc",
|
||||
main = "com.twitter.graph_feature_service.scalding.adhoc.RandomRequestGenerationApp",
|
||||
platform = "java8",
|
||||
runtime_platform = "java8",
|
||||
tags = [
|
||||
"bazel-compatible",
|
||||
"bazel-compatible:migrated",
|
||||
"bazel-only",
|
||||
],
|
||||
dependencies = [":adhoc"],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,77 +0,0 @@
|
|||
package com.twitter.graph_feature_service.scalding.adhoc
|
||||
|
||||
import com.twitter.bijection.Injection
|
||||
import com.twitter.frigate.common.constdb_util.Injections
|
||||
import com.twitter.ml.api.Feature.Discrete
|
||||
import com.twitter.ml.api.{DailySuffixFeatureSource, DataSetPipe, RichDataRecord}
|
||||
import com.twitter.scalding._
|
||||
import com.twitter.scalding_internal.job.TwitterExecutionApp
|
||||
import java.nio.ByteBuffer
|
||||
import java.util.TimeZone
|
||||
|
||||
object RandomRequestGenerationJob {
|
||||
implicit val timeZone: TimeZone = DateOps.UTC
|
||||
implicit val dateParser: DateParser = DateParser.default
|
||||
|
||||
val timelineRecapDataSetPath: String =
|
||||
"/atla/proc2/user/timelines/processed/suggests/recap/data_records"
|
||||
|
||||
val USER_ID = new Discrete("meta.user_id")
|
||||
val AUTHOR_ID = new Discrete("meta.author_id")
|
||||
|
||||
val timelineRecapOutPutPath: String = "/user/cassowary/gfs/adhoc/timeline_data"
|
||||
|
||||
implicit val inj: Injection[Long, ByteBuffer] = Injections.long2Varint
|
||||
|
||||
def run(
|
||||
dataSetPath: String,
|
||||
outPutPath: String,
|
||||
numOfPairsToTake: Int
|
||||
)(
|
||||
implicit dateRange: DateRange,
|
||||
uniqueID: UniqueID
|
||||
): Execution[Unit] = {
|
||||
|
||||
val NumUserAuthorPairs = Stat("NumUserAuthorPairs")
|
||||
|
||||
val dataSet: DataSetPipe = DailySuffixFeatureSource(dataSetPath).read
|
||||
|
||||
val userAuthorPairs: TypedPipe[(Long, Long)] = dataSet.records.map { record =>
|
||||
val richRecord = new RichDataRecord(record, dataSet.featureContext)
|
||||
|
||||
val userId = richRecord.getFeatureValue(USER_ID)
|
||||
val authorId = richRecord.getFeatureValue(AUTHOR_ID)
|
||||
NumUserAuthorPairs.inc()
|
||||
(userId, authorId)
|
||||
}
|
||||
|
||||
userAuthorPairs
|
||||
.limit(numOfPairsToTake)
|
||||
.writeExecution(
|
||||
TypedTsv[(Long, Long)](outPutPath)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* ./bazel bundle graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/adhoc:all
|
||||
*
|
||||
* oscar hdfs --screen --user cassowary --tee gfs_log --bundle gfs_random_request-adhoc \
|
||||
--tool com.twitter.graph_feature_service.scalding.adhoc.RandomRequestGenerationApp \
|
||||
-- --date 2018-08-11 \
|
||||
--input /atla/proc2/user/timelines/processed/suggests/recap/data_records \
|
||||
--output /user/cassowary/gfs/adhoc/timeline_data
|
||||
*/
|
||||
object RandomRequestGenerationApp extends TwitterExecutionApp {
|
||||
import RandomRequestGenerationJob._
|
||||
override def job: Execution[Unit] = Execution.withId { implicit uniqueId =>
|
||||
Execution.getArgs.flatMap { args: Args =>
|
||||
implicit val dateRange: DateRange = DateRange.parse(args.list("date"))(timeZone, dateParser)
|
||||
run(
|
||||
args.optional("input").getOrElse(timelineRecapDataSetPath),
|
||||
args.optional("output").getOrElse(timelineRecapOutPutPath),
|
||||
args.int("num_pairs", 3000)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
create_thrift_libraries(
|
||||
base_name = "graph_feature_service_thrift",
|
||||
sources = ["*.thrift"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
generate_languages = [
|
||||
"java",
|
||||
# ruby is added due to ruby dependees in timelines
|
||||
"ruby",
|
||||
"scala",
|
||||
"strato",
|
||||
],
|
||||
provides_java_name = "graph_feature_service_thrift_java",
|
||||
provides_scala_name = "graph_feature_service_thrift_scala",
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,123 +0,0 @@
|
|||
namespace java com.twitter.graph_feature_service.thriftjava
|
||||
#@namespace scala com.twitter.graph_feature_service.thriftscala
|
||||
#@namespace strato com.twitter.graph_feature_service.thriftscala
|
||||
|
||||
// edge type to differentiate different types of graphs (we can also add a lot of other types of edges)
|
||||
enum EdgeType {
|
||||
FOLLOWING,
|
||||
FOLLOWED_BY,
|
||||
FAVORITE,
|
||||
FAVORITED_BY,
|
||||
RETWEET,
|
||||
RETWEETED_BY,
|
||||
REPLY,
|
||||
REPLYED_BY,
|
||||
MENTION,
|
||||
MENTIONED_BY,
|
||||
MUTUAL_FOLLOW,
|
||||
SIMILAR_TO, // more edge types (like block, report, etc.) can be supported later.
|
||||
RESERVED_12,
|
||||
RESERVED_13,
|
||||
RESERVED_14,
|
||||
RESERVED_15,
|
||||
RESERVED_16,
|
||||
RESERVED_17,
|
||||
RESERVED_18,
|
||||
RESERVED_19,
|
||||
RESERVED_20
|
||||
}
|
||||
|
||||
enum PresetFeatureTypes {
|
||||
EMPTY,
|
||||
HTL_TWO_HOP,
|
||||
WTF_TWO_HOP,
|
||||
SQ_TWO_HOP,
|
||||
RUX_TWO_HOP,
|
||||
MR_TWO_HOP,
|
||||
USER_TYPEAHEAD_TWO_HOP
|
||||
}
|
||||
|
||||
struct UserWithCount {
|
||||
1: required i64 userId(personalDataType = 'UserId')
|
||||
2: required i32 count
|
||||
}(hasPersonalData = 'true')
|
||||
|
||||
struct UserWithScore {
|
||||
1: required i64 userId(personalDataType = 'UserId')
|
||||
2: required double score
|
||||
}(hasPersonalData = 'true')
|
||||
|
||||
// Feature Type
|
||||
// For example, to compute how many of source user's following's have favorited candidate user,
|
||||
// we need to compute the intersection between source user's FOLLOWING edges, and candidate user's
|
||||
// FAVORITED_BY edge. In this case, we should user FeatureType(FOLLOWING, FAVORITED_BY)
|
||||
struct FeatureType {
|
||||
1: required EdgeType leftEdgeType // edge type from source user
|
||||
2: required EdgeType rightEdgeType // edge type from candidate user
|
||||
}(persisted="true")
|
||||
|
||||
struct IntersectionValue {
|
||||
1: required FeatureType featureType
|
||||
2: optional i32 count
|
||||
3: optional list<i64> intersectionIds(personalDataType = 'UserId')
|
||||
4: optional i32 leftNodeDegree
|
||||
5: optional i32 rightNodeDegree
|
||||
}(persisted="true", hasPersonalData = 'true')
|
||||
|
||||
struct GfsIntersectionResult {
|
||||
1: required i64 candidateUserId(personalDataType = 'UserId')
|
||||
2: required list<IntersectionValue> intersectionValues
|
||||
}(hasPersonalData = 'true')
|
||||
|
||||
struct GfsIntersectionRequest {
|
||||
1: required i64 userId(personalDataType = 'UserId')
|
||||
2: required list<i64> candidateUserIds(personalDataType = 'UserId')
|
||||
3: required list<FeatureType> featureTypes
|
||||
4: optional i32 intersectionIdLimit
|
||||
}
|
||||
|
||||
struct GfsPresetIntersectionRequest {
|
||||
1: required i64 userId(personalDataType = 'UserId')
|
||||
2: required list<i64> candidateUserIds(personalDataType = 'UserId')
|
||||
3: required PresetFeatureTypes presetFeatureTypes
|
||||
4: optional i32 intersectionIdLimit
|
||||
}(hasPersonalData = 'true')
|
||||
|
||||
struct GfsIntersectionResponse {
|
||||
1: required list<GfsIntersectionResult> results
|
||||
}
|
||||
|
||||
service Server {
|
||||
GfsIntersectionResponse getIntersection(1: GfsIntersectionRequest request)
|
||||
GfsIntersectionResponse getPresetIntersection(1: GfsPresetIntersectionRequest request)
|
||||
}
|
||||
|
||||
###################################################################################################
|
||||
## For internal usage only
|
||||
###################################################################################################
|
||||
struct WorkerIntersectionRequest {
|
||||
1: required i64 userId(personalDataType = 'UserId')
|
||||
2: required list<i64> candidateUserIds(personalDataType = 'UserId')
|
||||
3: required list<FeatureType> featureTypes
|
||||
4: required PresetFeatureTypes presetFeatureTypes
|
||||
5: required i32 intersectionIdLimit
|
||||
}(hasPersonalData = 'true')
|
||||
|
||||
struct WorkerIntersectionResponse {
|
||||
1: required list<list<WorkerIntersectionValue>> results
|
||||
}
|
||||
|
||||
struct WorkerIntersectionValue {
|
||||
1: i32 count
|
||||
2: i32 leftNodeDegree
|
||||
3: i32 rightNodeDegree
|
||||
4: list<i64> intersectionIds(personalDataType = 'UserId')
|
||||
}(hasPersonalData = 'true')
|
||||
|
||||
struct CachedIntersectionResult {
|
||||
1: required list<WorkerIntersectionValue> values
|
||||
}
|
||||
|
||||
service Worker {
|
||||
WorkerIntersectionResponse getIntersection(1: WorkerIntersectionRequest request)
|
||||
}
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
jvm_binary(
|
||||
name = "bin",
|
||||
basename = "home-mixer",
|
||||
main = "com.twitter.home_mixer.HomeMixerServerMain",
|
||||
runtime_platform = "java11",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/ch/qos/logback:logback-classic",
|
||||
"finagle/finagle-zipkin-scribe/src/main/scala",
|
||||
"finatra/inject/inject-logback/src/main/scala",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer",
|
||||
"loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback",
|
||||
"twitter-server-internal/src/main/scala",
|
||||
"twitter-server/logback-classic/src/main/scala",
|
||||
],
|
||||
)
|
||||
|
||||
# Aurora Workflows build phase convention requires a jvm_app named with home-mixer-app
|
||||
jvm_app(
|
||||
name = "home-mixer-app",
|
||||
archive = "zip",
|
||||
binary = ":bin",
|
||||
bundles = [
|
||||
bundle(
|
||||
fileset = ["config/**/*"],
|
||||
owning_target = "home-mixer/config:files",
|
||||
),
|
||||
],
|
||||
tags = ["bazel-compatible"],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,101 +0,0 @@
|
|||
Home Mixer
|
||||
==========
|
||||
|
||||
Home Mixer is the main service used to construct and serve Twitter's Home Timelines. It currently
|
||||
powers:
|
||||
- For you - best Tweets from people you follow + recommended out-of-network content
|
||||
- Following - reverse chronological Tweets from people you follow
|
||||
- Lists - reverse chronological Tweets from List members
|
||||
|
||||
Home Mixer is built on Product Mixer, our custom Scala framework that facilitates building
|
||||
feeds of content.
|
||||
|
||||
## Overview
|
||||
|
||||
The For You recommendation algorithm in Home Mixer involves the following stages:
|
||||
|
||||
- Candidate Generation - fetch Tweets from various Candidate Sources. For example:
|
||||
- Earlybird Search Index
|
||||
- User Tweet Entity Graph
|
||||
- Cr Mixer
|
||||
- Follow Recommendations Service
|
||||
- Feature Hydration
|
||||
- Fetch the ~6000 features needed for ranking
|
||||
- Scoring and Ranking using ML model
|
||||
- Filters and Heuristics. For example:
|
||||
- Author Diversity
|
||||
- Content Balance (In network vs Out of Network)
|
||||
- Feedback fatigue
|
||||
- Deduplication / previously seen Tweets removal
|
||||
- Visibility Filtering (blocked, muted authors/tweets, NSFW settings)
|
||||
- Mixing - integrate Tweets with non-Tweet content
|
||||
- Ads
|
||||
- Who-to-follow modules
|
||||
- Prompts
|
||||
- Product Features and Serving
|
||||
- Conversation Modules for replies
|
||||
- Social Context
|
||||
- Timeline Navigation
|
||||
- Edited Tweets
|
||||
- Feedback options
|
||||
- Pagination and cursoring
|
||||
- Observability and logging
|
||||
- Client instructions and content marshalling
|
||||
|
||||
## Pipeline Structure
|
||||
|
||||
### General
|
||||
|
||||
Product Mixer services like Home Mixer are structured around Pipelines that split the execution
|
||||
into transparent and structured steps.
|
||||
|
||||
Requests first go to Product Pipelines, which are used to select which Mixer Pipeline or
|
||||
Recommendation Pipeline to run for a given request. Each Mixer or Recommendation
|
||||
Pipeline may run multiple Candidate Pipelines to fetch candidates to include in the response.
|
||||
|
||||
Mixer Pipelines combine the results of multiple heterogeneous Candidate Pipelines together
|
||||
(e.g. ads, tweets, users) while Recommendation Pipelines are used to score (via Scoring Pipelines)
|
||||
and rank the results of homogenous Candidate Pipelines so that the top ranked ones can be returned.
|
||||
These pipelines also marshall candidates into a domain object and then into a transport object
|
||||
to return to the caller.
|
||||
|
||||
Candidate Pipelines fetch candidates from underlying Candidate Sources and perform some basic
|
||||
operations on the Candidates, such as filtering out unwanted candidates, applying decorations,
|
||||
and hydrating features.
|
||||
|
||||
The sections below describe the high level pipeline structure (non-exhaustive) for the main Home
|
||||
Timeline tabs powered by Home Mixer.
|
||||
|
||||
### For You
|
||||
|
||||
- ForYouProductPipelineConfig
|
||||
- ForYouScoredTweetsMixerPipelineConfig (main orchestration layer - mixes Tweets with ads and users)
|
||||
- ForYouScoredTweetsCandidatePipelineConfig (fetch Tweets)
|
||||
- ScoredTweetsRecommendationPipelineConfig (main Tweet recommendation layer)
|
||||
- Fetch Tweet Candidates
|
||||
- ScoredTweetsInNetworkCandidatePipelineConfig
|
||||
- ScoredTweetsTweetMixerCandidatePipelineConfig
|
||||
- ScoredTweetsUtegCandidatePipelineConfig
|
||||
- ScoredTweetsFrsCandidatePipelineConfig
|
||||
- Feature Hydration and Scoring
|
||||
- ScoredTweetsScoringPipelineConfig
|
||||
- ForYouConversationServiceCandidatePipelineConfig (backup reverse chron pipeline in case Scored Tweets fails)
|
||||
- ForYouAdsCandidatePipelineConfig (fetch ads)
|
||||
- ForYouWhoToFollowCandidatePipelineConfig (fetch users to recommend)
|
||||
|
||||
### Following
|
||||
|
||||
- FollowingProductPipelineConfig
|
||||
- FollowingMixerPipelineConfig
|
||||
- FollowingEarlybirdCandidatePipelineConfig (fetch tweets from Search Index)
|
||||
- ConversationServiceCandidatePipelineConfig (fetch ancestors for conversation modules)
|
||||
- FollowingAdsCandidatePipelineConfig (fetch ads)
|
||||
- FollowingWhoToFollowCandidatePipelineConfig (fetch users to recommend)
|
||||
|
||||
### Lists
|
||||
|
||||
- ListTweetsProductPipelineConfig
|
||||
- ListTweetsMixerPipelineConfig
|
||||
- ListTweetsTimelineServiceCandidatePipelineConfig (fetch tweets from timeline service)
|
||||
- ConversationServiceCandidatePipelineConfig (fetch ancestors for conversation modules)
|
||||
- ListTweetsAdsCandidatePipelineConfig (fetch ads)
|
||||
|
|
@ -1,51 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"3rdparty/jvm/net/codingwell:scala-guice",
|
||||
"3rdparty/jvm/org/slf4j:slf4j-api",
|
||||
"finagle/finagle-core/src/main",
|
||||
"finagle/finagle-http/src/main/scala",
|
||||
"finagle/finagle-thriftmux/src/main/scala",
|
||||
"finatra-internal/mtls-http/src/main/scala",
|
||||
"finatra-internal/mtls-thriftmux/src/main/scala",
|
||||
"finatra/http-core/src/main/java/com/twitter/finatra/http",
|
||||
"finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations",
|
||||
"finatra/inject/inject-app/src/main/scala",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"finatra/inject/inject-server/src/main/scala",
|
||||
"finatra/inject/inject-utils/src/main/scala",
|
||||
"home-mixer/server/src/main/resources",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/controller",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/federated",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/module",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product",
|
||||
"home-mixer/thrift/src/main/thrift:thrift-scala",
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/module",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter",
|
||||
"product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala",
|
||||
"src/thrift/com/twitter/timelines/render:thrift-scala",
|
||||
"strato/config/columns/auth-context:auth-context-strato-client",
|
||||
"strato/config/columns/gizmoduck:gizmoduck-strato-client",
|
||||
"strato/src/main/scala/com/twitter/strato/fed",
|
||||
"strato/src/main/scala/com/twitter/strato/fed/server",
|
||||
"stringcenter/client",
|
||||
"stringcenter/client/src/main/java",
|
||||
"stringcenter/client/src/main/scala/com/twitter/stringcenter/client",
|
||||
"thrift-web-forms/src/main/scala/com/twitter/thriftwebforms/view",
|
||||
"timelines/src/main/scala/com/twitter/timelines/config",
|
||||
"timelines/src/main/scala/com/twitter/timelines/features/app",
|
||||
"twitter-server-internal",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"util/util-app/src/main/scala",
|
||||
"util/util-core:scala",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,18 +0,0 @@
|
|||
package com.twitter.home_mixer
|
||||
|
||||
import com.twitter.finatra.http.routing.HttpWarmup
|
||||
import com.twitter.finatra.httpclient.RequestBuilder._
|
||||
import com.twitter.util.logging.Logging
|
||||
import com.twitter.inject.utils.Handler
|
||||
import com.twitter.util.Try
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class HomeMixerHttpServerWarmupHandler @Inject() (warmup: HttpWarmup) extends Handler with Logging {
|
||||
|
||||
override def handle(): Unit = {
|
||||
Try(warmup.send(get("/admin/product-mixer/product-pipelines"), admin = true)())
|
||||
.onFailure(e => error(e.getMessage, e))
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,128 +0,0 @@
|
|||
package com.twitter.home_mixer
|
||||
|
||||
import com.google.inject.Module
|
||||
import com.twitter.finagle.Filter
|
||||
import com.twitter.finatra.annotations.DarkTrafficFilterType
|
||||
import com.twitter.finatra.http.HttpServer
|
||||
import com.twitter.finatra.http.routing.HttpRouter
|
||||
import com.twitter.finatra.mtls.http.{Mtls => HttpMtls}
|
||||
import com.twitter.finatra.mtls.thriftmux.Mtls
|
||||
import com.twitter.finatra.mtls.thriftmux.modules.MtlsThriftWebFormsModule
|
||||
import com.twitter.finatra.thrift.ThriftServer
|
||||
import com.twitter.finatra.thrift.filters._
|
||||
import com.twitter.finatra.thrift.routing.ThriftRouter
|
||||
import com.twitter.home_mixer.controller.HomeThriftController
|
||||
import com.twitter.home_mixer.federated.HomeMixerColumn
|
||||
import com.twitter.home_mixer.module._
|
||||
import com.twitter.home_mixer.param.GlobalParamConfigModule
|
||||
import com.twitter.home_mixer.product.HomeMixerProductModule
|
||||
import com.twitter.home_mixer.{thriftscala => st}
|
||||
import com.twitter.product_mixer.component_library.module.AccountRecommendationsMixerModule
|
||||
import com.twitter.product_mixer.component_library.module.DarkTrafficFilterModule
|
||||
import com.twitter.product_mixer.component_library.module.EarlybirdModule
|
||||
import com.twitter.product_mixer.component_library.module.ExploreRankerClientModule
|
||||
import com.twitter.product_mixer.component_library.module.GizmoduckClientModule
|
||||
import com.twitter.product_mixer.component_library.module.OnboardingTaskServiceModule
|
||||
import com.twitter.product_mixer.component_library.module.SocialGraphServiceModule
|
||||
import com.twitter.product_mixer.component_library.module.TimelineRankerClientModule
|
||||
import com.twitter.product_mixer.component_library.module.TimelineScorerClientModule
|
||||
import com.twitter.product_mixer.component_library.module.TimelineServiceClientModule
|
||||
import com.twitter.product_mixer.component_library.module.TweetImpressionStoreModule
|
||||
import com.twitter.product_mixer.component_library.module.TweetMixerClientModule
|
||||
import com.twitter.product_mixer.component_library.module.UserSessionStoreModule
|
||||
import com.twitter.product_mixer.core.controllers.ProductMixerController
|
||||
import com.twitter.product_mixer.core.module.LoggingThrowableExceptionMapper
|
||||
import com.twitter.product_mixer.core.module.ProductMixerModule
|
||||
import com.twitter.product_mixer.core.module.stringcenter.ProductScopeStringCenterModule
|
||||
import com.twitter.strato.fed.StratoFed
|
||||
import com.twitter.strato.fed.server.StratoFedServer
|
||||
|
||||
object HomeMixerServerMain extends HomeMixerServer
|
||||
|
||||
class HomeMixerServer
|
||||
extends StratoFedServer
|
||||
with ThriftServer
|
||||
with Mtls
|
||||
with HttpServer
|
||||
with HttpMtls {
|
||||
override val name = "home-mixer-server"
|
||||
|
||||
override val modules: Seq[Module] = Seq(
|
||||
AccountRecommendationsMixerModule,
|
||||
AdvertiserBrandSafetySettingsStoreModule,
|
||||
BlenderClientModule,
|
||||
ClientSentImpressionsPublisherModule,
|
||||
ConversationServiceModule,
|
||||
EarlybirdModule,
|
||||
ExploreRankerClientModule,
|
||||
FeedbackHistoryClientModule,
|
||||
GizmoduckClientModule,
|
||||
GlobalParamConfigModule,
|
||||
HomeAdsCandidateSourceModule,
|
||||
HomeMixerFlagsModule,
|
||||
HomeMixerProductModule,
|
||||
HomeMixerResourcesModule,
|
||||
ImpressionBloomFilterModule,
|
||||
InjectionHistoryClientModule,
|
||||
ManhattanClientsModule,
|
||||
ManhattanFeatureRepositoryModule,
|
||||
ManhattanTweetImpressionStoreModule,
|
||||
MemcachedFeatureRepositoryModule,
|
||||
NaviModelClientModule,
|
||||
OnboardingTaskServiceModule,
|
||||
OptimizedStratoClientModule,
|
||||
PeopleDiscoveryServiceModule,
|
||||
ProductMixerModule,
|
||||
RealGraphInNetworkScoresModule,
|
||||
RealtimeAggregateFeatureRepositoryModule,
|
||||
ScoredTweetsMemcacheModule,
|
||||
ScribeEventPublisherModule,
|
||||
SimClustersRecentEngagementsClientModule,
|
||||
SocialGraphServiceModule,
|
||||
StaleTweetsCacheModule,
|
||||
ThriftFeatureRepositoryModule,
|
||||
TimelineRankerClientModule,
|
||||
TimelineScorerClientModule,
|
||||
TimelineServiceClientModule,
|
||||
TimelinesPersistenceStoreClientModule,
|
||||
TopicSocialProofClientModule,
|
||||
TweetImpressionStoreModule,
|
||||
TweetMixerClientModule,
|
||||
TweetypieClientModule,
|
||||
TweetypieStaticEntitiesCacheClientModule,
|
||||
UserSessionStoreModule,
|
||||
new DarkTrafficFilterModule[st.HomeMixer.ReqRepServicePerEndpoint](),
|
||||
new MtlsThriftWebFormsModule[st.HomeMixer.MethodPerEndpoint](this),
|
||||
new ProductScopeStringCenterModule()
|
||||
)
|
||||
|
||||
override def configureThrift(router: ThriftRouter): Unit = {
|
||||
router
|
||||
.filter[LoggingMDCFilter]
|
||||
.filter[TraceIdMDCFilter]
|
||||
.filter[ThriftMDCFilter]
|
||||
.filter[StatsFilter]
|
||||
.filter[AccessLoggingFilter]
|
||||
.filter[ExceptionMappingFilter]
|
||||
.filter[Filter.TypeAgnostic, DarkTrafficFilterType]
|
||||
.exceptionMapper[LoggingThrowableExceptionMapper]
|
||||
.exceptionMapper[PipelineFailureExceptionMapper]
|
||||
.add[HomeThriftController]
|
||||
}
|
||||
|
||||
override def configureHttp(router: HttpRouter): Unit =
|
||||
router.add(
|
||||
ProductMixerController[st.HomeMixer.MethodPerEndpoint](
|
||||
this.injector,
|
||||
st.HomeMixer.ExecutePipeline))
|
||||
|
||||
override val dest: String = "/s/home-mixer/home-mixer:strato"
|
||||
|
||||
override val columns: Seq[Class[_ <: StratoFed.Column]] =
|
||||
Seq(classOf[HomeMixerColumn])
|
||||
|
||||
override protected def warmup(): Unit = {
|
||||
handle[HomeMixerThriftServerWarmupHandler]()
|
||||
handle[HomeMixerHttpServerWarmupHandler]()
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,73 +0,0 @@
|
|||
package com.twitter.home_mixer
|
||||
|
||||
import com.twitter.finagle.thrift.ClientId
|
||||
import com.twitter.finatra.thrift.routing.ThriftWarmup
|
||||
import com.twitter.home_mixer.{thriftscala => st}
|
||||
import com.twitter.util.logging.Logging
|
||||
import com.twitter.inject.utils.Handler
|
||||
import com.twitter.product_mixer.core.{thriftscala => pt}
|
||||
import com.twitter.scrooge.Request
|
||||
import com.twitter.scrooge.Response
|
||||
import com.twitter.util.Return
|
||||
import com.twitter.util.Throw
|
||||
import com.twitter.util.Try
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class HomeMixerThriftServerWarmupHandler @Inject() (warmup: ThriftWarmup)
|
||||
extends Handler
|
||||
with Logging {
|
||||
|
||||
private val clientId = ClientId("thrift-warmup-client")
|
||||
|
||||
def handle(): Unit = {
|
||||
val testIds = Seq(1, 2, 3)
|
||||
try {
|
||||
clientId.asCurrent {
|
||||
testIds.foreach { id =>
|
||||
val warmupReq = warmupQuery(id)
|
||||
info(s"Sending warm-up request to service with query: $warmupReq")
|
||||
warmup.sendRequest(
|
||||
method = st.HomeMixer.GetUrtResponse,
|
||||
req = Request(st.HomeMixer.GetUrtResponse.Args(warmupReq)))(assertWarmupResponse)
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
case e: Throwable => error(e.getMessage, e)
|
||||
}
|
||||
info("Warm-up done.")
|
||||
}
|
||||
|
||||
private def warmupQuery(userId: Long): st.HomeMixerRequest = {
|
||||
val clientContext = pt.ClientContext(
|
||||
userId = Some(userId),
|
||||
guestId = None,
|
||||
appId = Some(12345L),
|
||||
ipAddress = Some("0.0.0.0"),
|
||||
userAgent = Some("FAKE_USER_AGENT_FOR_WARMUPS"),
|
||||
countryCode = Some("US"),
|
||||
languageCode = Some("en"),
|
||||
isTwoffice = None,
|
||||
userRoles = None,
|
||||
deviceId = Some("FAKE_DEVICE_ID_FOR_WARMUPS")
|
||||
)
|
||||
st.HomeMixerRequest(
|
||||
clientContext = clientContext,
|
||||
product = st.Product.Following,
|
||||
productContext = Some(st.ProductContext.Following(st.Following())),
|
||||
maxResults = Some(3)
|
||||
)
|
||||
}
|
||||
|
||||
private def assertWarmupResponse(
|
||||
result: Try[Response[st.HomeMixer.GetUrtResponse.SuccessType]]
|
||||
): Unit = {
|
||||
result match {
|
||||
case Return(_) => // ok
|
||||
case Throw(exception) =>
|
||||
warn("Error performing warm-up request.")
|
||||
error(exception.getMessage, exception)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc",
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter",
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate",
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer",
|
||||
],
|
||||
exports = [
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,116 +0,0 @@
|
|||
package com.twitter.home_mixer.candidate_pipeline
|
||||
|
||||
import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator
|
||||
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
|
||||
import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
|
||||
import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter
|
||||
import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter
|
||||
import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter
|
||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
|
||||
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
||||
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
|
||||
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSourceRequest
|
||||
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata
|
||||
import com.twitter.product_mixer.component_library.filter.FeatureFilter
|
||||
import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter
|
||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||
import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource
|
||||
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
|
||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator
|
||||
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||
import com.twitter.product_mixer.core.functional_component.gate.BaseGate
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.DependentCandidatePipelineQueryTransformer
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig
|
||||
|
||||
/**
|
||||
* Candidate Pipeline Config that fetches tweets from the Conversation Service Candidate Source
|
||||
*/
|
||||
class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
|
||||
conversationServiceCandidateSource: ConversationServiceCandidateSource,
|
||||
tweetypieFeatureHydrator: TweetypieFeatureHydrator,
|
||||
namesFeatureHydrator: NamesFeatureHydrator,
|
||||
invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter,
|
||||
override val gates: Seq[BaseGate[Query]],
|
||||
override val decorator: Option[CandidateDecorator[Query, TweetCandidate]])
|
||||
extends DependentCandidatePipelineConfig[
|
||||
Query,
|
||||
ConversationServiceCandidateSourceRequest,
|
||||
TweetWithConversationMetadata,
|
||||
TweetCandidate
|
||||
] {
|
||||
|
||||
override val identifier: CandidatePipelineIdentifier =
|
||||
CandidatePipelineIdentifier("ConversationService")
|
||||
|
||||
private val TweetypieHydratedFilterId = "TweetypieHydrated"
|
||||
private val QuotedTweetDroppedFilterId = "QuotedTweetDropped"
|
||||
|
||||
override val candidateSource: BaseCandidateSource[
|
||||
ConversationServiceCandidateSourceRequest,
|
||||
TweetWithConversationMetadata
|
||||
] = conversationServiceCandidateSource
|
||||
|
||||
override val queryTransformer: DependentCandidatePipelineQueryTransformer[
|
||||
Query,
|
||||
ConversationServiceCandidateSourceRequest
|
||||
] = { (_, candidates) =>
|
||||
val tweetsWithConversationMetadata = candidates.map { candidate =>
|
||||
TweetWithConversationMetadata(
|
||||
tweetId = candidate.candidateIdLong,
|
||||
userId = candidate.features.getOrElse(AuthorIdFeature, None),
|
||||
sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None),
|
||||
sourceUserId = candidate.features.getOrElse(SourceUserIdFeature, None),
|
||||
inReplyToTweetId = candidate.features.getOrElse(InReplyToTweetIdFeature, None),
|
||||
conversationId = None,
|
||||
ancestors = Seq.empty
|
||||
)
|
||||
}
|
||||
ConversationServiceCandidateSourceRequest(tweetsWithConversationMetadata)
|
||||
}
|
||||
|
||||
override val featuresFromCandidateSourceTransformers: Seq[
|
||||
CandidateFeatureTransformer[TweetWithConversationMetadata]
|
||||
] = Seq(ConversationServiceResponseFeatureTransformer)
|
||||
|
||||
override val resultTransformer: CandidatePipelineResultsTransformer[
|
||||
TweetWithConversationMetadata,
|
||||
TweetCandidate
|
||||
] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) }
|
||||
|
||||
override val preFilterFeatureHydrationPhase1: Seq[
|
||||
BaseCandidateFeatureHydrator[Query, TweetCandidate, _]
|
||||
] = Seq(
|
||||
tweetypieFeatureHydrator,
|
||||
InNetworkFeatureHydrator,
|
||||
)
|
||||
|
||||
override def filters: Seq[Filter[Query, TweetCandidate]] = Seq(
|
||||
RetweetDeduplicationFilter,
|
||||
FeatureFilter.fromFeature(FilterIdentifier(TweetypieHydratedFilterId), IsHydratedFeature),
|
||||
PredicateFeatureFilter.fromPredicate(
|
||||
FilterIdentifier(QuotedTweetDroppedFilterId),
|
||||
shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) }
|
||||
),
|
||||
invalidSubscriptionTweetFilter,
|
||||
InvalidConversationModuleFilter
|
||||
)
|
||||
|
||||
override val postFilterFeatureHydration: Seq[
|
||||
BaseCandidateFeatureHydrator[Query, TweetCandidate, _]
|
||||
] = Seq(namesFeatureHydrator)
|
||||
|
||||
override val alerts = Seq(
|
||||
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(),
|
||||
HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert()
|
||||
)
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,34 +0,0 @@
|
|||
package com.twitter.home_mixer.candidate_pipeline
|
||||
|
||||
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
|
||||
import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
|
||||
import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter
|
||||
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
|
||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
|
||||
import com.twitter.product_mixer.core.functional_component.gate.BaseGate
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery] @Inject() (
|
||||
conversationServiceCandidateSource: ConversationServiceCandidateSource,
|
||||
tweetypieFeatureHydrator: TweetypieFeatureHydrator,
|
||||
invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter,
|
||||
namesFeatureHydrator: NamesFeatureHydrator) {
|
||||
|
||||
def build(
|
||||
gates: Seq[BaseGate[Query]] = Seq.empty,
|
||||
decorator: Option[CandidateDecorator[Query, TweetCandidate]] = None
|
||||
): ConversationServiceCandidatePipelineConfig[Query] = {
|
||||
new ConversationServiceCandidatePipelineConfig(
|
||||
conversationServiceCandidateSource,
|
||||
tweetypieFeatureHydrator,
|
||||
namesFeatureHydrator,
|
||||
invalidSubscriptionTweetFilter,
|
||||
gates,
|
||||
decorator
|
||||
)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,39 +0,0 @@
|
|||
package com.twitter.home_mixer.candidate_pipeline
|
||||
|
||||
import com.twitter.home_mixer.model.HomeFeatures._
|
||||
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer
|
||||
import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier
|
||||
import com.twitter.timelineservice.suggests.thriftscala.SuggestType
|
||||
|
||||
object ConversationServiceResponseFeatureTransformer
|
||||
extends CandidateFeatureTransformer[TweetWithConversationMetadata] {
|
||||
|
||||
override val identifier: TransformerIdentifier =
|
||||
TransformerIdentifier("ConversationServiceResponse")
|
||||
|
||||
override val features: Set[Feature[_, _]] = Set(
|
||||
AuthorIdFeature,
|
||||
InReplyToTweetIdFeature,
|
||||
IsRetweetFeature,
|
||||
SourceTweetIdFeature,
|
||||
SourceUserIdFeature,
|
||||
ConversationModuleFocalTweetIdFeature,
|
||||
AncestorsFeature,
|
||||
SuggestTypeFeature
|
||||
)
|
||||
|
||||
override def transform(candidate: TweetWithConversationMetadata): FeatureMap = FeatureMapBuilder()
|
||||
.add(AuthorIdFeature, candidate.userId)
|
||||
.add(InReplyToTweetIdFeature, candidate.inReplyToTweetId)
|
||||
.add(IsRetweetFeature, candidate.sourceTweetId.isDefined)
|
||||
.add(SourceTweetIdFeature, candidate.sourceTweetId)
|
||||
.add(SourceUserIdFeature, candidate.sourceUserId)
|
||||
.add(ConversationModuleFocalTweetIdFeature, candidate.conversationId)
|
||||
.add(AncestorsFeature, candidate.ancestors)
|
||||
.add(SuggestTypeFeature, Some(SuggestType.RankedOrganicTweet))
|
||||
.build()
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,84 +0,0 @@
|
|||
package com.twitter.home_mixer.candidate_pipeline
|
||||
|
||||
import com.twitter.home_mixer.functional_component.candidate_source.StaleTweetsCacheCandidateSource
|
||||
import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder
|
||||
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
|
||||
import com.twitter.home_mixer.functional_component.query_transformer.EditedTweetsCandidatePipelineQueryTransformer
|
||||
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
||||
import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator
|
||||
import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder
|
||||
import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder
|
||||
import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.EmptyClientEventInfoBuilder
|
||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||
import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource
|
||||
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
|
||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier
|
||||
import com.twitter.product_mixer.core.model.marshalling.response.rtf.safety_level.TimelineFocalTweetSafetyLevel
|
||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.contextual_ref.TweetHydrationContext
|
||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
/**
|
||||
* Candidate Pipeline Config that fetches edited tweets from the Stale Tweets Cache
|
||||
*/
|
||||
@Singleton
|
||||
case class EditedTweetsCandidatePipelineConfig @Inject() (
|
||||
staleTweetsCacheCandidateSource: StaleTweetsCacheCandidateSource,
|
||||
namesFeatureHydrator: NamesFeatureHydrator,
|
||||
homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder)
|
||||
extends DependentCandidatePipelineConfig[
|
||||
PipelineQuery,
|
||||
Seq[Long],
|
||||
Long,
|
||||
TweetCandidate
|
||||
] {
|
||||
|
||||
override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("EditedTweets")
|
||||
|
||||
override val candidateSource: BaseCandidateSource[Seq[Long], Long] =
|
||||
staleTweetsCacheCandidateSource
|
||||
|
||||
override val queryTransformer: CandidatePipelineQueryTransformer[
|
||||
PipelineQuery,
|
||||
Seq[Long]
|
||||
] = EditedTweetsCandidatePipelineQueryTransformer
|
||||
|
||||
override val resultTransformer: CandidatePipelineResultsTransformer[
|
||||
Long,
|
||||
TweetCandidate
|
||||
] = { candidate => TweetCandidate(id = candidate) }
|
||||
|
||||
override val postFilterFeatureHydration: Seq[
|
||||
BaseCandidateFeatureHydrator[PipelineQuery, TweetCandidate, _]
|
||||
] = Seq(namesFeatureHydrator)
|
||||
|
||||
override val decorator: Option[CandidateDecorator[PipelineQuery, TweetCandidate]] = {
|
||||
val tweetItemBuilder = TweetCandidateUrtItemBuilder[PipelineQuery, TweetCandidate](
|
||||
clientEventInfoBuilder = EmptyClientEventInfoBuilder,
|
||||
entryIdToReplaceBuilder = Some((_, candidate, _) =>
|
||||
Some(s"${TweetItem.TweetEntryNamespace}-${candidate.id.toString}")),
|
||||
contextualTweetRefBuilder = Some(
|
||||
ContextualTweetRefBuilder(
|
||||
TweetHydrationContext(
|
||||
// Apply safety level that includes canonical VF treatments that apply regardless of context.
|
||||
safetyLevelOverride = Some(TimelineFocalTweetSafetyLevel),
|
||||
outerTweetContext = None
|
||||
)
|
||||
)
|
||||
),
|
||||
feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder)
|
||||
)
|
||||
|
||||
Some(UrtItemCandidateDecorator(tweetItemBuilder))
|
||||
}
|
||||
|
||||
override val alerts = Seq(
|
||||
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.5, 50, 60, 60)
|
||||
)
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,123 +0,0 @@
|
|||
package com.twitter.home_mixer.candidate_pipeline
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.home_mixer.functional_component.gate.RequestContextNotGate
|
||||
import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature
|
||||
import com.twitter.home_mixer.model.request.DeviceContext
|
||||
import com.twitter.home_mixer.model.request.HasDeviceContext
|
||||
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
||||
import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator
|
||||
import com.twitter.product_mixer.component_library.decorator.urt.builder.item.alert.DurationParamBuilder
|
||||
import com.twitter.product_mixer.component_library.decorator.urt.builder.item.alert.ShowAlertCandidateUrtItemBuilder
|
||||
import com.twitter.product_mixer.component_library.decorator.urt.builder.item.alert.StaticShowAlertColorConfigurationBuilder
|
||||
import com.twitter.product_mixer.component_library.decorator.urt.builder.item.alert.StaticShowAlertDisplayLocationBuilder
|
||||
import com.twitter.product_mixer.component_library.decorator.urt.builder.item.alert.StaticShowAlertIconDisplayInfoBuilder
|
||||
import com.twitter.product_mixer.component_library.gate.FeatureGate
|
||||
import com.twitter.product_mixer.component_library.model.candidate.ShowAlertCandidate
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource
|
||||
import com.twitter.product_mixer.core.functional_component.candidate_source.StaticCandidateSource
|
||||
import com.twitter.product_mixer.core.functional_component.configapi.StaticParam
|
||||
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
|
||||
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.item.alert.BaseDurationBuilder
|
||||
import com.twitter.product_mixer.core.functional_component.gate.Gate
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier
|
||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.alert.NewTweets
|
||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.alert.ShowAlertColorConfiguration
|
||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.alert.ShowAlertIconDisplayInfo
|
||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.alert.Top
|
||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.alert.UpArrow
|
||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.color.TwitterBlueRosettaColor
|
||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.color.WhiteRosettaColor
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig
|
||||
import com.twitter.util.Duration
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
/**
|
||||
* Candidate Pipeline Config that creates the New Tweets Pill
|
||||
*/
|
||||
@Singleton
|
||||
class NewTweetsPillCandidatePipelineConfig[Query <: PipelineQuery with HasDeviceContext] @Inject() (
|
||||
) extends DependentCandidatePipelineConfig[
|
||||
Query,
|
||||
Unit,
|
||||
ShowAlertCandidate,
|
||||
ShowAlertCandidate
|
||||
] {
|
||||
import NewTweetsPillCandidatePipelineConfig._
|
||||
|
||||
override val identifier: CandidatePipelineIdentifier =
|
||||
CandidatePipelineIdentifier("NewTweetsPill")
|
||||
|
||||
override val gates: Seq[Gate[Query]] = Seq(
|
||||
RequestContextNotGate(Seq(DeviceContext.RequestContext.PullToRefresh)),
|
||||
FeatureGate.fromFeature(GetNewerFeature)
|
||||
)
|
||||
|
||||
override val candidateSource: CandidateSource[Unit, ShowAlertCandidate] =
|
||||
StaticCandidateSource(
|
||||
CandidateSourceIdentifier(identifier.name),
|
||||
Seq(ShowAlertCandidate(id = identifier.name, userIds = Seq.empty))
|
||||
)
|
||||
|
||||
override val queryTransformer: CandidatePipelineQueryTransformer[Query, Unit] = { _ => Unit }
|
||||
|
||||
override val resultTransformer: CandidatePipelineResultsTransformer[
|
||||
ShowAlertCandidate,
|
||||
ShowAlertCandidate
|
||||
] = { candidate => candidate }
|
||||
|
||||
override val decorator: Option[CandidateDecorator[Query, ShowAlertCandidate]] = {
|
||||
val triggerDelayBuilder = new BaseDurationBuilder[Query] {
|
||||
override def apply(
|
||||
query: Query,
|
||||
candidate: ShowAlertCandidate,
|
||||
features: FeatureMap
|
||||
): Option[Duration] = {
|
||||
val delay = query.deviceContext.flatMap(_.requestContextValue) match {
|
||||
case Some(DeviceContext.RequestContext.TweetSelfThread) => 0.millis
|
||||
case Some(DeviceContext.RequestContext.ManualRefresh) => 0.millis
|
||||
case _ => TriggerDelay
|
||||
}
|
||||
|
||||
Some(delay)
|
||||
}
|
||||
}
|
||||
|
||||
val homeShowAlertCandidateBuilder = ShowAlertCandidateUrtItemBuilder(
|
||||
alertType = NewTweets,
|
||||
colorConfigBuilder = StaticShowAlertColorConfigurationBuilder(DefaultColorConfig),
|
||||
displayLocationBuilder = StaticShowAlertDisplayLocationBuilder(Top),
|
||||
triggerDelayBuilder = Some(triggerDelayBuilder),
|
||||
displayDurationBuilder = Some(DurationParamBuilder(StaticParam(DisplayDuration))),
|
||||
iconDisplayInfoBuilder = Some(StaticShowAlertIconDisplayInfoBuilder(DefaultIconDisplayInfo))
|
||||
)
|
||||
|
||||
Some(UrtItemCandidateDecorator(homeShowAlertCandidateBuilder))
|
||||
}
|
||||
|
||||
override val alerts = Seq(
|
||||
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(),
|
||||
HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert()
|
||||
)
|
||||
}
|
||||
|
||||
object NewTweetsPillCandidatePipelineConfig {
|
||||
val DefaultColorConfig: ShowAlertColorConfiguration = ShowAlertColorConfiguration(
|
||||
background = TwitterBlueRosettaColor,
|
||||
text = WhiteRosettaColor,
|
||||
border = Some(WhiteRosettaColor)
|
||||
)
|
||||
|
||||
val DefaultIconDisplayInfo: ShowAlertIconDisplayInfo =
|
||||
ShowAlertIconDisplayInfo(icon = UpArrow, tint = WhiteRosettaColor)
|
||||
|
||||
// Unlimited display time (until user takes action)
|
||||
val DisplayDuration = -1.millisecond
|
||||
val TriggerDelay = 4.minutes
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,34 +0,0 @@
|
|||
package com.twitter.home_mixer.candidate_pipeline
|
||||
|
||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer
|
||||
import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier
|
||||
import com.twitter.timelineservice.{thriftscala => t}
|
||||
|
||||
object TimelineServiceResponseFeatureTransformer extends CandidateFeatureTransformer[t.Tweet] {
|
||||
|
||||
override val identifier: TransformerIdentifier = TransformerIdentifier("TimelineServiceResponse")
|
||||
|
||||
override val features: Set[Feature[_, _]] = Set(
|
||||
AuthorIdFeature,
|
||||
InReplyToTweetIdFeature,
|
||||
IsRetweetFeature,
|
||||
SourceTweetIdFeature,
|
||||
SourceUserIdFeature,
|
||||
)
|
||||
|
||||
override def transform(candidate: t.Tweet): FeatureMap = FeatureMapBuilder()
|
||||
.add(AuthorIdFeature, candidate.userId)
|
||||
.add(InReplyToTweetIdFeature, candidate.inReplyToStatusId)
|
||||
.add(IsRetweetFeature, candidate.sourceStatusId.isDefined)
|
||||
.add(SourceTweetIdFeature, candidate.sourceStatusId)
|
||||
.add(SourceUserIdFeature, candidate.sourceUserId)
|
||||
.build()
|
||||
}
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"finatra/thrift/src/main/scala/com/twitter/finatra/thrift:controller",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
|
||||
"home-mixer/thrift/src/main/thrift:thrift-scala",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt",
|
||||
"snowflake/src/main/scala/com/twitter/snowflake/id",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,51 +0,0 @@
|
|||
package com.twitter.home_mixer.controller
|
||||
|
||||
import com.twitter.finatra.thrift.Controller
|
||||
import com.twitter.home_mixer.marshaller.request.HomeMixerRequestUnmarshaller
|
||||
import com.twitter.home_mixer.model.request.HomeMixerRequest
|
||||
import com.twitter.home_mixer.service.ScoredTweetsService
|
||||
import com.twitter.home_mixer.{thriftscala => t}
|
||||
import com.twitter.product_mixer.core.controllers.DebugTwitterContext
|
||||
import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder
|
||||
import com.twitter.product_mixer.core.service.debug_query.DebugQueryService
|
||||
import com.twitter.product_mixer.core.service.urt.UrtService
|
||||
import com.twitter.snowflake.id.SnowflakeId
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.timelines.configapi.Params
|
||||
import javax.inject.Inject
|
||||
|
||||
class HomeThriftController @Inject() (
|
||||
homeRequestUnmarshaller: HomeMixerRequestUnmarshaller,
|
||||
urtService: UrtService,
|
||||
scoredTweetsService: ScoredTweetsService,
|
||||
paramsBuilder: ParamsBuilder)
|
||||
extends Controller(t.HomeMixer)
|
||||
with DebugTwitterContext {
|
||||
|
||||
handle(t.HomeMixer.GetUrtResponse) { args: t.HomeMixer.GetUrtResponse.Args =>
|
||||
val request = homeRequestUnmarshaller(args.request)
|
||||
val params = buildParams(request)
|
||||
Stitch.run(urtService.getUrtResponse[HomeMixerRequest](request, params))
|
||||
}
|
||||
|
||||
handle(t.HomeMixer.GetScoredTweetsResponse) { args: t.HomeMixer.GetScoredTweetsResponse.Args =>
|
||||
val request = homeRequestUnmarshaller(args.request)
|
||||
val params = buildParams(request)
|
||||
withDebugTwitterContext(request.clientContext) {
|
||||
Stitch.run(scoredTweetsService.getScoredTweetsResponse[HomeMixerRequest](request, params))
|
||||
}
|
||||
}
|
||||
|
||||
private def buildParams(request: HomeMixerRequest): Params = {
|
||||
val userAgeOpt = request.clientContext.userId.map { userId =>
|
||||
SnowflakeId.timeFromIdOpt(userId).map(_.untilNow.inDays).getOrElse(Int.MaxValue)
|
||||
}
|
||||
val fsCustomMapInput = userAgeOpt.map("account_age_in_days" -> _).toMap
|
||||
paramsBuilder.build(
|
||||
clientContext = request.clientContext,
|
||||
product = request.product,
|
||||
featureOverrides = request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty),
|
||||
fsCustomMapInput = fsCustomMapInput
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||
"home-mixer/thrift/src/main/thrift:thrift-scala",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry",
|
||||
"product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala",
|
||||
"src/thrift/com/twitter/gizmoduck:thrift-scala",
|
||||
"src/thrift/com/twitter/timelines/render:thrift-scala",
|
||||
"stitch/stitch-repo/src/main/scala",
|
||||
"strato/config/columns/auth-context:auth-context-strato-client",
|
||||
"strato/config/columns/gizmoduck:gizmoduck-strato-client",
|
||||
"strato/config/src/thrift/com/twitter/strato/graphql/timelines:graphql-timelines-scala",
|
||||
"strato/src/main/scala/com/twitter/strato/callcontext",
|
||||
"strato/src/main/scala/com/twitter/strato/fed",
|
||||
"strato/src/main/scala/com/twitter/strato/fed/server",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,217 +0,0 @@
|
|||
package com.twitter.home_mixer.federated
|
||||
|
||||
import com.twitter.gizmoduck.{thriftscala => gd}
|
||||
import com.twitter.home_mixer.marshaller.request.HomeMixerRequestUnmarshaller
|
||||
import com.twitter.home_mixer.model.request.HomeMixerRequest
|
||||
import com.twitter.home_mixer.{thriftscala => hm}
|
||||
import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder
|
||||
import com.twitter.product_mixer.core.pipeline.product.ProductPipelineRequest
|
||||
import com.twitter.product_mixer.core.pipeline.product.ProductPipelineResult
|
||||
import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry
|
||||
import com.twitter.product_mixer.core.{thriftscala => pm}
|
||||
import com.twitter.stitch.Arrow
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.strato.callcontext.CallContext
|
||||
import com.twitter.strato.catalog.OpMetadata
|
||||
import com.twitter.strato.config._
|
||||
import com.twitter.strato.data._
|
||||
import com.twitter.strato.fed.StratoFed
|
||||
import com.twitter.strato.generated.client.auth_context.AuditIpClientColumn
|
||||
import com.twitter.strato.generated.client.gizmoduck.CompositeOnUserClientColumn
|
||||
import com.twitter.strato.graphql.timelines.{thriftscala => gql}
|
||||
import com.twitter.strato.thrift.ScroogeConv
|
||||
import com.twitter.timelines.render.{thriftscala => tr}
|
||||
import com.twitter.util.Try
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class HomeMixerColumn @Inject() (
|
||||
homeMixerRequestUnmarshaller: HomeMixerRequestUnmarshaller,
|
||||
compositeOnUserClientColumn: CompositeOnUserClientColumn,
|
||||
auditIpClientColumn: AuditIpClientColumn,
|
||||
paramsBuilder: ParamsBuilder,
|
||||
productPipelineRegistry: ProductPipelineRegistry)
|
||||
extends StratoFed.Column(HomeMixerColumn.Path)
|
||||
with StratoFed.Fetch.Arrow {
|
||||
|
||||
override val contactInfo: ContactInfo = ContactInfo(
|
||||
contactEmail = "",
|
||||
ldapGroup = "",
|
||||
slackRoomId = ""
|
||||
)
|
||||
|
||||
override val metadata: OpMetadata =
|
||||
OpMetadata(
|
||||
lifecycle = Some(Lifecycle.Production),
|
||||
description =
|
||||
Some(Description.PlainText("Federated Strato column for Timelines served via Home Mixer"))
|
||||
)
|
||||
|
||||
private val bouncerAccess: Seq[Policy] = Seq(BouncerAccess())
|
||||
private val finatraTestServiceIdentifiers: Seq[Policy] = Seq(
|
||||
ServiceIdentifierPattern(
|
||||
role = "",
|
||||
service = "",
|
||||
env = "",
|
||||
zone = Seq(""))
|
||||
)
|
||||
|
||||
override val policy: Policy = AnyOf(bouncerAccess ++ finatraTestServiceIdentifiers)
|
||||
|
||||
override type Key = gql.TimelineKey
|
||||
override type View = gql.HomeTimelineView
|
||||
override type Value = tr.Timeline
|
||||
|
||||
override val keyConv: Conv[Key] = ScroogeConv.fromStruct[gql.TimelineKey]
|
||||
override val viewConv: Conv[View] = ScroogeConv.fromStruct[gql.HomeTimelineView]
|
||||
override val valueConv: Conv[Value] = ScroogeConv.fromStruct[tr.Timeline]
|
||||
|
||||
private def createHomeMixerRequestArrow(
|
||||
compositeOnUserClientColumn: CompositeOnUserClientColumn,
|
||||
auditIpClientColumn: AuditIpClientColumn
|
||||
): Arrow[(Key, View), hm.HomeMixerRequest] = {
|
||||
|
||||
val populateUserRolesAndIp: Arrow[(Key, View), (Option[Set[String]], Option[String])] = {
|
||||
val gizmoduckView: (gd.LookupContext, Set[gd.QueryFields]) =
|
||||
(gd.LookupContext(), Set(gd.QueryFields.Roles))
|
||||
|
||||
val populateUserRoles = Arrow
|
||||
.flatMap[(Key, View), Option[Set[String]]] { _ =>
|
||||
Stitch.collect {
|
||||
CallContext.twitterUserId.map { userId =>
|
||||
compositeOnUserClientColumn.fetcher
|
||||
.callStack(HomeMixerColumn.FetchCallstack)
|
||||
.fetch(userId, gizmoduckView).map(_.v)
|
||||
.map {
|
||||
_.flatMap(_.roles.map(_.roles.toSet)).getOrElse(Set.empty)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val populateIpAddress = Arrow
|
||||
.flatMap[(Key, View), Option[String]](_ =>
|
||||
auditIpClientColumn.fetcher
|
||||
.callStack(HomeMixerColumn.FetchCallstack)
|
||||
.fetch((), ()).map(_.v))
|
||||
|
||||
Arrow.join(
|
||||
populateUserRoles,
|
||||
populateIpAddress
|
||||
)
|
||||
}
|
||||
|
||||
Arrow.zipWithArg(populateUserRolesAndIp).map {
|
||||
case ((key, view), (roles, ipAddress)) =>
|
||||
val deviceContextOpt = Some(
|
||||
hm.DeviceContext(
|
||||
isPolling = CallContext.isPolling,
|
||||
requestContext = view.requestContext,
|
||||
latestControlAvailable = view.latestControlAvailable,
|
||||
autoplayEnabled = view.autoplayEnabled
|
||||
))
|
||||
val seenTweetIds = view.seenTweetIds.filter(_.nonEmpty)
|
||||
|
||||
val (product, productContext) = key match {
|
||||
case gql.TimelineKey.HomeTimeline(_) | gql.TimelineKey.HomeTimelineV2(_) =>
|
||||
(
|
||||
hm.Product.ForYou,
|
||||
hm.ProductContext.ForYou(
|
||||
hm.ForYou(
|
||||
deviceContextOpt,
|
||||
seenTweetIds,
|
||||
view.dspClientContext,
|
||||
view.pushToHomeTweetId
|
||||
)
|
||||
))
|
||||
case gql.TimelineKey.HomeLatestTimeline(_) | gql.TimelineKey.HomeLatestTimelineV2(_) =>
|
||||
(
|
||||
hm.Product.Following,
|
||||
hm.ProductContext.Following(
|
||||
hm.Following(deviceContextOpt, seenTweetIds, view.dspClientContext)))
|
||||
case gql.TimelineKey.CreatorSubscriptionsTimeline(_) =>
|
||||
(
|
||||
hm.Product.Subscribed,
|
||||
hm.ProductContext.Subscribed(hm.Subscribed(deviceContextOpt, seenTweetIds)))
|
||||
case _ => throw new UnsupportedOperationException(s"Unknown product: $key")
|
||||
}
|
||||
|
||||
val clientContext = pm.ClientContext(
|
||||
userId = CallContext.twitterUserId,
|
||||
guestId = CallContext.guestId,
|
||||
guestIdAds = CallContext.guestIdAds,
|
||||
guestIdMarketing = CallContext.guestIdMarketing,
|
||||
appId = CallContext.clientApplicationId,
|
||||
ipAddress = ipAddress,
|
||||
userAgent = CallContext.userAgent,
|
||||
countryCode = CallContext.requestCountryCode,
|
||||
languageCode = CallContext.requestLanguageCode,
|
||||
isTwoffice = CallContext.isInternalOrTwoffice,
|
||||
userRoles = roles,
|
||||
deviceId = CallContext.deviceId,
|
||||
mobileDeviceId = CallContext.mobileDeviceId,
|
||||
mobileDeviceAdId = CallContext.adId,
|
||||
limitAdTracking = CallContext.limitAdTracking
|
||||
)
|
||||
|
||||
hm.HomeMixerRequest(
|
||||
clientContext = clientContext,
|
||||
product = product,
|
||||
productContext = Some(productContext),
|
||||
maxResults = Try(view.count.get.toInt).toOption.orElse(HomeMixerColumn.MaxCount),
|
||||
cursor = view.cursor.filter(_.nonEmpty)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
override val fetch: Arrow[(Key, View), Result[Value]] = {
|
||||
val transformThriftIntoPipelineRequest: Arrow[
|
||||
(Key, View),
|
||||
ProductPipelineRequest[HomeMixerRequest]
|
||||
] = {
|
||||
Arrow
|
||||
.identity[(Key, View)]
|
||||
.andThen {
|
||||
createHomeMixerRequestArrow(compositeOnUserClientColumn, auditIpClientColumn)
|
||||
}
|
||||
.map {
|
||||
case thriftRequest =>
|
||||
val request = homeMixerRequestUnmarshaller(thriftRequest)
|
||||
val params = paramsBuilder.build(
|
||||
clientContext = request.clientContext,
|
||||
product = request.product,
|
||||
featureOverrides =
|
||||
request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty),
|
||||
)
|
||||
ProductPipelineRequest(request, params)
|
||||
}
|
||||
}
|
||||
|
||||
val underlyingProduct: Arrow[
|
||||
ProductPipelineRequest[HomeMixerRequest],
|
||||
ProductPipelineResult[tr.TimelineResponse]
|
||||
] = Arrow
|
||||
.identity[ProductPipelineRequest[HomeMixerRequest]]
|
||||
.map { pipelineRequest =>
|
||||
val pipelineArrow = productPipelineRegistry
|
||||
.getProductPipeline[HomeMixerRequest, tr.TimelineResponse](
|
||||
pipelineRequest.request.product)
|
||||
.arrow
|
||||
(pipelineArrow, pipelineRequest)
|
||||
}.applyArrow
|
||||
|
||||
transformThriftIntoPipelineRequest.andThen(underlyingProduct).map {
|
||||
_.result match {
|
||||
case Some(result) => found(result.timeline)
|
||||
case _ => missing
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object HomeMixerColumn {
|
||||
val Path = "home-mixer/homeMixer.Timeline"
|
||||
private val FetchCallstack = s"$Path:fetch"
|
||||
private val MaxCount: Option[Int] = Some(100)
|
||||
}
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"finagle/finagle-memcached/src/main/scala",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||
"src/thrift/com/twitter/search:earlybird-scala",
|
||||
"stitch/stitch-timelineservice/src/main/scala",
|
||||
],
|
||||
exports = [
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,44 +0,0 @@
|
|||
package com.twitter.home_mixer.functional_component.candidate_source
|
||||
|
||||
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
||||
import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSourceWithExtractedFeatures
|
||||
import com.twitter.product_mixer.core.functional_component.candidate_source.CandidatesWithSourceFeatures
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier
|
||||
import com.twitter.search.earlybird.{thriftscala => t}
|
||||
import com.twitter.stitch.Stitch
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
case object EarlybirdResponseTruncatedFeature
|
||||
extends FeatureWithDefaultOnFailure[t.EarlybirdRequest, Boolean] {
|
||||
override val defaultValue: Boolean = false
|
||||
}
|
||||
|
||||
case object EarlybirdBottomTweetFeature
|
||||
extends FeatureWithDefaultOnFailure[t.EarlybirdRequest, Option[Long]] {
|
||||
override val defaultValue: Option[Long] = None
|
||||
}
|
||||
|
||||
@Singleton
|
||||
case class EarlybirdCandidateSource @Inject() (
|
||||
earlybird: t.EarlybirdService.MethodPerEndpoint)
|
||||
extends CandidateSourceWithExtractedFeatures[t.EarlybirdRequest, t.ThriftSearchResult] {
|
||||
|
||||
override val identifier = CandidateSourceIdentifier("Earlybird")
|
||||
|
||||
override def apply(
|
||||
request: t.EarlybirdRequest
|
||||
): Stitch[CandidatesWithSourceFeatures[t.ThriftSearchResult]] = {
|
||||
Stitch.callFuture(earlybird.search(request)).map { response =>
|
||||
val candidates = response.searchResults.map(_.results).getOrElse(Seq.empty)
|
||||
|
||||
val features = FeatureMapBuilder()
|
||||
.add(EarlybirdResponseTruncatedFeature, candidates.size == request.searchQuery.numResults)
|
||||
.add(EarlybirdBottomTweetFeature, candidates.lastOption.map(_.id))
|
||||
.build()
|
||||
|
||||
CandidatesWithSourceFeatures(candidates, features)
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,30 +0,0 @@
|
|||
package com.twitter.home_mixer.functional_component.candidate_source
|
||||
|
||||
import com.google.inject.name.Named
|
||||
import com.twitter.finagle.memcached.{Client => MemcachedClient}
|
||||
import com.twitter.home_mixer.param.HomeMixerInjectionNames.StaleTweetsCache
|
||||
import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier
|
||||
import com.twitter.stitch.Stitch
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class StaleTweetsCacheCandidateSource @Inject() (
|
||||
@Named(StaleTweetsCache) staleTweetsCache: MemcachedClient)
|
||||
extends CandidateSource[Seq[Long], Long] {
|
||||
|
||||
override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("StaleTweetsCache")
|
||||
|
||||
private val StaleTweetsCacheKeyPrefix = "v1_"
|
||||
|
||||
override def apply(request: Seq[Long]): Stitch[Seq[Long]] = {
|
||||
val keys = request.map(StaleTweetsCacheKeyPrefix + _)
|
||||
|
||||
Stitch.callFuture(staleTweetsCache.get(keys).map { tweets =>
|
||||
tweets.map {
|
||||
case (k, _) => k.replaceFirst(StaleTweetsCacheKeyPrefix, "").toLong
|
||||
}.toSeq
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -1,28 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"finagle/finagle-core/src/main",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
||||
"product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||
"src/scala/com/twitter/suggests/controller_data",
|
||||
"src/thrift/com/twitter/suggests/controller_data:controller_data-scala",
|
||||
"src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala",
|
||||
"src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala",
|
||||
"stringcenter/client",
|
||||
"stringcenter/client/src/main/java",
|
||||
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,51 +0,0 @@
|
|||
package com.twitter.home_mixer.functional_component.decorator
|
||||
|
||||
import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder
|
||||
import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder
|
||||
import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder
|
||||
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature
|
||||
import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator
|
||||
import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator
|
||||
import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder
|
||||
import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder
|
||||
import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.StaticModuleDisplayTypeBuilder
|
||||
import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder
|
||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace
|
||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.VerticalConversation
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.timelines.injection.scribe.InjectionScribeUtil
|
||||
import com.twitter.timelineservice.suggests.{thriftscala => st}
|
||||
|
||||
object HomeConversationServiceCandidateDecorator {
|
||||
|
||||
private val ConversationModuleNamespace = EntryNamespace("home-conversation")
|
||||
|
||||
def apply(
|
||||
homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder
|
||||
): Some[UrtMultipleModulesDecorator[PipelineQuery, TweetCandidate, Long]] = {
|
||||
val suggestType = st.SuggestType.RankedOrganicTweet
|
||||
val component = InjectionScribeUtil.scribeComponent(suggestType).get
|
||||
val clientEventInfoBuilder = ClientEventInfoBuilder(component)
|
||||
val tweetItemBuilder = TweetCandidateUrtItemBuilder(
|
||||
clientEventInfoBuilder = clientEventInfoBuilder,
|
||||
timelinesScoreInfoBuilder = Some(HomeTimelinesScoreInfoBuilder),
|
||||
feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder)
|
||||
)
|
||||
|
||||
val moduleBuilder = TimelineModuleBuilder(
|
||||
entryNamespace = ConversationModuleNamespace,
|
||||
clientEventInfoBuilder = clientEventInfoBuilder,
|
||||
displayTypeBuilder = StaticModuleDisplayTypeBuilder(VerticalConversation),
|
||||
metadataBuilder = Some(HomeConversationModuleMetadataBuilder())
|
||||
)
|
||||
|
||||
Some(
|
||||
UrtMultipleModulesDecorator(
|
||||
urtItemCandidateDecorator = UrtItemCandidateDecorator(tweetItemBuilder),
|
||||
moduleBuilder = moduleBuilder,
|
||||
groupByKey = (_, _, candidateFeatures) =>
|
||||
candidateFeatures.getOrElse(ConversationModuleFocalTweetIdFeature, None)
|
||||
))
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,18 +0,0 @@
|
|||
package com.twitter.home_mixer.functional_component.decorator
|
||||
|
||||
import com.twitter.home_mixer.model.HomeFeatures._
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
|
||||
object HomeQueryTypePredicates {
|
||||
private[this] val QueryPredicates: Seq[(String, FeatureMap => Boolean)] = Seq(
|
||||
("request", _ => true),
|
||||
("get_initial", _.getOrElse(GetInitialFeature, false)),
|
||||
("get_newer", _.getOrElse(GetNewerFeature, false)),
|
||||
("get_older", _.getOrElse(GetOlderFeature, false)),
|
||||
("pull_to_refresh", _.getOrElse(PullToRefreshFeature, false)),
|
||||
("request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)),
|
||||
("request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false))
|
||||
)
|
||||
|
||||
val PredicateMap = QueryPredicates.toMap
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue